Bump version to 0.6.1

.dockerignore

@@ -2,9 +2,3 @@
 .codeclimate.yml
 *.png
 *.md
-logs
-.venv
-start
-config.yaml
-registration.yaml
-*.db

.editorconfig

@@ -13,6 +13,3 @@ max_line_length = 99
 
 [*.{yaml,yml,py}]
 indent_style = space
-
-[.gitlab-ci.yml]
-indent_size = 2

.gitignore

@@ -1,18 +1,17 @@
-/.idea/
+.idea/
 
-/.venv
-/env/
+.venv
+env/
 pip-selfcheck.json
 *.pyc
 __pycache__
-/build
-/dist
-/*.egg-info
-/.eggs
+build
+dist
+*.egg-info
+.eggs
 
-/config.yaml
-/registration.yaml
+config.yaml
+registration.yaml
 *.log*
 *.db
-*.pickle
 *.bak

.gitlab-ci.yml

@@ -2,47 +2,37 @@ image: docker:stable
 
 stages:
 - build
-- manifest
+- push
 
 default:
   before_script:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
-build amd64:
+build:
   stage: build
-  tags:
-  - amd64
   script:
   - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  after_script:
-  - |
-    if [ "$CI_COMMIT_BRANCH" = "master" ]; then
-      apk add --update curl
-      rm -rf /var/cache/apk/*
-      curl "$NOVA_ADMIN_API_URL" -H "Content-Type: application/json" -d '{"password":"'"$NOVA_ADMIN_NIGHTLY_PASS"'","bridge":"'$NOVA_BRIDGE_TYPE'","image":"'$CI_REGISTRY_IMAGE':'$CI_COMMIT_SHA'-amd64"}'
-    fi
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
 
-build arm64:
-  stage: build
-  tags:
-  - arm64
+push latest:
+  stage: push
+  only:
+  - master
+  variables:
+    GIT_STRATEGY: none
   script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:latest
 
-manifest:
-  stage: manifest
-  before_script:
-  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+push tag:
+  stage: push
+  variables:
+    GIT_STRATEGY: none
+  except:
+  - master
   script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
-  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME

.isort.cfg

@@ -1,16 +0,0 @@
-[settings]
-line_length=99
-indent=4
-
-multi_line_output=5
-
-sections=FUTURE,STDLIB,THIRDPARTY,TELETHON,MAUTRIX,FIRSTPARTY,LOCALFOLDER
-no_lines_before=LOCALFOLDER
-default_section=FIRSTPARTY
-
-known_thirdparty=aiohttp,sqlalchemy,alembic,commonmark,ruamel.yaml,PIL,moviepy,prometheus_client,yarl,mako,pkg_resources
-known_telethon=telethon,alchemysession,cryptg
-known_mautrix=mautrix
-
-balanced_wrapping=True
-length_sort=True

Dockerfile

@@ -1,73 +1,44 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.12
+FROM docker.io/alpine:3.10
 
-ARG TARGETARCH=amd64
-
-RUN echo $'\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
+ENV UID=1337 \
+    GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
 
+COPY . /opt/mautrix-telegram
+WORKDIR /opt/mautrix-telegram
 RUN apk add --no-cache \
-      python3 py3-pip py3-setuptools py3-wheel \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-magic \
       py3-sqlalchemy \
-      py3-telethon-session-sqlalchemy@edge \
-      py3-alembic@edge \
+      py3-markdown \
       py3-psycopg2 \
       py3-ruamel.yaml \
-      py3-commonmark@edge \
       # Indirect dependencies
-      py3-idna \
+      #commonmark
+        py3-future \
+      #alembic
+        py3-mako \
+        py3-dateutil \
+        py3-markupsafe \
       #moviepy
         py3-decorator \
-        py3-tqdm \
+        #py3-tqdm \
         py3-requests \
         #imageio
           py3-numpy \
-      #py3-telethon@edge \ (outdated)
-        # Optional for socks proxies
-        py3-pysocks \
-        # cryptg
-          py3-cffi \
-	  py3-qrcode@edge \
-      py3-brotli \
+      #telethon
+        py3-rsa \
       # Other dependencies
-      ffmpeg \
-      ca-certificates \
-      su-exec \
-      netcat-openbsd \
-      # encryption
-      olm-dev \
-      py3-pycryptodome \
-      py3-unpaddedbase64 \
-      py3-future \
-      bash \
-      curl \
-      jq && \
-  curl -sLo yq https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${TARGETARCH} && \
-  chmod +x yq && mv yq /usr/bin/yq
-
-COPY requirements.txt /opt/mautrix-telegram/requirements.txt
-COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
-WORKDIR /opt/mautrix-telegram
-RUN apk add --virtual .build-deps \
       python3-dev \
       libffi-dev \
       build-base \
- && sed -Ei 's/psycopg2-binary.+//' optional-requirements.txt \
- && pip3 install -r requirements.txt -r optional-requirements.txt \
- && apk del .build-deps
-
-COPY . /opt/mautrix-telegram
-RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics,e2be] && apk del git \
-  # This doesn't make the image smaller, but it's needed so that the `version` command works properly
-  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
+      ffmpeg \
+      ca-certificates \
+      su-exec \
+ && pip3 install .[all]
 
 VOLUME /data
-ENV UID=1337 GID=1337 \
-    FFMPEG_BINARY=/usr/bin/ffmpeg
 
 CMD ["/opt/mautrix-telegram/docker-run.sh"]

MANIFEST.in

@@ -1,4 +0,0 @@
-include README.md
-include LICENSE
-include requirements.txt
-include optional-requirements.txt

README.md

@@ -1,28 +1,9 @@
 # mautrix-telegram
-![Languages](https://img.shields.io/github/languages/top/tulir/mautrix-telegram.svg)
-[![License](https://img.shields.io/github/license/tulir/mautrix-telegram.svg)](LICENSE)
-[![Release](https://img.shields.io/github/release/tulir/mautrix-telegram/all.svg)](https://github.com/tulir/mautrix-telegram/releases)
-[![GitLab CI](https://mau.dev/tulir/mautrix-telegram/badges/master/pipeline.svg)](https://mau.dev/tulir/mautrix-telegram/container_registry)
-[![Maintainability](https://img.shields.io/codeclimate/maintainability/tulir/mautrix-telegram.svg)](https://codeclimate.com/github/tulir/mautrix-telegram)
-
 A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
-## Sponsors
-* [Joel Lehtonen / Zouppen](https://github.com/zouppen)
+### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)
 
-### Wiki
-All setup and usage instructions are located in the GitHub
-[wiki](https://github.com/tulir/mautrix-telegram/wiki). Some quick links:
-
-* [Bridge setup](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup)
-  (or [with Docker](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup-with-Docker))
-* Basic usage: [Authentication](https://github.com/tulir/mautrix-telegram/wiki/Authentication),
-  [Creating chats](https://github.com/tulir/mautrix-telegram/wiki/Creating-and-managing-chats),
-  [Relaybot setup](https://github.com/tulir/mautrix-telegram/wiki/Relay-bot)
-
-### Features & Roadmap
-[ROADMAP.md](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
-contains a general overview of what is supported by the bridge.
+### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
 
 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)
@@ -30,4 +11,4 @@ Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net
 Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
 
 ## Preview
-![Preview](preview.png)
+![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)

ROADMAP.md

@@ -28,10 +28,7 @@
 	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
-  * [x] Message history
-    * [x] Manually (`!tg backfill`)
-    * [x] Automatically when creating portal
-    * [x] Automatically for missed messages
+  * [ ] Message history
   * [x] Avatars
   * [x] Presence
   * [x] Typing notifications
@@ -53,11 +50,11 @@
     * [x] At startup
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
-  * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
-  * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
-  * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
+  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
-† Information not automatically sent from source, i.e. implementation may not be possible
+† Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/env.py

@@ -7,8 +7,7 @@ from os.path import abspath, dirname
 
 sys.path.insert(0, dirname(dirname(abspath(__file__))))
 
-from mautrix.util.db import Base
-import mautrix_telegram.db
+from mautrix_telegram.db import Base
 from mautrix_telegram.config import Config
 from alchemysession import AlchemySessionContainer
 
@@ -19,9 +18,17 @@ config = context.config
 mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
 mxtg_config = Config(mxtg_config_path, None, None)
 mxtg_config.load()
-config.set_main_option("sqlalchemy.url", mxtg_config["appservice.database"].replace("%", "%%"))
+config.set_main_option("sqlalchemy.url",
+                       mxtg_config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
 
-AlchemySessionContainer.create_table_classes(None, "telethon_", Base)
+
+class FakeDB:
+    @staticmethod
+    def query_property():
+        return None
+
+
+AlchemySessionContainer.create_table_classes(FakeDB(), "telethon_", Base)
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
@@ -54,8 +61,7 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True,
-        render_as_batch=True)
+        url=url, target_metadata=target_metadata, literal_binds=True)
 
     with context.begin_transaction():
         context.run_migrations()
@@ -76,8 +82,7 @@ def run_migrations_online():
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
-            target_metadata=target_metadata,
-            render_as_batch=True
+            target_metadata=target_metadata
         )
 
         with context.begin_transaction():

alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py

@@ -1,27 +0,0 @@
-"""Add encrypted field for portals
-
-Revision ID: 24f31fc8a72b
-Revises: a7c04a56041b
-Create Date: 2020-03-28 20:14:29.046699
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "24f31fc8a72b"
-down_revision = "a7c04a56041b"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("portal") as batch_op:
-        batch_op.add_column(sa.Column("encrypted", sa.Boolean(), nullable=False,
-                                      server_default=sa.sql.expression.false()))
-
-
-def downgrade():
-    with op.batch_alter_table("portal") as batch_op:
-        batch_op.drop_column("encrypted")

alembic/versions/3e3745baa458_store_matrix_avatar_url_in_database.py

@@ -1,32 +0,0 @@
-"""Store Matrix avatar URL in database
-
-Revision ID: 3e3745baa458
-Revises: dff56c93da8d
-Create Date: 2020-06-15 14:32:10.454033
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '3e3745baa458'
-down_revision = 'dff56c93da8d'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('portal', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('avatar_url', sa.String(), nullable=True))
-
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('portal', schema=None) as batch_op:
-        batch_op.drop_column('avatar_url')
-
-    # ### end Alembic commands ###

alembic/versions/4f7d7ed5792a_switch_mx_user_profile_to_native_enum.py

@@ -1,26 +0,0 @@
-"""Switch mx_user_profile to native enum
-
-Revision ID: 4f7d7ed5792a
-Revises: 9e9c89b0b877
-Create Date: 2019-08-04 17:47:36.568120
-
-"""
-from alembic import op
-
-
-# revision identifiers, used by Alembic.
-revision = '4f7d7ed5792a'
-down_revision = '9e9c89b0b877'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    conn = op.get_bind()
-    conn.execute("UPDATE mx_user_profile SET membership=UPPER(membership)")
-    conn.execute("UPDATE mx_user_profile SET membership='LEAVE' WHERE membership='LEFT'")
-
-
-def downgrade():
-    conn = op.get_bind()
-    conn.execute("UPDATE mx_user_profile SET membership=LOWER(membership)")

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -5,16 +5,14 @@ Revises: 2228d49c383f
 Create Date: 2018-06-26 21:31:26.911307
 
 """
-import json
-import re
-
 from alembic import context, op
 import sqlalchemy.orm as orm
 import sqlalchemy as sa
-
-from mautrix.util.db import Base
+import json
+import re
 
 from mautrix_telegram.config import Config
+from mautrix_telegram.db import Base
 
 # revision identifiers, used by Alembic.
 revision = "6ca3d74d51e4"
@@ -24,6 +22,7 @@ depends_on = None
 
 
 class RoomState(Base):
+    query = None
     __tablename__ = "mx_room_state"
     __table_args__ = {"extend_existing": True}
 
@@ -32,6 +31,7 @@ class RoomState(Base):
 
 
 class UserProfile(Base):
+    query = None
     __tablename__ = "mx_user_profile"
     __table_args__ = {"extend_existing": True}
 
@@ -43,6 +43,7 @@ class UserProfile(Base):
 
 
 class Puppet(Base):
+    query = None
     __tablename__ = "puppet"
     __table_args__ = {"extend_existing": True}
 
@@ -82,7 +83,7 @@ def upgrade():
 
 def migrate_state_store():
     conn = op.get_bind()
-    session: orm.Session = orm.sessionmaker(bind=conn)()
+    session = orm.sessionmaker(bind=conn)()  # type: orm.Session
 
     try:
         with open("mx-state.json") as file:

alembic/versions/888275d58e57_add_double_puppet_base_url_to_puppet_.py

@@ -1,30 +0,0 @@
-"""Add double puppet base URL to puppet table
-
-Revision ID: 888275d58e57
-Revises: a328bf4f0932
-Create Date: 2020-10-14 18:52:00.730666
-
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = '888275d58e57'
-down_revision = 'a328bf4f0932'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('puppet', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('base_url', sa.Text(), nullable=True))
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('puppet', schema=None) as batch_op:
-        batch_op.drop_column('base_url')
-    # ### end Alembic commands ###

alembic/versions/a328bf4f0932_store_encryption_state_event_in_db.py

@@ -1,38 +0,0 @@
-"""Store encryption state event in db
-
-Revision ID: a328bf4f0932
-Revises: ccbaff858240
-Create Date: 2020-07-11 21:31:27.059813
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-from mautrix.client.state_store.sqlalchemy import SerializableType
-from mautrix.types import RoomEncryptionStateEventContent
-
-# revision identifiers, used by Alembic.
-revision = 'a328bf4f0932'
-down_revision = 'ccbaff858240'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('encryption',
-                                      SerializableType(RoomEncryptionStateEventContent),
-                                      nullable=True))
-        batch_op.add_column(sa.Column('has_full_member_list', sa.Boolean(), nullable=True))
-        batch_op.add_column(sa.Column('is_encrypted', sa.Boolean(), nullable=True))
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
-        batch_op.drop_column('is_encrypted')
-        batch_op.drop_column('has_full_member_list')
-        batch_op.drop_column('encryption')
-    # ### end Alembic commands ###

alembic/versions/a7c04a56041b_store_custom_puppet_next_batch_in_.py

@@ -1,26 +0,0 @@
-"""Store custom puppet next_batch in database
-
-Revision ID: a7c04a56041b
-Revises: 4f7d7ed5792a
-Create Date: 2019-08-06 23:08:51.087651
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "a7c04a56041b"
-down_revision = "4f7d7ed5792a"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.add_column(sa.Column("next_batch", sa.String(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column("next_batch")

alembic/versions/ccbaff858240_switch_to_mautrix_python_crypto.py

@@ -1,71 +0,0 @@
-"""Switch to mautrix-python crypto
-
-Revision ID: ccbaff858240
-Revises: 3e3745baa458
-Create Date: 2020-07-08 19:06:12.588047
-
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = 'ccbaff858240'
-down_revision = '3e3745baa458'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('nio_account')
-    op.drop_table('nio_device_key')
-    op.drop_table('nio_outgoing_key_request')
-    op.drop_table('nio_olm_session')
-    op.drop_table('nio_megolm_inbound_session')
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('nio_megolm_inbound_session',
-    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('fp_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.Column('forwarded_chains', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('session_id', name='nio_megolm_inbound_session_pkey')
-    )
-    op.create_table('nio_olm_session',
-    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
-    sa.Column('last_used', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('session_id', name='nio_olm_session_pkey')
-    )
-    op.create_table('nio_outgoing_key_request',
-    sa.Column('request_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('algorithm', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('request_id', name='nio_outgoing_key_request_pkey')
-    )
-    op.create_table('nio_device_key',
-    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('display_name', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('deleted', sa.BOOLEAN(), autoincrement=False, nullable=False),
-    sa.Column('keys', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_device_key_pkey')
-    )
-    op.create_table('nio_account',
-    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('shared', sa.BOOLEAN(), autoincrement=False, nullable=False),
-    sa.Column('sync_token', sa.TEXT(), autoincrement=False, nullable=False),
-    sa.Column('account', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_account_pkey')
-    )
-    # ### end Alembic commands ###

alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py

@@ -1,26 +0,0 @@
-"""Add decryption info field for reuploaded telegram files
-
-Revision ID: d3c922a6acd2
-Revises: 24f31fc8a72b
-Create Date: 2020-03-30 20:07:17.340346
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'd3c922a6acd2'
-down_revision = '24f31fc8a72b'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("telegram_file") as batch_op:
-        batch_op.add_column(sa.Column("decryption_info", sa.Text(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("telegram_file") as batch_op:
-        batch_op.drop_column("decryption_info")

alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py

@@ -1,71 +0,0 @@
-"""Add matrix-nio state store to main db
-
-Revision ID: dff56c93da8d
-Revises: d3c922a6acd2
-Create Date: 2020-03-31 22:04:04.014048
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'dff56c93da8d'
-down_revision = 'd3c922a6acd2'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('nio_account',
-    sa.Column('user_id', sa.String(length=255), nullable=False),
-    sa.Column('device_id', sa.String(length=255), nullable=False),
-    sa.Column('shared', sa.Boolean(), nullable=False),
-    sa.Column('sync_token', sa.Text(), nullable=False),
-    sa.Column('account', sa.LargeBinary(), nullable=False),
-    sa.PrimaryKeyConstraint('user_id', 'device_id')
-    )
-    op.create_table('nio_device_key',
-    sa.Column('user_id', sa.String(length=255), nullable=False),
-    sa.Column('device_id', sa.String(length=255), nullable=False),
-    sa.Column('display_name', sa.String(length=255), nullable=False),
-    sa.Column('deleted', sa.Boolean(), nullable=False),
-    sa.Column('keys', sa.PickleType(), nullable=False),
-    sa.PrimaryKeyConstraint('user_id', 'device_id')
-    )
-    op.create_table('nio_megolm_inbound_session',
-    sa.Column('session_id', sa.String(length=255), nullable=False),
-    sa.Column('sender_key', sa.String(length=255), nullable=False),
-    sa.Column('fp_key', sa.String(length=255), nullable=False),
-    sa.Column('room_id', sa.String(length=255), nullable=False),
-    sa.Column('session', sa.LargeBinary(), nullable=False),
-    sa.Column('forwarded_chains', sa.PickleType(), nullable=False),
-    sa.PrimaryKeyConstraint('session_id')
-    )
-    op.create_table('nio_olm_session',
-    sa.Column('session_id', sa.String(length=255), nullable=False),
-    sa.Column('sender_key', sa.String(length=255), nullable=False),
-    sa.Column('session', sa.LargeBinary(), nullable=False),
-    sa.Column('created_at', sa.DateTime(), nullable=False),
-    sa.Column('last_used', sa.DateTime(), nullable=False),
-    sa.PrimaryKeyConstraint('session_id')
-    )
-    op.create_table('nio_outgoing_key_request',
-    sa.Column('request_id', sa.String(length=255), nullable=False),
-    sa.Column('session_id', sa.String(length=255), nullable=False),
-    sa.Column('room_id', sa.String(length=255), nullable=False),
-    sa.Column('algorithm', sa.String(length=255), nullable=False),
-    sa.PrimaryKeyConstraint('request_id')
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('nio_outgoing_key_request')
-    op.drop_table('nio_olm_session')
-    op.drop_table('nio_megolm_inbound_session')
-    op.drop_table('nio_device_key')
-    op.drop_table('nio_account')
-    # ### end Alembic commands ###

docker-run.sh

@@ -13,6 +13,8 @@ sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /d
 if [ -f /data/mx-state.json ]; then
 	ln -s /data/mx-state.json
 fi
+# Check that database is in the right state
+alembic -x config=/data/config.yaml upgrade head
 
 if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
@@ -33,8 +35,5 @@ if [ ! -f /data/registration.yaml ]; then
 	exit
 fi
 
-# Check that database is in the right state
-alembic -x config=/data/config.yaml upgrade head
-
 fixperms
 exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

example-config.yaml

@@ -1,26 +1,22 @@
 # Homeserver details
 homeserver:
     # The address that this appservice can use to connect to the homeserver.
-    address: https://example.com
+    address: https://matrix.org
     # The domain of the homeserver (for MXIDs, etc).
-    domain: example.com
+    domain: matrix.org
     # Whether or not to verify the SSL certificate of the homeserver.
     # Only applies if address starts with https://
     verify_ssl: true
-    asmux: false
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
 appservice:
     # The address that the homeserver can use to connect to this appservice.
-    address: http://localhost:29317
-    # When using https:// the TLS certificate and key files for the address.
-    tls_cert: false
-    tls_key: false
+    address: http://localhost:8080
 
     # The hostname and port where this appservice should listen.
     hostname: 0.0.0.0
-    port: 29317
+    port: 8080
     # The maximum body size of appservice API requests (from the homeserver) in mebibytes
     # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
     max_body_size: 1
@@ -31,15 +27,13 @@ appservice:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
     database: sqlite:///mautrix-telegram.db
-    # Optional extra arguments for SQLAlchemy's create_engine
-    database_opts: {}
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in
     # the HS database.
     public:
         # Whether or not the public-facing endpoints should be enabled.
-        enabled: false
+        enabled: true
         # The prefix to use in the public-facing endpoints.
         prefix: /public
         # The base URL where the public-facing endpoints are available. The prefix is not added
@@ -47,7 +41,7 @@ appservice:
         external: https://example.com/public
 
     # Provisioning API part of the web server for automated portal creation and fetching information.
-    # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager).
+    # Used by things like Dimension (https://dimension.t2bot.io/).
     provisioning:
         # Whether or not the provisioning API should be enabled.
         enabled: true
@@ -68,15 +62,8 @@ appservice:
 
     # Community ID for bridged users (changes registration file) and rooms.
     # Must be created manually.
-    #
-    # Example: "+telegram:example.com". Set to false to disable.
     community_id: false
 
-    # Whether or not to receive ephemeral events via appservice transactions.
-    # Requires MSC2409 support (i.e. Synapse 1.22+).
-    # You should disable bridge -> sync_with_custom_puppets when this is enabled.
-    ephemeral_events: false
-
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
@@ -86,17 +73,6 @@ metrics:
     enabled: false
     listen_port: 8000
 
-# Manhole config.
-manhole:
-    # Whether or not opening the manhole is allowed.
-    enabled: false
-    # The path for the unix socket.
-    path: /var/tmp/mautrix-telegram.manhole
-    # The list of UIDs who can be added to the whitelist.
-    # If empty, any UIDs can be specified in the open-manhole command.
-    whitelist:
-    - 0
-
 # Bridge config
 bridge:
     # Localpart template of MXIDs for Telegram users.
@@ -127,18 +103,12 @@ bridge:
     - full name
     - username
     - phone number
-    # Maximum length of displayname
-    displayname_max_length: 100
-    # Remove avatars from Telegram ghost users when removed on Telegram. This is disabled by default
-    # as there's no way to determine whether an avatar is removed or just hidden from some users. If
-    # you're on a single-user instance, this should be safe to enable.
-    allow_avatar_remove: false
 
     # Maximum number of members to sync per portal when starting up. Other members will be
     # synced when they send messages. The maximum is 10000, after which the Telegram server
     # will not send any more members.
-    # -1 means no limit (which means it's limited to 10000 by the server)
-    max_initial_member_sync: 100
+    # Defaults to no local limit (-> limited to 10000 by server)
+    max_initial_member_sync: -1
     # Whether or not to sync the member list in channels.
     # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
     # list regardless of this setting.
@@ -149,21 +119,17 @@ bridge:
     # their Telegram account at startup.
     startup_sync: true
     # Number of most recently active dialogs to check when syncing chats.
+    # Dialogs include groups and private chats, but only groups are synced.
     # Set to 0 to remove limit.
-    sync_update_limit: 0
-    # Number of most recently active dialogs to create portals for when syncing chats.
-    # Set to 0 to remove limit.
-    sync_create_limit: 30
-    # Whether or not to sync and create portals for direct chats at startup.
-    sync_direct_chats: false
+    sync_dialog_limit: 30
     # The maximum number of simultaneous Telegram deletions to handle.
     # A large number of simultaneous redactions could put strain on your homeserver.
     max_telegram_delete: 10
     # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
     # at startup and when creating a bridge.
     sync_matrix_state: true
-    # Allow logging in within Matrix. If false, users can only log in using login-qr or the
-    # out-of-Matrix login website (see appservice.public config section)
+    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
+    # login website (see appservice.public config section)
     allow_matrix_login: true
     # Whether or not to bridge plaintext highlights.
     # Only enable this if your displayname_template has some static part that the bridge can use to
@@ -171,27 +137,13 @@ bridge:
     plaintext_highlights: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
-    # Whether or not to use /sync to get presence, read receipts and typing notifications
-    # when double puppeting is enabled
+    # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
+    # Currently only works for private chats and normal groups.
+    # WARNING: This feature seems to be broken in the telegram library.
+    catch_up: false
+    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # your own Matrix account as the Matrix puppet for your Telegram account.
     sync_with_custom_puppets: true
-    # Whether or not to update the m.direct account data event when double puppeting is enabled.
-    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
-    # and is therefore prone to race conditions.
-    sync_direct_chat_list: false
-    # Servers to always allow double puppeting from
-    double_puppet_server_map:
-        example.com: https://example.com
-    # Allow using double puppeting from any server with a valid client .well-known file.
-    double_puppet_allow_discovery: false
-    # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth
-    #
-    # If set, custom puppets will be enabled automatically for local users
-    # instead of users having to find an access token and run `login-matrix`
-    # manually.
-    # If using this for other servers than the bridge's server,
-    # you must also set the URL in the double_puppet_server_map.
-    login_shared_secret_map:
-        example.com: foobar
     # Set to false to disable link previews in messages sent to Telegram.
     telegram_link_preview: true
     # Use inline images instead of a separate message for the caption.
@@ -201,106 +153,6 @@ bridge:
     image_as_file_size: 10
     # Maximum size of Telegram documents in megabytes to bridge.
     max_document_size: 100
-    # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
-    # streaming from/to Matrix and using many connections for Telegram.
-    # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
-    parallel_file_transfer: false
-    # Whether or not created rooms should have federation enabled.
-    # If false, created portal rooms will never be federated.
-    federate_rooms: true
-    # Settings for converting animated stickers.
-    animated_sticker:
-        # Format to which animated stickers should be converted.
-        # disable - No conversion, send as-is (gzipped lottie)
-        # png - converts to non-animated png (fastest),
-        # gif - converts to animated gif, but loses transparency
-        # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support
-        target: gif
-        # Arguments for converter. All converters take width and height.
-        # GIF converter takes background as a hex color.
-        args:
-            width: 256
-            height: 256
-            background: "020202"  # only for gif
-            fps: 30               # only for webm
-    # End-to-bridge encryption support options. These require matrix-nio to be installed with pip
-    # and login_shared_secret to be configured in order to get a device for the bridge bot.
-    #
-    # Additionally, https://github.com/matrix-org/synapse/pull/5758 is required if using a normal
-    # application service.
-    encryption:
-        # Allow encryption, work in group chat rooms with e2ee enabled
-        allow: false
-        # Default to encryption, force-enable encryption in all portals the bridge creates
-        # This will cause the bridge bot to be in private chats for the encryption to work properly.
-        default: false
-        # Database for the encryption data. Currently only supports Postgres and an in-memory
-        # store that's persisted as a pickle.
-        # If set to `default`, will use the appservice postgres database
-        # or a pickle file if the appservice database is sqlite.
-        #
-        # Format examples:
-        #   Pickle:   pickle:///filename.pickle
-        #   Postgres: postgres://username:password@hostname/dbname
-        database: default
-        # Options for automatic key sharing.
-        key_sharing:
-            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
-            # You must use a client that supports requesting keys from other users to use this feature.
-            allow: false
-            # Require the requesting device to have a valid cross-signing signature?
-            # This doesn't require that the bridge has verified the device, only that the user has verified it.
-            # Not yet implemented.
-            require_cross_signing: false
-            # Require devices to be verified by the bridge?
-            # Verification by the bridge is not yet implemented.
-            require_verification: true
-    # Whether or not to explicitly set the avatar and room name for private
-    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
-    private_chat_portal_meta: false
-    # Whether or not the bridge should send a read receipt from the bridge bot when a message has
-    # been sent to Telegram.
-    delivery_receipts: false
-    # Whether or not delivery errors should be reported as messages in the Matrix room.
-    delivery_error_reports: false
-    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
-    # This field will automatically be changed back to false after it,
-    # except if the config file is not writable.
-    resend_bridge_info: false
-    # Settings for backfilling messages from Telegram.
-    backfill:
-        # Whether or not the Telegram ghosts of logged in Matrix users should be
-        # invited to private chats when backfilling history from Telegram. This is
-        # usually needed to prevent rate limits and to allow timestamp massaging.
-        invite_own_puppet: true
-        # Maximum number of messages to backfill without using a takeout.
-        # The first time a takeout is used, the user has to manually approve it from a different
-        # device. If initial_limit or missed_limit are higher than this value, the bridge will ask
-        # the user to accept the takeout after logging in before syncing any chats.
-        takeout_limit: 100
-        # Maximum number of messages to backfill initially.
-        # Set to 0 to disable backfilling when creating portal, or -1 to disable the limit.
-        #
-        # N.B. Initial backfill will only start after member sync. Make sure your
-        #      max_initial_member_sync is set to a low enough value so it doesn't take forever.
-        initial_limit: 0
-        # Maximum number of messages to backfill if messages were missed while the bridge was
-        # disconnected. Note that this only works for logged in users and only if the chat isn't
-        # older than sync_update_limit
-        # Set to 0 to disable backfilling missed messages.
-        missed_limit: 50
-        # If using double puppeting, should notifications be disabled
-        # while the initial backfill is in progress?
-        disable_notifications: false
-        # Whether or not to enable backfilling in normal groups.
-        # Normal groups have numerous technical problems in Telegram, and backfilling normal groups
-        # will likely cause problems if there are multiple Matrix users in the group.
-        normal_groups: false
-
-    # Overrides for base power levels.
-    initial_power_level_overrides:
-        user: {}
-        group: {}
 
     # Whether to bridge Telegram bot messages as m.notices or m.texts.
     bot_messages_as_notices: true
@@ -323,30 +175,24 @@ bridge:
         # You might need to increase this on high-traffic bridge instances.
         cache_queue_length: 20
 
+
     # The formats to use when sending messages to Telegram via the relay bot.
-    # Text msgtypes (m.text, m.notice and m.emote) support HTML, media msgtypes don't.
+    #
+    # Telegram doesn't have built-in emotes, so the m.emote format is also used for non-relaybot users.
     #
     # Available variables:
-    #   $sender_displayname - The display name of the sender (e.g. Example User)
-    #   $sender_username    - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
-    #   $sender_mxid        - The Matrix ID of the sender (e.g. @exampleuser:example.com)
-    #   $message            - The message content
+    #   $sender_displayname    - The display name of the sender (e.g. Example User)
+    #   $sender_username       - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
+    #   $sender_mxid           - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $message               - The message content as HTML
     message_formats:
         m.text: "<b>$sender_displayname</b>: $message"
-        m.notice: "<b>$sender_displayname</b>: $message"
         m.emote: "* <b>$sender_displayname</b> $message"
         m.file: "<b>$sender_displayname</b> sent a file: $message"
         m.image: "<b>$sender_displayname</b> sent an image: $message"
         m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
         m.video: "<b>$sender_displayname</b> sent a video: $message"
         m.location: "<b>$sender_displayname</b> sent a location: $message"
-    # Telegram doesn't have built-in emotes, this field specifies how m.emote's from authenticated
-    # users are sent to telegram. All fields in message_formats are supported. Additionally, the
-    # Telegram user info is available in the following variables:
-    #    $displayname - Telegram displayname
-    #    $username    - Telegram username (may not exist)
-    #    $mention     - Telegram @username or displayname mention (depending on which exists)
-    emote_format: "* $mention $formatted_body"
 
     # The formats to use when sending state events to Telegram via the relay bot.
     #
@@ -394,22 +240,6 @@ bridge:
 
     # Options related to the message relay Telegram bot.
     relaybot:
-        private_chat:
-            # List of users to invite to the portal when someone starts a private chat with the bot.
-            # If empty, private chats with the bot won't create a portal.
-            invite: []
-            # Whether or not to bridge state change messages in relaybot private chats.
-            state_changes: true
-            # When private_chat_invite is empty, this message is sent to users /starting the
-            # relaybot. Telegram's "markdown" is supported.
-            message: This is a Matrix bridge relaybot and does not support direct chats
-        # List of users to invite to all group chat portals created by the bridge.
-        group_chat_invite: []
-        # Whether or not the relaybot should not bridge events in unbridged group chats.
-        # If false, portals will be created when the relaybot receives messages, just like normal
-        # users. This behavior is usually not desirable, as it interferes with manually bridging
-        # the chat to another room.
-        ignore_unbridged_group_chat: true
         # Whether or not to allow creating portals from Telegram.
         authless_portals: true
         # Whether or not to allow Telegram group admins to use the bot commands.
@@ -477,14 +307,14 @@ telegram:
     # Telethon proxy configuration.
     # You must install PySocks from pip for proxies to work.
     proxy:
-        # Allowed types: disabled, socks4, socks5, http, mtproxy
+        # Allowed types: disabled, socks4, socks5, http
         type: disabled
         # Proxy IP address and port.
         address: 127.0.0.1
         port: 1080
-        # Whether or not to perform DNS resolving remotely. Only for socks/http proxies.
+        # Whether or not to perform DNS resolving remotely.
         rdns: true
-        # Proxy authentication (optional). Put MTProxy secret in password field.
+        # Proxy authentication (optional).
         username: ""
         password: ""
 
@@ -495,26 +325,23 @@ telegram:
 logging:
     version: 1
     formatters:
-        colored:
-            (): mautrix_telegram.util.ColorFormatter
-            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
-        normal:
+        precise:
             format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
     handlers:
         file:
             class: logging.handlers.RotatingFileHandler
-            formatter: normal
+            formatter: precise
             filename: ./mautrix-telegram.log
             maxBytes: 10485760
             backupCount: 10
         console:
             class: logging.StreamHandler
-            formatter: colored
+            formatter: precise
     loggers:
         mau:
             level: DEBUG
         telethon:
-            level: INFO
+            level: DEBUG
         aiohttp:
             level: INFO
     root:

helm/mautrix-telegram/.editorconfig

@@ -1,2 +0,0 @@
-[*.{yaml,yml}]
-indent_size = 2

helm/mautrix-telegram/.gitignore

@@ -1 +0,0 @@
-charts/*

helm/mautrix-telegram/.helmignore

@@ -1,22 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

helm/mautrix-telegram/Chart.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-name: mautrix-telegram
-version: 0.1.0
-appVersion: "0.7.0"
-description: A Matrix-Telegram hybrid puppeting/relaybot bridge.
-keywords:
-  - matrix
-  - bridge
-  - telegram
-maintainers:
-  - name: Tulir Asokan
-    email: tulir@maunium.net
-sources:
-  - https://github.com/tulir/mautrix-telegram

helm/mautrix-telegram/requirements.lock

@@ -1,6 +0,0 @@
-dependencies:
-- name: postgresql
-  repository: https://kubernetes-charts.storage.googleapis.com/
-  version: 6.5.0
-digest: sha256:85139e9d4207e49c11c5f84d7920d0135cffd3d427f3f3638d4e51258990de2a
-generated: "2019-10-23T22:11:37.005827507+03:00"

helm/mautrix-telegram/requirements.yaml

@@ -1,5 +0,0 @@
-dependencies:
-  - name: postgresql
-    version: 6.5.0
-    repository: https://kubernetes-charts.storage.googleapis.com/
-    condition: postgresql.enabled

helm/mautrix-telegram/templates/NOTES.txt

@@ -1,21 +0,0 @@
-Your registration file is below. Save it into a YAML file and give the path to that file to synapse:
-
-id: {{ .Values.appservice.id }}
-as_token: {{ .Values.appservice.asToken }}
-hs_token: {{ .Values.appservice.hsToken }}
-namespaces:
-    users:
-        - exclusive: true
-          regex: "@{{ .Values.bridge.username_template | replace "{userid}" ".+"}}:{{ .Values.homeserver.domain }}"
-          {{- if .Values.appservice.communityID }}
-          group_id: {{ .Values.appservice.communityID }}
-          {{- end }}
-    aliases:
-        - exclusive: true
-          regex: "@{{ .Values.bridge.alias_template | replace "{groupname}" ".+"}}:{{ .Values.homeserver.domain }}"
-          {{- if .Values.appservice.communityID }}
-          group_id: {{ .Values.appservice.communityID }}
-          {{- end }}
-url: {{ .Values.appservice.address }}
-sender_localpart: {{ .Values.appservice.botUsername }}
-rate_limited: false

helm/mautrix-telegram/templates/_helpers.tpl

@@ -1,55 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "mautrix-telegram.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "mautrix-telegram.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name -}}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "mautrix-telegram.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Common labels
-*/}}
-{{- define "mautrix-telegram.labels" -}}
-app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
-helm.sh/chart: {{ include "mautrix-telegram.chart" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end -}}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "mautrix-telegram.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create -}}
-    {{ default (include "mautrix-telegram.fullname" .) .Values.serviceAccount.name }}
-{{- else -}}
-    {{ default "default" .Values.serviceAccount.name }}
-{{- end -}}
-{{- end -}}

helm/mautrix-telegram/templates/configmap.yaml

@@ -1,57 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ template "mautrix-telegram.fullname" . }}
-  labels:
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/name: {{ template "mautrix-telegram.name" . }}
-data:
-  config.yaml: |
-    homeserver:
-      address: {{ .Values.homeserver.address }}
-      domain: {{ .Values.homeserver.domain }}
-      verify_ssl: {{ .Values.homeserver.verifySSL }}
-
-    appservice:
-      address: http://{{ include "mautrix-telegram.fullname" . }}:{{ .Values.service.port }}
-
-      hostname: 0.0.0.0
-      port: {{ .Values.service.port }}
-      max_body_size: {{ .Values.appservice.maxBodySize }}
-
-      {{- if .Values.postgresql.enabled }}
-      database: "postgres://postgres:{{ .Values.postgresql.postgresqlPassword }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.postgresqlDatabase }}"
-      {{- else }}
-      database: {{ .Values.appservice.database | quote }}
-      {{- end }}
-
-      public:
-        {{- toYaml .Values.appservice.public | nindent 8 }}
-
-      provisioning:
-        {{- toYaml .Values.appservice.provisioning | nindent 8 }}
-
-      id: {{ .Values.appservice.id }}
-      bot_username: {{ .Values.appservice.botUsername }}
-      bot_displayname: {{ .Values.appservice.botDisplayname }}
-      bot_avatar:  {{ .Values.appservice.botAvatar }}
-
-      community_id: {{ .Values.appservice.communityID }}
-
-      as_token: {{ .Values.appservice.asToken }}
-      hs_token: {{ .Values.appservice.hsToken }}
-
-    metrics:
-      {{- toYaml .Values.metrics | nindent 6 }}
-
-    bridge:
-      {{- toYaml .Values.bridge | nindent 6 }}
-
-    telegram:
-      {{- toYaml .Values.telegram | nindent 6 }}
-
-    logging:
-      {{- toYaml .Values.logging | nindent 6 }}
-  registration.yaml: ""

helm/mautrix-telegram/templates/deployment.yaml

@@ -1,69 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "mautrix-telegram.fullname" . }}
-  labels:
-    {{- include "mautrix-telegram.labels" . | nindent 4 }}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
-      app.kubernetes.io/instance: {{ .Release.Name }}
-  template:
-    {{- if .Values.podAnnotations }}
-    annotations:
-      {{- toYaml .Values.podAnnotations | nindent 6 }}
-    {{- end }}
-    metadata:
-      labels:
-        app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
-        app.kubernetes.io/instance: {{ .Release.Name }}
-    spec:
-      serviceAccountName: {{ template "mautrix-telegram.serviceAccountName" . }}
-      containers:
-        - name: {{ .Chart.Name }}
-          securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          volumeMounts:
-            - mountPath: /data
-              name: config-volume
-          ports:
-            - name: http
-              containerPort: {{ .Values.service.port }}
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              path: /_matrix/mau/live
-              port: http
-            initialDelaySeconds: 60
-            periodSeconds: 5
-          readinessProbe:
-            httpGet:
-              path: /_matrix/mau/ready
-              port: http
-            initialDelaySeconds: 60
-            periodSeconds: 5
-          resources:
-            {{- toYaml .Values.resources | nindent 12 }}
-      volumes:
-        - name: config-volume
-          configMap:
-            name: {{ template "mautrix-telegram.fullname" . }}
-
-      securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}

helm/mautrix-telegram/templates/service.yaml

@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "mautrix-telegram.fullname" . }}
-  labels:
-{{ include "mautrix-telegram.labels" . | indent 4 }}
-spec:
-  type: {{ .Values.service.type }}
-  ports:
-    - port: {{ .Values.service.port }}
-      targetPort: http
-      protocol: TCP
-      name: http
-  selector:
-    app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}

helm/mautrix-telegram/templates/serviceaccount.yaml

@@ -1,8 +0,0 @@
-{{- if .Values.serviceAccount.create -}}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ template "mautrix-telegram.serviceAccountName" . }}
-  labels:
-{{ include "mautrix-telegram.labels" . | indent 4 }}
-{{- end -}}

helm/mautrix-telegram/values.yaml

@@ -1,141 +0,0 @@
-image:
-  repository: dock.mau.dev/tulir/mautrix-telegram
-  tag: latest
-  pullPolicy: IfNotPresent
-
-nameOverride: ""
-fullnameOverride: ""
-
-serviceAccount:
-  # Specifies whether a service account should be created
-  create: true
-  # The name of the service account to use.
-  # If not set and create is true, a name is generated using the fullname template
-  name:
-
-service:
-  type: ClusterIP
-  port: 29317
-
-resources: {}
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
-
-nodeSelector: {}
-
-tolerations: []
-
-affinity: {}
-
-# Postgres pod configs
-postgresql:
-  enabled: true
-  postgresqlDatabase: mxtg
-  postgresqlPassword: SET TO RANDOM STRING
-  persistence:
-    size: 2Gi
-  resources:
-    requests:
-      memory: 256Mi
-      cpu: 100m
-
-# Homeserver details
-homeserver:
-  # The address that this appservice can use to connect to the homeserver.
-  address: https://example.com
-  # The domain of the homeserver (for MXIDs, etc).
-  domain: example.com
-  # Whether or not to verify the SSL certificate of the homeserver.
-  # Only applies if address starts with https://
-  verifySSL: true
-
-# Application service host/registration related details
-# Changing these values requires regeneration of the registration.
-appservice:
-  # The maximum body size of appservice API requests (from the homeserver) in mebibytes
-  # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
-  maxBodySize: 1
-
-  # Public part of web server for out-of-Matrix interaction with the bridge.
-  # Used for things like login if the user wants to make sure the 2FA password isn't stored in
-  # the HS database.
-  public:
-    # Whether or not the public-facing endpoints should be enabled.
-    enabled: true
-    # The prefix to use in the public-facing endpoints.
-    prefix: /public
-    # The base URL where the public-facing endpoints are available. The prefix is not added
-    # implicitly.
-    external: https://example.com/public
-
-  # Provisioning API part of the web server for automated portal creation and fetching information.
-  # Used by things like Dimension (https://dimension.t2bot.io/).
-  provisioning:
-    # Whether or not the provisioning API should be enabled.
-    enabled: true
-    # The prefix to use in the provisioning API endpoints.
-    prefix: /_matrix/provision/v1
-    # The shared secret to authorize users of the API.
-    shared_secret: SET TO RANDOM STRING
-
-  id: telegram
-  botUsername: telegrambot
-  # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
-  # to leave display name/avatar as-is.
-  botDisplayname: Telegram bridge bot
-  botAvatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
-
-  # Community ID for bridged users (changes registration file) and rooms.
-  # Must be created manually.
-  communityID: false
-
-  # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
-  asToken: SET TO RANDOM STRING
-  hsToken: SET TO RANDOM STRING
-
-# The keys below can be used to override the configs in the base config:
-# https://github.com/tulir/mautrix-telegram/blob/master/example-config.yaml
-# Note that the "appservice" and "homeserver" sections are above and slightly different than the base.
-
-# Bridge config
-bridge:
-  # Localpart template of MXIDs for Telegram users.
-  # {userid} is replaced with the user ID of the Telegram user.
-  username_template: "telegram_{userid}"
-  # Localpart template of room aliases for Telegram portal rooms.
-  # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} )
-  alias_template: "telegram_{groupname}"
-
-  # Permissions for using the bridge.
-  # Permitted values:
-  #   relaybot - Only use the bridge via the relaybot, no access to commands.
-  #       user - Relaybot level + access to commands to create bridges.
-  #  puppeting - User level + logging in with a Telegram account.
-  #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
-  #      admin - Full access to use the bridge and some extra administration commands.
-  # Permitted keys:
-  #        * - All Matrix users
-  #   domain - All users on that homeserver
-  #     mxid - Specific user
-  permissions:
-    "*": "relaybot"
-    "public.example.com": "user"
-    "example.com": "full"
-    "@admin:example.com": "admin"
-
-# Prometheus telemetry config.
-metrics:
-  enabled: false
-  listen_port: 8000
-
-# Telegram config
-telegram:
-  # Get your own API keys at https://my.telegram.org/apps
-  api_id: 12345
-  api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
-  # (Optional) Create your own bot at https://t.me/BotFather
-  # bot_token: 123456789:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.9.0rc3"
+__version__ = "0.6.1"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,121 +14,150 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from typing import Awaitable, List, Any
+from time import time
+import argparse
+import asyncio
+import logging.config
+import sys
+import copy
+import signal
+import os
 
+import sqlalchemy as sql
+
+from mautrix_appservice import AppService
 from alchemysession import AlchemySessionContainer
 
-from mautrix.types import UserID, RoomID
-from mautrix.bridge import Bridge
-from mautrix.util.db import Base
-
 from .web.provisioning import ProvisioningAPI
 from .web.public import PublicBridgeWebsite
-from .commands.manhole import ManholeState
 from .abstract_user import init as init_abstract_user
-from .bot import Bot, init as init_bot
+from .bot import init as init_bot
 from .config import Config
 from .context import Context
-from .db import init as init_db
+from .db import Base, init as init_db
 from .formatter import init as init_formatter
 from .matrix import MatrixHandler
-from .portal import Portal, init as init_portal
-from .puppet import Puppet, init as init_puppet
+from .portal import init as init_portal
+from .puppet import init as init_puppet
+from .sqlstatestore import SQLStateStore
 from .user import User, init as init_user
-from .version import version, linkified_version
+from . import __version__
 
 try:
     import prometheus_client as prometheus
 except ImportError:
     prometheus = None
 
+parser = argparse.ArgumentParser(
+    description="A Matrix-Telegram puppeting bridge.",
+    prog="python -m mautrix-telegram")
+parser.add_argument("-c", "--config", type=str, default="config.yaml",
+                    metavar="<path>", help="the path to your config file")
+parser.add_argument("-b", "--base-config", type=str, default="example-config.yaml",
+                    metavar="<path>", help="the path to the example config "
+                                           "(for automatic config updates)")
+parser.add_argument("-g", "--generate-registration", action="store_true",
+                    help="generate registration and quit")
+parser.add_argument("-r", "--registration", type=str, default="registration.yaml",
+                    metavar="<path>", help="the path to save the generated registration to")
+args = parser.parse_args()
 
-class TelegramBridge(Bridge):
-    module = "mautrix_telegram"
-    name = "mautrix-telegram"
-    command = "python -m mautrix-telegram"
-    description = "A Matrix-Telegram puppeting bridge."
-    repo_url = "https://github.com/tulir/mautrix-telegram"
-    real_user_content_key = "net.maunium.telegram.puppet"
-    version = version
-    markdown_version = linkified_version
-    config_class = Config
-    matrix_class = MatrixHandler
+config = Config(args.config, args.registration, args.base_config, os.environ)
+config.load()
+config.update()
 
-    config: Config
-    session_container: AlchemySessionContainer
-    bot: Bot
-    manhole: Optional[ManholeState]
+if args.generate_registration:
+    config.generate_registration()
+    config.save()
+    print(f"Registration generated and saved to {config.registration_path}")
+    sys.exit(0)
 
-    def prepare_db(self) -> None:
-        super().prepare_db()
-        init_db(self.db)
-        self.session_container = AlchemySessionContainer(
-            engine=self.db, table_base=Base, session=False,
-            table_prefix="telethon_", manage_tables=False)
+logging.config.dictConfig(copy.deepcopy(config["logging"]))
+log = logging.getLogger("mau.init")  # type: logging.Logger
+log.debug(f"Initializing mautrix-telegram {__version__}")
 
-    def _prepare_website(self, context: Context) -> None:
-        if self.config["appservice.public.enabled"]:
-            public_website = PublicBridgeWebsite(self.loop)
-            self.az.app.add_subapp(self.config["appservice.public.prefix"], public_website.app)
-            context.public_website = public_website
+db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
+Base.metadata.bind = db_engine
 
-        if self.config["appservice.provisioning.enabled"]:
-            provisioning_api = ProvisioningAPI(context)
-            self.az.app.add_subapp(self.config["appservice.provisioning.prefix"],
-                                   provisioning_api.app)
-            context.provisioning_api = provisioning_api
+session_container = AlchemySessionContainer(engine=db_engine, table_base=Base, session=False,
+                                            table_prefix="telethon_", manage_tables=False)
+session_container.core_mode = True
 
-    def prepare_bridge(self) -> None:
-        self.bot = init_bot(self.config)
-        context = Context(self.az, self.config, self.loop, self.session_container, self, self.bot)
-        self._prepare_website(context)
-        self.matrix = context.mx = MatrixHandler(context)
-        self.manhole = None
+try:
+    import uvloop
 
-        init_abstract_user(context)
-        init_formatter(context)
-        init_portal(context)
-        self.add_startup_actions(init_puppet(context))
-        self.add_startup_actions(init_user(context))
-        if self.bot:
-            self.add_startup_actions(self.bot.start())
-        if self.config["bridge.resend_bridge_info"]:
-            self.add_startup_actions(self.resend_bridge_info())
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+    log.debug("Using uvloop for asyncio")
+except ImportError:
+    pass
 
-    async def resend_bridge_info(self) -> None:
-        self.config["bridge.resend_bridge_info"] = False
-        self.config.save()
-        self.log.info("Re-sending bridge info state event to all portals")
-        for portal in Portal.all():
-            await portal.update_bridge_info()
-        self.log.info("Finished re-sending bridge info state events")
+loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
 
-    def prepare_stop(self) -> None:
-        for puppet in Puppet.by_custom_mxid.values():
-            puppet.stop()
-        self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
-        if self.manhole:
-            self.manhole.close()
-            self.manhole = None
+state_store = SQLStateStore()
+mebibyte = 1024 ** 2
+appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
+                     config["appservice.as_token"], config["appservice.hs_token"],
+                     config["appservice.bot_username"], log="mau.as", loop=loop,
+                     verify_ssl=config["homeserver.verify_ssl"], state_store=state_store,
+                     real_user_content_key="net.maunium.telegram.puppet",
+                     aiohttp_params={
+                         "client_max_size": config["appservice.max_body_size"] * mebibyte
+                     })
+bot = init_bot(config)
+context = Context(appserv, config, loop, session_container, bot)
 
-    async def get_user(self, user_id: UserID, create: bool = True) -> User:
-        user = User.get_by_mxid(user_id, create=create)
-        if user:
-            await user.ensure_started()
-        return user
+if config["appservice.public.enabled"]:
+    public_website = PublicBridgeWebsite(loop)
+    appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public_website.app)
+    context.public_website = public_website
 
-    async def get_portal(self, room_id: RoomID) -> Portal:
-        return Portal.get_by_mxid(room_id)
+if config["appservice.provisioning.enabled"]:
+    provisioning_api = ProvisioningAPI(context)
+    appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning",
+                           provisioning_api.app)
+    context.provisioning_api = provisioning_api
 
-    async def get_puppet(self, user_id: UserID, create: bool = False) -> Puppet:
-        return await Puppet.get_by_mxid(user_id, create=create)
+context.mx = MatrixHandler(context)
 
-    async def get_double_puppet(self, user_id: UserID) -> Puppet:
-        return await Puppet.get_by_custom_mxid(user_id)
+if config["metrics.enabled"]:
+    if prometheus:
+        prometheus.start_http_server(config["metrics.listen_port"])
+    else:
+        log.warn("Metrics are enabled in the config, but prometheus-async is not installed.")
 
-    def is_bridge_ghost(self, user_id: UserID) -> bool:
-        return bool(Puppet.get_id_from_mxid(user_id))
+with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
+    start_ts = time()
+    init_db(db_engine)
+    init_abstract_user(context)
+    init_formatter(context)
+    init_portal(context)
+    startup_actions = (init_puppet(context) +
+                       init_user(context) +
+                       [start, context.mx.init_as_bot()])  # type: List[Awaitable[Any]]
 
+    if context.bot:
+        startup_actions.append(context.bot.start())
 
-TelegramBridge().run()
+    signal.signal(signal.SIGINT, signal.default_int_handler)
+    signal.signal(signal.SIGTERM, signal.default_int_handler)
+
+    end_ts = time()
+    try:
+        log.debug(f"Initialization complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " running startup actions")
+        start_ts = time()
+        loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
+        end_ts = time()
+        log.debug(f"Startup actions complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " now running forever")
+        loop.run_forever()
+    except KeyboardInterrupt:
+        log.debug("Interrupt received, stopping clients")
+        loop.run_until_complete(
+            asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
+        log.debug("Clients stopped, shutting down")
+        sys.exit(0)
+    except Exception as e:
+        log.exception("Unexpected error")
+        sys.exit(1)

mautrix_telegram/abstract_user.py

@@ -1,5 +1,6 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,36 +14,28 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional, Union, Dict, Type, Any, TYPE_CHECKING
+from typing import Tuple, Optional, List, Union, Dict, TYPE_CHECKING
 from abc import ABC, abstractmethod
 import asyncio
 import logging
 import platform
 import time
 
-from telethon.sessions import Session
-from telethon.network import (ConnectionTcpMTProxyRandomizedIntermediate, ConnectionTcpFull,
-                              Connection)
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
-    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdateChatPinnedMessage,
-    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
-    UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
-    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
-    UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
-    UpdateUserTyping, User, UserStatusOffline, UserStatusOnline, UpdateReadHistoryInbox,
-    UpdateReadChannelInbox, MessageEmpty)
+    Channel, ChannelForbidden, Chat, ChatForbidden, MessageActionChannelMigrateFrom, PeerUser,
+    TypeUpdate, UpdateChannelPinnedMessage, UpdateChatPinnedMessage, UpdateChatParticipantAdmin,
+    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateDeleteMessages,
+    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateNewMessage,
+    UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, UpdateUserName,
+    UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, UserStatusOnline)
 
-from mautrix.types import UserID, PresenceState
-from mautrix.errors import MatrixError
-from mautrix.appservice import AppService
-from mautrix.util.logging import TraceLogger
-from mautrix.util.opt_prometheus import Histogram, Counter
+from mautrix_appservice import MatrixRequestError, AppService
 from alchemysession import AlchemySessionContainer
 
 from . import portal as po, puppet as pu, __version__
 from .db import Message as DBMessage
-from .types import TelegramID
+from .types import TelegramID, MatrixUserID
 from .tgclient import MautrixTelegramClient
 
 if TYPE_CHECKING:
@@ -50,79 +43,64 @@ if TYPE_CHECKING:
     from .config import Config
     from .bot import Bot
 
-config: Optional['Config'] = None
+config = None  # type: Config
 # Value updated from config in init()
-MAX_DELETIONS: int = 10
+MAX_DELETIONS = 10  # type: int
 
 UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
-UPDATE_TIME = Histogram("bridge_telegram_update", "Time spent processing Telegram updates",
-                        ("update_type",))
-UPDATE_ERRORS = Counter("bridge_telegram_update_error",
-                        "Number of fatal errors while handling Telegram updates", ("update_type",))
+try:
+    from prometheus_client import Histogram
 
+    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
+                            ["update_type"])
+except ImportError:
+    Histogram = None
+    UPDATE_TIME = None
 
 class AbstractUser(ABC):
-    session_container: AlchemySessionContainer = None
-    loop: asyncio.AbstractEventLoop = None
-    log: TraceLogger
-    az: AppService
-    relaybot: Optional['Bot']
-    ignore_incoming_bot_events: bool = True
-
-    client: Optional[MautrixTelegramClient]
-    mxid: Optional[UserID]
-
-    tgid: Optional[TelegramID]
-    username: Optional['str']
-    is_bot: bool
-
-    is_relaybot: bool
-
-    puppet_whitelisted: bool
-    whitelisted: bool
-    relaybot_whitelisted: bool
-    matrix_puppet_whitelisted: bool
-    is_admin: bool
+    session_container = None  # type: AlchemySessionContainer
+    loop = None  # type: asyncio.AbstractEventLoop
+    log = None  # type: logging.Logger
+    az = None  # type: AppService
+    bot = None  # type: Bot
+    ignore_incoming_bot_events = True  # type: bool
 
     def __init__(self) -> None:
-        self.is_admin = False
-        self.matrix_puppet_whitelisted = False
-        self.puppet_whitelisted = False
-        self.whitelisted = False
-        self.relaybot_whitelisted = False
-        self.client = None
-        self.is_relaybot = False
-        self.is_bot = False
+        self.is_admin = False  # type: bool
+        self.matrix_puppet_whitelisted = False  # type: bool
+        self.puppet_whitelisted = False  # type: bool
+        self.whitelisted = False  # type: bool
+        self.relaybot_whitelisted = False  # type: bool
+        self.client = None  # type: MautrixTelegramClient
+        self.tgid = None  # type: TelegramID
+        self.mxid = None  # type: MatrixUserID
+        self.is_relaybot = False  # type: bool
+        self.is_bot = False  # type: bool
+        self.relaybot = None  # type: Optional[Bot]
 
     @property
     def connected(self) -> bool:
         return self.client and self.client.is_connected()
 
     @property
-    def _proxy_settings(self) -> Tuple[Type[Connection], Optional[Tuple[Any, ...]]]:
+    def _proxy_settings(self) -> Optional[Tuple[int, str, str, str, str, str]]:
         proxy_type = config["telegram.proxy.type"].lower()
-        connection = ConnectionTcpFull
-        connection_data = (config["telegram.proxy.address"],
-                           config["telegram.proxy.port"],
-                           config["telegram.proxy.rdns"],
-                           config["telegram.proxy.username"],
-                           config["telegram.proxy.password"])
         if proxy_type == "disabled":
-            connection_data = None
+            return None
         elif proxy_type == "socks4":
-            connection_data = (1,) + connection_data
+            proxy_type = 1
         elif proxy_type == "socks5":
-            connection_data = (2,) + connection_data
+            proxy_type = 2
         elif proxy_type == "http":
-            connection_data = (3,) + connection_data
-        elif proxy_type == "mtproxy":
-            connection = ConnectionTcpMTProxyRandomizedIntermediate
-            connection_data = (connection_data[0], connection_data[1], connection_data[4])
+            proxy_type = 3
 
-        return connection, connection_data
+        return (proxy_type,
+                config["telegram.proxy.address"], config["telegram.proxy.port"],
+                config["telegram.proxy.rdns"],
+                config["telegram.proxy.username"], config["telegram.proxy.password"])
 
     def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
@@ -141,9 +119,6 @@ class AbstractUser(ABC):
         device = config["telegram.device_info.device_model"]
         sysversion = config["telegram.device_info.system_version"]
         appversion = config["telegram.device_info.app_version"]
-        connection, proxy = self._proxy_settings
-
-        assert isinstance(self.session, Session)
 
         self.client = MautrixTelegramClient(
             session=self.session,
@@ -152,19 +127,16 @@ class AbstractUser(ABC):
             api_hash=config["telegram.api_hash"],
 
             app_version=__version__ if appversion == "auto" else appversion,
-            system_version=(MautrixTelegramClient.__version__
-                            if sysversion == "auto" else sysversion),
-            device_model=(f"{platform.system()} {platform.release()}"
-                          if device == "auto" else device),
+            system_version=MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion,
+            device_model=f"{platform.system()} {platform.release()}" if device == "auto" else device,
 
             timeout=config["telegram.connection.timeout"],
             connection_retries=config["telegram.connection.retries"],
             retry_delay=config["telegram.connection.retry_delay"],
             flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
             request_retries=config["telegram.connection.request_retries"],
-            connection=connection,
-            proxy=proxy,
-            raise_last_call_error=True,
+
+            proxy=self._proxy_settings,
 
             loop=self.loop,
             base_logger=base_logger
@@ -180,23 +152,31 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     @abstractmethod
-    async def register_portal(self, portal: po.Portal) -> None:
+    def register_portal(self, portal: po.Portal) -> None:
         raise NotImplementedError()
 
     @abstractmethod
-    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
+    def unregister_portal(self, portal: po.Portal) -> None:
         raise NotImplementedError()
 
     async def _update_catch(self, update: TypeUpdate) -> None:
         start_time = time.time()
-        update_type = type(update).__name__
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
-            self.log.exception(f"Failed to handle Telegram update {update}")
-            UPDATE_ERRORS.labels(update_type=update_type).inc()
-        UPDATE_TIME.labels(update_type=update_type).observe(time.time() - start_time)
+            self.log.exception("Failed to handle Telegram update")
+        if UPDATE_TIME:
+            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
+
+    async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
+        if self.is_bot:
+            return []
+        dialogs = await self.client.get_dialogs(limit=limit)
+        return [dialog.entity for dialog in dialogs if (
+            not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden))
+            and not (isinstance(dialog.entity, Chat)
+                     and (dialog.entity.deactivated or dialog.entity.left)))]
 
     @property
     @abstractmethod
@@ -204,8 +184,7 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     async def is_logged_in(self) -> bool:
-        return (self.client and self.client.is_connected()
-                and await self.client.is_user_authorized())
+        return self.client and self.client.is_connected() and await self.client.is_user_authorized()
 
     async def has_full_access(self, allow_bot: bool = False) -> bool:
         return (self.puppet_whitelisted
@@ -216,15 +195,14 @@ class AbstractUser(ABC):
         if not self.client:
             self._init_client()
         await self.client.connect()
-        self.log.debug(f"{'Bot' if self.is_relaybot else self.mxid} connected: {self.connected}")
+        self.log.debug("%s connected: %s", self.mxid, self.connected)
         return self
 
     async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
-        if self.connected:
+        if not self.puppet_whitelisted or self.connected:
             return self
+        self.log.debug("ensure_started(%s, even_if_no_session=%s)", self.mxid, even_if_no_session)
         if even_if_no_session or self.session_container.has_session(self.mxid):
-            self.log.debug("Starting client due to ensure_started"
-                           f"(even_if_no_session={even_if_no_session})")
             await self.start(delete_unless_authenticated=not even_if_no_session)
         return self
 
@@ -258,10 +236,8 @@ class AbstractUser(ABC):
             await self.update_others_info(update)
         elif isinstance(update, UpdateReadHistoryOutbox):
             await self.update_read_receipt(update)
-        elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
-            await self.update_own_read_receipt(update)
         else:
-            self.log.trace("Unhandled update: %s", update)
+            self.log.debug("Unhandled update: %s", update)
 
     async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
                                                          UpdateChatPinnedMessage]) -> None:
@@ -276,7 +252,7 @@ class AbstractUser(ABC):
     async def update_participants(update: UpdateChatParticipants) -> None:
         portal = po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
         if portal and portal.mxid:
-            await portal.update_power_levels(update.participants.participants)
+            await portal.update_telegram_participants(update.participants.participants)
 
     async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
         if not isinstance(update.peer, PeerUser):
@@ -295,35 +271,9 @@ class AbstractUser(ABC):
         puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
-    async def update_own_read_receipt(self, update: Union[UpdateReadHistoryInbox,
-                                                          UpdateReadChannelInbox]) -> None:
-        puppet = pu.Puppet.get(self.tgid)
-        if not puppet.is_real_user:
-            return
-
-        if isinstance(update, UpdateReadChannelInbox):
-            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
-        elif isinstance(update.peer, PeerChat):
-            portal = po.Portal.get_by_tgid(TelegramID(update.peer.chat_id))
-        elif isinstance(update.peer, PeerUser):
-            portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
-        else:
-            self.log.debug("Unexpected own read receipt peer: %s", update.peer)
-            return
-
-        if not portal or not portal.mxid:
-            return
-
-        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
-        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), tg_space, edit_index=-1)
-        if not message:
-            return
-
-        await puppet.intent.mark_read(portal.mxid, message.mxid)
-
     async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
         # TODO duplication not checked
-        portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
         if not portal or not portal.mxid:
             return
 
@@ -333,7 +283,7 @@ class AbstractUser(ABC):
         if isinstance(update, UpdateUserTyping):
             portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
         else:
-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
 
         if not portal or not portal.mxid:
             return
@@ -346,7 +296,7 @@ class AbstractUser(ABC):
         try:
             users = (entity for entity in entities.values() if isinstance(entity, User))
             puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
-            await asyncio.gather(*[puppet.try_update_info(self, info)
+            await asyncio.gather(*[puppet.update_info(self, info)
                                    for puppet, info in puppets if puppet])
         except Exception:
             self.log.exception("Failed to handle entity updates")
@@ -357,30 +307,28 @@ class AbstractUser(ABC):
         if isinstance(update, UpdateUserName):
             puppet.username = update.username
             if await puppet.update_displayname(self, update):
-                await puppet.save()
+                puppet.save()
         elif isinstance(update, UpdateUserPhoto):
             if await puppet.update_avatar(self, update.photo):
-                await puppet.save()
+                puppet.save()
         else:
-            self.log.warning(f"Unexpected other user info update: {type(update)}")
+            self.log.warning("Unexpected other user info update: %s", update)
 
     async def update_status(self, update: UpdateUserStatus) -> None:
         puppet = pu.Puppet.get(TelegramID(update.user_id))
         if isinstance(update.status, UserStatusOnline):
-            await puppet.default_mxid_intent.set_presence(PresenceState.ONLINE)
+            await puppet.default_mxid_intent.set_presence("online")
         elif isinstance(update.status, UserStatusOffline):
-            await puppet.default_mxid_intent.set_presence(PresenceState.OFFLINE)
+            await puppet.default_mxid_intent.set_presence("offline")
         else:
-            self.log.warning(f"Unexpected user status update: type({update})")
+            self.log.warning("Unexpected user status update: %s", update)
         return
 
     def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
                                                                   Optional[pu.Puppet],
                                                                   Optional[po.Portal]]:
         if isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
-            if not portal:
-                self.log.warning(f"Received message in chat with unknown type {update.chat_id}")
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
             sender = pu.Puppet.get(TelegramID(update.from_id))
         elif isinstance(update, UpdateShortMessage):
             portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
@@ -388,18 +336,15 @@ class AbstractUser(ABC):
         elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
                                  UpdateEditMessage, UpdateEditChannelMessage)):
             update = update.message
-            if isinstance(update, MessageEmpty):
-                return update, None, None
-            portal = po.Portal.get_by_entity(update.peer_id, receiver_id=self.tgid)
-            if update.out:
-                sender = pu.Puppet.get(self.tgid)
-            elif isinstance(update.from_id, PeerUser):
-                sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
+            if isinstance(update.to_id, PeerUser) and not update.out:
+                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
             else:
-                sender = None
+                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
+            sender = pu.Puppet.get(update.from_id) if update.from_id else None
         else:
-            self.log.warning("Unexpected message type in User#get_message_details: "
-                             f"{type(update)}")
+            self.log.warning(
+                f"Unexpected message type in User#get_message_details: {type(update)}")
             return update, None, None
         return update, sender, portal
 
@@ -410,7 +355,7 @@ class AbstractUser(ABC):
             return
         try:
             await portal.main_intent.redact(message.mx_room, message.mxid)
-        except MatrixError:
+        except MatrixRequestError:
             pass
 
     async def delete_message(self, update: UpdateDeleteMessages) -> None:
@@ -418,10 +363,12 @@ class AbstractUser(ABC):
             return
 
         for message_id in update.messages:
-            for message in DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
+            for message in messages:
                 message.delete()
                 number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
                 if number_left == 0:
+                    portal = po.Portal.get_by_mxid(message.mx_room)
                     await self._try_redact(message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
@@ -431,54 +378,44 @@ class AbstractUser(ABC):
         channel_id = TelegramID(update.channel_id)
 
         for message_id in update.messages:
-            for message in DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
+            for message in messages:
                 message.delete()
                 await self._try_redact(message)
 
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
-        if not portal:
-            return
-        elif portal and not portal.allow_bridging:
-            self.log.debug(f"Ignoring message in portal {portal.tgid_log} (bridging disallowed)")
+
+        if self.is_bot and not portal.mxid:
+            self.log.debug(f"Ignoring message received by bot in unbridged chat %s",
+                           portal.tgid_log)
             return
 
-        if self.is_relaybot:
-            if update.is_private:
-                if not config["bridge.relaybot.private_chat.invite"]:
-                    self.log.debug(f"Ignoring private message to bot from {sender.id}")
-                    return
-            elif not portal.mxid and config["bridge.relaybot.ignore_unbridged_group_chat"]:
-                self.log.debug("Ignoring message received by bot"
-                               f" in unbridged chat {portal.tgid_log}")
-                return
-
-        if ((self.ignore_incoming_bot_events and self.relaybot
-             and sender and sender.id == self.relaybot.tgid)):
-            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
+        if self.ignore_incoming_bot_events and self.bot and sender.id == self.bot.tgid:
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
             return
 
-        await portal.backfill_lock.wait(update.id)
-
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.trace(f"Received %s in %s by %d, unregistering portal...",
-                               update.action, portal.tgid_log, sender.id)
-                await self.unregister_portal(update.action.chat_id, update.action.chat_id)
-                await self.register_portal(portal)
+                self.log.debug(f"Ignoring action %s to %s by %d", update.action,
+                               portal.tgid_log,
+                               sender.id)
                 return
-            self.log.trace("Handling action %s to %s by %d", update.action, portal.tgid_log,
+            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
                            sender.id)
             return await portal.handle_telegram_action(self, sender, update)
 
+        user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
             return await portal.handle_telegram_edit(self, sender, update)
+
+        self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
         return await portal.handle_telegram_message(self, sender, update)
 
     # endregion
 
 
-def init(context: 'Context') -> None:
+def init(context: "Context") -> None:
     global config, MAX_DELETIONS
     AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
     AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]

mautrix_telegram/bot.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,21 +14,21 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING
+from typing import Awaitable, Callable, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
 import logging
+import re
 
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
     ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
-    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser, PeerUser,
+    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
     MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
     UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
 
-from mautrix.types import UserID
-
+from .types import MatrixUserID
 from .abstract_user import AbstractUser
 from .db import BotChat
 from .types import TelegramID
@@ -35,41 +36,34 @@ from . import puppet as pu, portal as po, user as u
 
 if TYPE_CHECKING:
     from .config import Config
+    from .context import Context
 
-config: Optional['Config'] = None
+config = None  # type: Config
 
 ReplyFunc = Callable[[str], Awaitable[Message]]
 
 
 class Bot(AbstractUser):
-    log: logging.Logger = logging.getLogger("mau.user.bot")
-
-    token: str
-    chats: Dict[int, str]
-    tg_whitelist: List[int]
-    whitelist_group_admins: bool
-    _me_info: Optional[User]
-    _me_mxid: Optional[UserID]
+    log = logging.getLogger("mau.bot")  # type: logging.Logger
+    mxid_regex = re.compile("@.+:.+")  # type: Pattern
 
     def __init__(self, token: str) -> None:
         super().__init__()
-        self.token = token
-        self.tgid = None
-        self.mxid = None
-        self.puppet_whitelisted = True
-        self.whitelisted = True
-        self.relaybot_whitelisted = True
-        self.username = None
-        self.is_relaybot = True
-        self.is_bot = True
-        self.chats = {}
-        self.tg_whitelist = []
+        self.token = token  # type: str
+        self.puppet_whitelisted = True  # type: bool
+        self.whitelisted = True  # type: bool
+        self.relaybot_whitelisted = True  # type: bool
+        self.username = None  # type: str
+        self.is_relaybot = True  # type: bool
+        self.is_bot = True  # type: bool
+        self.chats = {}  # type: Dict[int, str]
+        self.tg_whitelist = []  # type: List[int]
         self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
-                                       or False)
-        self._me_info = None
-        self._me_mxid = None
+                                       or False)  # type: bool
+        self._me_info = None  # type: Optional[User]
+        self._me_mxid = None  # type: Optional[MatrixUserID]
 
-    async def get_me(self, use_cache: bool = True) -> Tuple[User, UserID]:
+    async def get_me(self, use_cache: bool = True) -> Tuple[User, MatrixUserID]:
         if not use_cache or not self._me_mxid:
             self._me_info = await self.client.get_me()
             self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
@@ -98,7 +92,7 @@ class Bot(AbstractUser):
     async def post_login(self) -> None:
         await self.init_permissions()
         info = await self.client.get_me()
-        self.tgid = TelegramID(info.id)
+        self.tgid = info.id
         self.username = info.username
         self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
 
@@ -117,11 +111,17 @@ class Bot(AbstractUser):
             except (ChannelPrivateError, ChannelInvalidError):
                 self.remove_chat(TelegramID(channel_id.channel_id))
 
-    async def register_portal(self, portal: po.Portal) -> None:
+        if config["bridge.catch_up"]:
+            try:
+                await self.client.catch_up()
+            except Exception:
+                self.log.exception("Failed to run catch_up() for bot")
+
+    def register_portal(self, portal: po.Portal) -> None:
         self.add_chat(portal.tgid, portal.peer_type)
 
-    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
-        self.remove_chat(tgid)
+    def unregister_portal(self, portal: po.Portal) -> None:
+        self.remove_chat(portal.tgid)
 
     def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
         if chat_id not in self.chats:
@@ -133,7 +133,7 @@ class Bot(AbstractUser):
             del self.chats[chat_id]
         except KeyError:
             pass
-        BotChat.delete_by_id(chat_id)
+        BotChat.delete(chat_id)
 
     async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
         if tgid in self.tg_whitelist:
@@ -147,7 +147,7 @@ class Bot(AbstractUser):
         if self.whitelist_group_admins:
             if isinstance(chat, PeerChannel):
                 p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin))
+                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
             elif isinstance(chat, PeerChat):
                 chat = await self.client(GetFullChatRequest(chat.chat_id))
                 participants = chat.full_chat.participants.participants
@@ -157,7 +157,7 @@ class Bot(AbstractUser):
         return False
 
     async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
-        if not await self._can_use_commands(event.to_id, TelegramID(event.from_id)):
+        if not await self._can_use_commands(event.to_id, event.from_id):
             await reply("You do not have the permission to use that command.")
             return False
         return True
@@ -166,7 +166,7 @@ class Bot(AbstractUser):
         if not config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
-        if not portal.allow_bridging:
+        if not portal.allow_bridging():
             return await reply("This bridge doesn't allow bridging this chat.")
 
         await portal.create_matrix_room(self)
@@ -179,15 +179,15 @@ class Bot(AbstractUser):
                     "Portal is not public. Use `/invite <mxid>` to get an invite.")
 
     async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc,
-                                    mxid_input: UserID) -> Message:
+                                    mxid_input: MatrixUserID) -> Message:
         if len(mxid_input) == 0:
             return await reply("Usage: `/invite <mxid>`")
         elif not portal.mxid:
             return await reply("Portal does not have Matrix room. "
                                "Create one with /portal first.")
-        if mxid_input[0] != '@' or mxid_input.find(':') < 2:
+        if not self.mxid_regex.match(mxid_input):
             return await reply("That doesn't look like a Matrix ID.")
-        user = await u.User.get_by_mxid(mxid_input).ensure_started()
+        user = await u.User.get_by_mxid(MatrixUserID(mxid_input)).ensure_started()
         if not user.relaybot_whitelisted:
             return await reply("That user is not whitelisted to use the bridge.")
         elif await user.is_logged_in():
@@ -195,7 +195,7 @@ class Bot(AbstractUser):
             return await reply("That user seems to be logged in. "
                                f"Just invite [{displayname}](tg://user?id={user.tgid})")
         else:
-            await portal.main_intent.invite_user(portal.mxid, user.mxid)
+            await portal.main_intent.invite(portal.mxid, user.mxid)
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
     @staticmethod
@@ -204,12 +204,7 @@ class Bot(AbstractUser):
         # chat is a normal group or a supergroup/channel when using the ID.
         if isinstance(message.to_id, PeerChannel):
             return reply(f"-100{message.to_id.channel_id}")
-        elif isinstance(message.to_id, PeerChat):
-            return reply(str(-message.to_id.chat_id))
-        elif isinstance(message.to_id, PeerUser):
-            return reply(f"Your user ID is {message.from_id}.")
-        else:
-            return reply("Failed to find chat ID.")
+        return reply(str(-message.to_id.chat_id))
 
     def match_command(self, text: str, command: str) -> bool:
         text = text.lower()
@@ -228,44 +223,36 @@ class Bot(AbstractUser):
 
     async def handle_command(self, message: Message) -> None:
         def reply(reply_text: str) -> Awaitable[Message]:
-            return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)
+            return self.client.send_message(message.to_id, reply_text, reply_to=message.id)
 
         text = message.message
 
-        if self.match_command(text, "start"):
-            pcm = config["bridge.relaybot.private_chat.message"]
-            if pcm:
-                await reply(pcm)
-            return
-        elif self.match_command(text, "id"):
+        if self.match_command(text, "id"):
             await self.handle_command_id(message, reply)
             return
-        elif message.is_private:
-            return
 
         portal = po.Portal.get_by_entity(message.to_id)
 
-        is_portal_cmd = self.match_command(text, "portal")
-        is_invite_cmd = self.match_command(text, "invite")
-        if is_portal_cmd or is_invite_cmd:
+        if self.match_command(text, "portal"):
             if not await self.check_can_use_commands(message, reply):
                 return
-            if is_portal_cmd:
-                await self.handle_command_portal(portal, reply)
-            elif is_invite_cmd:
-                try:
-                    mxid = text[text.index(" ") + 1:]
-                except ValueError:
-                    mxid = ""
-                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
+            await self.handle_command_portal(portal, reply)
+        elif self.match_command(text, "invite"):
+            if not await self.check_can_use_commands(message, reply):
+                return
+            try:
+                mxid = text[text.index(" ") + 1:]
+            except ValueError:
+                mxid = ""
+            await self.handle_command_invite(portal, reply, mxid_input=mxid)
 
     def handle_service_message(self, message: MessageService) -> None:
-        to_peer = message.to_id
-        if isinstance(to_peer, PeerChannel):
-            to_id = TelegramID(to_peer.channel_id)
+        to_id = message.to_id  # type: TelegramID
+        if isinstance(to_id, PeerChannel):
+            to_id = to_id.channel_id
             chat_type = "channel"
-        elif isinstance(to_peer, PeerChat):
-            to_id = TelegramID(to_peer.chat_id)
+        elif isinstance(to_id, PeerChat):
+            to_id = to_id.chat_id
             chat_type = "chat"
         else:
             return
@@ -288,10 +275,10 @@ class Bot(AbstractUser):
 
         is_command = (isinstance(update.message, Message)
                       and update.message.entities and len(update.message.entities) > 0
-                      and isinstance(update.message.entities[0], MessageEntityBotCommand)
-                      and update.message.entities[0].offset == 0)
+                      and isinstance(update.message.entities[0], MessageEntityBotCommand))
         if is_command:
             await self.handle_command(update.message)
+            return True
         return False
 
     def is_in_chat(self, peer_id) -> bool:

mautrix_telegram/commands/__init__.py

@@ -1,8 +1,5 @@
-from .handler import (command_handler, CommandHandler, CommandProcessor, CommandEvent,
-                      SECTION_AUTH, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT,
-                      SECTION_MISC, SECTION_ADMIN)
-from . import portal, telegram, matrix_auth, manhole
-
-__all__ = ["command_handler", "CommandHandler", "CommandProcessor", "CommandEvent",
-           "SECTION_AUTH", "SECTION_MISC", "SECTION_ADMIN", "SECTION_CREATING_PORTALS",
-           "SECTION_PORTAL_MANAGEMENT"]
+from .handler import (command_handler, command_handlers as _command_handlers,
+                      CommandHandler, CommandProcessor, CommandEvent,
+                      SECTION_GENERAL, SECTION_AUTH, SECTION_CREATING_PORTALS,
+                      SECTION_PORTAL_MANAGEMENT, SECTION_MISC, SECTION_ADMIN)
+from . import portal, telegram, clean_rooms, matrix_auth, meta

mautrix_telegram/commands/clean_rooms.py

@@ -0,0 +1,178 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, NewType, Optional, Tuple, Union
+
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from ..types import MatrixRoomID, MatrixUserID
+from . import command_handler, CommandEvent, SECTION_ADMIN
+from .. import puppet as pu, portal as po
+
+ManagementRoom = NewType('ManagementRoom', Tuple[MatrixRoomID, MatrixUserID])
+
+
+async def _find_rooms(intent: IntentAPI) -> Tuple[List[ManagementRoom], List[MatrixRoomID],
+                                                  List['po.Portal'], List['po.Portal']]:
+    management_rooms = []  # type: List[ManagementRoom]
+    unidentified_rooms = []  # type: List[MatrixRoomID]
+    portals = []  # type: List[po.Portal]
+    empty_portals = []  # type: List[po.Portal]
+
+    rooms = await intent.get_joined_rooms()
+    for room_str in rooms:
+        room = MatrixRoomID(room_str)
+        portal = po.Portal.get_by_mxid(room)
+        if not portal:
+            try:
+                members = await intent.get_room_members(room)
+            except MatrixRequestError:
+                members = []
+            if len(members) == 2:
+                other_member = MatrixUserID(members[0] if members[0] != intent.mxid else members[1])
+                if pu.Puppet.get_id_from_mxid(other_member):
+                    unidentified_rooms.append(room)
+                else:
+                    management_rooms.append(ManagementRoom((room, other_member)))
+            else:
+                unidentified_rooms.append(room)
+        else:
+            members = await portal.get_authenticated_matrix_users()
+            if len(members) == 0:
+                empty_portals.append(portal)
+            else:
+                portals.append(portal)
+
+    return management_rooms, unidentified_rooms, portals, empty_portals
+
+
+@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
+                 help_section=SECTION_ADMIN,
+                 help_text="Clean up unused portal/management rooms.")
+async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
+    management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
+
+    reply = ["#### Management rooms (M)"]
+    reply += ([f"{n+1}. [M{n+1}](https://matrix.to/#/{room}) (with {other_member}"
+               for n, (room, other_member) in enumerate(management_rooms)]
+              or ["No management rooms found."])
+    reply.append("#### Active portal rooms (A)")
+    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
+               f"(to Telegram chat \"{portal.title}\")"
+               for n, portal in enumerate(portals)]
+              or ["No active portal rooms found."])
+    reply.append("#### Unidentified rooms (U)")
+    reply += ([f"{n+1}. [U{n+1}](https://matrix.to/#/{room})"
+               for n, room in enumerate(unidentified_rooms)]
+              or ["No unidentified rooms found."])
+    reply.append("#### Inactive portal rooms (I)")
+    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
+               f"(to Telegram chat \"{portal.title}\")"
+               for n, portal in enumerate(empty_portals)]
+              or ["No inactive portal rooms found."])
+
+    reply += ["#### Usage",
+              ("To clean the recommended set of rooms (unidentified & inactive portals), "
+               "type `$cmdprefix+sp clean-recommended`"),
+              "",
+              ("To clean other groups of rooms, type `$cmdprefix+sp clean-groups <letters>` "
+               "where `letters` are the first letters of the group names (M, A, U, I)"),
+              "",
+              ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
+               "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
+               "the group name. (e.g. `I2-6`)"),
+              "",
+              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
+               "between each use of the commands above.")]
+
+    evt.sender.command_status = {
+        "next": lambda clean_evt: set_rooms_to_clean(clean_evt, management_rooms,
+                                                     unidentified_rooms, portals, empty_portals),
+        "action": "Room cleaning",
+    }
+
+    return await evt.reply("\n".join(reply))
+
+
+async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
+                             unidentified_rooms: List[MatrixRoomID], portals: List["po.Portal"],
+                             empty_portals: List["po.Portal"]) -> None:
+    command = evt.args[0]
+    rooms_to_clean = []  # type: List[Union[po.Portal, MatrixRoomID]]
+    if command == "clean-recommended":
+        rooms_to_clean += empty_portals
+        rooms_to_clean += unidentified_rooms
+    elif command == "clean-groups":
+        if len(evt.args) < 2:
+            return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
+        groups_to_clean = evt.args[1].upper()
+        if "M" in groups_to_clean:
+            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
+        if "A" in groups_to_clean:
+            rooms_to_clean += portals
+        if "U" in groups_to_clean:
+            rooms_to_clean += unidentified_rooms
+        if "I" in groups_to_clean:
+            rooms_to_clean += empty_portals
+    elif command == "clean-range":
+        try:
+            clean_range = evt.args[1]
+            group, clean_range = clean_range[0], clean_range[1:]
+            start, end = clean_range.split("-")
+            start, end = int(start), int(end)
+            if group == "M":
+                group = [room_id for (room_id, user_id) in management_rooms]
+            elif group == "A":
+                group = portals
+            elif group == "U":
+                group = unidentified_rooms
+            elif group == "I":
+                group = empty_portals
+            else:
+                raise ValueError("Unknown group")
+            rooms_to_clean = group[start - 1:end]
+        except (KeyError, ValueError):
+            return await evt.reply(
+                "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
+    else:
+        return await evt.reply(f"Unknown room cleaning action `{command}`. "
+                               "Use `$cmdprefix+sp cancel` to cancel room "
+                               "cleaning.")
+
+    evt.sender.command_status = {
+        "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
+        "action": "Room cleaning",
+    }
+    await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type"
+                    "`$cmdprefix+sp confirm-clean`.")
+
+
+async def execute_room_cleanup(evt, rooms_to_clean: List[Union[po.Portal, MatrixRoomID]]) -> None:
+    if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
+        await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
+                        "This might take a while.")
+        cleaned = 0
+        for room in rooms_to_clean:
+            if isinstance(room, po.Portal):
+                await room.cleanup_and_delete()
+                cleaned += 1
+            elif isinstance(room, str):  # str is aliased by MatrixRoomID
+                await po.Portal.cleanup_room(evt.az.intent, room, message="Room deleted")
+                cleaned += 1
+        evt.sender.command_status = None
+        await evt.reply(f"{cleaned} rooms cleaned up successfully.")
+    else:
+        await evt.reply("Room cleaning cancelled.")

mautrix_telegram/commands/handler.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -14,29 +15,23 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """This module contains classes handling commands issued by Matrix users."""
-from typing import Awaitable, Callable, List, Optional, NamedTuple, Any
+from typing import Awaitable, Callable, Dict, List, NamedTuple, Optional
+import logging
+import traceback
+
+import commonmark
 
 from telethon.errors import FloodWaitError
 
-from mautrix.types import RoomID, EventID, MessageEventContent
-from mautrix.bridge.commands import (HelpSection, CommandEvent as BaseCommandEvent,
-                                     CommandHandler as BaseCommandHandler,
-                                     CommandProcessor as BaseCommandProcessor,
-                                     CommandHandlerFunc, command_handler as base_command_handler)
-
+from ..types import MatrixRoomID, MatrixEventID
 from ..util import format_duration
-from .. import user as u, context as c, portal as po
+from .. import user as u, context as c
 
+command_handlers = {}  # type: Dict[str, CommandHandler]
 
-class HelpCacheKey(NamedTuple):
-    is_management: bool
-    is_portal: bool
-    puppet_whitelisted: bool
-    matrix_puppet_whitelisted: bool
-    is_admin: bool
-    is_logged_in: bool
-
+HelpSection = NamedTuple('HelpSection', [('name', str), ('order', int), ('description', str)])
 
+SECTION_GENERAL = HelpSection("General", 0, "")
 SECTION_AUTH = HelpSection("Authentication", 10, "")
 SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
 SECTION_PORTAL_MANAGEMENT = HelpSection("Portal management", 30, "")
@@ -44,80 +39,327 @@ SECTION_MISC = HelpSection("Miscellaneous", 40, "")
 SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 
-class CommandEvent(BaseCommandEvent):
-    sender: u.User
-    portal: po.Portal
+class HtmlEscapingRenderer(commonmark.HtmlRenderer):
+    def __init__(self, allow_html: bool = False):
+        super().__init__()
+        self.allow_html = allow_html
 
-    def __init__(self, processor: 'CommandProcessor', room_id: RoomID, event_id: EventID,
-                 sender: u.User, command: str, args: List[str], content: MessageEventContent,
-                 portal: Optional['po.Portal'], is_management: bool, has_bridge_bot: bool) -> None:
-        super().__init__(processor, room_id, event_id, sender, command, args, content,
-                         portal, is_management, has_bridge_bot)
-        self.bridge = processor.bridge
+    def lit(self, s):
+        if self.allow_html:
+            return super().lit(s)
+        return super().lit(s.replace("<", "&lt;").replace(">", "&gt;"))
+
+    def image(self, node, entering):
+        prev = self.allow_html
+        self.allow_html = True
+        super().image(node, entering)
+        self.allow_html = prev
+
+
+md_parser = commonmark.Parser()
+md_renderer = HtmlEscapingRenderer()
+
+
+def ensure_trailing_newline(s: str) -> str:
+    """Returns the passed string, but with a guaranteed trailing newline."""
+    return s + ("" if s[-1] == "\n" else "\n")
+
+
+class CommandEvent:
+    """Holds information about a command issued in a Matrix room.
+
+    When a Matrix command was issued to the bot, CommandEvent will hold
+    information regarding the event.
+
+    Attributes:
+        room_id: The id of the Matrix room in which the command was issued.
+        event_id: The id of the matrix event which contained the command.
+        sender: The user who issued the command.
+        command: The issued command.
+        args: Arguments given with the issued command.
+        is_management: Determines whether the room in which the command wa
+            issued is a management room.
+        is_portal: Determines whether the room in which the command was issued
+            is a portal.
+    """
+
+    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, event: MatrixEventID,
+                 sender: u.User, command: str, args: List[str], is_management: bool,
+                 is_portal: bool) -> None:
+        self.az = processor.az
+        self.log = processor.log
+        self.loop = processor.loop
         self.tgbot = processor.tgbot
         self.config = processor.config
         self.public_website = processor.public_website
+        self.command_prefix = processor.command_prefix
+        self.room_id = room
+        self.event_id = event
+        self.sender = sender
+        self.command = command
+        self.args = args
+        self.is_management = is_management
+        self.is_portal = is_portal
 
-    @property
-    def print_error_traceback(self) -> bool:
-        return self.sender.is_admin
+    def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True
+              ) -> Awaitable[Dict]:
+        """Write a reply to the room in which the command was issued.
 
-    async def get_help_key(self) -> HelpCacheKey:
-        return HelpCacheKey(self.is_management, self.portal is not None,
-                            self.sender.puppet_whitelisted, self.sender.matrix_puppet_whitelisted,
-                            self.sender.is_admin, await self.sender.is_logged_in())
+        Replaces occurences of "$cmdprefix" in the message with the command
+        prefix and replaces occurences of "$cmdprefix+sp " with the command
+        prefix if the command was not issued in a management room.
+        If allow_html and render_markdown are both False, the message will not
+        be rendered to html and sending of html is disabled.
+
+        Args:
+            message: The message to post in the room.
+            allow_html: Escape html in the message or don't render html at all
+                if markdown is disabled.
+            render_markdown: Use markdown formatting to render the passed
+                message to html.
+
+        Returns:
+            Handler for the message sending function.
+        """
+        message_cmd = self._replace_command_prefix(message)
+        html = self._render_message(message_cmd, allow_html=allow_html,
+                                    render_markdown=render_markdown)
+
+        return self.az.intent.send_notice(self.room_id, message_cmd, html=html)
+
+    def mark_read(self) -> Awaitable[Dict]:
+        """Marks the command as read by the bot."""
+        return self.az.intent.mark_read(self.room_id, self.event_id)
+
+    def _replace_command_prefix(self, message: str) -> str:
+        """Returns the string with the proper command prefix entered."""
+        message = message.replace(
+            "$cmdprefix+sp ", "" if self.is_management else f"{self.command_prefix} "
+        )
+        return message.replace("$cmdprefix", self.command_prefix)
+
+    @staticmethod
+    def _render_message(message: str, allow_html: bool, render_markdown: bool) -> Optional[str]:
+        """Renders the message as HTML.
+
+        Args:
+            allow_html: Flag to allow custom HTML in the message.
+            render_markdown: If true, markdown styling is applied to the message.
+
+        Returns:
+            The message rendered as HTML.
+            None is returned if no styled output is required.
+        """
+        html = ""
+        if render_markdown:
+            md_renderer.allow_html = allow_html
+            html = md_renderer.render(md_parser.parse(message))
+        elif allow_html:
+            html = message
+        return ensure_trailing_newline(html) if html else None
 
 
-class CommandHandler(BaseCommandHandler):
-    name: str
+class CommandHandler:
+    """A command which can be executed from a Matrix room.
 
-    needs_puppeting: bool
-    needs_matrix_puppeting: bool
+    The command manages its permission and help texts.
+    When called, it will check the permission of the command event and execute
+    the command or, in case of error, report back to the user.
 
-    def __init__(self, handler: Callable[[CommandEvent], Awaitable[EventID]],
+    Attributes:
+        needs_auth: Flag indicating if the sender is required to be logged in.
+        needs_puppeting: Flag indicating if the sender is required to use
+            Telegram puppeteering for this command.
+        needs_matrix_puppeting: Flag indicating if the sender is required to use
+            Matrix pupeteering.
+        needs_admin: Flag for whether only admin users can issue this command.
+        management_only: Whether the command can exclusively be issued in a
+            management room.
+        name: The name of this command.
+        help_section: Section of the help in which this command will appear.
+    """
+
+    def __init__(self, handler: Callable[[CommandEvent], Awaitable[Dict]], needs_auth: bool,
+                 needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
                  management_only: bool, name: str, help_text: str, help_args: str,
-                 help_section: HelpSection, needs_auth: bool, needs_puppeting: bool,
-                 needs_matrix_puppeting: bool, needs_admin: bool) -> None:
-        super().__init__(handler, management_only, name, help_text, help_args, help_section,
-                         needs_auth=needs_auth, needs_puppeting=needs_puppeting,
-                         needs_matrix_puppeting=needs_matrix_puppeting, needs_admin=needs_admin)
+                 help_section: HelpSection) -> None:
+        """
+        Args:
+            handler: The function handling the execution of this command.
+            needs_auth: Flag indicating if the sender is required to be logged in.
+            needs_puppeting: Flag indicating if the sender is required to use
+                Telegram puppeteering for this command.
+            needs_matrix_puppeting: Flag indicating if the sender is required to
+                use Matrix pupeteering.
+            needs_admin: Flag for whether only admin users can issue this command.
+            management_only: Whether the command can exclusively be issued
+                in a management room.
+            name: The name of this command.
+            help_text: The text displayed in the help for this command.
+            help_args: Help text for the arguments of this command.
+            help_section: Section of the help in which this command will appear.
+        """
+        self._handler = handler
+        self.needs_auth = needs_auth
+        self.needs_puppeting = needs_puppeting
+        self.needs_matrix_puppeting = needs_matrix_puppeting
+        self.needs_admin = needs_admin
+        self.management_only = management_only
+        self.name = name
+        self._help_text = help_text
+        self._help_args = help_args
+        self.help_section = help_section
 
     async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
-        if self.needs_puppeting and not evt.sender.puppet_whitelisted:
+        """Returns the reason why the command could not be issued.
+
+        Args:
+            evt: The event for which to get the error information.
+
+        Returns:
+            A string describing the error or None if there was no error.
+        """
+        if self.management_only and not evt.is_management:
+            return (f"`{evt.command}` is a restricted command: "
+                    "you may only run it in management rooms.")
+        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
             return "This command requires puppeting privileges."
         elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
             return "This command requires Matrix puppeting privileges."
-        return await super().get_permission_error(evt)
+        elif self.needs_admin and not evt.sender.is_admin:
+            return "This command requires administrator privileges."
+        elif self.needs_auth and not await evt.sender.is_logged_in():
+            return "This command requires you to be logged in."
+        return None
 
-    def has_permission(self, key: HelpCacheKey) -> bool:
-        return (super().has_permission(key) and
-                (not self.needs_puppeting or key.puppet_whitelisted) and
-                (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted))
+    def has_permission(self, is_management: bool, puppet_whitelisted: bool,
+                       matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
+        """Checks the permission for this command with the given status.
+
+        Args:
+            is_management: If the room in which the command will be issued is a
+                management room.
+            puppet_whitelisted: If the connected Telegram account puppet is
+                allowed to issue the command.
+            matrix_puppet_whitelisted: If the connected Matrix account puppet is
+                allowed to issue the command.
+            is_admin: If the issuing user is an admin.
+            is_logged_in: If the issuing user is logged in.
+
+        Returns:
+            True if a user with the given state is allowed to issue the
+            command.
+        """
+        return ((not self.management_only or is_management) and
+                (not self.needs_puppeting or puppet_whitelisted) and
+                (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
+                (not self.needs_admin or is_admin) and
+                (not self.needs_auth or is_logged_in))
+
+    async def __call__(self, evt: CommandEvent) -> Dict:
+        """Executes the command if evt was issued with proper rights.
+
+        Args:
+            evt: The CommandEvent for which to check permissions.
+
+        Returns:
+            The result of the command or the error message function.
+
+        Raises:
+            FloodWaitError
+        """
+        error = await self.get_permission_error(evt)
+        if error is not None:
+            return await evt.reply(error)
+        return await self._handler(evt)
+
+    @property
+    def has_help(self) -> bool:
+        """Returns true if this command has a help text."""
+        return bool(self.help_section) and bool(self._help_text)
+
+    @property
+    def help(self) -> str:
+        """Returns the help text to this command."""
+        return f"**{self.name}** {self._help_args} - {self._help_text}"
 
 
-def command_handler(_func: Optional[CommandHandlerFunc] = None, *, needs_auth: bool = True,
-                    needs_puppeting: bool = True, needs_matrix_puppeting: bool = False,
-                    needs_admin: bool = False, management_only: bool = False,
-                    name: Optional[str] = None, help_text: str = "", help_args: str = "",
-                    help_section: HelpSection = None) -> Callable[[CommandHandlerFunc],
-                                                                  CommandHandler]:
-    return base_command_handler(
-        _func, _handler_class=CommandHandler, name=name, help_text=help_text, help_args=help_args,
-        help_section=help_section, management_only=management_only, needs_auth=needs_auth,
-        needs_admin=needs_admin, needs_puppeting=needs_puppeting,
-        needs_matrix_puppeting=needs_matrix_puppeting)
+def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
+                    needs_auth: bool = True, needs_puppeting: bool = True,
+                    needs_matrix_puppeting: bool = False, needs_admin: bool = False,
+                    management_only: bool = False, name: Optional[str] = None,
+                    help_text: str = "", help_args: str = "", help_section: HelpSection = None
+                    ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
+                                  CommandHandler]:
+    def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:
+        actual_name = name or func.__name__.replace("_", "-")
+        handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
+                                 needs_admin, management_only, actual_name, help_text, help_args,
+                                 help_section)
+        command_handlers[handler.name] = handler
+        return handler
+
+    return decorator if _func is None else decorator(_func)
 
 
-class CommandProcessor(BaseCommandProcessor):
+class CommandProcessor:
+    """Handles the raw commands issued by a user to the Matrix bot."""
+    log = logging.getLogger("mau.commands")
+
     def __init__(self, context: c.Context) -> None:
-        super().__init__(event_class=CommandEvent, bridge=context.bridge)
-        self.tgbot = context.bot
+        self.az, self.config, self.loop, self.tgbot = context.core
         self.public_website = context.public_website
+        self.command_prefix = self.config["bridge.command_prefix"]
 
-    @staticmethod
-    async def _run_handler(handler: Callable[[CommandEvent], Awaitable[Any]], evt: CommandEvent
-                           ) -> Any:
+    async def handle(self, room: MatrixRoomID, event_id: MatrixEventID, sender: u.User,
+                     command: str, args: List[str], is_management: bool, is_portal: bool
+                     ) -> Optional[Dict]:
+        """Handles the raw commands issued by a user to the Matrix bot.
+
+        If the command is not known, it might be a followup command and is
+        delegated to a command handler registered for that purpose in the
+        senders command_status as "next".
+
+        Args:
+            room: ID of the Matrix room in which the command was issued.
+            event_id: ID of the event by which the command was issued.
+            sender: The sender who issued the command.
+            command: The issued command, case insensitive.
+            args: Arguments given with the command.
+            is_management: Whether the room is a management room.
+            is_portal: Whether the room is a portal.
+
+        Returns:
+            The result of the error message function or None if no error
+            occured. Unknown and delegated commands do not count as errors.
+        """
+        if not command_handlers or "unknown-command" not in command_handlers:
+            raise ValueError("command_handlers are not properly initialized.")
+
+        evt = CommandEvent(self, room, event_id, sender, command, args, is_management, is_portal)
+        orig_command = command
+        command = command.lower()
         try:
-            return await handler(evt)
+            handler = command_handlers[command]
+        except KeyError:
+            if sender.command_status and "next" in sender.command_status:
+                args.insert(0, orig_command)
+                evt.command = ""
+                handler = sender.command_status["next"]
+            else:
+                handler = command_handlers["unknown-command"]
+        try:
+            await handler(evt)
         except FloodWaitError as e:
             return await evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
+        except Exception:
+            self.log.exception("Unhandled error while handling command "
+                               f"{evt.command} {' '.join(args)} from {sender.mxid}")
+            if evt.sender.is_admin and evt.is_management:
+                return await evt.reply("Unhandled error while handling command:\n\n"
+                                       "```traceback\n"
+                                       f"{traceback.format_exc()}"
+                                       "```")
+            return await evt.reply("Unhandled error while handling command. "
+                                   "Check logs for more details.")
+        return None

mautrix_telegram/commands/manhole.py

@@ -1,128 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Set, Callable
-import asyncio
-import sys
-import os
-
-from attr import dataclass
-
-from telethon import __version__ as __telethon_version__
-
-from mautrix import __version__ as __mautrix_version__
-from mautrix.types import UserID
-from mautrix.errors import MatrixConnectionError
-from mautrix.util.manhole import start_manhole
-
-from .. import __version__
-from . import command_handler, CommandEvent, SECTION_ADMIN
-
-
-@dataclass
-class ManholeState:
-    server: asyncio.AbstractServer
-    opened_by: UserID
-    close: Callable[[], None]
-    whitelist: Set[int]
-
-
-@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
-                 help_text="Open a manhole into the bridge.", help_args="<_uid..._>")
-async def open_manhole(evt: CommandEvent) -> None:
-    if not evt.config["manhole.enabled"]:
-        await evt.reply("The manhole has been disabled in the config.")
-        return
-    elif len(evt.args) == 0:
-        await evt.reply("**Usage:** `$cmdprefix+sp open-manhole <uid...>`")
-        return
-
-    whitelist = set()
-    whitelist_whitelist = evt.config["manhole.whitelist"]
-    for arg in evt.args:
-        try:
-            uid = int(arg)
-        except ValueError:
-            await evt.reply(f"{arg} is not an integer.")
-            return
-        if whitelist_whitelist and uid not in whitelist_whitelist:
-            await evt.reply(f"{uid} is not in the list of allowed UIDs.")
-            return
-        whitelist.add(uid)
-
-    if evt.bridge.manhole:
-        added = [uid for uid in whitelist
-                 if uid not in evt.bridge.manhole.whitelist]
-        evt.bridge.manhole.whitelist |= set(added)
-        if len(added) == 0:
-            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
-                            " and all the given UIDs are already whitelisted.")
-        else:
-            added_str = (f"{', '.join(str(uid) for uid in added[:-1])} and {added[-1]}"
-                         if len(added) > 1 else added[0])
-            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
-                            f". Added {added_str} to the whitelist.")
-            evt.log.info(f"{evt.sender.mxid} added {added_str} to the manhole whitelist.")
-        return
-
-    from ..portal import Portal
-    from ..puppet import Puppet
-    from ..user import User
-    namespace = {
-        "bridge": evt.bridge,
-        "User": User,
-        "Portal": Portal,
-        "Puppet": Puppet,
-    }
-    banner = (f"Python {sys.version} on {sys.platform}\n"
-              f"mautrix-telegram {__version__} with mautrix-python {__mautrix_version__} "
-              f"and Telethon {__telethon_version__}\n\nManhole opened by {evt.sender.mxid}\n")
-    path = evt.config["manhole.path"]
-
-    wl_list = list(whitelist)
-    whitelist_str = (f"{', '.join(str(uid) for uid in wl_list[:-1])} and {wl_list[-1]}"
-                     if len(wl_list) > 1 else wl_list[0])
-    evt.log.info(f"{evt.sender.mxid} opened a manhole with {whitelist_str} whitelisted.")
-    server, close = await start_manhole(path=path, banner=banner, namespace=namespace,
-                                        loop=evt.loop, whitelist=whitelist)
-    evt.bridge.manhole = ManholeState(server=server, opened_by=evt.sender.mxid, close=close,
-                                      whitelist=whitelist)
-    plrl = "s" if len(whitelist) != 1 else ""
-    await evt.reply(f"Opened manhole at unix://{path} with UID{plrl} {whitelist_str} whitelisted")
-    await server.wait_closed()
-    evt.bridge.manhole = None
-    try:
-        os.unlink(path)
-    except FileNotFoundError:
-        pass
-    evt.log.info(f"{evt.sender.mxid}'s manhole was closed.")
-    try:
-        await evt.reply("Your manhole was closed.")
-    except (AttributeError, MatrixConnectionError) as e:
-        evt.log.warning(f"Failed to send manhole close notification: {e}")
-
-
-@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
-                 help_text="Close an open manhole.")
-async def close_manhole(evt: CommandEvent) -> None:
-    if not evt.bridge.manhole:
-        await evt.reply("There is no open manhole.")
-        return
-
-    opened_by = evt.bridge.manhole.opened_by
-    evt.bridge.manhole.close()
-    evt.bridge.manhole = None
-    if opened_by != evt.sender.mxid:
-        await evt.reply(f"Closed manhole opened by {opened_by}")

mautrix_telegram/commands/matrix_auth.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,17 +14,17 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.types import EventID
-from mautrix.bridge import InvalidAccessToken, OnlyLoginSelf
+from typing import Dict, Optional
 
 from . import command_handler, CommandEvent, SECTION_AUTH
 from .. import puppet as pu
 
 
 @command_handler(needs_auth=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH, help_text="Revert your Telegram account's Matrix "
-                                                      "puppet to use the default Matrix account.")
-async def logout_matrix(evt: CommandEvent) -> EventID:
+                 help_section=SECTION_AUTH,
+                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
+                           "account.")
+async def logout_matrix(evt: CommandEvent) -> Optional[Dict]:
     puppet = pu.Puppet.get(evt.sender.tgid)
     if not puppet.is_real_user:
         return await evt.reply("You are not logged in with your Matrix account.")
@@ -35,7 +36,7 @@ async def logout_matrix(evt: CommandEvent) -> EventID:
                  help_section=SECTION_AUTH,
                  help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
                            "account.")
-async def login_matrix(evt: CommandEvent) -> EventID:
+async def login_matrix(evt: CommandEvent) -> Optional[Dict]:
     puppet = pu.Puppet.get(evt.sender.tgid)
     if puppet.is_real_user:
         return await evt.reply("You have already logged in with your Matrix account. "
@@ -70,44 +71,31 @@ async def login_matrix(evt: CommandEvent) -> EventID:
 @command_handler(needs_auth=True, needs_matrix_puppeting=True,
                  help_section=SECTION_AUTH,
                  help_text="Pings the server with the stored matrix authentication.")
-async def ping_matrix(evt: CommandEvent) -> EventID:
+async def ping_matrix(evt: CommandEvent) -> Optional[Dict]:
     puppet = pu.Puppet.get(evt.sender.tgid)
     if not puppet.is_real_user:
         return await evt.reply("You are not logged in with your Matrix account.")
-    try:
-        await puppet.start()
-    except InvalidAccessToken:
+    resp = await puppet.init_custom_mxid()
+    if resp == pu.PuppetError.InvalidAccessToken:
         return await evt.reply("Your access token is invalid.")
-    return await evt.reply("Your Matrix login is working.")
+    elif resp == pu.PuppetError.Success:
+        return await evt.reply("Your Matrix login is working.")
+    return await evt.reply(f"Unknown response while checking your Matrix login: {resp}.")
 
 
-@command_handler(needs_auth=True, needs_matrix_puppeting=True, help_section=SECTION_AUTH,
-                 help_text="Clear the Matrix sync token stored for your custom puppet.")
-async def clear_cache_matrix(evt: CommandEvent) -> EventID:
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if not puppet.is_real_user:
-        return await evt.reply("You are not logged in with your Matrix account.")
-    try:
-        puppet.stop()
-        puppet.next_batch = None
-        await puppet.start()
-    except InvalidAccessToken:
-        return await evt.reply("Your access token is invalid.")
-    return await evt.reply("Cleared cache successfully.")
-
-
-async def enter_matrix_token(evt: CommandEvent) -> EventID:
+async def enter_matrix_token(evt: CommandEvent) -> Dict:
     evt.sender.command_status = None
 
     puppet = pu.Puppet.get(evt.sender.tgid)
     if puppet.is_real_user:
         return await evt.reply("You have already logged in with your Matrix account. "
                                "Log out with `$cmdprefix+sp logout-matrix` first.")
-    try:
-        await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
-    except OnlyLoginSelf:
+
+    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    if resp == pu.PuppetError.OnlyLoginSelf:
         return await evt.reply("You can only log in as your own Matrix user.")
-    except InvalidAccessToken:
+    elif resp == pu.PuppetError.InvalidAccessToken:
         return await evt.reply("Failed to verify access token.")
-    return await evt.reply("Replaced your Telegram account's Matrix puppet "
-                           f"with {puppet.custom_mxid}.")
+    assert resp == pu.PuppetError.Success, "Encountered an unhandled PuppetError."
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")

mautrix_telegram/commands/meta.py

@@ -0,0 +1,72 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Optional, Tuple
+
+from . import command_handler, CommandEvent, _command_handlers, SECTION_GENERAL
+from .handler import HelpSection
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Cancel an ongoing action (such as login)")
+async def cancel(evt: CommandEvent) -> Optional[Dict]:
+    if evt.sender.command_status:
+        action = evt.sender.command_status["action"]
+        evt.sender.command_status = None
+        return await evt.reply(f"{action} cancelled.")
+    else:
+        return await evt.reply("No ongoing command.")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False)
+async def unknown_command(evt: CommandEvent) -> Optional[Dict]:
+    return await evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
+
+
+help_cache = {}  # type: Dict[Tuple[bool, bool, bool, bool, bool], str]
+
+
+async def _get_help_text(evt: CommandEvent) -> str:
+    cache_key = (evt.is_management, evt.sender.puppet_whitelisted,
+                 evt.sender.matrix_puppet_whitelisted, evt.sender.is_admin,
+                 await evt.sender.is_logged_in())
+    if cache_key not in help_cache:
+        help_sections = {}  # type: Dict[HelpSection, List[str]]
+        for handler in _command_handlers.values():
+            if handler.has_help and handler.has_permission(*cache_key):
+                help_sections.setdefault(handler.help_section, [])
+                help_sections[handler.help_section].append(handler.help + "  ")
+        help_sorted = sorted(help_sections.items(), key=lambda item: item[0].order)
+        helps = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help_sorted]
+        help_cache[cache_key] = "\n".join(helps)
+    return help_cache[cache_key]
+
+
+def _get_management_status(evt: CommandEvent) -> str:
+    if evt.is_management:
+        return "This is a management room: prefixing commands with `$cmdprefix` is not required."
+    elif evt.is_portal:
+        return ("**This is a portal room**: you must always prefix commands with `$cmdprefix`.\n"
+                "Management commands will not be sent to Telegram.")
+    return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
+
+
+@command_handler(name="help", needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Show this help message.")
+async def help_cmd(evt: CommandEvent) -> Optional[Dict]:
+    return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))

mautrix_telegram/commands/portal/admin.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,19 +14,42 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
 import asyncio
 
-from mautrix.types import EventID
+from mautrix_appservice import MatrixRequestError
 
 from ... import portal as po, puppet as pu, user as u
 from .. import command_handler, CommandEvent, SECTION_ADMIN
 
 
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
+                 help_section=SECTION_ADMIN,
+                 help_args="<_level_> [_mxid_]",
+                 help_text="Set a temporary power level without affecting Telegram.")
+async def set_power_level(evt: CommandEvent) -> Dict:
+    try:
+        level = int(evt.args[0])
+    except KeyError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp set-pl <level> [mxid]`")
+    except ValueError:
+        return await evt.reply("The level must be an integer.")
+    levels = await evt.az.intent.get_power_levels(evt.room_id)
+    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
+    levels["users"][mxid] = level
+    try:
+        await evt.az.intent.set_power_levels(evt.room_id, levels)
+    except MatrixRequestError:
+        evt.log.exception("Failed to set power level.")
+        return await evt.reply("Failed to set power level.")
+    return {}
+
+
 @command_handler(needs_admin=True, needs_auth=False,
                  help_section=SECTION_ADMIN,
                  help_args="<`portal`|`puppet`|`user`>",
                  help_text="Clear internal bridge caches")
-async def clear_db_cache(evt: CommandEvent) -> EventID:
+async def clear_db_cache(evt: CommandEvent) -> Dict:
     try:
         section = evt.args[0].lower()
     except IndexError:
@@ -39,8 +63,9 @@ async def clear_db_cache(evt: CommandEvent) -> EventID:
         for puppet in pu.Puppet.by_custom_mxid.values():
             puppet.sync_task.cancel()
         pu.Puppet.by_custom_mxid = {}
-        await asyncio.gather(*[puppet.try_start() for puppet in pu.Puppet.all_with_custom_mxid()],
-                             loop=evt.loop)
+        await asyncio.gather(
+            *[puppet.init_custom_mxid() for puppet in pu.Puppet.all_with_custom_mxid()],
+            loop=evt.loop)
         await evt.reply("Cleared puppet cache and restarted custom puppet syncers")
     elif section == "user":
         u.User.by_mxid = {
@@ -56,7 +81,7 @@ async def clear_db_cache(evt: CommandEvent) -> EventID:
                  help_section=SECTION_ADMIN,
                  help_args="[_mxid_]",
                  help_text="Reload and reconnect a user")
-async def reload_user(evt: CommandEvent) -> EventID:
+async def reload_user(evt: CommandEvent) -> Dict:
     if len(evt.args) > 0:
         mxid = evt.args[0]
     else:
@@ -64,7 +89,7 @@ async def reload_user(evt: CommandEvent) -> EventID:
     user = u.User.get_by_mxid(mxid, create=False)
     if not user:
         return await evt.reply("User not found")
-    puppet = await pu.Puppet.get_by_custom_mxid(mxid)
+    puppet = pu.Puppet.get_by_custom_mxid(mxid)
     if puppet:
         puppet.sync_task.cancel()
     await user.stop()
@@ -72,5 +97,5 @@ async def reload_user(evt: CommandEvent) -> EventID:
     user = u.User.get_by_mxid(mxid)
     await user.ensure_started()
     if puppet:
-        await puppet.start()
-    return await evt.reply(f"Reloaded and reconnected {user.mxid} (telegram: {user.human_tg_id})")
+        await puppet.init_custom_mxid()
+    await evt.reply(f"Reloaded and reconnected {user.mxid} (telegram: {user.human_tg_id})")

mautrix_telegram/commands/portal/bridge.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,14 +14,13 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Tuple, Awaitable
+from typing import Dict, Optional, Tuple, Coroutine
 import asyncio
 
 from telethon.tl.types import ChatForbidden, ChannelForbidden
 
-from mautrix.types import EventID, RoomID
-
-from ...types import TelegramID
+from ...types import MatrixRoomID, TelegramID
+from ...util import ignore_coro
 from ... import portal as po
 from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
 from .util import user_has_power_level, get_initial_state
@@ -32,7 +32,7 @@ from .util import user_has_power_level, get_initial_state
                  help_text="Bridge the current Matrix room to the Telegram chat with the given "
                            "ID. The ID must be the prefixed version that you get with the `/id` "
                            "command of the Telegram-side bot.")
-async def bridge(evt: CommandEvent) -> EventID:
+async def bridge(evt: CommandEvent) -> Dict:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** "
                                "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
@@ -40,7 +40,7 @@ async def bridge(evt: CommandEvent) -> EventID:
     if evt.args[0] == "--usebot" and evt.sender.is_admin:
         force_use_bot = True
         evt.args = evt.args[1:]
-    room_id = RoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
+    room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
     that_this = "This" if room_id == evt.room_id else "That"
 
     portal = po.Portal.get_by_mxid(room_id)
@@ -65,7 +65,7 @@ async def bridge(evt: CommandEvent) -> EventID:
                                "Bridging private chats to existing rooms is not allowed.")
 
     portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
-    if not portal.allow_bridging:
+    if not portal.allow_bridging():
         return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
                                "If you're the bridge admin, try "
                                "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
@@ -105,17 +105,19 @@ async def bridge(evt: CommandEvent) -> EventID:
 
 
 async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
-                                            ) -> Tuple[bool, Optional[Awaitable[None]]]:
+                                            ) -> Tuple[bool, Optional[Coroutine[None, None, None]]]:
     if not portal.mxid:
         await evt.reply("The portal seems to have lost its Matrix room between you"
                         "calling `$cmdprefix+sp bridge` and this command.\n\n"
                         "Continuing without touching previous Matrix room...")
         return True, None
     elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_portal("Portal deleted (moving to another room)", delete=False)
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Portal deleted (moving to another room)")
     elif evt.args[0] == "unbridge-and-continue":
-        return True, portal.cleanup_portal("Room unbridged (portal moving to another room)",
-                                           puppets_only=True, delete=False)
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Room unbridged (portal moving to another room)",
+                                         puppets_only=True)
     else:
         await evt.reply(
             "The chat you were trying to bridge already has a Matrix portal room.\n\n"
@@ -126,7 +128,7 @@ async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Porta
         return False, None
 
 
-async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
+async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
     status = evt.sender.command_status
     try:
         portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
@@ -136,15 +138,12 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
         return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
                                "This shouldn't happen unless you're messing with the command "
                                "handler code.")
-
-    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
-
     if "mxid" in status:
         ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
         if not ok:
             return None
         elif coro:
-            asyncio.ensure_future(coro, loop=evt.loop)
+            ignore_coro(asyncio.ensure_future(coro, loop=evt.loop))
             await evt.reply("Cleaning up previous portal room...")
     elif portal.mxid:
         evt.sender.command_status = None
@@ -156,13 +155,7 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
                                "`$cmdprefix+sp cancel` to cancel.")
 
     evt.sender.command_status = None
-    async with portal._room_create_lock:
-        await _locked_confirm_bridge(evt, portal=portal, room_id=bridge_to_mxid,
-                                     is_logged_in=is_logged_in)
-
-
-async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id: RoomID,
-                                 is_logged_in: bool) -> Optional[EventID]:
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)
@@ -180,14 +173,15 @@ async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id
         else:
             return await evt.reply("The bot doesn't seem to be in that chat.")
 
-    portal.mxid = room_id
-    portal.by_mxid[portal.mxid] = portal
-    (portal.title, portal.about, levels,
-     portal.encrypted) = await get_initial_state(evt.az.intent, evt.room_id)
-    portal.photo_id = ""
-    await portal.save()
+    direct = False
 
-    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
-                          loop=evt.loop)
+    portal.mxid = bridge_to_mxid
+    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    portal.photo_id = ""
+    portal.save()
+
+    ignore_coro(asyncio.ensure_future(portal.update_matrix_room(user, entity, direct,
+                                                                levels=levels),
+                                      loop=evt.loop))
 
     return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,19 +14,15 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Any
+from typing import Dict, Awaitable
 from io import StringIO
 
-from ruamel.yaml import YAMLError
-
-from mautrix.util.config import yaml
-from mautrix.types import EventID
-
+from ...config import yaml
 from ... import portal as po, util
 from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
 
 
-@command_handler(needs_auth=False, help_section=SECTION_PORTAL_MANAGEMENT,
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
                  help_text="View or change per-portal settings.",
                  help_args="<`help`|_subcommand_> [...]")
 async def config(evt: CommandEvent) -> None:
@@ -45,16 +42,8 @@ async def config(evt: CommandEvent) -> None:
         await config_view(evt, portal)
         return
 
-    if not await portal.can_user_perform(evt.sender, "config"):
-        await evt.reply("You do not have the permissions to configure this room.")
-        return
-
     key = evt.args[1] if len(evt.args) > 1 else None
-    try:
-        value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
-    except YAMLError as e:
-        await evt.reply(f"Invalid value provided. Values must be valid YAML.\n{e}")
-        return
+    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
     if cmd == "set":
         await config_set(evt, portal, key, value)
     elif cmd == "unset":
@@ -63,10 +52,10 @@ async def config(evt: CommandEvent) -> None:
         await config_add_del(evt, portal, key, value, cmd)
     else:
         return
-    await portal.save()
+    portal.save()
 
 
-def config_help(evt: CommandEvent) -> Awaitable[EventID]:
+def config_help(evt: CommandEvent) -> Awaitable[Dict]:
     return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
 
 * **help** - View this help text.
@@ -79,12 +68,15 @@ def config_help(evt: CommandEvent) -> Awaitable[EventID]:
 """)
 
 
-def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[EventID]:
-    return evt.reply(f"Room-specific config:\n{_str_value(portal.local_config).rstrip()}")
+def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump(portal.local_config, stream)
+    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
 
 
-def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
-    value = _str_value({
+def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump({
         "bridge_notices": {
             "default": evt.config["bridge.bridge_notices.default"],
             "exceptions": evt.config["bridge.bridge_notices.exceptions"],
@@ -92,34 +84,23 @@ def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
         "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
         "inline_images": evt.config["bridge.inline_images"],
         "message_formats": evt.config["bridge.message_formats"],
-        "emote_format": evt.config["bridge.emote_format"],
         "state_event_formats": evt.config["bridge.state_event_formats"],
         "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
-    })
-    return evt.reply(f"Bridge instance wide config:\n{value.rstrip()}")
+    }, stream)
+    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
 
 
-def _str_value(value: Any) -> str:
-    stream = StringIO()
-    yaml.dump(value, stream)
-    value_str = stream.getvalue()
-    if "\n" in value_str:
-        return f"\n```yaml\n{value_str}\n```\n"
-    else:
-        return f"`{value_str}`"
-
-
-def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: Any) -> Awaitable[EventID]:
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[Dict]:
     if not key or value is None:
         return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
     elif util.recursive_set(portal.local_config, key, value):
-        return evt.reply(f"Successfully set the value of `{key}` to {_str_value(value)}".rstrip())
+        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
     else:
         return evt.reply(f"Failed to set value of `{key}`. "
                          "Does the path contain non-map types?")
 
 
-def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[EventID]:
+def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Dict]:
     if not key:
         return evt.reply(f"**Usage:** `$cmdprefix+sp config unset <key>`")
     elif util.recursive_del(portal.local_config, key):
@@ -129,7 +110,7 @@ def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Ev
 
 
 def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
-                   ) -> Awaitable[EventID]:
+                   ) -> Awaitable[Dict]:
     if not key or value is None:
         return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
 
@@ -141,11 +122,11 @@ def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, c
         return evt.reply("`{key}` does not seem to be an array.")
     elif cmd == "add":
         if value in arr:
-            return evt.reply(f"The array at `{key}` already contains {_str_value(value)}".rstrip())
+            return evt.reply(f"The array at `{key}` already contains `{value}`.")
         arr.append(value)
-        return evt.reply(f"Successfully added {_str_value(value)} to the array at `{key}`")
+        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
     else:
         if value not in arr:
-            return evt.reply(f"The array at `{key}` does not contain {_str_value(value)}")
+            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
         arr.remove(value)
-        return evt.reply(f"Successfully removed {_str_value(value)} from the array at `{key}`")
+        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.types import EventID
+from typing import Dict
 
 from ... import portal as po
 from ...types import TelegramID
@@ -25,10 +26,10 @@ from .util import user_has_power_level, get_initial_state
                  help_args="[_type_]",
                  help_text="Create a Telegram chat of the given type for the current Matrix room. "
                            "The type is either `group`, `supergroup` or `channel` (defaults to "
-                           "`supergroup`).")
-async def create(evt: CommandEvent) -> EventID:
-    type = evt.args[0] if len(evt.args) > 0 else "supergroup"
-    if type not in ("chat", "group", "supergroup", "channel"):
+                           "`group`).")
+async def create(evt: CommandEvent) -> Dict:
+    type = evt.args[0] if len(evt.args) > 0 else "group"
+    if type not in {"chat", "group", "supergroup", "channel"}:
         return await evt.reply(
             "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
 
@@ -38,7 +39,7 @@ async def create(evt: CommandEvent) -> EventID:
     if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
         return await evt.reply("You do not have the permissions to bridge this room.")
 
-    title, about, levels, encrypted = await get_initial_state(evt.az.intent, evt.room_id)
+    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
     if not title:
         return await evt.reply("Please set a title before creating a Telegram chat.")
 
@@ -50,11 +51,11 @@ async def create(evt: CommandEvent) -> EventID:
         "group": "chat",
     }[type]
 
-    portal = po.Portal(tgid=TelegramID(0), peer_type=type, mxid=evt.room_id,
-                       title=title, about=about, encrypted=encrypted)
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+                       mxid=evt.room_id, title=title, about=about)
     try:
         await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
     except ValueError as e:
-        await portal.delete()
+        portal.delete()
         return await evt.reply(e.args[0])
     return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/filter.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.types import EventID
+from typing import Dict, Optional
 
 from ... import portal as po
 from .. import command_handler, CommandEvent, SECTION_ADMIN
@@ -24,7 +25,7 @@ from .. import command_handler, CommandEvent, SECTION_ADMIN
                  help_args="<`whitelist`|`blacklist`>",
                  help_text="Change whether the bridge will allow or disallow bridging rooms by "
                            "default.")
-async def filter_mode(evt: CommandEvent) -> EventID:
+async def filter_mode(evt: CommandEvent) -> Dict:
     try:
         mode = evt.args[0]
         if mode not in ("whitelist", "blacklist"):
@@ -49,7 +50,7 @@ async def filter_mode(evt: CommandEvent) -> EventID:
                  help_section=SECTION_ADMIN,
                  help_args="<`whitelist`|`blacklist`> <_chat ID_>",
                  help_text="Allow or disallow bridging a specific chat.")
-async def edit_filter(evt: CommandEvent) -> EventID:
+async def edit_filter(evt: CommandEvent) -> Optional[Dict]:
     try:
         action = evt.args[0]
         if action not in ("whitelist", "blacklist", "add", "remove"):
@@ -91,5 +92,4 @@ async def edit_filter(evt: CommandEvent) -> EventID:
         filter_id_list.remove(filter_id)
         save()
         return await evt.reply(f"Chat ID removed from {mode}.")
-    else:
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
+    return None

mautrix_telegram/commands/portal/misc.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,12 +14,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from telethon.tl.functions.channels import GetFullChannelRequest
-from telethon.tl.functions.messages import GetFullChatRequest
-from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
-                             UsernameNotModifiedError, UsernameOccupiedError, RPCError)
+from typing import Dict
 
-from mautrix.types import EventID
+from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
+                             UsernameNotModifiedError, UsernameOccupiedError)
 
 from ... import portal as po
 from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT, SECTION_MISC
@@ -28,47 +27,21 @@ from .util import user_has_power_level
 @command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
                  help_section=SECTION_MISC,
                  help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.")
-async def sync_state(evt: CommandEvent) -> EventID:
+async def sync_state(evt: CommandEvent) -> Dict:
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
     elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
         return await evt.reply(f"You do not have the permissions to synchronize this room.")
 
-    await portal.main_intent.get_joined_members(portal.mxid)
+    await portal.sync_matrix_members()
     await evt.reply("Synchronization complete")
 
 
-@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
-                 help_section=SECTION_MISC)
-async def sync_full(evt: CommandEvent) -> EventID:
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if len(evt.args) > 0 and evt.args[0] == "--usebot" and evt.sender.is_admin:
-        src = evt.tgbot
-    else:
-        src = evt.tgbot if await evt.sender.needs_relaybot(portal) else evt.sender
-
-    try:
-        if portal.peer_type == "channel":
-            res = await src.client(GetFullChannelRequest(portal.peer))
-        elif portal.peer_type == "chat":
-            res = await src.client(GetFullChatRequest(portal.tgid))
-        else:
-            return await evt.reply("This is not a channel or chat portal.")
-    except (ValueError, RPCError):
-        return await evt.reply("Failed to get portal info from Telegram.")
-
-    await portal.update_matrix_room(src, res.full_chat)
-    return await evt.reply("Portal synced successfully.")
-
-
 @command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
                  help_section=SECTION_MISC,
                  help_text="Get the ID of the Telegram chat where this room is bridged.")
-async def get_id(evt: CommandEvent) -> EventID:
+async def get_id(evt: CommandEvent) -> Dict:
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
@@ -82,7 +55,7 @@ async def get_id(evt: CommandEvent) -> EventID:
 
 @command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
                  help_text="Get a Telegram invite link to the current chat.")
-async def invite_link(evt: CommandEvent) -> EventID:
+async def invite_link(evt: CommandEvent) -> Dict:
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
@@ -101,7 +74,7 @@ async def invite_link(evt: CommandEvent) -> EventID:
 
 @command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
                  help_text="Upgrade a normal Telegram group to a supergroup.")
-async def upgrade(evt: CommandEvent) -> EventID:
+async def upgrade(evt: CommandEvent) -> Dict:
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
@@ -123,7 +96,7 @@ async def upgrade(evt: CommandEvent) -> EventID:
                  help_args="<_name_|`-`>",
                  help_text="Change the username of a supergroup/channel. "
                            "To disable, use a dash (`-`) as the name.")
-async def group_name(evt: CommandEvent) -> EventID:
+async def group_name(evt: CommandEvent) -> Dict:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
 

mautrix_telegram/commands/portal/unbridge.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -15,15 +16,16 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Dict, Callable, Optional
 
-from mautrix.types import RoomID, EventID
-
+from ...types import MatrixRoomID
 from ... import portal as po
 from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
 from .util import user_has_power_level
 
 
-async def _get_portal_and_check_permission(evt: CommandEvent) -> Optional[po.Portal]:
-    room_id = RoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
+async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
+                                           action: Optional[str] = None
+                                           ) -> Optional[po.Portal]:
+    room_id = MatrixRoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
 
     portal = po.Portal.get_by_mxid(room_id)
     if not portal:
@@ -31,21 +33,16 @@ async def _get_portal_and_check_permission(evt: CommandEvent) -> Optional[po.Por
         await evt.reply(f"{that_this} is not a portal room.")
         return None
 
-    if portal.peer_type == "user":
-        if portal.tg_receiver != evt.sender.tgid:
-            await evt.reply("You do not have the permissions to unbridge that portal.")
-            return None
-        return portal
-
-    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-        await evt.reply("You do not have the permissions to unbridge that portal.")
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
+        action = action or f"{permission.replace('_', ' ')}s"
+        await evt.reply(f"You do not have the permissions to {action} that portal.")
         return None
     return portal
 
 
 def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
                                 completed_message: str) -> Dict:
-    async def post_confirm(confirm) -> Optional[EventID]:
+    async def post_confirm(confirm) -> Optional[Dict]:
         confirm.sender.command_status = None
         if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
             await function()
@@ -66,8 +63,8 @@ def _get_portal_murder_function(action: str, room_id: str, function: Callable, c
                  help_text="Remove all users from the current portal room and forget the portal. "
                            "Only works for group chats; to delete a private chat portal, simply "
                            "leave the room.")
-async def delete_portal(evt: CommandEvent) -> Optional[EventID]:
-    portal = await _get_portal_and_check_permission(evt)
+async def delete_portal(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
     if not portal:
         return None
 
@@ -87,8 +84,8 @@ async def delete_portal(evt: CommandEvent) -> Optional[EventID]:
 @command_handler(needs_auth=False, needs_puppeting=False,
                  help_section=SECTION_PORTAL_MANAGEMENT,
                  help_text="Remove puppets from the current portal room and forget the portal.")
-async def unbridge(evt: CommandEvent) -> Optional[EventID]:
-    portal = await _get_portal_and_check_permission(evt)
+async def unbridge(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
     if not portal:
         return None
 

mautrix_telegram/commands/portal/util.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,50 +14,43 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional
+from typing import Dict, Tuple
 
-from mautrix.errors import MatrixRequestError
-from mautrix.appservice import IntentAPI
-from mautrix.types import RoomID, EventType, PowerLevelStateEventContent
+from mautrix_appservice import MatrixRequestError, IntentAPI
 
 from ... import user as u
 
-OptStr = Optional[str]
 
-
-async def get_initial_state(intent: IntentAPI, room_id: RoomID
-                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent], bool]:
-    state = await intent.get_state(room_id)
-    title: OptStr = None
-    about: OptStr = None
-    levels: Optional[PowerLevelStateEventContent] = None
-    encrypted: bool = False
+async def get_initial_state(intent: IntentAPI, room_id: str) -> Tuple[str, str, Dict]:
+    state = await intent.get_room_state(room_id)
+    title = None
+    about = None
+    levels = None
     for event in state:
         try:
-            if event.type == EventType.ROOM_NAME:
-                title = event.content.name
-            elif event.type == EventType.ROOM_TOPIC:
-                about = event.content.topic
-            elif event.type == EventType.ROOM_POWER_LEVELS:
-                levels = event.content
-            elif event.type == EventType.ROOM_CANONICAL_ALIAS:
-                title = title or event.content.canonical_alias
-            elif event.type == EventType.ROOM_ENCRYPTION:
-                encrypted = True
+            if event["type"] == "m.room.name":
+                title = event["content"]["name"]
+            elif event["type"] == "m.room.topic":
+                about = event["content"]["topic"]
+            elif event["type"] == "m.room.power_levels":
+                levels = event["content"]
+            elif event["type"] == "m.room.canonical_alias":
+                title = title or event["content"]["alias"]
         except KeyError:
             # Some state event probably has empty content
             pass
-    return title, about, levels, encrypted
+    return title, about, levels
 
 
-async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.User,
-                               event: str) -> bool:
+async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50
+                               ) -> bool:
     if sender.is_admin:
         return True
     # Make sure the state store contains the power levels.
     try:
-        await intent.get_power_levels(room_id)
+        await intent.get_power_levels(room)
     except MatrixRequestError:
         return False
-    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
-    return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)
+    return intent.state_store.has_power_level(room, sender.mxid,
+                                              event=f"net.maunium.telegram.{event}",
+                                              default=default)

mautrix_telegram/commands/telegram/account.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from typing import Dict, Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
                              HashInvalidError, AuthKeyError, FirstNameInvalidError)
@@ -21,8 +22,6 @@ from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
                                            ResetAuthorizationRequest, UpdateProfileRequest)
 
-from mautrix.types import EventID
-
 from .. import command_handler, CommandEvent, SECTION_AUTH
 
 
@@ -30,7 +29,7 @@ from .. import command_handler, CommandEvent, SECTION_AUTH
                  help_section=SECTION_AUTH,
                  help_args="<_new username_>",
                  help_text="Change your Telegram username.")
-async def username(evt: CommandEvent) -> EventID:
+async def username(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp username <new username>`")
     if evt.sender.is_bot:
@@ -56,7 +55,7 @@ async def username(evt: CommandEvent) -> EventID:
 
 @command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
                  help_text="Change your Telegram displayname.")
-async def displayname(evt: CommandEvent) -> EventID:
+async def displayname(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
     if evt.sender.is_bot:
@@ -70,7 +69,7 @@ async def displayname(evt: CommandEvent) -> EventID:
     except FirstNameInvalidError:
         return await evt.reply("Invalid first name")
     await evt.sender.update_info()
-    return await evt.reply("Displayname updated")
+    await evt.reply("Displayname updated")
 
 
 def _format_session(sess: Authorization) -> str:
@@ -84,7 +83,7 @@ def _format_session(sess: Authorization) -> str:
                  help_section=SECTION_AUTH,
                  help_args="<`list`|`terminate`> [_hash_]",
                  help_text="View or delete other Telegram sessions.")
-async def session(evt: CommandEvent) -> EventID:
+async def session(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")
     elif evt.sender.is_bot:

mautrix_telegram/commands/telegram/auth.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -15,37 +16,22 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Any, Dict, Optional
 import asyncio
-import io
 
-from telethon.errors import (  # isort: skip
+from telethon.errors import (
     AccessTokenExpiredError, AccessTokenInvalidError, FirstNameInvalidError, FloodWaitError,
     PasswordHashInvalidError, PhoneCodeExpiredError, PhoneCodeInvalidError,
     PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
-    PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError,
-    PhoneNumberInvalidError)
-from telethon.tl.types import User
+    PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError)
 
-from mautrix.types import (EventID, UserID, MediaMessageEventContent, ImageInfo, MessageType,
-                           TextMessageEventContent)
-
-from ... import user as u
-from ...types import TelegramID
+from ... import puppet as pu, user as u
 from ...commands import command_handler, CommandEvent, SECTION_AUTH
-from ...util import format_duration as fmt_duration
-
-try:
-    import qrcode
-    import PIL as _
-    from telethon.tl.custom import QRLogin
-except ImportError:
-    qrcode = None
-    QRLogin = None
+from ...util import format_duration, ignore_coro
 
 
 @command_handler(needs_auth=False,
                  help_section=SECTION_AUTH,
                  help_text="Check if you're logged into Telegram.")
-async def ping(evt: CommandEvent) -> EventID:
+async def ping(evt: CommandEvent) -> Optional[Dict]:
     me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
     if me:
         human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
@@ -57,7 +43,7 @@ async def ping(evt: CommandEvent) -> EventID:
 @command_handler(needs_auth=False, needs_puppeting=False,
                  help_section=SECTION_AUTH,
                  help_text="Get the info of the message relay Telegram bot.")
-async def ping_bot(evt: CommandEvent) -> EventID:
+async def ping_bot(evt: CommandEvent) -> Optional[Dict]:
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
     info, mxid = await evt.tgbot.get_me(use_cache=False)
@@ -70,7 +56,7 @@ async def ping_bot(evt: CommandEvent) -> EventID:
                  help_section=SECTION_AUTH,
                  help_args="<_phone_> <_full name_>",
                  help_text="Register to Telegram")
-async def register(evt: CommandEvent) -> EventID:
+async def register(evt: CommandEvent) -> Optional[Dict]:
     if await evt.sender.is_logged_in():
         return await evt.reply("You are already logged in.")
     elif len(evt.args) < 1:
@@ -87,18 +73,17 @@ async def register(evt: CommandEvent) -> EventID:
         "action": "Register",
         "full_name": full_name,
     })
-    return await evt.reply("By signing up for Telegram, you agree to "
-                           "the terms of service: https://telegram.org/tos")
+    return None
 
 
-async def enter_code_register(evt: CommandEvent) -> EventID:
+async def enter_code_register(evt: CommandEvent) -> Dict:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
         first_name, last_name = evt.sender.command_status["full_name"]
         user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
-        asyncio.ensure_future(evt.sender.post_login(user, first_login=True), loop=evt.loop)
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
         evt.sender.command_status = None
         return await evt.reply(f"Successfully registered to Telegram.")
     except PhoneNumberOccupiedError:
@@ -117,76 +102,18 @@ async def enter_code_register(evt: CommandEvent) -> EventID:
                                "Check console for more details.")
 
 
-@command_handler(needs_auth=False, management_only=True, help_section=SECTION_AUTH,
-                 help_text="Log in by scanning a QR code.")
-async def login_qr(evt: CommandEvent) -> EventID:
-    login_as = evt.sender
-    if len(evt.args) > 0 and evt.sender.is_admin:
-        login_as = u.User.get_by_mxid(UserID(evt.args[0]))
-    if not qrcode or not QRLogin:
-        return await evt.reply("This bridge instance does not support logging in with a QR code.")
-    if await login_as.is_logged_in():
-        return await evt.reply(f"You are already logged in as {login_as.human_tg_id}.")
-
-    await login_as.ensure_started(even_if_no_session=True)
-    qr_login = QRLogin(login_as.client, ignored_ids=[])
-    qr_event_id: Optional[EventID] = None
-
-    async def upload_qr() -> None:
-        nonlocal qr_event_id
-        buffer = io.BytesIO()
-        image = qrcode.make(qr_login.url)
-        size = image.pixel_size
-        image.save(buffer, "PNG")
-        qr = buffer.getvalue()
-        mxc = await evt.az.intent.upload_media(qr, "image/png", "login-qr.png", len(qr))
-        content = MediaMessageEventContent(body=qr_login.url, url=mxc, msgtype=MessageType.IMAGE,
-                                           info=ImageInfo(mimetype="image/png", size=len(qr),
-                                                          width=size, height=size))
-        if qr_event_id:
-            content.set_edit(qr_event_id)
-            await evt.az.intent.send_message(evt.room_id, content)
-        else:
-            content.set_reply(evt.event_id)
-            qr_event_id = await evt.az.intent.send_message(evt.room_id, content)
-
-    retries = 4
-    while retries > 0:
-        await qr_login.recreate()
-        await upload_qr()
-        try:
-            user = await qr_login.wait()
-            break
-        except asyncio.TimeoutError:
-            retries -= 1
-        except SessionPasswordNeededError:
-            evt.sender.command_status = {
-                "next": enter_password,
-                "login_as": login_as if login_as != evt.sender else None,
-                "action": "Login (password entry)",
-            }
-            return await evt.reply("Your account has two-factor authentication. "
-                                   "Please send your password here.")
-    else:
-        timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
-        timeout.set_edit(qr_event_id)
-        return await evt.az.intent.send_message(evt.room_id, timeout)
-
-    return await _finish_sign_in(evt, user, login_as=login_as)
-
-
 @command_handler(needs_auth=False, management_only=True,
                  help_section=SECTION_AUTH,
                  help_text="Get instructions on how to log in.")
-async def login(evt: CommandEvent) -> EventID:
+async def login(evt: CommandEvent) -> Optional[Dict]:
     override_sender = False
     if len(evt.args) > 0 and evt.sender.is_admin:
-        evt.sender = await u.User.get_by_mxid(UserID(evt.args[0])).ensure_started()
+        evt.sender = await u.User.get_by_mxid(evt.args[0]).ensure_started()
         override_sender = True
     if await evt.sender.is_logged_in():
         return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 
-    allow_matrix_login = evt.config["bridge.allow_matrix_login"]
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
     if allow_matrix_login and not override_sender:
         evt.sender.command_status = {
             "next": enter_phone_or_token,
@@ -215,7 +142,7 @@ async def login(evt: CommandEvent) -> EventID:
 
 
 async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
-                        ) -> EventID:
+                        ) -> Dict:
     ok = False
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
@@ -223,20 +150,21 @@ async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[
         ok = True
         return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
     except PhoneNumberAppSignupForbiddenError:
-        return await evt.reply("Your phone number does not allow 3rd party apps to sign in.")
+        return await evt.reply(
+            "Your phone number does not allow 3rd party apps to sign in.")
     except PhoneNumberFloodError:
-        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
-                               "The ban is usually applied for around a day.")
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            "The ban is usually applied for around a day.")
     except FloodWaitError as e:
-        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
-                               f"Please wait for {fmt_duration(e.seconds)} before trying again.")
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            f"Please wait for {format_duration(e.seconds)} before trying again.")
     except PhoneNumberBannedError:
-        return await evt.reply("Your phone number has been banned from Telegram.")
+        return await  evt.reply("Your phone number has been banned from Telegram.")
     except PhoneNumberUnoccupiedError:
-        return await evt.reply("That phone number has not been registered. "
-                               "Please register with `$cmdprefix+sp register <phone>`.")
-    except PhoneNumberInvalidError:
-        return await evt.reply("That phone number is not valid.")
+        return await  evt.reply("That phone number has not been registered. "
+                                "Please register with `$cmdprefix+sp register <phone>`.")
     except Exception:
         evt.log.exception("Error requesting phone code")
         return await evt.reply("Unhandled exception while requesting code. "
@@ -246,7 +174,7 @@ async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[
 
 
 @command_handler(needs_auth=False)
-async def enter_phone_or_token(evt: CommandEvent) -> Optional[EventID]:
+async def enter_phone_or_token(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
@@ -270,7 +198,7 @@ async def enter_phone_or_token(evt: CommandEvent) -> Optional[EventID]:
 
 
 @command_handler(needs_auth=False)
-async def enter_code(evt: CommandEvent) -> Optional[EventID]:
+async def enter_code(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
@@ -286,15 +214,14 @@ async def enter_code(evt: CommandEvent) -> Optional[EventID]:
 
 
 @command_handler(needs_auth=False)
-async def enter_password(evt: CommandEvent) -> Optional[EventID]:
+async def enter_password(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await _sign_in(evt, login_as=evt.sender.command_status.get("login_as", None),
-                       password=" ".join(evt.args))
+        await _sign_in(evt, password=" ".join(evt.args))
     except AccessTokenInvalidError:
         return await evt.reply("That bot token is not valid.")
     except AccessTokenExpiredError:
@@ -306,12 +233,20 @@ async def enter_password(evt: CommandEvent) -> Optional[EventID]:
     return None
 
 
-async def _sign_in(evt: CommandEvent, login_as: 'u.User' = None, **sign_in_info) -> EventID:
-    login_as = login_as or evt.sender
+async def _sign_in(evt: CommandEvent, **sign_in_info) -> Dict:
     try:
-        await login_as.ensure_started(even_if_no_session=True)
-        user = await login_as.client.sign_in(**sign_in_info)
-        await _finish_sign_in(evt, user)
+        await evt.sender.ensure_started(even_if_no_session=True)
+        user = await evt.sender.client.sign_in(**sign_in_info)
+        existing_user = u.User.get_by_tgid(user.id)
+        if existing_user and existing_user != evt.sender:
+            await existing_user.log_out()
+            await evt.reply(f"[{existing_user.displayname}]"
+                            f"(https://matrix.to/#/{existing_user.mxid})"
+                            " was logged out from the account.")
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
+        evt.sender.command_status = None
+        name = f"@{user.username}" if user.username else f"+{user.phone}"
+        return await evt.reply(f"Successfully logged in as {name}")
     except PhoneCodeExpiredError:
         return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
     except PhoneCodeInvalidError:
@@ -327,29 +262,10 @@ async def _sign_in(evt: CommandEvent, login_as: 'u.User' = None, **sign_in_info)
                                "Please send your password here.")
 
 
-async def _finish_sign_in(evt: CommandEvent, user: User, login_as: 'u.User' = None) -> EventID:
-    login_as = login_as or evt.sender
-    existing_user = u.User.get_by_tgid(TelegramID(user.id))
-    if existing_user and existing_user != login_as:
-        await existing_user.log_out()
-        await evt.reply(f"[{existing_user.displayname}]"
-                        f"(https://matrix.to/#/{existing_user.mxid})"
-                        " was logged out from the account.")
-    asyncio.ensure_future(login_as.post_login(user, first_login=True), loop=evt.loop)
-    evt.sender.command_status = None
-    name = f"@{user.username}" if user.username else f"+{user.phone}"
-    if login_as != evt.sender:
-        msg = (f"Successfully logged in [{login_as.mxid}](https://matrix.to/#/{login_as.mxid})"
-               f" as {name}")
-    else:
-        msg = f"Successfully logged in as {name}"
-    return await evt.reply(msg)
-
-
 @command_handler(needs_auth=True,
                  help_section=SECTION_AUTH,
                  help_text="Log out from Telegram.")
-async def logout(evt: CommandEvent) -> EventID:
+async def logout(evt: CommandEvent) -> Optional[Dict]:
     if await evt.sender.log_out():
         return await evt.reply("Logged out successfully.")
     return await evt.reply("Failed to log out.")

mautrix_telegram/commands/telegram/misc.py

@@ -1,5 +1,6 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,53 +14,32 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Optional, Tuple, cast
-import logging
+from typing import Dict, List, Optional, Tuple
 import codecs
 import base64
 import re
 
 from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
-                             UserAlreadyParticipantError, ChatIdInvalidError,
-                             TakeoutInitDelayError, EmoticonInvalidError)
+                             UserAlreadyParticipantError)
 from telethon.tl.patched import Message
 from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
-                               TypeInputPeer, InputMediaDice)
+                               TypePeer)
 from telethon.tl.types.messages import BotCallbackAnswer
 from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
                                             GetBotCallbackAnswerRequest, SendVoteRequest)
 from telethon.tl.functions.channels import JoinChannelRequest
 
-from mautrix.types import EventID, Format
-
 from ... import puppet as pu, portal as po
 from ...abstract_user import AbstractUser
 from ...db import Message as DBMessage
 from ...types import TelegramID
-from ...commands import (command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS,
-                         SECTION_PORTAL_MANAGEMENT)
-
-
-@command_handler(needs_auth=False,
-                 help_section=SECTION_MISC, help_args="<_caption_>",
-                 help_text="Set a caption for the next image you send")
-async def caption(evt: CommandEvent) -> EventID:
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp caption <caption>`")
-
-    prefix = f"{evt.command_prefix} caption "
-    if evt.content.format == Format.HTML:
-        evt.content.formatted_body = evt.content.formatted_body.replace(prefix, "", 1)
-    evt.content.body = evt.content.body.replace(prefix, "", 1)
-    evt.sender.command_status = {"caption": evt.content, "action": "Caption"}
-    return await evt.reply("Your next image or file will be sent with that caption. "
-                           "Use `$cmdprefix+sp cancel` to cancel the caption.")
+from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
 
 
 @command_handler(help_section=SECTION_MISC,
                  help_args="[_-r|--remote_] <_query_>",
                  help_text="Search your contacts or the Telegram servers for users.")
-async def search(evt: CommandEvent) -> EventID:
+async def search(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
 
@@ -80,7 +60,7 @@ async def search(evt: CommandEvent) -> EventID:
                                    "Minimum length of remote query is 5 characters.")
         return await evt.reply("No results 3:")
 
-    reply: List[str] = []
+    reply = []  # type: List[str]
     if remote:
         reply += ["**Results from Telegram server:**", ""]
     else:
@@ -94,18 +74,18 @@ async def search(evt: CommandEvent) -> EventID:
     return await evt.reply("\n".join(reply))
 
 
-@command_handler(help_section=SECTION_CREATING_PORTALS, help_args="<_identifier_>",
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_identifier_>",
                  help_text="Open a private chat with the given Telegram user. The identifier is "
                            "either the internal user ID, the username or the phone number. "
                            "**N.B.** The phone numbers you start chats with must already be in "
                            "your contacts.")
-async def pm(evt: CommandEvent) -> EventID:
+async def pm(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
 
     try:
-        id = "".join(evt.args).translate({ord(c): None for c in "+()- "})
-        user = await evt.sender.client.get_entity(id)
+        user = await evt.sender.client.get_entity(evt.args[0])
     except ValueError:
         return await evt.reply("Invalid user identifier or user not found.")
 
@@ -119,7 +99,7 @@ async def pm(evt: CommandEvent) -> EventID:
                            f"{pu.Puppet.get_displayname(user, False)}")
 
 
-async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
+async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[Dict]]:
     if arg.startswith("joinchat/"):
         invite_hash = arg[len("joinchat/"):]
         try:
@@ -142,7 +122,7 @@ async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Opt
 @command_handler(help_section=SECTION_CREATING_PORTALS,
                  help_args="<_link_>",
                  help_text="Join a chat with an invite link.")
-async def join(evt: CommandEvent) -> Optional[EventID]:
+async def join(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
 
@@ -162,13 +142,7 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
             return await evt.reply(f"Invited you to portal of {portal.title}")
         else:
             await evt.reply(f"Creating room for {chat.title}... This might take a while.")
-            try:
-                await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
-            except ChatIdInvalidError as e:
-                logging.getLogger("mau.commands").trace("ChatIdInvalidError while creating portal "
-                                                        "from !tg join command: %s",
-                                                        updates.stringify())
-                raise e
+            await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
             return await evt.reply(f"Created room for {portal.title}")
     return None
 
@@ -176,7 +150,7 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
 @command_handler(help_section=SECTION_MISC,
                  help_args="[`chats`|`contacts`|`me`]",
                  help_text="Synchronize your chat portals, contacts and/or own info.")
-async def sync(evt: CommandEvent) -> EventID:
+async def sync(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) > 0:
         sync_only = evt.args[0]
         if sync_only not in ("chats", "contacts", "me"):
@@ -185,10 +159,8 @@ async def sync(evt: CommandEvent) -> EventID:
         sync_only = None
 
     if not sync_only or sync_only == "chats":
-        await evt.reply("Synchronizing chats...")
-        await evt.sender.sync_dialogs()
+        await evt.sender.sync_dialogs(synchronous_create=True)
     if not sync_only or sync_only == "contacts":
-        await evt.reply("Synchronizing contacts...")
         await evt.sender.sync_contacts()
     if not sync_only or sync_only == "me":
         await evt.sender.update_info()
@@ -205,7 +177,7 @@ class MessageIDError(ValueError):
 
 
 async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
-                               ) -> Tuple[TypeInputPeer, Message]:
+                               ) -> Tuple[TypePeer, Message]:
     try:
         enc_id += (4 - len(enc_id) % 4) * "="
         enc_id = base64.b64decode(enc_id)
@@ -234,13 +206,13 @@ async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
     msg = await user.client.get_messages(entity=peer, ids=msg_id)
     if not msg:
         raise MessageIDError(f"Invalid {type_name} ID (message not found)")
-    return peer, cast(Message, msg)
+    return peer, msg
 
 
 @command_handler(help_section=SECTION_MISC,
                  help_args="<_play ID_>",
                  help_text="Play a Telegram game.")
-async def play(evt: CommandEvent) -> EventID:
+async def play(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) < 1:
         return await evt.reply("**Usage:** `$cmdprefix+sp play <play ID>`")
     elif not await evt.sender.is_logged_in():
@@ -256,19 +228,18 @@ async def play(evt: CommandEvent) -> EventID:
     if not isinstance(msg.media, MessageMediaGame):
         return await evt.reply("Invalid play ID (message doesn't look like a game)")
 
-    game = await evt.sender.client(
-        GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
+    game = await evt.sender.client(GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
     if not isinstance(game, BotCallbackAnswer):
         return await evt.reply("Game request response invalid")
 
-    return await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
-                           f"{msg.media.game.description}")
+    await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
+                    f"{msg.media.game.description}")
 
 
 @command_handler(help_section=SECTION_MISC,
                  help_args="<_poll ID_> <_choice number_>",
                  help_text="Vote in a Telegram poll.")
-async def vote(evt: CommandEvent) -> EventID:
+async def vote(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) < 1:
         return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
     elif not await evt.sender.is_logged_in():
@@ -310,52 +281,3 @@ async def vote(evt: CommandEvent) -> EventID:
         return await evt.reply("You passed too many options.")
     # TODO use response
     return await evt.mark_read()
-
-
-@command_handler(help_section=SECTION_MISC, help_args="<_emoji_>",
-                 help_text="Roll a dice (\U0001F3B2), kick a football (\u26BD\uFE0F) or throw a "
-                           "dart (\U0001F3AF) or basketball (\U0001F3C0) on the Telegram servers.")
-async def random(evt: CommandEvent) -> EventID:
-    if not evt.is_portal:
-        return await evt.reply("You can only randomize values in portal rooms")
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    arg = evt.args[0] if len(evt.args) > 0 else "dice"
-    emoticon = {
-        "dart": "\U0001F3AF",
-        "dice": "\U0001F3B2",
-        "ball": "\U0001F3C0",
-        "basketball": "\U0001F3C0",
-        "football": "\u26BD",
-        "soccer": "\u26BD",
-    }.get(arg, arg)
-    try:
-        await evt.sender.client.send_media(await portal.get_input_entity(evt.sender),
-                                           InputMediaDice(emoticon))
-    except EmoticonInvalidError:
-        return await evt.reply("Invalid emoji for randomization")
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT, help_args="[_limit_]",
-                 help_text="Backfill messages from Telegram history.")
-async def backfill(evt: CommandEvent) -> None:
-    if not evt.is_portal:
-        await evt.reply("You can only use backfill in portal rooms")
-        return
-    try:
-        limit = int(evt.args[0])
-    except (ValueError, IndexError):
-        limit = -1
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
-        await evt.reply("Backfilling normal groups is disabled in the bridge config")
-        return
-    try:
-        await portal.backfill(evt.sender, limit=limit)
-    except TakeoutInitDelayError:
-        msg = ("Please accept the data export request from a mobile device, "
-               "then re-run the backfill command.")
-        if portal.peer_type == "user":
-            from mautrix.appservice import IntentAPI
-            await portal.main_intent.send_notice(evt.room_id, msg)
-        else:
-            await evt.reply(msg)

mautrix_telegram/config.py

@@ -1,5 +1,6 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,52 +14,165 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Any, List, NamedTuple
+from typing import Any, Dict, Optional, Tuple
+from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
-import os
+import random
+import string
 
-from mautrix.types import UserID
-from mautrix.client import Client
-from mautrix.bridge.config import BaseBridgeConfig
-from mautrix.util.config import ForbiddenKey, ForbiddenDefault, ConfigUpdateHelper
-
-Permissions = NamedTuple("Permissions", relaybot=bool, user=bool, puppeting=bool,
-                         matrix_puppeting=bool, admin=bool, level=str)
+yaml = YAML()  # type: YAML
+yaml.indent(4)
 
 
-class Config(BaseBridgeConfig):
+class DictWithRecursion:
+    def __init__(self, data: Optional[CommentedMap] = None) -> None:
+        self._data = data or CommentedMap()  # type: CommentedMap
+
+    @staticmethod
+    def _parse_key(key: str) -> Tuple[str, Optional[str]]:
+        if '.' not in key:
+            return key, None
+        key, next_key = key.split('.', 1)
+        if len(key) > 0 and key[0] == "[":
+            end_index = next_key.index("]")
+            key = key[1:] + "." + next_key[:end_index]
+            next_key = next_key[end_index + 2:] if len(next_key) > end_index + 1 else None
+        return key, next_key
+
+    def _recursive_get(self, data: CommentedMap, key: str, default_value: Any) -> Any:
+        key, next_key = self._parse_key(key)
+        if next_key is not None:
+            next_data = data.get(key, CommentedMap())
+            return self._recursive_get(next_data, next_key, default_value)
+        return data.get(key, default_value)
+
+    def get(self, key: str, default_value: Any, allow_recursion: bool = True) -> Any:
+        if allow_recursion and '.' in key:
+            return self._recursive_get(self._data, key, default_value)
+        return self._data.get(key, default_value)
+
+    def __getitem__(self, key: str) -> Any:
+        return self.get(key, None)
+
+    def __contains__(self, key: str) -> bool:
+        return self[key] is not None
+
+    def _recursive_set(self, data: CommentedMap, key: str, value: Any) -> None:
+        key, next_key = self._parse_key(key)
+        if next_key is not None:
+            if key not in data:
+                data[key] = CommentedMap()
+            next_data = data.get(key, CommentedMap())
+            return self._recursive_set(next_data, next_key, value)
+        data[key] = value
+
+    def set(self, key: str, value: Any, allow_recursion: bool = True) -> None:
+        if allow_recursion and '.' in key:
+            self._recursive_set(self._data, key, value)
+            return
+        self._data[key] = value
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        self.set(key, value)
+
+    def _recursive_del(self, data: CommentedMap, key: str) -> None:
+        key, next_key = self._parse_key(key)
+        if next_key is not None:
+            if key not in data:
+                return
+            next_data = data[key]
+            return self._recursive_del(next_data, next_key)
+        try:
+            del data[key]
+            del data.ca.items[key]
+        except KeyError:
+            pass
+
+    def delete(self, key: str, allow_recursion: bool = True) -> None:
+        if allow_recursion and '.' in key:
+            self._recursive_del(self._data, key)
+            return
+        try:
+            del self._data[key]
+            del self._data.ca.items[key]
+        except KeyError:
+            pass
+
+    def __delitem__(self, key: str) -> None:
+        self.delete(key)
+
+
+class Config(DictWithRecursion):
+    def __init__(self, path: str, registration_path: str, base_path: str,
+                 overrides: Dict[str, Any] = None) -> None:
+        super().__init__()
+        self.path = path  # type: str
+        self.registration_path = registration_path  # type: str
+        self.base_path = base_path  # type: str
+        self._registration = None  # type: Optional[Dict]
+        self._overrides = overrides or {}  # type: Dict[str, Any]
+
     def __getitem__(self, key: str) -> Any:
         try:
-            return os.environ[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
+            return self._overrides[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
         except KeyError:
             return super().__getitem__(key)
 
-    @property
-    def forbidden_defaults(self) -> List[ForbiddenDefault]:
-        return [
-            *super().forbidden_defaults,
-            ForbiddenDefault("appservice.public.external", "https://example.com/public",
-                             condition="appservice.public.enabled"),
-            ForbiddenDefault("bridge.permissions", ForbiddenKey("example.com")),
-            ForbiddenDefault("telegram.api_id", 12345),
-            ForbiddenDefault("telegram.api_hash", "tjyd5yge35lbodk1xwzw2jstp90k55qz"),
-        ]
+    def load(self) -> None:
+        with open(self.path, 'r') as stream:
+            self._data = yaml.load(stream)
 
-    def do_update(self, helper: ConfigUpdateHelper) -> None:
-        super().do_update(helper)
-        copy, copy_dict, base = helper
+    def load_base(self) -> Optional[DictWithRecursion]:
+        try:
+            with open(self.base_path, 'r') as stream:
+                return DictWithRecursion(yaml.load(stream))
+        except OSError:
+            pass
+        return None
 
-        copy("homeserver.asmux")
+    def save(self) -> None:
+        with open(self.path, 'w') as stream:
+            yaml.dump(self._data, stream)
+        if self._registration and self.registration_path:
+            with open(self.registration_path, 'w') as stream:
+                yaml.dump(self._registration, stream)
+
+    @staticmethod
+    def _new_token() -> str:
+        return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
+
+    def update(self) -> None:
+        base = self.load_base()
+        if not base:
+            return
+
+        def copy(from_path, to_path=None) -> None:
+            if from_path in self:
+                base[to_path or from_path] = self[from_path]
+
+        def copy_dict(from_path, to_path=None, override_existing_map=True) -> None:
+            if from_path in self:
+                to_path = to_path or from_path
+                if override_existing_map or to_path not in base:
+                    base[to_path] = CommentedMap()
+                for key, value in self[from_path].items():
+                    base[to_path][key] = value
+
+        copy("homeserver.address")
+        copy("homeserver.domain")
+        copy("homeserver.verify_ssl")
 
         if "appservice.protocol" in self and "appservice.address" not in self:
             protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
                                         self["appservice.port"])
             base["appservice.address"] = f"{protocol}://{hostname}:{port}"
-        if "appservice.debug" in self and "logging" not in self:
-            level = "DEBUG" if self["appservice.debug"] else "INFO"
-            base["logging.root.level"] = level
-            base["logging.loggers.mau.level"] = level
-            base["logging.loggers.telethon.level"] = level
+        else:
+            copy("appservice.address")
+        copy("appservice.hostname")
+        copy("appservice.port")
+        copy("appservice.max_body_size")
+
+        copy("appservice.database")
 
         copy("appservice.public.enabled")
         copy("appservice.public.prefix")
@@ -70,76 +184,41 @@ class Config(BaseBridgeConfig):
         if base["appservice.provisioning.shared_secret"] == "generate":
             base["appservice.provisioning.shared_secret"] = self._new_token()
 
+        copy("appservice.id")
+        copy("appservice.bot_username")
+        copy("appservice.bot_displayname")
+        copy("appservice.bot_avatar")
+
         copy("appservice.community_id")
 
+        copy("appservice.as_token")
+        copy("appservice.hs_token")
+
         copy("metrics.enabled")
         copy("metrics.listen_port")
 
-        copy("manhole.enabled")
-        copy("manhole.path")
-        copy("manhole.whitelist")
-
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
 
         copy("bridge.displayname_preference")
-        copy("bridge.displayname_max_length")
-        copy("bridge.allow_avatar_remove")
 
         copy("bridge.max_initial_member_sync")
         copy("bridge.sync_channel_members")
         copy("bridge.skip_deleted_members")
         copy("bridge.startup_sync")
-        if "bridge.sync_dialog_limit" in self:
-            base["bridge.sync_create_limit"] = self["bridge.sync_dialog_limit"]
-            base["bridge.sync_update_limit"] = self["bridge.sync_dialog_limit"]
-        else:
-            copy("bridge.sync_update_limit")
-            copy("bridge.sync_create_limit")
-        copy("bridge.sync_direct_chats")
+        copy("bridge.sync_dialog_limit")
         copy("bridge.max_telegram_delete")
         copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
         copy("bridge.plaintext_highlights")
         copy("bridge.public_portals")
+        copy("bridge.catch_up")
         copy("bridge.sync_with_custom_puppets")
-        copy("bridge.sync_direct_chat_list")
-        copy("bridge.double_puppet_server_map")
-        copy("bridge.double_puppet_allow_discovery")
-        if "bridge.login_shared_secret" in self:
-            base["bridge.login_shared_secret_map"] = {
-                base["homeserver.domain"]: self["bridge.login_shared_secret"]
-            }
-        else:
-            copy("bridge.login_shared_secret_map")
         copy("bridge.telegram_link_preview")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
         copy("bridge.max_document_size")
-        copy("bridge.parallel_file_transfer")
-        copy("bridge.federate_rooms")
-        copy("bridge.animated_sticker.target")
-        copy("bridge.animated_sticker.args")
-        copy("bridge.encryption.allow")
-        copy("bridge.encryption.default")
-        copy("bridge.encryption.database")
-        copy("bridge.encryption.key_sharing.allow")
-        copy("bridge.encryption.key_sharing.require_cross_signing")
-        copy("bridge.encryption.key_sharing.require_verification")
-        copy("bridge.private_chat_portal_meta")
-        copy("bridge.delivery_receipts")
-        copy("bridge.delivery_error_reports")
-        copy("bridge.resend_bridge_info")
-        copy("bridge.backfill.invite_own_puppet")
-        copy("bridge.backfill.takeout_limit")
-        copy("bridge.backfill.initial_limit")
-        copy("bridge.backfill.missed_limit")
-        copy("bridge.backfill.disable_notifications")
-        copy("bridge.backfill.normal_groups")
-
-        copy("bridge.initial_power_level_overrides.group")
-        copy("bridge.initial_power_level_overrides.user")
 
         copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):
@@ -156,7 +235,6 @@ class Config(BaseBridgeConfig):
         if "bridge.message_formats.m_text" in self:
             del self["bridge.message_formats"]
         copy_dict("bridge.message_formats", override_existing_map=False)
-        copy("bridge.emote_format")
 
         copy("bridge.state_event_formats.join")
         copy("bridge.state_event_formats.leave")
@@ -183,11 +261,6 @@ class Config(BaseBridgeConfig):
         if "bridge.relaybot" not in self:
             copy("bridge.authless_relaybot_portals", "bridge.relaybot.authless_portals")
         else:
-            copy("bridge.relaybot.private_chat.invite")
-            copy("bridge.relaybot.private_chat.state_changes")
-            copy("bridge.relaybot.private_chat.message")
-            copy("bridge.relaybot.group_chat_invite")
-            copy("bridge.relaybot.ignore_unbridged_group_chat")
             copy("bridge.relaybot.authless_portals")
             copy("bridge.relaybot.whitelist_group_admins")
             copy("bridge.relaybot.whitelist")
@@ -221,22 +294,66 @@ class Config(BaseBridgeConfig):
         copy("telegram.proxy.username")
         copy("telegram.proxy.password")
 
-    def _get_permissions(self, key: str) -> Permissions:
+        if "appservice.debug" in self and "logging" not in self:
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
+            base["logging.root.level"] = level
+            base["logging.loggers.mau.level"] = level
+            base["logging.loggers.telethon.level"] = level
+        else:
+            copy("logging")
+
+        self._data = base._data
+        self.save()
+
+    def _get_permissions(self, key: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
         level = self["bridge.permissions"].get(key, "")
         admin = level == "admin"
         matrix_puppeting = level == "full" or admin
         puppeting = level == "puppeting" or matrix_puppeting
         user = level == "user" or puppeting
         relaybot = level == "relaybot" or user
-        return Permissions(relaybot, user, puppeting, matrix_puppeting, admin, level)
+        return relaybot, user, puppeting, matrix_puppeting, admin, level
 
-    def get_permissions(self, mxid: UserID) -> Permissions:
-        permissions = self["bridge.permissions"]
+    def get_permissions(self, mxid: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
+        permissions = self["bridge.permissions"] or {}
         if mxid in permissions:
             return self._get_permissions(mxid)
 
-        _, homeserver = Client.parse_user_id(mxid)
+        homeserver = mxid[mxid.index(":") + 1:]
         if homeserver in permissions:
             return self._get_permissions(homeserver)
 
         return self._get_permissions("*")
+
+    def generate_registration(self) -> None:
+        homeserver = self["homeserver.domain"]
+
+        username_format = self.get("bridge.username_template", "telegram_{userid}") \
+            .format(userid=".+")
+        alias_format = self.get("bridge.alias_template", "telegram_{groupname}") \
+            .format(groupname=".+")
+
+        self.set("appservice.as_token", self._new_token())
+        self.set("appservice.hs_token", self._new_token())
+
+        self._registration = {
+            "id": self["appservice.id"] or "telegram",
+            "as_token": self["appservice.as_token"],
+            "hs_token": self["appservice.hs_token"],
+            "namespaces": {
+                "users": [{
+                    "exclusive": True,
+                    "regex": f"@{username_format}:{homeserver}"
+                }],
+                "aliases": [{
+                    "exclusive": True,
+                    "regex": f"#{alias_format}:{homeserver}"
+                }]
+            },
+            "url": self["appservice.address"],
+            "sender_localpart": self["appservice.bot_username"],
+            "rate_limited": False
+        }
+        if self["appservice.community_id"]:
+            self._registration["namespaces"]["users"][0]["group_id"] \
+                = self["appservice.community_id"]

mautrix_telegram/context.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -14,44 +15,31 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Tuple, TYPE_CHECKING
-import asyncio
-
-from alchemysession import AlchemySessionContainer
-
-from mautrix.appservice import AppService
 
 if TYPE_CHECKING:
+    import asyncio
+
+    from alchemysession import AlchemySessionContainer
+    from mautrix_appservice import AppService
+
     from .web import PublicBridgeWebsite, ProvisioningAPI
     from .config import Config
     from .bot import Bot
     from .matrix import MatrixHandler
-    from .__main__ import TelegramBridge
 
 
 class Context:
-    az: AppService
-    config: 'Config'
-    loop: asyncio.AbstractEventLoop
-    bridge: 'TelegramBridge'
-    bot: Optional['Bot']
-    mx: Optional['MatrixHandler']
-    session_container: AlchemySessionContainer
-    public_website: Optional['PublicBridgeWebsite']
-    provisioning_api: Optional['ProvisioningAPI']
-
-    def __init__(self, az: AppService, config: 'Config', loop: asyncio.AbstractEventLoop,
-                 session_container: AlchemySessionContainer, bridge: 'TelegramBridge',
-                 bot: Optional['Bot']) -> None:
-        self.az = az
-        self.config = config
-        self.loop = loop
-        self.bridge = bridge
-        self.bot = bot
-        self.mx = None
-        self.session_container = session_container
-        self.public_website = None
-        self.provisioning_api = None
+    def __init__(self, az: 'AppService', config: 'Config', loop: 'asyncio.AbstractEventLoop',
+                 session_container: 'AlchemySessionContainer', bot: Optional['Bot']) -> None:
+        self.az = az  # type: AppService
+        self.config = config  # type: Config
+        self.loop = loop  # type: asyncio.AbstractEventLoop
+        self.bot = bot  # type: Optional[Bot]
+        self.mx = None  # type: Optional[MatrixHandler]
+        self.session_container = session_container  # type: AlchemySessionContainer
+        self.public_website = None  # type: Optional[PublicBridgeWebsite]
+        self.provisioning_api = None  # type: Optional[ProvisioningAPI]
 
     @property
-    def core(self) -> Tuple[AppService, 'Config', asyncio.AbstractEventLoop, Optional['Bot']]:
+    def core(self) -> Tuple['AppService', 'Config', 'asyncio.AbstractEventLoop', Optional['Bot']]:
         return self.az, self.config, self.loop, self.bot

mautrix_telegram/db/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,19 +14,20 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy.engine.base import Engine
-
-from mautrix.client.state_store.sqlalchemy import UserProfile, RoomState
-
+from .base import Base
 from .bot_chat import BotChat
 from .message import Message
 from .portal import Portal
 from .puppet import Puppet
+from .room_state import RoomState
 from .telegram_file import TelegramFile
 from .user import User, UserPortal, Contact
+from .user_profile import UserProfile
 
 
-def init(db_engine: Engine) -> None:
+def init(db_engine) -> None:
     for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
                   RoomState, BotChat):
-        table.bind(db_engine)
+        table.db = db_engine
+        table.t = table.__table__
+        table.c = table.t.c

mautrix_telegram/db/base.py

@@ -0,0 +1,58 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+
+class BaseBase:
+    db = None  # type: Engine
+    t = None  # type: Table
+    __table__ = None  # type: Table
+    c = None  # type: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy):
+        pass
+
+    @classmethod
+    def _select_one_or_none(cls, *args):
+        return cls._one_or_none(cls.db.execute(cls.t.select().where(*args)))
+
+    @property
+    @abstractmethod
+    def _edit_identity(self):
+        pass
+
+    def update(self, **values) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self._edit_identity)
+                         .values(**values))
+        for key, value in values.items():
+            setattr(self, key, value)
+
+    def delete(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.delete().where(self._edit_identity))
+
+Base = declarative_base(cls=BaseBase)

mautrix_telegram/db/base.pyi

@@ -0,0 +1,26 @@
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+class Base(declarative_base):
+	db: Engine
+	t: Table
+	__table__: Table
+	c: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy): ...
+
+	@classmethod
+	def _select_one_or_none(cls, *args): ...
+
+    def _edit_identity(self): ...
+
+    def update(self, **values) -> None: ...
+
+    def delete(self) -> None: ...

mautrix_telegram/db/bot_chat.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -17,22 +18,28 @@ from typing import Iterable
 
 from sqlalchemy import Column, Integer, String
 
-from mautrix.util.db import Base
-
 from ..types import TelegramID
+from .base import Base
 
 
 # Fucking Telegram not telling bots what chats they are in 3:<
 class BotChat(Base):
     __tablename__ = "bot_chat"
-    id: TelegramID = Column(Integer, primary_key=True)
-    type: str = Column(String, nullable=False)
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    type = Column(String, nullable=False)
 
     @classmethod
-    def delete_by_id(cls, chat_id: TelegramID) -> None:
+    def delete(cls, chat_id: TelegramID) -> None:
         with cls.db.begin() as conn:
             conn.execute(cls.t.delete().where(cls.c.id == chat_id))
 
     @classmethod
     def all(cls) -> Iterable['BotChat']:
-        return cls._select_all()
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            chat_id, chat_type = row
+            yield cls(id=chat_id, type=chat_type)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(id=self.id, type=self.type))

mautrix_telegram/db/message.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,46 +14,62 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Iterator
-
 from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional, List
 
-from mautrix.types import RoomID, EventID
-from mautrix.util.db import Base
-
-from ..types import TelegramID
+from ..types import MatrixRoomID, MatrixEventID, TelegramID
+from .base import Base
 
 
 class Message(Base):
     __tablename__ = "message"
 
-    mxid: EventID = Column(String)
-    mx_room: RoomID = Column(String)
-    tgid: TelegramID = Column(Integer, primary_key=True)
-    tg_space: TelegramID = Column(Integer, primary_key=True)
-    edit_index: int = Column(Integer, primary_key=True)
+    mxid = Column(String)  # type: MatrixEventID
+    mx_room = Column(String)  # type: MatrixRoomID
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+    edit_index = Column(Integer, primary_key=True)  # type: int
 
-    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"),)
+    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
 
     @classmethod
-    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Iterator['Message']:
-        return cls._select_all(cls.c.tgid == tgid, cls.c.tg_space == tg_space)
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
+        try:
+            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
+                       edit_index=edit_index)
+        except StopIteration:
+            return None
+
+    @staticmethod
+    def _all(rows: RowProxy) -> List['Message']:
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
+                        edit_index=row[4])
+                for row in rows]
+
+    @classmethod
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
+        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
+                                                                 cls.c.tg_space == tg_space))))
 
     @classmethod
     def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
                         ) -> Optional['Message']:
+        query = cls.t.select()
         if edit_index < 0:
-            return cls._one_or_none(cls.db.execute(
-                cls.t.select()
-                    .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
-                    .order_by(desc(cls.c.edit_index))
-                    .limit(1).offset(-edit_index - 1)))
+            query = (query
+                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                     .order_by(desc(cls.c.edit_index))
+                     .limit(1)
+                     .offset(-edit_index - 1))
         else:
-            return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
-                                           cls.c.edit_index == edit_index)
+            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                     cls.c.edit_index == edit_index))
+        return cls._one_or_none(cls.db.execute(query))
 
     @classmethod
-    def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
+    def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
         rows = cls.db.execute(select([func.count(cls.c.tg_space)])
                               .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
         try:
@@ -62,20 +79,11 @@ class Message(Base):
             return 0
 
     @classmethod
-    def find_last(cls, mx_room: RoomID, tg_space: TelegramID) -> Optional['Message']:
-        return cls._one_or_none(cls.db.execute(
-            cls._make_simple_select(cls.c.mx_room == mx_room, cls.c.tg_space == tg_space)
-                .order_by(desc(cls.c.tgid)).limit(1)))
-
-    @classmethod
-    def delete_all(cls, mx_room: RoomID) -> None:
-        cls.db.execute(cls.t.delete().where(cls.c.mx_room == mx_room))
-
-    @classmethod
-    def get_by_mxid(cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
+    def get_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID, tg_space: TelegramID
                     ) -> Optional['Message']:
-        return cls._select_one_or_none(cls.c.mxid == mxid, cls.c.mx_room == mx_room,
-                                       cls.c.tg_space == tg_space)
+        return cls._select_one_or_none(and_(cls.c.mxid == mxid,
+                                            cls.c.mx_room == mx_room,
+                                            cls.c.tg_space == tg_space))
 
     @classmethod
     def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
@@ -87,8 +95,19 @@ class Message(Base):
                          .values(**values))
 
     @classmethod
-    def update_by_mxid(cls, s_mxid: EventID, s_mx_room: RoomID, **values) -> None:
+    def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values) -> None:
         with cls.db.begin() as conn:
             conn.execute(cls.t.update()
                          .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
                          .values(**values))
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
+                    self.c.edit_index == self.edit_index)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
+                                                tgid=self.tgid, tg_space=self.tg_space,
+                                                edit_index=self.edit_index))

mautrix_telegram/db/portal.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,54 +14,68 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Iterable
+from sqlalchemy import Column, Integer, String, Boolean, Text, and_
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional
 
-from sqlalchemy import Column, Integer, String, Boolean, Text, func, sql
-
-from mautrix.types import RoomID, ContentURI
-from mautrix.util.db import Base
-
-from ..types import TelegramID
+from ..types import MatrixRoomID, TelegramID
+from .base import Base
 
 
 class Portal(Base):
     __tablename__ = "portal"
 
     # Telegram chat information
-    tgid: TelegramID = Column(Integer, primary_key=True)
-    tg_receiver: TelegramID = Column(Integer, primary_key=True)
-    peer_type: str = Column(String, nullable=False)
-    megagroup: bool = Column(Boolean)
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_receiver = Column(Integer, primary_key=True)  # type: TelegramID
+    peer_type = Column(String, nullable=False)
+    megagroup = Column(Boolean)
 
     # Matrix portal information
-    mxid: Optional[RoomID] = Column(String, unique=True, nullable=True)
-    avatar_url: Optional[ContentURI] = Column(String, nullable=True)
-    encrypted: bool = Column(Boolean, nullable=False, server_default=sql.expression.false())
+    mxid = Column(String, unique=True, nullable=True)  # type: Optional[MatrixRoomID]
 
-    config: str = Column(Text, nullable=True)
+    config = Column(Text, nullable=True)
 
     # Telegram chat metadata
-    username: str = Column(String, nullable=True)
-    title: str = Column(String, nullable=True)
-    about: str = Column(String, nullable=True)
-    photo_id: str = Column(String, nullable=True)
+    username = Column(String, nullable=True)
+    title = Column(String, nullable=True)
+    about = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+
+    @classmethod
+    def scan(cls, row) -> Optional['Portal']:
+        (tgid, tg_receiver, peer_type, megagroup, mxid, config, username, title, about,
+         photo_id) = row
+        return cls(tgid=tgid, tg_receiver=tg_receiver, peer_type=peer_type, megagroup=megagroup,
+                   mxid=mxid, config=config, username=username, title=title, about=about,
+                   photo_id=photo_id)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Portal']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
 
     @classmethod
     def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
-        return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver)
+        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver))
 
     @classmethod
-    def find_private_chats(cls, tg_receiver: TelegramID) -> Iterable['Portal']:
-        yield from cls._select_all(cls.c.tg_receiver == tg_receiver, cls.c.peer_type == "user")
-
-    @classmethod
-    def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
+    def get_by_mxid(cls, mxid: MatrixRoomID) -> Optional['Portal']:
         return cls._select_one_or_none(cls.c.mxid == mxid)
 
     @classmethod
     def get_by_username(cls, username: str) -> Optional['Portal']:
-        return cls._select_one_or_none(func.lower(cls.c.username) == username)
+        return cls._select_one_or_none(cls.c.username == username)
 
-    @classmethod
-    def all(cls) -> Iterable['Portal']:
-        yield from cls._select_all()
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_receiver == self.tg_receiver)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
+                megagroup=self.megagroup, mxid=self.mxid, config=self.config,
+                username=self.username, title=self.title, about=self.about, photo_id=self.photo_id))

mautrix_telegram/db/puppet.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,49 +14,75 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql import expression
 from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Text, Boolean
-from sqlalchemy.sql import expression, func
-
-from mautrix.types import UserID, SyncToken
-from mautrix.util.db import Base
-
-from ..types import TelegramID
+from ..types import MatrixUserID, MatrixRoomID, TelegramID
+from .base import Base
 
 
 class Puppet(Base):
     __tablename__ = "puppet"
 
-    id: TelegramID = Column(Integer, primary_key=True)
-    custom_mxid: UserID = Column(String, nullable=True)
-    access_token: str = Column(String, nullable=True)
-    next_batch: SyncToken = Column(String, nullable=True)
-    base_url: str = Column(Text, nullable=True)
-    displayname: str = Column(String, nullable=True)
-    displayname_source: TelegramID = Column(Integer, nullable=True)
-    username: str = Column(String, nullable=True)
-    photo_id: str = Column(String, nullable=True)
-    is_bot: bool = Column(Boolean, nullable=True)
-    matrix_registered: bool = Column(Boolean, nullable=False, server_default=expression.false())
-    disable_updates: bool = Column(Boolean, nullable=False, server_default=expression.false())
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    custom_mxid = Column(String, nullable=True)  # type: Optional[MatrixUserID]
+    access_token = Column(String, nullable=True)
+    displayname = Column(String, nullable=True)
+    displayname_source = Column(Integer, nullable=True)  # type: Optional[TelegramID]
+    username = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+    is_bot = Column(Boolean, nullable=True)
+    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
+    disable_updates = Column(Boolean, nullable=False, server_default=expression.false())
+
+    @classmethod
+    def scan(cls, row) -> Optional['Puppet']:
+        (id, custom_mxid, access_token, displayname, displayname_source, username, photo_id,
+         is_bot, matrix_registered, disable_updates) = row
+        return cls(id=id, custom_mxid=custom_mxid, access_token=access_token,
+                   displayname=displayname, displayname_source=displayname_source,
+                   username=username, photo_id=photo_id, is_bot=is_bot,
+                   matrix_registered=matrix_registered, disable_updates=disable_updates)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Puppet']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
 
     @classmethod
     def all_with_custom_mxid(cls) -> Iterable['Puppet']:
-        yield from cls._select_all(cls.c.custom_mxid != None)
+        rows = cls.db.execute(cls.t.select().where(cls.c.custom_mxid != None))
+        for row in rows:
+            yield cls.scan(row)
 
     @classmethod
     def get_by_tgid(cls, tgid: TelegramID) -> Optional['Puppet']:
         return cls._select_one_or_none(cls.c.id == tgid)
 
     @classmethod
-    def get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
+    def get_by_custom_mxid(cls, mxid: MatrixUserID) -> Optional['Puppet']:
         return cls._select_one_or_none(cls.c.custom_mxid == mxid)
 
     @classmethod
     def get_by_username(cls, username: str) -> Optional['Puppet']:
-        return cls._select_one_or_none(func.lower(cls.c.username) == username)
+        return cls._select_one_or_none(cls.c.username == username)
 
     @classmethod
     def get_by_displayname(cls, displayname: str) -> Optional['Puppet']:
         return cls._select_one_or_none(cls.c.displayname == displayname)
+
+    @property
+    def _edit_identity(self):
+        return self.c.id == self.id
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
+                displayname=self.displayname, displayname_source=self.displayname_source,
+                username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
+                matrix_registered=self.matrix_registered, disable_updates=self.disable_updates))

mautrix_telegram/db/room_state.py

@@ -0,0 +1,62 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, Text
+from typing import Dict, Optional
+import json
+
+from ..types import MatrixRoomID
+from .base import Base
+
+
+class RoomState(Base):
+    __tablename__ = "mx_room_state"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    power_levels = Column("power_levels", Text, nullable=True)  # type: Optional[Dict]
+
+    @property
+    def _power_levels_text(self) -> Optional[str]:
+        return json.dumps(self.power_levels) if self.power_levels else None
+
+    @property
+    def has_power_levels(self) -> bool:
+        return bool(self.power_levels)
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID) -> Optional['RoomState']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.room_id == room_id))
+        try:
+            room_id, power_levels_text = next(rows)
+            return cls(room_id=room_id, power_levels=(json.loads(power_levels_text)
+                                                      if power_levels_text else None))
+        except StopIteration:
+            return None
+
+    def update(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self.c.room_id == self.room_id)
+                         .values(power_levels=self._power_levels_text))
+
+    @property
+    def _edit_identity(self):
+        return self.c.room_id == self.room_id
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id,
+                                                power_levels=self._power_levels_text))

mautrix_telegram/db/telegram_file.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,67 +14,43 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, cast, Dict, Any
+from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
+from typing import Optional
 
-from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
-                        TypeDecorator)
-from sqlalchemy.engine.result import RowProxy
-
-from mautrix.types import ContentURI, EncryptedFile
-from mautrix.util.db import Base
-
-
-class DBEncryptedFile(TypeDecorator):
-    impl = Text
-
-    @property
-    def python_type(self):
-        return EncryptedFile
-
-    def process_bind_param(self, value: EncryptedFile, dialect) -> Optional[str]:
-        if value is not None:
-            return value.json()
-        return None
-
-    def process_result_value(self, value: str, dialect) -> Optional[EncryptedFile]:
-        if value is not None:
-            return EncryptedFile.parse_json(value)
-        return None
-
-    def process_literal_param(self, value, dialect):
-        return value
+from .base import Base
 
 
 class TelegramFile(Base):
     __tablename__ = "telegram_file"
 
-    id: str = Column(String, primary_key=True)
-    mxc: ContentURI = Column(String)
-    mime_type: str = Column(String)
-    was_converted: bool = Column(Boolean)
-    timestamp: int = Column(BigInteger)
-    size: Optional[int] = Column(Integer, nullable=True)
-    width: Optional[int] = Column(Integer, nullable=True)
-    height: Optional[int] = Column(Integer, nullable=True)
-    decryption_info: Optional[Dict[str, Any]] = Column(DBEncryptedFile, nullable=True)
-    thumbnail_id: str = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
-    thumbnail: Optional['TelegramFile'] = None
-
-    @classmethod
-    def scan(cls, row: RowProxy) -> 'TelegramFile':
-        telegram_file = cast(TelegramFile, super().scan(row))
-        if isinstance(telegram_file.thumbnail, str):
-            telegram_file.thumbnail = cls.get(telegram_file.thumbnail)
-        return telegram_file
+    id = Column(String, primary_key=True)
+    mxc = Column(String)
+    mime_type = Column(String)
+    was_converted = Column(Boolean)
+    timestamp = Column(BigInteger)
+    size = Column(Integer, nullable=True)
+    width = Column(Integer, nullable=True)
+    height = Column(Integer, nullable=True)
+    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+    thumbnail = None  # type: Optional[TelegramFile]
 
     @classmethod
     def get(cls, loc_id: str) -> Optional['TelegramFile']:
-        return cls._select_one_or_none(cls.c.id == loc_id)
+        rows = cls.db.execute(cls.t.select().where(cls.c.id == loc_id))
+        try:
+            loc_id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
+            thumb = None
+            if thumb_id:
+                thumb = cls.get(thumb_id)
+            return cls(id=loc_id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
+                       size=s, width=w, height=h, thumbnail_id=thumb_id, thumbnail=thumb)
+        except StopIteration:
+            return None
 
     def insert(self) -> None:
         with self.db.begin() as conn:
             conn.execute(self.t.insert().values(
                 id=self.id, mxc=self.mxc, mime_type=self.mime_type,
                 was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
-                width=self.width, height=self.height, decryption_info=self.decryption_info,
+                width=self.width, height=self.height,
                 thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/user.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,40 +14,61 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String
+from sqlalchemy.engine.result import RowProxy
 from typing import Optional, Iterable, Tuple
 
-from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String, func
-
-from mautrix.types import UserID
-from mautrix.util.db import Base
-
-from ..types import TelegramID
+from ..types import MatrixUserID, TelegramID
+from .base import Base
 
 
 class User(Base):
     __tablename__ = "user"
 
-    mxid: UserID = Column(String, primary_key=True)
-    tgid: Optional[TelegramID] = Column(Integer, nullable=True, unique=True)
-    tg_username: str = Column(String, nullable=True)
-    tg_phone: str = Column(String, nullable=True)
-    saved_contacts: int = Column(Integer, default=0, nullable=False)
+    mxid = Column(String, primary_key=True)  # type: MatrixUserID
+    tgid = Column(Integer, nullable=True, unique=True)  # type: Optional[TelegramID]
+    tg_username = Column(String, nullable=True)
+    tg_phone = Column(String, nullable=True)
+    saved_contacts = Column(Integer, default=0, nullable=False)
 
     @classmethod
-    def all_with_tgid(cls) -> Iterable['User']:
-        return cls._select_all(cls.c.tgid != None)
+    def _one_or_none(cls, rows: RowProxy) -> Optional['User']:
+        try:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = next(rows)
+            return cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                       saved_contacts=saved_contacts)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def all(cls) -> Iterable['User']:
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = row
+            yield cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                      saved_contacts=saved_contacts)
 
     @classmethod
     def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
         return cls._select_one_or_none(cls.c.tgid == tgid)
 
     @classmethod
-    def get_by_mxid(cls, mxid: UserID) -> Optional['User']:
+    def get_by_mxid(cls, mxid: MatrixUserID) -> Optional['User']:
         return cls._select_one_or_none(cls.c.mxid == mxid)
 
     @classmethod
     def get_by_username(cls, username: str) -> Optional['User']:
-        return cls._select_one_or_none(func.lower(cls.c.tg_username) == username)
+        return cls._select_one_or_none(cls.c.tg_username == username)
+
+    @property
+    def _edit_identity(self):
+        return self.c.mxid == self.mxid
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                mxid=self.mxid, tgid=self.tgid, tg_username=self.tg_username,
+                tg_phone=self.tg_phone, saved_contacts=self.saved_contacts))
 
     @property
     def contacts(self) -> Iterable[TelegramID]:
@@ -91,10 +113,10 @@ class User(Base):
 class UserPortal(Base):
     __tablename__ = "user_portal"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE",
-                                                  ondelete="CASCADE"), primary_key=True)
-    portal: TelegramID = Column(Integer, primary_key=True)
-    portal_receiver: TelegramID = Column(Integer, primary_key=True)
+    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
+                  primary_key=True)  # type: TelegramID
+    portal = Column(Integer, primary_key=True)  # type: TelegramID
+    portal_receiver = Column(Integer, primary_key=True)  # type: TelegramID
 
     __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
                                            ("portal.tgid", "portal.tg_receiver"),
@@ -104,5 +126,5 @@ class UserPortal(Base):
 class Contact(Base):
     __tablename__ = "contact"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
-    contact: TelegramID = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
+    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)  # type: TelegramID
+    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)  # type: TelegramID

mautrix_telegram/db/user_profile.py

@@ -0,0 +1,69 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, and_
+from typing import Dict, Optional
+
+from ..types import MatrixUserID, MatrixRoomID
+from .base import Base
+
+
+class UserProfile(Base):
+    __tablename__ = "mx_user_profile"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    user_id = Column(String, primary_key=True)  # type: MatrixUserID
+    membership = Column(String, nullable=False, default="leave")
+    displayname = Column(String, nullable=True)
+    avatar_url = Column(String, nullable=True)
+
+    def dict(self) -> Dict[str, str]:
+        return {
+            "membership": self.membership,
+            "displayname": self.displayname,
+            "avatar_url": self.avatar_url,
+        }
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID, user_id: MatrixUserID) -> Optional['UserProfile']:
+        rows = cls.db.execute(
+            cls.t.select().where(and_(cls.c.room_id == room_id, cls.c.user_id == user_id)))
+        try:
+            room_id, user_id, membership, displayname, avatar_url = next(rows)
+            return cls(room_id=room_id, user_id=user_id, membership=membership,
+                       displayname=displayname, avatar_url=avatar_url)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def delete_all(cls, room_id: MatrixRoomID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.room_id == room_id))
+
+    def update(self) -> None:
+        super().update(membership=self.membership, displayname=self.displayname,
+                       avatar_url=self.avatar_url)
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.room_id == self.room_id, self.c.user_id == self.user_id)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id, user_id=self.user_id,
+                                                membership=self.membership,
+                                                displayname=self.displayname,
+                                                avatar_url=self.avatar_url))

mautrix_telegram/formatter/__init__.py

@@ -1,4 +1,5 @@
-from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram, init_mx
+from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
+                          init_mx)
 from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,49 +14,52 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, List, Tuple, Callable, Pattern, Match, TYPE_CHECKING
+from typing import Optional, List, Tuple, Callable, Pattern, Match, TYPE_CHECKING, Dict, Any
 import re
 import logging
 
 from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
-                               TypeMessageEntity, InputMessageEntityMentionName)
-from telethon.helpers import add_surrogate, del_surrogate
-from telethon import TelegramClient
-
-from mautrix.types import RoomID, MessageEventContent
-from mautrix.util.logging import TraceLogger
+                               TypeMessageEntity)
 
 from ... import puppet as pu
-from ...types import TelegramID
+from ...types import TelegramID, MatrixRoomID
 from ...db import Message as DBMessage
+from ..util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                    trim_reply_fallback_text)
 from .parser import ParsedMessage, parse_html
 
 if TYPE_CHECKING:
     from ...context import Context
 
-log: TraceLogger = logging.getLogger("mau.fmt.mx")
-should_bridge_plaintext_highlights: bool = False
+log = logging.getLogger("mau.fmt.mx")  # type: logging.Logger
+should_bridge_plaintext_highlights = False  # type: bool
 
-command_regex: Pattern = re.compile(r"^!([A-Za-z0-9@]+)")
-not_command_regex: Pattern = re.compile(r"^\\(![A-Za-z0-9@]+)")
-plain_mention_regex: Optional[Pattern] = None
-
-MAX_LENGTH = 4096
-CUTOFF_TEXT = " [message cut]"
-CUT_MAX_LENGTH = MAX_LENGTH - len(CUTOFF_TEXT)
+command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
+not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
+plain_mention_regex = None  # type: Optional[Pattern]
 
 
-def _cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
-    if len(message) > MAX_LENGTH:
-        message = message[0:CUT_MAX_LENGTH] + CUTOFF_TEXT
+def plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
+def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+    if len(message) > 4096:
+        message = message[0:4082] + " [message cut]"
         new_entities = []
         for entity in entities:
-            if entity.offset > CUT_MAX_LENGTH:
+            if entity.offset > 4082:
                 continue
-            if entity.offset + entity.length > CUT_MAX_LENGTH:
-                entity.length = CUT_MAX_LENGTH - entity.offset
+            if entity.offset + entity.length > 4082:
+                entity.length = 4082 - entity.offset
             new_entities.append(entity)
-        new_entities.append(MessageEntityItalic(CUT_MAX_LENGTH, len(CUTOFF_TEXT)))
+        new_entities.append(MessageEntityItalic(4082, len(" [message cut]")))
         entities = new_entities
     return message, entities
 
@@ -64,12 +68,43 @@ class FormatError(Exception):
     pass
 
 
-def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
-                             room_id: Optional[RoomID] = None) -> Optional[TelegramID]:
-    event_id = content.get_reply_to()
+def matrix_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(plain_mention_to_html, html)
+
+        text, entities = parse_html(add_surrogates(html))
+        text = remove_surrogates(text.strip())
+        text, entities = cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
+                             room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
+    relates_to = content.get("m.relates_to", None) or {}
+    if not relates_to:
+        return None
+    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
+             else relates_to.get("m.in_reply_to", None) or {})
+    if not reply:
+        return None
+    room_id = room_id or reply.get("room_id", None)
+    event_id = reply.get("event_id", None)
     if not event_id:
         return
-    content.trim_reply_fallback()
+
+    try:
+        if content["format"] == "org.matrix.custom.html":
+            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
+    except KeyError:
+        pass
+    content["body"] = trim_reply_fallback_text(content["body"])
 
     message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
     if message:
@@ -77,64 +112,22 @@ def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
     return None
 
 
-async def matrix_to_telegram(client: TelegramClient, *, text: Optional[str] = None,
-                             html: Optional[str] = None) -> ParsedMessage:
-    if html is not None:
-        text, entities = _matrix_html_to_telegram(html)
-    elif text is not None:
-        text, entities = _matrix_text_to_telegram(text)
-    else:
-        raise ValueError("text or html must be provided to convert formatting")
-    await _fix_name_mentions(client, entities)
-    return text, entities
-
-
-def _matrix_html_to_telegram(html: str) -> ParsedMessage:
-    try:
-        html = command_regex.sub(r"<command>\1</command>", html)
-        html = html.replace("\t", " " * 4)
-        html = not_command_regex.sub(r"\1", html)
-        if should_bridge_plaintext_highlights:
-            html = plain_mention_regex.sub(_plain_mention_to_html, html)
-
-        text, entities = parse_html(add_surrogate(html))
-        text = del_surrogate(text.strip())
-        text, entities = _cut_long_message(text, entities)
-
-        return text, entities
-    except Exception as e:
-        raise FormatError(f"Failed to convert Matrix format: {html}") from e
-
-
-def _matrix_text_to_telegram(text: str) -> ParsedMessage:
+def matrix_text_to_telegram(text: str) -> ParsedMessage:
     text = command_regex.sub(r"/\1", text)
     text = text.replace("\t", " " * 4)
     text = not_command_regex.sub(r"\1", text)
     if should_bridge_plaintext_highlights:
-        entities, pmr_replacer = _plain_mention_to_text()
+        entities, pmr_replacer = plain_mention_to_text()
         text = plain_mention_regex.sub(pmr_replacer, text)
     else:
         entities = []
     return text, entities
 
 
-async def _fix_name_mentions(client: TelegramClient, entities: List[TypeMessageEntity]) -> None:
-    for index in reversed(range(len(entities))):
-        entity = entities[index]
-        if isinstance(entity, (MessageEntityMentionName, InputMessageEntityMentionName)):
-            try:
-                user = await client.get_input_entity(entity.user_id)
-            except (ValueError, TypeError) as e:
-                log.trace(f"Dropping mention of {entity.user_id}: {e}")
-                del entities[index]
-            else:
-                entities[index] = InputMessageEntityMentionName(entity.offset, entity.length, user)
-
-
-def _plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
+def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
     entities = []
 
-    def replacer(match: Match) -> str:
+    def replacer(match) -> str:
         puppet = pu.Puppet.find_by_displayname(match.group(2))
         if puppet:
             offset = match.start()
@@ -152,20 +145,10 @@ def _plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match],
     return entities, replacer
 
 
-def _plain_mention_to_html(match: Match) -> str:
-    puppet = pu.Puppet.find_by_displayname(match.group(2))
-    if puppet:
-        return (f"{match.group(1)}"
-                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                f"{puppet.displayname}"
-                "</a>")
-    return "".join(match.groups())
-
-
 def init_mx(context: "Context") -> None:
     global plain_mention_regex, should_bridge_plaintext_highlights
     config = context.config
-    dn_template = config["bridge.displayname_template"]
+    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
     dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
     plain_mention_regex = re.compile(f"^({dn_template})")
-    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"]
+    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -0,0 +1,66 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Tuple
+
+from html.parser import HTMLParser
+
+
+class HTMLNode(list):
+    def __init__(self, tag: str, attrs: List[Tuple[str, str]]):
+        super().__init__()
+        self.tag = tag  # type: str
+        self.text = ""  # type: str
+        self.tail = ""  # type: str
+        self.attrib = dict(attrs)  # type: Dict[str, str]
+
+
+class NodeifyingParser(HTMLParser):
+    # From https://www.w3.org/TR/html5/syntax.html#writing-html-documents-elements
+    void_tags = ("area", "base", "br", "col", "command", "embed", "hr", "img", "input", "link",
+                 "meta", "param", "source", "track", "wbr")
+
+    def __init__(self):
+        super().__init__()
+        self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
+
+    def handle_starttag(self, tag, attrs):
+        node = HTMLNode(tag, attrs)
+        self.stack[-1].append(node)
+        if tag not in self.void_tags:
+            self.stack.append(node)
+
+    def handle_startendtag(self, tag, attrs):
+        self.stack[-1].append(HTMLNode(tag, attrs))
+
+    def handle_endtag(self, tag):
+        if tag == self.stack[-1].tag:
+            self.stack.pop()
+
+    def handle_data(self, data):
+        if len(self.stack[-1]) > 0:
+            self.stack[-1][-1].tail += data
+        else:
+            self.stack[-1].text += data
+
+    def error(self, message):
+        pass
+
+
+def read_html(data: str) -> HTMLNode:
+    parser = NodeifyingParser()
+    parser.feed(data)
+    return parser.stack[0]

mautrix_telegram/formatter/from_matrix/html_reader.pyi

@@ -0,0 +1,11 @@
+from typing import Dict, List
+
+
+class HTMLNode(List['HTMLNode']):
+    tag: str
+    text: str
+    tail: str
+    attrib: Dict[str, str]
+
+
+def read_html(data: str) -> HTMLNode: ...

mautrix_telegram/formatter/from_matrix/parser.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,77 +14,240 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Tuple, Optional
+from typing import List, Tuple, Pattern
+import re
 
-from telethon.tl.types import TypeMessageEntity
-
-from mautrix.types import UserID, RoomID
-from mautrix.util.formatter import MatrixParser as BaseMatrixParser, RecursionContext
-from mautrix.util.formatter.html_reader_htmlparser import read_html, HTMLNode
+from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
+                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
+                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
+                               MessageEntityBold as Bold, MessageEntityItalic as Italic,
+                               MessageEntityCode as Code, MessageEntityPre as Pre,
+                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
+                               MessageEntityBlockquote as Blockquote, TypeMessageEntity)
 
 from ... import user as u, puppet as pu, portal as po
-from .telegram_message import TelegramMessage, TelegramEntityType
+from ...types import MatrixUserID
+from .telegram_message import TelegramMessage, Entity, offset_length_multiply
 
+from .html_reader import HTMLNode, read_html
 
 ParsedMessage = Tuple[str, List[TypeMessageEntity]]
 
 
 def parse_html(input_html: str) -> ParsedMessage:
-    msg = MatrixParser.parse(input_html)
-    return msg.text, msg.telegram_entities
+    return MatrixParser.parse(input_html)
 
 
-class MatrixParser(BaseMatrixParser[TelegramMessage]):
-    e = TelegramEntityType
-    fs = TelegramMessage
-    read_html = read_html
+class RecursionContext:
+    def __init__(self, strip_linebreaks: bool = True, ul_depth: int = 0):
+        self.strip_linebreaks = strip_linebreaks  # type: bool
+        self.ul_depth = ul_depth  # type: int
+        self._inited = True  # type: bool
+
+    def __setattr__(self, key, value):
+        if getattr(self, "_inited", False) is True:
+            raise TypeError("'RecursionContext' object is immutable")
+        super(RecursionContext, self).__setattr__(key, value)
+
+    def enter_list(self) -> 'RecursionContext':
+        return RecursionContext(strip_linebreaks=self.strip_linebreaks, ul_depth=self.ul_depth + 1)
+
+    def enter_code_block(self) -> 'RecursionContext':
+        return RecursionContext(strip_linebreaks=False, ul_depth=self.ul_depth)
+
+
+class MatrixParser:
+    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")  # type: Pattern
+    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")  # type: Pattern
+    block_tags = ("p", "pre", "blockquote",
+                  "ol", "ul", "li",
+                  "h1", "h2", "h3", "h4", "h5", "h6",
+                  "div", "hr", "table")  # type: Tuple[str, ...]
+    list_bullets = ("●", "○", "■", "‣")  # type: Tuple[str, ...]
 
     @classmethod
-    def custom_node_to_fstring(cls, node: HTMLNode, ctx: RecursionContext
-                               ) -> Optional[TelegramMessage]:
+    def list_bullet(cls, depth: int) -> str:
+        return cls.list_bullets[(depth - 1) % len(cls.list_bullets)] + " "
+
+    @classmethod
+    def list_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        ordered = node.tag == "ol"
+        tagged_children = cls.node_to_tagged_tmessages(node, ctx)
+        counter = 1
+        indent_length = 0
+        if ordered:
+            try:
+                counter = int(node.attrib.get("start", "1"))
+            except ValueError:
+                counter = 1
+
+            longest_index = counter - 1 + len(tagged_children)
+            indent_length = len(str(longest_index))
+        indent = (indent_length + 4) * " "
+        children = []  # type: List[TelegramMessage]
+        for child, tag in tagged_children:
+            if tag != "li":
+                continue
+
+            if ordered:
+                prefix = f"{counter}. "
+                counter += 1
+            else:
+                prefix = cls.list_bullet(ctx.ul_depth)
+            child = child.prepend(prefix)
+            parts = child.split("\n")
+            parts = parts[:1] + [part.prepend(indent) for part in parts[1:]]
+            child = TelegramMessage.join(parts, "\n")
+            children.append(child)
+        return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def header_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        children = cls.node_to_tmessages(node, ctx)
+        length = int(node.tag[1])
+        prefix = "#" * length + " "
+        return TelegramMessage.join(children, "").prepend(prefix).format(Bold)
+
+    @classmethod
+    def basic_format_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         msg = cls.tag_aware_parse_node(node, ctx)
-        if node.tag == "command":
-            msg.format(TelegramEntityType.COMMAND)
-        return None
+        if node.tag in ("b", "strong"):
+            msg.format(Bold)
+        elif node.tag in ("i", "em"):
+            msg.format(Italic)
+        elif node.tag in ("s", "strike", "del"):
+            msg.format(Strike)
+        elif node.tag in ("u", "ins"):
+            msg.format(Underline)
+        elif node == "blockquote":
+            msg.format(Blockquote)
+        elif node.tag == "command":
+            msg.format(Command)
 
-    @classmethod
-    def user_pill_to_fstring(cls, msg: TelegramMessage, user_id: UserID) -> TelegramMessage:
-        user = (pu.Puppet.deprecated_sync_get_by_mxid(user_id)
-                or u.User.get_by_mxid(user_id, create=False))
-        if not user:
-            return msg
-        if user.username:
-            return TelegramMessage(f"@{user.username}").format(TelegramEntityType.MENTION)
-        elif user.tgid:
-            displayname = user.plain_displayname or msg.text
-            return TelegramMessage(displayname).format(TelegramEntityType.MENTION_NAME,
-                                                       user_id=user.tgid)
         return msg
 
     @classmethod
-    def url_to_fstring(cls, msg: TelegramMessage, url: str) -> TelegramMessage:
-        if url == msg.text:
-            return msg.format(cls.e.URL)
-        else:
-            return msg.format(cls.e.INLINE_URL, url=url)
+    def link_to_tstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        href = node.attrib.get("href", "")
+        if not href:
+            return msg
+
+        if href.startswith("mailto:"):
+            return TelegramMessage(href[len("mailto:"):]).format(Email)
+
+        mention = cls.mention_regex.match(href)
+        if mention:
+            mxid = MatrixUserID(mention.group(1))
+            user = (pu.Puppet.get_by_mxid(mxid)
+                    or u.User.get_by_mxid(mxid, create=False))
+            if not user:
+                return msg
+            if user.username:
+                return TelegramMessage(f"@{user.username}").format(Mention)
+            elif user.tgid:
+                displayname = user.plain_displayname or msg.text
+                return TelegramMessage(displayname).format(MentionName, user_id=user.tgid)
+            return msg
+
+        room = cls.room_regex.match(href)
+        if room:
+            username = po.Portal.get_username_from_mx_alias(room.group(1))
+            portal = po.Portal.find_by_username(username)
+            if portal and portal.username:
+                return TelegramMessage(f"@{portal.username}").format(Mention)
+
+        return (msg.format(URL)
+                if msg.text == href
+                else msg.format(TextURL, url=href))
 
     @classmethod
-    def room_pill_to_fstring(cls, msg: TelegramMessage, room_id: RoomID) -> TelegramMessage:
-        username = po.Portal.get_username_from_mx_alias(room_id)
-        portal = po.Portal.find_by_username(username)
-        if portal and portal.username:
-            return TelegramMessage(f"@{portal.username}").format(TelegramEntityType.MENTION)
-
-    @classmethod
-    def header_to_fstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        children = cls.node_to_fstrings(node, ctx)
-        length = int(node.tag[1])
-        prefix = "#" * length + " "
-        return TelegramMessage.join(children, "").prepend(prefix).format(TelegramEntityType.BOLD)
-
-    @classmethod
-    def blockquote_to_fstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         msg = cls.tag_aware_parse_node(node, ctx)
         children = msg.trim().split("\n")
         children = [child.prepend("> ") for child in children]
         return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def node_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        if node.tag == "mx-reply":
+            return TelegramMessage("")
+        elif node.tag == "ol":
+            return cls.list_to_tmessage(node, ctx)
+        elif node.tag == "ul":
+            return cls.list_to_tmessage(node, ctx.enter_list())
+        elif node.tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            return cls.header_to_tmessage(node, ctx)
+        elif node.tag == "br":
+            return TelegramMessage("\n")
+        elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
+            return cls.basic_format_to_tmessage(node, ctx)
+        elif node.tag == "blockquote":
+            # Telegram already has blockquote entities in the protocol schema, but it strips them
+            # server-side and none of the official clients support them.
+            # TODO once Telegram changes that, use the above if block for blockquotes too.
+            return cls.blockquote_to_tmessage(node, ctx)
+        elif node.tag == "a":
+            return cls.link_to_tstring(node, ctx)
+        elif node.tag == "p":
+            return cls.tag_aware_parse_node(node, ctx).append("\n")
+        elif node.tag == "pre":
+            lang = ""
+            try:
+                if node[0].tag == "code":
+                    node = node[0]
+                    lang = node.attrib["class"][len("language-"):]
+            except (IndexError, KeyError):
+                pass
+            return cls.parse_node(node, ctx.enter_code_block()).format(Pre, language=lang)
+        elif node.tag == "code":
+            return cls.parse_node(node, ctx.enter_code_block()).format(Code)
+        return cls.tag_aware_parse_node(node, ctx)
+
+    @staticmethod
+    def text_to_tmessage(text: str, ctx: RecursionContext) -> TelegramMessage:
+        if ctx.strip_linebreaks:
+            text = text.replace("\n", "")
+        return TelegramMessage(text)
+
+    @classmethod
+    def node_to_tagged_tmessages(cls, node: HTMLNode, ctx: RecursionContext
+                                 ) -> List[Tuple[TelegramMessage, str]]:
+        output = []
+
+        if node.text:
+            output.append((cls.text_to_tmessage(node.text, ctx), "text"))
+        for child in node:
+            output.append((cls.node_to_tmessage(child, ctx), child.tag))
+            if child.tail:
+                output.append((cls.text_to_tmessage(child.tail, ctx), "text"))
+        return output
+
+    @classmethod
+    def node_to_tmessages(cls, node: HTMLNode, ctx: RecursionContext
+                          ) -> List[TelegramMessage]:
+        return [msg for (msg, tag) in cls.node_to_tagged_tmessages(node, ctx)]
+
+    @classmethod
+    def tag_aware_parse_node(cls, node: HTMLNode, ctx: RecursionContext
+                             ) -> TelegramMessage:
+        msgs = cls.node_to_tagged_tmessages(node, ctx)
+        output = TelegramMessage()
+        prev_was_block = False
+        for msg, tag in msgs:
+            if tag in cls.block_tags:
+                msg = msg.append("\n")
+                if not prev_was_block:
+                    msg = msg.prepend("\n")
+                prev_was_block = True
+            output = output.append(msg)
+        return output.trim()
+
+    @classmethod
+    def parse_node(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        return TelegramMessage.join(cls.node_to_tmessages(node, ctx))
+
+    @classmethod
+    def parse(cls, data: str) -> ParsedMessage:
+        msg = cls.node_to_tmessage(read_html(f"<body>{data}</body>"), RecursionContext())
+        return msg.text, msg.entities

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,87 +14,145 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Union, Any, List, Type, Dict
-from enum import Enum
+from typing import Callable, List, Optional, Sequence, Type, Union
 
-from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
-                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
-                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
-                               MessageEntityBold as Bold, MessageEntityItalic as Italic,
-                               MessageEntityCode as Code, MessageEntityPre as Pre,
-                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
-                               MessageEntityBlockquote as Blockquote, TypeMessageEntity,
-                               InputMessageEntityMentionName as InputMentionName)
-
-from mautrix.util.formatter import EntityString, SemiAbstractEntity
+from telethon.tl.types import (MessageEntityMentionName as MentionName,
+                               MessageEntityTextUrl as TextURL, MessageEntityPre as Pre,
+                               TypeMessageEntity, InputMessageEntityMentionName as InputMentionName)
 
 
-class TelegramEntityType(Enum):
-    """EntityType is a Matrix formatting entity type."""
-    BOLD = Bold
-    ITALIC = Italic
-    STRIKETHROUGH = Strike
-    UNDERLINE = Underline
-    URL = URL
-    INLINE_URL = TextURL
-    EMAIL = Email
-    PREFORMATTED = Pre
-    INLINE_CODE = Code
-    BLOCKQUOTE = Blockquote
-    MENTION = Mention
-    MENTION_NAME = MentionName
-    COMMAND = Command
+class Entity:
+    @staticmethod
+    def copy(entity: TypeMessageEntity) -> Optional[TypeMessageEntity]:
+        if not entity:
+            return None
+        kwargs = {
+            "offset": entity.offset,
+            "length": entity.length,
+        }
+        if isinstance(entity, Pre):
+            kwargs["language"] = entity.language
+        elif isinstance(entity, TextURL):
+            kwargs["url"] = entity.url
+        elif isinstance(entity, (MentionName, InputMentionName)):
+            kwargs["user_id"] = entity.user_id
+        return entity.__class__(**kwargs)
 
-    USER_MENTION = 1
-    ROOM_MENTION = 2
-    HEADER = 3
+    @classmethod
+    def adjust(cls, entity: Union[TypeMessageEntity, List[TypeMessageEntity]],
+               func: Callable[[TypeMessageEntity], None]
+               ) -> Union[Optional[TypeMessageEntity], List[TypeMessageEntity]]:
+        if isinstance(entity, list):
+            return [Entity.adjust(element, func) for element in entity if entity]
+        elif not entity:
+            return None
+        entity = cls.copy(entity)
+        func(entity)
+        if entity.offset < 0:
+            entity.length += entity.offset
+            entity.offset = 0
+        return entity
 
 
-class TelegramEntity(SemiAbstractEntity):
-    internal: TypeMessageEntity
+def offset_diff(amount: int) -> Callable[[TypeMessageEntity], None]:
+    def func(entity: TypeMessageEntity) -> None:
+        entity.offset += amount
 
-    def __init__(self, type: Union[TelegramEntityType, Type[TypeMessageEntity]],
-                 offset: int, length: int, extra_info: Dict[str, Any]) -> None:
-        if isinstance(type, TelegramEntityType):
-            if isinstance(type.value, int):
-                raise ValueError(f"Can't create Entity with non-Telegram EntityType {type}")
-            type = type.value
-        self.internal = type(offset=offset, length=length, **extra_info)
-
-    def copy(self) -> Optional['TelegramEntity']:
-        extra_info = {}
-        if isinstance(self.internal, Pre):
-            extra_info["language"] = self.internal.language
-        elif isinstance(self.internal, TextURL):
-            extra_info["url"] = self.internal.url
-        elif isinstance(self.internal, (MentionName, InputMentionName)):
-            extra_info["user_id"] = self.internal.user_id
-        return TelegramEntity(type(self.internal), offset=self.internal.offset,
-                              length=self.internal.length, extra_info=extra_info)
-
-    def __repr__(self) -> str:
-        return str(self.internal)
-
-    @property
-    def offset(self) -> int:
-        return self.internal.offset
-
-    @offset.setter
-    def offset(self, value: int) -> None:
-        self.internal.offset = value
-
-    @property
-    def length(self) -> int:
-        return self.internal.length
-
-    @length.setter
-    def length(self, value: int) -> None:
-        self.internal.length = value
+    return func
 
 
-class TelegramMessage(EntityString[TelegramEntity, TelegramEntityType]):
-    entity_class = TelegramEntity
+def offset_length_multiply(amount: int) -> Callable[[TypeMessageEntity], None]:
+    def func(entity: TypeMessageEntity) -> None:
+        entity.offset *= amount
+        entity.length *= amount
 
-    @property
-    def telegram_entities(self) -> List[TypeMessageEntity]:
-        return [entity.internal for entity in self.entities]
+    return func
+
+
+class TelegramMessage:
+    def __init__(self, text: str = "", entities: Optional[List[TypeMessageEntity]] = None) -> None:
+        self.text = text  # type: str
+        self.entities = entities or []  # type: List[TypeMessageEntity]
+
+    def offset_entities(self, offset: int) -> 'TelegramMessage':
+        def apply_offset(entity: TypeMessageEntity, inner_offset: int
+                         ) -> Optional[TypeMessageEntity]:
+            entity = Entity.copy(entity)
+            entity.offset += inner_offset
+            if entity.offset < 0:
+                entity.offset = 0
+            elif entity.offset > len(self.text):
+                return None
+            elif entity.offset + entity.length > len(self.text):
+                entity.length = len(self.text) - entity.offset
+            return entity
+
+        self.entities = [apply_offset(entity, offset) for entity in self.entities if entity]
+        self.entities = [x for x in self.entities if x is not None]
+        return self
+
+    def append(self, *args: Union[str, 'TelegramMessage']) -> 'TelegramMessage':
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities += Entity.adjust(msg.entities, offset_diff(len(self.text)))
+            self.text += msg.text
+        return self
+
+    def prepend(self, *args: Union[str, 'TelegramMessage']) -> 'TelegramMessage':
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities = msg.entities + Entity.adjust(self.entities, offset_diff(len(msg.text)))
+            self.text = msg.text + self.text
+        return self
+
+    def format(self, entity_type: Type[TypeMessageEntity], offset: int = None, length: int = None,
+               **kwargs) -> 'TelegramMessage':
+        self.entities.append(entity_type(offset=offset or 0,
+                                         length=length if length is not None else len(self.text),
+                                         **kwargs))
+        return self
+
+    def concat(self, *args: Union[str, 'TelegramMessage']) -> 'TelegramMessage':
+        return TelegramMessage().append(self, *args)
+
+    def trim(self) -> 'TelegramMessage':
+        orig_len = len(self.text)
+        self.text = self.text.lstrip()
+        diff = orig_len - len(self.text)
+        self.text = self.text.rstrip()
+        self.offset_entities(-diff)
+        return self
+
+    def split(self, separator, max_items: int = 0) -> List['TelegramMessage']:
+        text_parts = self.text.split(separator, max_items - 1)
+        output = []  # type: List[TelegramMessage]
+
+        offset = 0
+        for part in text_parts:
+            msg = TelegramMessage(part)
+            for entity in self.entities:
+                start_in_range = len(part) > entity.offset - offset >= 0
+                end_in_range = len(part) >= entity.offset - offset + entity.length > 0
+                if start_in_range and end_in_range:
+                    msg.entities.append(Entity.adjust(entity, offset_diff(-offset)))
+            output.append(msg)
+
+            offset += len(part)
+            offset += len(separator)
+
+        return output
+
+    @staticmethod
+    def join(items: Sequence[Union[str, 'TelegramMessage']],
+             separator: str = " ") -> 'TelegramMessage':
+        main = TelegramMessage()
+        for msg in items:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            main.entities += Entity.adjust(msg.entities, offset_diff(len(main.text)))
+            main.text += msg.text + separator
+        if len(separator) > 0:
+            main.text = main.text[:-len(separator)]
+        return main

mautrix_telegram/formatter/from_telegram.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Optional, TYPE_CHECKING
+from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
 from html import escape
 import logging
 import re
@@ -22,120 +23,148 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, M
                                MessageEntityEmail, MessageEntityTextUrl, MessageEntityBold,
                                MessageEntityItalic, MessageEntityCode, MessageEntityPre,
                                MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
-                               MessageEntityPhone, TypeMessageEntity, PeerChannel, PeerChat,
+                               MessageEntityPhone, TypeMessageEntity, Message, PeerChannel,
                                MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
                                MessageEntityUnderline, PeerUser)
-from telethon.tl.custom import Message
-from telethon.errors import RPCError
-from telethon.helpers import add_surrogate, del_surrogate
 
-from mautrix.errors import MatrixRequestError
-from mautrix.appservice import IntentAPI
-from mautrix.types import (TextMessageEventContent, RelatesTo, RelationType, Format, MessageType,
-                           MessageEvent)
+from mautrix_appservice import MatrixRequestError
+from mautrix_appservice.intent_api import IntentAPI
 
 from .. import user as u, puppet as pu, portal as po
 from ..types import TelegramID
 from ..db import Message as DBMessage
+from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                   trim_reply_fallback_text)
 
 if TYPE_CHECKING:
     from ..abstract_user import AbstractUser
 
-log: logging.Logger = logging.getLogger("mau.fmt.tg")
+log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
 
 
-def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Optional[RelatesTo]:
-    if evt.reply_to:
-        space = (evt.peer_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
+def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
+    if evt.reply_to_msg_id:
+        space = (evt.to_id.channel_id
+                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
+        msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
         if msg:
-            return RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
-    return None
+            return {
+                "m.in_reply_to": {
+                    "event_id": msg.mxid,
+                    "room_id": msg.mx_room,
+                },
+                "rel_type": "m.reference",
+                "event_id": msg.mxid,
+                "room_id": msg.mx_room,
+            }
+    return {}
 
 
-async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventContent,
-                              fwd_from: MessageFwdHeader) -> None:
-    if not content.formatted_body or content.format != Format.HTML:
-        content.format = Format.HTML
-        content.formatted_body = escape(content.body)
+async def _add_forward_header(source, text: str, html: Optional[str],
+                              fwd_from: MessageFwdHeader) -> Tuple[str, str]:
+    if not html:
+        html = escape(text)
     fwd_from_html, fwd_from_text = None, None
-    if isinstance(fwd_from.from_id, PeerUser):
-        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id.user_id))
+    if fwd_from.from_id:
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
             fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
                              f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
-            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id.user_id), create=False)
+            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
                 fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
                                  f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
-            try:
-                user = await source.client.get_entity(fwd_from.from_id)
-                if user:
-                    fwd_from_text = pu.Puppet.get_displayname(user, False)
-                    fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
-            except (ValueError, RPCError):
-                fwd_from_text = fwd_from_html = "unknown user"
-    elif isinstance(fwd_from.from_id, (PeerChannel, PeerChat)):
-        from_id = (fwd_from.from_id.chat_id if isinstance(fwd_from.from_id, PeerChat)
-                   else fwd_from.from_id.channel_id)
-        portal = po.Portal.get_by_tgid(TelegramID(from_id))
+            user = await source.client.get_entity(PeerUser(fwd_from.from_id))
+            if user:
+                fwd_from_text = pu.Puppet.get_displayname(user, False)
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+    else:
+        portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
         if portal:
             fwd_from_text = portal.title
             if portal.alias:
                 fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
                                  f"{escape(fwd_from_text)}</a>")
             else:
-                fwd_from_html = f"channel <b>{escape(fwd_from_text)}</b>"
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
         else:
-            try:
-                channel = await source.client.get_entity(fwd_from.from_id)
-                if channel:
-                    fwd_from_text = f"channel {channel.title}"
-                    fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
-            except (ValueError, RPCError):
-                fwd_from_text = fwd_from_html = "unknown channel"
-    elif fwd_from.from_name:
-        fwd_from_text = fwd_from.from_name
-        fwd_from_html = f"<b>{escape(fwd_from.from_name)}</b>"
-    else:
-        fwd_from_text = "unknown source"
-        fwd_from_html = f"unknown source"
+            channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
+            if channel:
+                fwd_from_text = channel.title
+                fwd_from_html = f"<b>{fwd_from_text}</b>"
 
-    content.body = "\n".join([f"> {line}" for line in content.body.split("\n")])
-    content.body = f"Forwarded from {fwd_from_text}:\n{content.body}"
-    content.formatted_body = (
-        f"Forwarded message from {fwd_from_html}<br/>"
-        f"<tg-forward><blockquote>{content.formatted_body}</blockquote></tg-forward>")
+    if not fwd_from_text:
+        if fwd_from.from_id:
+            fwd_from_text = "Unknown user"
+        else:
+            fwd_from_text = "Unknown source"
+        fwd_from_html = f"<b>{fwd_from_text}</b>"
+
+    text = "\n".join([f"> {line}" for line in text.split("\n")])
+    text = f"Forwarded from {fwd_from_text}:\n{text}"
+    html = (f"Forwarded message from {fwd_from_html}<br/>"
+            f"<tg-forward><blockquote>{html}</blockquote></tg-forward>")
+    return text, html
 
 
-async def _add_reply_header(source: 'AbstractUser', content: TextMessageEventContent, evt: Message,
-                            main_intent: IntentAPI):
-    space = (evt.peer_id.channel_id
-             if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
+async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
+                            relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]:
+    space = (evt.to_id.channel_id
+             if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
+    msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
     if not msg:
-        return
+        return text, html
 
-    content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
+    relates_to["rel_type"] = "m.reference"
+    relates_to["event_id"] = msg.mxid
+    relates_to["room_id"] = msg.mx_room
+    relates_to["m.in_reply_to"] = {
+        "event_id": msg.mxid,
+        "room_id": msg.mx_room,
+    }
 
     try:
-        event: MessageEvent = await main_intent.get_event(msg.mx_room, msg.mxid)
-        if isinstance(event.content, TextMessageEventContent):
-            event.content.trim_reply_fallback()
-        puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
-        content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
-    except MatrixRequestError:
-        log.exception("Failed to get event to add reply fallback")
+        event = await main_intent.get_event(msg.mx_room, msg.mxid)
+
+        content = event["content"]
+        r_sender = event["sender"]
+
+        r_text_body = trim_reply_fallback_text(content["body"])
+        r_html_body = trim_reply_fallback_html(content["formatted_body"]
+                                               if "formatted_body" in content
+                                               else escape(content["body"]))
+
+        puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
+        r_displayname = puppet.displayname if puppet else r_sender
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{escape(r_displayname)}</a>"
+    except (ValueError, KeyError, MatrixRequestError):
+        r_sender_link = "unknown user"
+        r_displayname = "unknown user"
+        r_text_body = "Failed to fetch message"
+        r_html_body = "<em>Failed to fetch message</em>"
+
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>In reply to</a>"
+    html = (
+        f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
+        + (html or escape(text)))
+
+    lines = r_text_body.strip().split("\n")
+    text_with_quote = f"> <{r_displayname}> {lines.pop(0)}"
+    for line in lines:
+        if line:
+            text_with_quote += f"\n> {line}"
+    text_with_quote += "\n\n"
+    text_with_quote += text
+    return text_with_quote, html
 
 
 async def telegram_to_matrix(evt: Message, source: "AbstractUser",
@@ -143,42 +172,33 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
                              prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
                              override_text: str = None,
                              override_entities: List[TypeMessageEntity] = None,
-                             no_reply_fallback: bool = False) -> TextMessageEventContent:
-    content = TextMessageEventContent(
-        msgtype=MessageType.TEXT,
-        body=add_surrogate(override_text or evt.message),
-    )
+                             no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
+    text = add_surrogates(override_text or evt.message)
     entities = override_entities or evt.entities
-    if entities:
-        content.format = Format.HTML
-        content.formatted_body = _telegram_entities_to_matrix_catch(content.body, entities)
+    html = _telegram_entities_to_matrix_catch(text, entities) if entities else None
+    relates_to = {}  # type: Dict
 
     if prefix_html:
-        if not content.formatted_body:
-            content.format = Format.HTML
-            content.formatted_body = escape(content.body)
-        content.formatted_body = prefix_html + content.formatted_body
+        html = prefix_html + (html or escape(text))
     if prefix_text:
-        content.body = prefix_text + content.body
+        text = prefix_text + text
 
     if evt.fwd_from:
-        await _add_forward_header(source, content, evt.fwd_from)
+        text, html = await _add_forward_header(source, text, html, evt.fwd_from)
 
-    if evt.reply_to and not no_reply_fallback:
-        await _add_reply_header(source, content, evt, main_intent)
+    if evt.reply_to_msg_id and not no_reply_fallback:
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
-        if not content.formatted_body:
-            content.formatted_body = escape(content.body)
-        content.body += f"\n- {evt.post_author}"
-        content.formatted_body += f"<br/><i>- <u>{evt.post_author}</u></i>"
+        if not html:
+            html = escape(text)
+        text += f"\n- {evt.post_author}"
+        html += f"<br/><i>- <u>{evt.post_author}</u></i>"
 
-    content.body = del_surrogate(content.body)
+    if html:
+        html = html.replace("\n", "<br/>")
 
-    if content.formatted_body:
-        content.formatted_body = del_surrogate(content.formatted_body.replace("\n", "<br/>"))
-
-    return content
+    return remove_surrogates(text), remove_surrogates(html), relates_to
 
 
 def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEntity]) -> str:
@@ -293,8 +313,8 @@ def _parse_name_mention(html: List[str], entity_text: str, user_id: TelegramID)
     return False
 
 
-message_link_regex = re.compile(r"https?://t(?:elegram)?\.(?:me|dog)/"
-                                r"([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})")
+message_link_regex = re.compile(
+    r"https?://t(?:elegram)?\.(?:me|dog)/([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})")
 
 
 def _parse_url(html: List[str], entity_text: str, url: str) -> bool:

mautrix_telegram/formatter/util.py

@@ -0,0 +1,57 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Pattern
+from html import escape
+import struct
+import re
+
+
+# add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
+# Licensed under the MIT license.
+# https://github.com/LonamiWebs/Telethon/blob/7cce7aa3e4c6c7019a55530391b1761d33e5a04e/telethon/helpers.py
+def add_surrogates(text: Optional[str]) -> Optional[str]:
+    if text is None:
+        return None
+    return "".join("".join(chr(y) for y in struct.unpack("<HH", x.encode("utf-16-le")))
+                   if (0x10000 <= ord(x) <= 0x10FFFF) else x for x in text)
+
+
+def remove_surrogates(text: Optional[str]) -> Optional[str]:
+    if text is None:
+        return None
+    return text.encode("utf-16", "surrogatepass").decode("utf-16")
+
+
+# trim_reply_fallback_text, html_reply_fallback_regex and trim_reply_fallback_html are Matrix
+# reply fallback utility functions.
+# You may copy and use them under any OSI-approved license.
+def trim_reply_fallback_text(text: str) -> str:
+    if not text.startswith("> ") or "\n" not in text:
+        return text
+    lines = text.split("\n")
+    while len(lines) > 0 and lines[0].startswith("> "):
+        lines.pop(0)
+    return "\n".join(lines)
+
+
+html_reply_fallback_regex = re.compile("^<mx-reply>"
+                                       r"[\s\S]+?"
+                                       "</mx-reply>")  # type: Pattern
+
+
+def trim_reply_fallback_html(html: str) -> str:
+    return html_reply_fallback_regex.sub("", html)

mautrix_telegram/get_version.py

@@ -1,49 +0,0 @@
-import subprocess
-import shutil
-import os
-
-from . import __version__
-
-cmd_env = {
-    "PATH": os.environ["PATH"],
-    "HOME": os.environ["HOME"],
-    "LANG": "C",
-    "LC_ALL": "C",
-}
-
-
-def run(cmd):
-    return subprocess.check_output(cmd, stderr=subprocess.DEVNULL, env=cmd_env)
-
-if os.path.exists(".git") and shutil.which("git"):
-    try:
-        git_revision = run(["git", "rev-parse", "HEAD"]).strip().decode("ascii")
-        git_revision_url = f"https://github.com/tulir/mautrix-telegram/commit/{git_revision}"
-        git_revision = git_revision[:8]
-    except (subprocess.SubprocessError, OSError):
-        git_revision = "unknown"
-        git_revision_url = None
-
-    try:
-        git_tag = run(["git", "describe", "--exact-match", "--tags"]).strip().decode("ascii")
-    except (subprocess.SubprocessError, OSError):
-        git_tag = None
-else:
-    git_revision = "unknown"
-    git_revision_url = None
-    git_tag = None
-
-git_tag_url = (f"https://github.com/tulir/mautrix-telegram/releases/tag/{git_tag}"
-               if git_tag else None)
-
-if git_tag and __version__ == git_tag[1:].replace("-", ""):
-    version = __version__
-    linkified_version = f"[{version}]({git_tag_url})"
-else:
-    if not __version__.endswith("+dev"):
-        __version__ += "+dev"
-    version = f"{__version__}.{git_revision}"
-    if git_revision_url:
-        linkified_version = f"{__version__}.[{git_revision}]({git_revision_url})"
-    else:
-        linkified_version = version

mautrix_telegram/matrix.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -13,47 +14,60 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, Set, Tuple, Union, Iterable, TYPE_CHECKING
+from typing import Dict, List, Match, Optional, Set, Tuple, TYPE_CHECKING
+import logging
+import asyncio
+import time
+import re
 
-from mautrix.bridge import BaseMatrixHandler
-from mautrix.types import (Event, EventType, RoomID, UserID, EventID, ReceiptEvent, ReceiptType,
-                           ReceiptEventContent, PresenceEvent, PresenceState, TypingEvent,
-                           MessageEvent, StateEvent, RedactionEvent, RoomNameStateEventContent,
-                           RoomAvatarStateEventContent, RoomTopicStateEventContent,
-                           MemberStateEventContent, EncryptedEvent, TextMessageEventContent,
-                           MessageType)
-from mautrix.errors import MatrixError
+from mautrix_appservice import MatrixRequestError, IntentError
 
+from .types import MatrixEvent, MatrixEventID, MatrixRoomID, MatrixUserID
 from . import user as u, portal as po, puppet as pu, commands as com
 
 if TYPE_CHECKING:
     from .context import Context
-    from .bot import Bot
 
-RoomMetaStateEventContent = Union[RoomNameStateEventContent, RoomAvatarStateEventContent,
-                                  RoomTopicStateEventContent]
+try:
+    from prometheus_client import Histogram
+
+    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events",
+                           ["event_type"])
+except ImportError:
+    Histogram = None
+    EVENT_TIME = None
 
 
-class MatrixHandler(BaseMatrixHandler):
-    bot: 'Bot'
-    commands: 'com.CommandProcessor'
-    previously_typing: Dict[RoomID, Set[UserID]]
+class MatrixHandler:
+    log = logging.getLogger("mau.mx")  # type: logging.Logger
 
     def __init__(self, context: 'Context') -> None:
-        prefix, suffix = context.config["bridge.username_template"].format(userid=":").split(":")
-        homeserver = context.config["homeserver.domain"]
-        self.user_id_prefix = f"@{prefix}"
-        self.user_id_suffix = f"{suffix}:{homeserver}"
+        self.az, self.config, _, self.tgbot = context.core
+        self.commands = com.CommandProcessor(context)  # type: com.CommandProcessor
+        self.previously_typing = []  # type: List[MatrixUserID]
 
-        super().__init__(command_processor=com.CommandProcessor(context), bridge=context.bridge)
+        self.az.matrix_event_handler(self.handle_event)
 
-        self.bot = context.bot
-        self.previously_typing = {}
+    async def init_as_bot(self) -> None:
+        displayname = self.config["appservice.bot_displayname"]
+        if displayname:
+            try:
+                await self.az.intent.set_display_name(
+                    displayname if displayname != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set displayname")
 
-    async def handle_puppet_invite(self, room_id: RoomID, puppet: pu.Puppet, inviter: u.User,
-                                   event_id: EventID) -> None:
+        avatar = self.config["appservice.bot_avatar"]
+        if avatar:
+            try:
+                await self.az.intent.set_avatar(avatar if avatar != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set avatar")
+
+    async def handle_puppet_invite(self, room_id: MatrixRoomID, puppet: pu.Puppet, inviter: u.User
+                                   ) -> None:
         intent = puppet.default_mxid_intent
-        self.log.debug(f"{inviter.mxid} invited puppet for {puppet.tgid} to {room_id}")
+        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}")
         if not await inviter.is_logged_in():
             await intent.error_and_leave(
                 room_id, text="Please log in before inviting Telegram puppets.")
@@ -68,12 +82,11 @@ class MatrixHandler(BaseMatrixHandler):
             await intent.join_room(room_id)
             return
         try:
-            members = await intent.get_room_members(room_id)
-        except MatrixError:
-            self.log.exception(f"Failed to get members after joining {room_id} as {intent.mxid}")
-            return
+            members = await self.az.intent.get_room_members(room_id)
+        except MatrixRequestError:
+            members = []
         if self.az.bot_mxid not in members:
-            if len(members) > 2:
+            if len(members) > 1:
                 await intent.error_and_leave(room_id, text=None, html=(
                     f"Please invite "
                     f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
@@ -82,59 +95,89 @@ class MatrixHandler(BaseMatrixHandler):
 
             await intent.join_room(room_id)
             portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
+            # TODO: if portal is None:
             if portal.mxid:
                 try:
-                    await intent.invite_user(portal.mxid, inviter.mxid)
-                    await intent.send_notice(
-                        room_id, text=f"You already have a private chat with me: {portal.mxid}",
-                        html=("You already have a private chat with me: "
-                              f"<a href='https://matrix.to/#/{portal.mxid}'>Link to room</a>"))
+                    await intent.invite(portal.mxid, inviter.mxid)
+                    await intent.send_notice(room_id, text=None, html=(
+                        "You already have a private chat with me: "
+                        f"<a href='https://matrix.to/#/{portal.mxid}'>"
+                        "Link to room"
+                        "</a>"))
                     await intent.leave_room(room_id)
                     return
-                except MatrixError:
+                except MatrixRequestError:
                     pass
             portal.mxid = room_id
-            e2be_ok = None
-            if self.config["bridge.encryption.default"] and self.e2ee:
-                e2be_ok = await portal.enable_dm_encryption()
-            await portal.save()
-            await inviter.register_portal(portal)
-            if e2be_ok is True:
-                evt_type, content = await self.e2ee.encrypt(
-                    room_id, EventType.ROOM_MESSAGE,
-                    TextMessageEventContent(msgtype=MessageType.NOTICE,
-                                            body="Portal to private chat created and end-to-bridge"
-                                                 " encryption enabled."))
-                await intent.send_message_event(room_id, evt_type, content)
-            else:
-                message = "Portal to private chat created."
-                if e2be_ok is False:
-                    message += "\n\nWarning: Failed to enable end-to-bridge encryption"
-                await intent.send_notice(room_id, message)
+            portal.save()
+            inviter.register_portal(portal)
+            await intent.send_notice(room_id, "Portal to private chat created.")
         else:
             await intent.join_room(room_id)
             await intent.send_notice(room_id, "This puppet will remain inactive until a "
                                               "Telegram chat is created for this room.")
 
-    async def send_welcome_message(self, room_id: RoomID, inviter: 'u.User') -> None:
+    async def accept_bot_invite(self, room_id: MatrixRoomID, inviter: u.User) -> None:
+        tries = 0
+        while tries < 5:
+            try:
+                await self.az.intent.join_room(room_id)
+                break
+            except (IntentError, MatrixRequestError):
+                tries += 1
+                wait_for_seconds = (tries + 1) * 10
+                if tries < 5:
+                    self.log.exception(f"Failed to join room {room_id} with bridge bot, "
+                                       f"retrying in {wait_for_seconds} seconds...")
+                    await asyncio.sleep(wait_for_seconds)
+                else:
+                    self.log.exception("Failed to join room {room}, giving up.")
+                    return
+
+        if not inviter.whitelisted:
+            await self.az.intent.send_notice(
+                room_id,
+                text="You are not whitelisted to use this bridge.\n\n"
+                     "If you are the owner of this bridge, see the "
+                     "`bridge.permissions` section in your config file.",
+                html="<p>You are not whitelisted to use this bridge.</p>"
+                     "<p>If you are the owner of this bridge, see the "
+                     "<code>bridge.permissions</code> section in your config file.</p>")
+            await self.az.intent.leave_room(room_id)
+
         try:
             is_management = len(await self.az.intent.get_room_members(room_id)) == 2
-        except MatrixError:
-            # The AS bot is not in the room.
-            return
+        except MatrixRequestError:
+            is_management = False
         cmd_prefix = self.commands.command_prefix
         text = html = "Hello, I'm a Telegram bridge bot. "
         if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
             text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
             html += (f"Use <code>{cmd_prefix} help</code> for help"
                      f" or <code>{cmd_prefix} login</code> to log in.")
+            pass
         else:
             text += f"Use `{cmd_prefix} help` for help."
             html += f"Use <code>{cmd_prefix} help</code> for help."
         await self.az.intent.send_notice(room_id, text=text, html=html)
 
-    async def handle_invite(self, room_id: RoomID, user_id: UserID, inviter: 'u.User',
-                            event_id: EventID) -> None:
+    async def handle_invite(self, room_id: MatrixRoomID, user_id: MatrixUserID,
+                            inviter_mxid: MatrixUserID) -> None:
+        self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
+        inviter = u.User.get_by_mxid(inviter_mxid)
+        if inviter is None:
+            self.log.exception("Failed to find user with Matrix ID {inviter_mxid}")
+        await inviter.ensure_started()
+        if user_id == self.az.bot_mxid:
+            return await self.accept_bot_invite(room_id, inviter)
+        elif not inviter.whitelisted:
+            return
+
+        puppet = pu.Puppet.get_by_mxid(user_id)
+        if puppet:
+            await self.handle_puppet_invite(room_id, puppet, inviter)
+            return
+
         user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
@@ -142,8 +185,12 @@ class MatrixHandler(BaseMatrixHandler):
         portal = po.Portal.get_by_mxid(room_id)
         if user and await user.has_full_access(allow_bot=True) and portal:
             await portal.invite_telegram(inviter, user)
+            return
 
-    async def handle_join(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None:
+        # The rest can probably be ignored
+
+    async def handle_join(self, room_id: MatrixRoomID, user_id: MatrixUserID,
+                          event_id: MatrixEventID) -> None:
         user = await u.User.get_by_mxid(user_id).ensure_started()
 
         portal = po.Portal.get_by_mxid(room_id)
@@ -151,58 +198,35 @@ class MatrixHandler(BaseMatrixHandler):
             return
 
         if not user.relaybot_whitelisted:
-            await portal.main_intent.kick_user(room_id, user.mxid,
-                                               "You are not whitelisted on this Telegram bridge.")
+            await portal.main_intent.kick(room_id, user.mxid,
+                                          "You are not whitelisted on this Telegram bridge.")
             return
         elif not await user.is_logged_in() and not portal.has_bot:
-            await portal.main_intent.kick_user(room_id, user.mxid,
-                                               "This chat does not have a bot relaying "
-                                               "messages for unauthenticated users.")
+            await portal.main_intent.kick(room_id, user.mxid,
+                                          "This chat does not have a bot relaying "
+                                          "messages for unauthenticated users.")
             return
 
-        self.log.debug(f"{user.mxid} joined {room_id}")
+        self.log.debug(f"{user} joined {room_id}")
         if await user.is_logged_in() or portal.has_bot:
             await portal.join_matrix(user, event_id)
 
-    async def get_leave_handle_info(self) -> Tuple[po.Portal, u.User]:
-        pass
-
-    async def handle_leave(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None:
+    async def handle_part(self, room_id: MatrixRoomID, user_id: MatrixUserID,
+                          sender_mxid: MatrixUserID, event_id: MatrixEventID) -> None:
         self.log.debug(f"{user_id} left {room_id}")
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
-            return
 
-        user = u.User.get_by_mxid(user_id, create=False)
-        if not user:
-            return
-        await user.ensure_started()
-        await portal.leave_matrix(user, event_id)
-
-    async def handle_kick_ban(self, ban: bool, room_id: RoomID, user_id: UserID, sender: UserID,
-                              reason: str, event_id: EventID) -> None:
-        action = "banned" if ban else "kicked"
-        self.log.debug(f"{user_id} was {action} from {room_id} by {sender} for {reason}")
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
-            return
-
-        if user_id == self.az.bot_mxid:
-            # Direct chat portal unbridging is handled in portal.kick_matrix
-            if portal.peer_type != "user":
-                await portal.unbridge()
-            return
-
-        sender = u.User.get_by_mxid(sender, create=False)
+        sender = u.User.get_by_mxid(sender_mxid, create=False)
         if not sender:
             return
         await sender.ensure_started()
 
-        puppet = await pu.Puppet.get_by_mxid(user_id)
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        puppet = pu.Puppet.get_by_mxid(user_id)
         if puppet:
-            if ban:
-                await portal.ban_matrix(puppet, sender)
-            else:
+            if sender:
                 await portal.kick_matrix(puppet, sender)
             return
 
@@ -210,193 +234,244 @@ class MatrixHandler(BaseMatrixHandler):
         if not user:
             return
         await user.ensure_started()
-        if ban:
-            await portal.ban_matrix(user, sender)
-        else:
-            await portal.kick_matrix(user, sender)
+        if await user.is_logged_in() or portal.has_bot:
+            await portal.leave_matrix(user, sender, event_id)
 
-    async def handle_kick(self, room_id: RoomID, user_id: UserID, kicked_by: UserID, reason: str,
-                          event_id: EventID) -> None:
-        await self.handle_kick_ban(False, room_id, user_id, kicked_by, reason, event_id)
+    def is_command(self, message: Dict) -> Tuple[bool, str]:
+        text = message.get("body", "")
+        prefix = self.config["bridge.command_prefix"]
+        is_command = text.startswith(prefix)
+        if is_command:
+            text = text[len(prefix) + 1:].lstrip()
+        return is_command, text
 
-    async def handle_unban(self, room_id: RoomID, user_id: UserID, unbanned_by: UserID,
-                           reason: str, event_id: EventID) -> None:
-        # TODO handle unbans properly instead of handling it as a kick
-        await self.handle_kick_ban(False, room_id, user_id, unbanned_by, reason, event_id)
+    async def handle_message(self, room: MatrixRoomID, sender_id: MatrixUserID, message: Dict,
+                             event_id: MatrixEventID) -> None:
+        is_command, text = self.is_command(message)
+        sender = await u.User.get_by_mxid(sender_id).ensure_started()
+        if not sender.relaybot_whitelisted:
+            self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:"
+                           " User is not whitelisted.")
+            return
+        self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}")
 
-    async def handle_ban(self, room_id: RoomID, user_id: UserID, banned_by: UserID, reason: str,
-                         event_id: EventID) -> None:
-        await self.handle_kick_ban(True, room_id, user_id, banned_by, reason, event_id)
+        portal = po.Portal.get_by_mxid(room)
+        if not is_command and portal and (await sender.is_logged_in() or portal.has_bot):
+            await portal.handle_matrix_message(sender, message, event_id)
+            return
+
+        if not sender.whitelisted or message.get("msgtype", "m.unknown") != "m.text":
+            return
+
+        try:
+            is_management = len(await self.az.intent.get_room_members(room)) == 2
+        except MatrixRequestError:
+            # The AS bot is not in the room.
+            return
+
+        if is_command or is_management:
+            try:
+                command, arguments = text.split(" ", 1)
+                args = arguments.split(" ")
+            except ValueError:
+                # Not enough values to unpack, i.e. no arguments
+                command = text
+                args = []
+            await self.commands.handle(room, event_id, sender, command, args, is_management,
+                                       is_portal=portal is not None)
 
     @staticmethod
-    async def allow_message(user: 'u.User') -> bool:
-        return user.relaybot_whitelisted
-
-    @staticmethod
-    async def allow_command(user: 'u.User') -> bool:
-        return user.whitelisted
-
-    @staticmethod
-    async def allow_bridging_message(user: 'u.User', portal: 'po.Portal') -> bool:
-        return await user.is_logged_in() or portal.has_bot
-
-    @staticmethod
-    async def handle_redaction(evt: RedactionEvent) -> None:
-        sender = await u.User.get_by_mxid(evt.sender).ensure_started()
+    async def handle_redaction(room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                               event_id: MatrixEventID) -> None:
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if not sender.relaybot_whitelisted:
             return
 
-        portal = po.Portal.get_by_mxid(evt.room_id)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
-        await portal.handle_matrix_deletion(sender, evt.redacts, evt.event_id)
+        await portal.handle_matrix_deletion(sender, event_id)
 
     @staticmethod
-    async def handle_power_levels(evt: StateEvent) -> None:
-        portal = po.Portal.get_by_mxid(evt.room_id)
-        sender = await u.User.get_by_mxid(evt.sender).ensure_started()
-        if await sender.has_full_access(allow_bot=True) and portal:
-            await portal.handle_matrix_power_levels(sender, evt.content.users,
-                                                    evt.unsigned.prev_content.users,
-                                                    evt.event_id)
-
-    @staticmethod
-    async def handle_room_meta(evt_type: EventType, room_id: RoomID, sender_mxid: UserID,
-                               content: RoomMetaStateEventContent, event_id: EventID) -> None:
+    async def handle_power_levels(room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                                  new: Dict, old: Dict) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
-            handler, content_type, content_key = {
-                EventType.ROOM_NAME: (portal.handle_matrix_title, RoomNameStateEventContent, "name"),
-                EventType.ROOM_TOPIC: (portal.handle_matrix_about, RoomTopicStateEventContent, "topic"),
-                EventType.ROOM_AVATAR: (portal.handle_matrix_avatar, RoomAvatarStateEventContent, "url"),
-            }[evt_type]
-            if not isinstance(content, content_type):
-                return
-            await handler(sender, content[content_key], event_id)
+            await portal.handle_matrix_power_levels(sender, new["users"], old["users"])
 
     @staticmethod
-    async def handle_room_pin(room_id: RoomID, sender_mxid: UserID,
-                              new_events: Set[str], old_events: Set[str],
-                              event_id: EventID) -> None:
+    async def handle_room_meta(evt_type: str, room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                               content: dict) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
+            handler, content_key = {
+                "m.room.name": (portal.handle_matrix_title, "name"),
+                "m.room.topic": (portal.handle_matrix_about, "topic"),
+                "m.room.avatar": (portal.handle_matrix_avatar, "url"),
+            }[evt_type]
+            if content_key not in content:
+                return
+            await handler(sender, content[content_key])
+
+    @staticmethod
+    async def handle_room_pin(room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                              new_events: Set[str], old_events: Set[str]) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
             events = new_events - old_events
             if len(events) > 0:
                 # New event pinned, set that as pinned in Telegram.
-                await portal.handle_matrix_pin(sender, EventID(events.pop()), event_id)
+                await portal.handle_matrix_pin(sender, MatrixEventID(events.pop()))
             elif len(new_events) == 0:
                 # All pinned events removed, remove pinned event in Telegram.
-                await portal.handle_matrix_pin(sender, None, event_id)
+                await portal.handle_matrix_pin(sender, None)
 
     @staticmethod
-    async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID,
-                                  event_id: EventID) -> None:
+    async def handle_room_upgrade(room_id: MatrixRoomID, new_room_id: MatrixRoomID) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         if portal:
-            await portal.handle_matrix_upgrade(sender, new_room_id, event_id)
-
-    async def handle_member_info_change(self, room_id: RoomID, user_id: UserID,
-                                        profile: MemberStateEventContent,
-                                        prev_profile: MemberStateEventContent,
-                                        event_id: EventID) -> None:
-        if profile.displayname == prev_profile.displayname:
-            return
+            await portal.handle_matrix_upgrade(new_room_id)
 
+    @staticmethod
+    async def handle_name_change(room_id: MatrixRoomID, user_id: MatrixUserID, displayname: str,
+                                 prev_displayname: str, event_id: MatrixEventID) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         if not portal or not portal.has_bot:
             return
 
         user = await u.User.get_by_mxid(user_id).ensure_started()
         if await user.needs_relaybot(portal):
-            await portal.name_change_matrix(user, profile.displayname, prev_profile.displayname,
-                                            event_id)
+            await portal.name_change_matrix(user, displayname, prev_displayname, event_id)
 
     @staticmethod
-    def parse_read_receipts(content: ReceiptEventContent) -> Iterable[Tuple[UserID, EventID]]:
-        return ((user_id, event_id)
+    def parse_read_receipts(content: Dict) -> Dict[MatrixUserID, MatrixEventID]:
+        return {user_id: event_id
                 for event_id, receipts in content.items()
-                for user_id in receipts.get(ReceiptType.READ, {}))
+                for user_id in receipts.get("m.read", {})}
 
     @staticmethod
-    async def handle_read_receipts(room_id: RoomID, receipts: Iterable[Tuple[UserID, EventID]]
-                                   ) -> None:
+    async def handle_read_receipts(room_id: MatrixRoomID,
+                                   receipts: Dict[MatrixUserID, MatrixEventID]) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
-        for user_id, event_id in receipts:
-            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
-            if user and await user.is_logged_in():
-                await portal.mark_read(user, event_id)
+        for user_id, event_id in receipts.items():
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+            await portal.mark_read(user, event_id)
 
     @staticmethod
-    async def handle_presence(user_id: UserID, presence: PresenceState) -> None:
-        user = u.User.get_by_mxid(user_id, check_db=False, create=False)
-        if user and await user.is_logged_in():
-            await user.set_presence(presence == PresenceState.ONLINE)
+    async def handle_presence(user_id: MatrixUserID, presence: str) -> None:
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if not await user.is_logged_in():
+            return
+        await user.set_presence(presence == "online")
 
-    async def handle_typing(self, room_id: RoomID, now_typing: Set[UserID]) -> None:
+    async def handle_typing(self, room_id: MatrixRoomID, now_typing: List[MatrixUserID]) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
-        previously_typing = self.previously_typing.get(room_id, set())
-
-        for user_id in set(previously_typing | now_typing):
+        for user_id in set(self.previously_typing + now_typing):
             is_typing = user_id in now_typing
-            was_typing = user_id in previously_typing
+            was_typing = user_id in self.previously_typing
             if is_typing and was_typing:
                 continue
 
-            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
-            if user and await user.is_logged_in():
-                await portal.set_typing(user, is_typing)
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
 
-        self.previously_typing[room_id] = now_typing
+            await portal.set_typing(user, is_typing)
 
-    def filter_matrix_event(self, evt: Event) -> bool:
-        if isinstance(evt, (TypingEvent, ReceiptEvent, PresenceEvent)):
+        self.previously_typing = now_typing
+
+    def filter_matrix_event(self, event: MatrixEvent) -> bool:
+        sender = event.get("sender", None)
+        if not sender:
             return False
-        elif not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent, EncryptedEvent)):
-            return True
-        if evt.content.get(self.az.real_user_content_key, False):
-            puppet = pu.Puppet.deprecated_sync_get_by_custom_mxid(evt.sender)
-            if puppet:
-                self.log.debug("Ignoring puppet-sent event %s", evt.event_id)
-                return True
-        return evt.sender and (evt.sender == self.az.bot_mxid
-                               or pu.Puppet.get_id_from_mxid(evt.sender) is not None)
+        return (sender == self.az.bot_mxid
+                or pu.Puppet.get_id_from_mxid(sender) is not None)
 
-    async def handle_ephemeral_event(self, evt: Union[ReceiptEvent, PresenceEvent, TypingEvent]
-                                     ) -> None:
-        if evt.type == EventType.RECEIPT:
-            await self.handle_read_receipts(evt.room_id, self.parse_read_receipts(evt.content))
-        elif evt.type == EventType.PRESENCE:
-            await self.handle_presence(evt.sender, evt.content.presence)
-        elif evt.type == EventType.TYPING:
-            await self.handle_typing(evt.room_id, set(evt.content.user_ids))
+    async def try_handle_ephemeral_event(self, evt: MatrixEvent) -> None:
+        try:
+            await self.handle_ephemeral_event(evt)
+        except Exception:
+            self.log.exception("Error handling manually received Matrix event")
 
-    async def handle_event(self, evt: Event) -> None:
-        if evt.type == EventType.ROOM_REDACTION:
-            await self.handle_redaction(evt)
+    async def handle_ephemeral_event(self, evt: MatrixEvent) -> None:
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
+        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        content = evt.get("content", {})  # type: Dict
+        if evt_type == "m.receipt":
+            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
+        elif evt_type == "m.presence":
+            await self.handle_presence(sender, content.get("presence", "offline"))
+        elif evt_type == "m.typing":
+            await self.handle_typing(room_id, content.get("user_ids", []))
 
-    async def handle_state_event(self, evt: StateEvent) -> None:
-        if evt.type == EventType.ROOM_POWER_LEVELS:
-            await self.handle_power_levels(evt)
-        elif evt.type in (EventType.ROOM_NAME, EventType.ROOM_AVATAR, EventType.ROOM_TOPIC):
-            await self.handle_room_meta(evt.type, evt.room_id, evt.sender, evt.content,
-                                        evt.event_id)
-        elif evt.type == EventType.ROOM_PINNED_EVENTS:
-            new_events = set(evt.content.pinned)
-            try:
-                old_events = set(evt.unsigned.prev_content.pinned)
-            except (KeyError, ValueError, TypeError, AttributeError):
-                old_events = set()
-            await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events,
-                                       evt.event_id)
-        elif evt.type == EventType.ROOM_TOMBSTONE:
-            await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room,
-                                           evt.event_id)
+    async def handle_event(self, evt: MatrixEvent) -> None:
+        if self.filter_matrix_event(evt):
+            return
+        start_time = time.time()
+        self.log.debug("Received event: %s", evt)
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
+        event_id = evt.get("event_id", None)  # type: Optional[MatrixEventID]
+        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        state_key = evt.get("state_key", None)
+        content = evt.get("content", {})  # type: Dict
+        if state_key is not None:
+            if evt_type == "m.room.member":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
+                membership = content.get("membership", "")  # type: str
+                prev_membership = prev_content.get("membership", "leave")  # type: str
+                if membership == prev_membership:
+                    match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
+                    mxid = match.group(0)  # type: str
+                    displayname = content.get("displayname", None) or mxid  # type: str
+                    prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
+                    if displayname != prev_displayname:
+                        await self.handle_name_change(room_id, state_key, displayname,
+                                                      prev_displayname, event_id)
+                elif membership == "invite":
+                    await self.handle_invite(room_id, state_key, sender)
+                elif prev_membership == "join" and membership == "leave":
+                    await self.handle_part(room_id, state_key, sender, event_id)
+                elif membership == "join":
+                    await self.handle_join(room_id, state_key, event_id)
+            elif evt_type == "m.room.power_levels":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})
+                await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
+            elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
+                await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
+            elif evt_type == "m.room.pinned_events":
+                new_events = set(evt["content"]["pinned"])
+                try:
+                    old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+                except KeyError:
+                    old_events = set()
+                await self.handle_room_pin(room_id, sender, new_events, old_events)
+            elif evt_type == "m.room.tombstone":
+                await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
+            else:
+                return
+        else:
+            if evt_type in ("m.room.message", "m.sticker"):
+                if evt_type != "m.room.message":
+                    content["msgtype"] = evt_type
+                await self.handle_message(room_id, sender, content, event_id)
+            elif evt_type == "m.room.redaction":
+                await self.handle_redaction(room_id, sender, evt["redacts"])
+            else:
+                return
+
+        if EVENT_TIME:
+            EVENT_TIME.labels(event_type=evt_type).observe(time.time() - start_time)

mautrix_telegram/portal/__init__.py

@@ -1,21 +0,0 @@
-from .base import BasePortal, init as init_base
-from .matrix import PortalMatrix, init as init_matrix
-from .metadata import PortalMetadata, init as init_metadata
-from .telegram import PortalTelegram, init as init_telegram
-from .deduplication import init as init_dedup
-from ..context import Context
-
-
-class Portal(PortalMatrix, PortalTelegram, PortalMetadata):
-    pass
-
-
-def init(context: Context) -> None:
-    init_base(context)
-    init_dedup(context)
-    init_metadata(context)
-    init_telegram(context)
-    init_matrix(context)
-
-
-__all__ = ["Portal", "init"]

mautrix_telegram/portal/__init__.pyi

@@ -1,15 +0,0 @@
-from typing import Union
-from .base import BasePortal
-from .matrix import PortalMatrix
-from .metadata import PortalMetadata
-from .telegram import PortalTelegram
-from ..context import Context
-
-Portal = Union[BasePortal, PortalMatrix, PortalMetadata, PortalTelegram]
-
-
-def init(context: Context) -> None:
-	pass
-
-
-__all__ = ["Portal", "init"]

mautrix_telegram/portal/base.py

@@ -1,536 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, Set, Iterable, TYPE_CHECKING
-from abc import ABC, abstractmethod
-import asyncio
-import logging
-import json
-
-from telethon.tl.functions.messages import ExportChatInviteRequest
-from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, ChatInviteEmpty, InputChannel,
-                               InputPeerChannel, InputPeerChat, InputPeerUser, InputUser,
-                               PeerChannel, PeerChat, PeerUser, TypeChat, TypeInputPeer, TypePeer,
-                               TypeUser, TypeUserFull, User, UserFull, TypeInputChannel, Photo,
-                               Document, TypePhotoSize, PhotoSize, InputPhotoFileLocation,
-                               TypeChatParticipant, TypeChannelParticipant, PhotoEmpty, ChatPhoto,
-                               ChatPhotoEmpty)
-
-from mautrix.errors import MatrixRequestError, IntentError
-from mautrix.appservice import AppService, IntentAPI
-from mautrix.types import (RoomID, RoomAlias, UserID, EventID, EventType, MessageEventContent,
-                           PowerLevelStateEventContent, ContentURI)
-from mautrix.util.simple_template import SimpleTemplate
-from mautrix.util.simple_lock import SimpleLock
-from mautrix.util.logging import TraceLogger
-from mautrix.bridge import BasePortal as MautrixBasePortal
-
-from ..types import TelegramID
-from ..context import Context
-from ..db import Portal as DBPortal, Message as DBMessage
-from .. import puppet as p, user as u, util
-from .deduplication import PortalDedup
-from .send_lock import PortalSendLock
-
-if TYPE_CHECKING:
-    from ..bot import Bot
-    from ..abstract_user import AbstractUser
-    from ..config import Config
-    from ..matrix import MatrixHandler
-    from . import Portal
-
-TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
-TypeChatPhoto = Union[ChatPhoto, ChatPhotoEmpty, Photo, PhotoEmpty]
-InviteList = Union[UserID, List[UserID]]
-
-config: Optional['Config'] = None
-
-
-class BasePortal(MautrixBasePortal, ABC):
-    base_log: TraceLogger = logging.getLogger("mau.portal")
-    az: AppService = None
-    bot: 'Bot' = None
-    loop: asyncio.AbstractEventLoop = None
-    matrix: 'MatrixHandler' = None
-
-    # Config cache
-    filter_mode: str = None
-    filter_list: List[str] = None
-
-    max_initial_member_sync: int = -1
-    sync_channel_members: bool = True
-    sync_matrix_state: bool = True
-    public_portals: bool = False
-    private_chat_portal_meta: bool = False
-
-    alias_template: SimpleTemplate[str]
-    hs_domain: str
-
-    # Instance cache
-    by_mxid: Dict[RoomID, 'Portal'] = {}
-    by_tgid: Dict[Tuple[TelegramID, TelegramID], 'Portal'] = {}
-
-    mxid: Optional[RoomID]
-    tgid: TelegramID
-    tg_receiver: TelegramID
-    peer_type: str
-    username: str
-    megagroup: bool
-    title: Optional[str]
-    about: Optional[str]
-    photo_id: Optional[str]
-    local_config: Dict[str, Any]
-    avatar_url: Optional[ContentURI]
-    encrypted: bool
-    deleted: bool
-    backfill_lock: SimpleLock
-    backfill_method_lock: asyncio.Lock
-    backfill_leave: Optional[Set[IntentAPI]]
-    log: TraceLogger
-
-    alias: Optional[RoomAlias]
-
-    dedup: PortalDedup
-    send_lock: PortalSendLock
-
-    _db_instance: DBPortal
-    _main_intent: Optional[IntentAPI]
-    _room_create_lock: asyncio.Lock
-
-    def __init__(self, tgid: TelegramID, peer_type: str, tg_receiver: Optional[TelegramID] = None,
-                 mxid: Optional[RoomID] = None, username: Optional[str] = None,
-                 megagroup: Optional[bool] = False, title: Optional[str] = None,
-                 about: Optional[str] = None, photo_id: Optional[str] = None,
-                 local_config: Optional[str] = None, avatar_url: Optional[ContentURI] = None,
-                 encrypted: Optional[bool] = False, db_instance: DBPortal = None) -> None:
-        self.mxid = mxid
-        self.tgid = tgid
-        self.tg_receiver = tg_receiver or tgid
-        self.peer_type = peer_type
-        self.username = username
-        self.megagroup = megagroup
-        self.title = title
-        self.about = about
-        self.photo_id = photo_id
-        self.local_config = json.loads(local_config or "{}")
-        self.avatar_url = avatar_url
-        self.encrypted = encrypted
-        self._db_instance = db_instance
-        self._main_intent = None
-        self.deleted = False
-        self.log = self.base_log.getChild(self.tgid_log if self.tgid else self.mxid)
-        self.backfill_lock = SimpleLock("Waiting for backfilling to finish before handling %s",
-                                        log=self.log)
-        self.backfill_method_lock = asyncio.Lock()
-        self.backfill_leave = None
-
-        self.dedup = PortalDedup(self)
-        self.send_lock = PortalSendLock()
-
-        if tgid:
-            self.by_tgid[self.tgid_full] = self
-        if mxid:
-            self.by_mxid[mxid] = self
-
-    # region Properties
-
-    @property
-    def tgid_full(self) -> Tuple[TelegramID, TelegramID]:
-        return self.tgid, self.tg_receiver
-
-    @property
-    def tgid_log(self) -> str:
-        if self.tgid == self.tg_receiver:
-            return str(self.tgid)
-        return f"{self.tg_receiver}<->{self.tgid}"
-
-    @property
-    def name(self) -> str:
-        return self.title
-
-    @property
-    def alias(self) -> Optional[RoomAlias]:
-        if not self.username:
-            return None
-        return RoomAlias(f"#{self.alias_localpart}:{self.hs_domain}")
-
-    @property
-    def alias_localpart(self) -> Optional[str]:
-        if not self.username:
-            return None
-        return self.alias_template.format(self.username)
-
-    @property
-    def peer(self) -> Union[TypePeer, TypeInputPeer]:
-        if self.peer_type == "user":
-            return PeerUser(user_id=self.tgid)
-        elif self.peer_type == "chat":
-            return PeerChat(chat_id=self.tgid)
-        elif self.peer_type == "channel":
-            return PeerChannel(channel_id=self.tgid)
-
-    @property
-    def has_bot(self) -> bool:
-        return (bool(self.bot)
-                and (self.bot.is_in_chat(self.tgid)
-                     or (self.peer_type == "user" and self.tg_receiver == self.bot.tgid)))
-
-    @property
-    def main_intent(self) -> IntentAPI:
-        if not self._main_intent:
-            direct = self.peer_type == "user"
-            puppet = p.Puppet.get(self.tgid) if direct else None
-            self._main_intent = puppet.intent_for(self) if direct else self.az.intent
-        return self._main_intent
-
-    @property
-    def allow_bridging(self) -> bool:
-        if self.peer_type == "user":
-            return True
-        elif self.filter_mode == "whitelist":
-            return self.tgid in self.filter_list
-        elif self.filter_mode == "blacklist":
-            return self.tgid not in self.filter_list
-        return True
-
-    # endregion
-    # region Miscellaneous getters
-
-    def get_config(self, key: str) -> Any:
-        local = util.recursive_get(self.local_config, key)
-        if local is not None:
-            return local
-        return config[f"bridge.{key}"]
-
-    @staticmethod
-    def _get_largest_photo_size(photo: Union[Photo, Document]
-                                ) -> Tuple[Optional[InputPhotoFileLocation],
-                                           Optional[TypePhotoSize]]:
-        if not photo or isinstance(photo, PhotoEmpty) or (isinstance(photo, Document)
-                                                          and not photo.thumbs):
-            return None, None
-
-        largest = max(photo.thumbs if isinstance(photo, Document) else photo.sizes,
-                      key=(lambda photo2: (len(photo2.bytes)
-                                           if not isinstance(photo2, PhotoSize)
-                                           else photo2.size)))
-        return InputPhotoFileLocation(
-            id=photo.id,
-            access_hash=photo.access_hash,
-            file_reference=photo.file_reference,
-            thumb_size=largest.type,
-        ), largest
-
-    async def can_user_perform(self, user: 'u.User', event: str) -> bool:
-        if user.is_admin:
-            return True
-        if not self.mxid:
-            # No room for anybody to perform actions in
-            return False
-        try:
-            await self.main_intent.get_power_levels(self.mxid)
-        except MatrixRequestError:
-            return False
-        evt_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
-        return await self.main_intent.state_store.has_power_level(self.mxid, user.mxid, evt_type)
-
-    def get_input_entity(self, user: 'AbstractUser'
-                         ) -> Awaitable[Union[TypeInputPeer, TypeInputChannel]]:
-        return user.client.get_input_entity(self.peer)
-
-    async def get_entity(self, user: 'AbstractUser') -> TypeChat:
-        try:
-            return await user.client.get_entity(self.peer)
-        except ValueError:
-            if user.is_bot:
-                self.log.warning(f"Could not find entity with bot {user.tgid}. Failing...")
-                raise
-            self.log.warning(f"Could not find entity with user {user.tgid}. "
-                             "falling back to get_dialogs.")
-            async for dialog in user.client.iter_dialogs():
-                if dialog.entity.id == self.tgid:
-                    return dialog.entity
-            raise
-
-    async def get_invite_link(self, user: 'u.User') -> str:
-        if self.peer_type == "user":
-            raise ValueError("You can't invite users to private chats.")
-        if self.username:
-            return f"https://t.me/{self.username}"
-        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user)))
-        if isinstance(link, ChatInviteEmpty):
-            raise ValueError("Failed to get invite link.")
-        return link.link
-
-    # endregion
-    # region Matrix room cleanup
-
-    async def get_authenticated_matrix_users(self) -> List[UserID]:
-        try:
-            members = await self.main_intent.get_room_members(self.mxid)
-        except MatrixRequestError:
-            return []
-        authenticated: List[UserID] = []
-        has_bot = self.has_bot
-        for member in members:
-            if p.Puppet.get_id_from_mxid(member) or member == self.az.bot_mxid:
-                continue
-            user = await u.User.get_by_mxid(member).ensure_started()
-            authenticated_through_bot = has_bot and user.relaybot_whitelisted
-            if authenticated_through_bot or await user.has_full_access(allow_bot=True):
-                authenticated.append(user.mxid)
-        return authenticated
-
-    async def cleanup_portal(self, message: str, puppets_only: bool = False, delete: bool = True
-                             ) -> None:
-        if self.username:
-            try:
-                await self.main_intent.remove_room_alias(self.alias_localpart)
-            except (MatrixRequestError, IntentError):
-                self.log.warning("Failed to remove alias when cleaning up room", exc_info=True)
-        await self.cleanup_room(self.main_intent, self.mxid, message, puppets_only)
-        if delete:
-            await self.delete()
-
-    # endregion
-    # region Database conversion
-
-    @property
-    def db_instance(self) -> DBPortal:
-        if not self._db_instance:
-            self._db_instance = self.new_db_instance()
-        return self._db_instance
-
-    def new_db_instance(self) -> DBPortal:
-        return DBPortal(tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
-                        mxid=self.mxid, username=self.username, megagroup=self.megagroup,
-                        title=self.title, about=self.about, photo_id=self.photo_id,
-                        config=json.dumps(self.local_config), avatar_url=self.avatar_url,
-                        encrypted=self.encrypted)
-
-    async def save(self) -> None:
-        self.db_instance.edit(mxid=self.mxid, username=self.username, title=self.title,
-                              about=self.about, photo_id=self.photo_id, megagroup=self.megagroup,
-                              config=json.dumps(self.local_config), avatar_url=self.avatar_url,
-                              encrypted=self.encrypted)
-
-    async def delete(self) -> None:
-        self.delete_sync()
-
-    def delete_sync(self) -> None:
-        try:
-            del self.by_tgid[self.tgid_full]
-        except KeyError:
-            pass
-        try:
-            del self.by_mxid[self.mxid]
-        except KeyError:
-            pass
-        if self._db_instance:
-            self._db_instance.delete()
-        DBMessage.delete_all(self.mxid)
-        self.deleted = True
-
-    @classmethod
-    def from_db(cls, db_portal: DBPortal) -> 'Portal':
-        return cls(tgid=db_portal.tgid, tg_receiver=db_portal.tg_receiver,
-                   peer_type=db_portal.peer_type, mxid=db_portal.mxid, username=db_portal.username,
-                   megagroup=db_portal.megagroup, title=db_portal.title, about=db_portal.about,
-                   photo_id=db_portal.photo_id, local_config=db_portal.config,
-                   avatar_url=db_portal.avatar_url, encrypted=db_portal.encrypted,
-                   db_instance=db_portal)
-
-    # endregion
-    # region Class instance lookup
-
-    @classmethod
-    def all(cls) -> Iterable['Portal']:
-        for db_portal in DBPortal.all():
-            try:
-                yield cls.by_tgid[(db_portal.tgid, db_portal.tg_receiver)]
-            except KeyError:
-                yield cls.from_db(db_portal)
-
-    @classmethod
-    def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
-        try:
-            return cls.by_mxid[mxid]
-        except KeyError:
-            pass
-
-        portal = DBPortal.get_by_mxid(mxid)
-        if portal:
-            return cls.from_db(portal)
-
-        return None
-
-    @classmethod
-    def get_username_from_mx_alias(cls, alias: str) -> Optional[str]:
-        return cls.alias_template.parse(alias)
-
-    @classmethod
-    def find_by_username(cls, username: str) -> Optional['Portal']:
-        if not username:
-            return None
-
-        username = username.lower()
-
-        for _, portal in cls.by_tgid.items():
-            if portal.username and portal.username.lower() == username:
-                return portal
-
-        dbportal = DBPortal.get_by_username(username)
-        if dbportal:
-            return cls.from_db(dbportal)
-
-        return None
-
-    @classmethod
-    def get_by_tgid(cls, tgid: TelegramID, tg_receiver: Optional[TelegramID] = None,
-                    peer_type: str = None) -> Optional['Portal']:
-        if peer_type == "user" and tg_receiver is None:
-            raise ValueError("tg_receiver is required when peer_type is \"user\"")
-        tg_receiver = tg_receiver or tgid
-        tgid_full = (tgid, tg_receiver)
-        try:
-            return cls.by_tgid[tgid_full]
-        except KeyError:
-            pass
-
-        db_portal = DBPortal.get_by_tgid(tgid, tg_receiver)
-        if db_portal:
-            return cls.from_db(db_portal)
-
-        if peer_type:
-            cls.log.info(f"Creating portal for {peer_type} {tgid} (receiver {tg_receiver})")
-            # TODO enable this for non-release builds
-            #      (or add better wrong peer type error handling)
-            # if peer_type == "chat":
-            #     import traceback
-            #     cls.log.info("Chat portal stack trace:\n" + "".join(traceback.format_stack()))
-            portal = cls(tgid, peer_type=peer_type, tg_receiver=tg_receiver)
-            portal.db_instance.insert()
-            return portal
-
-        return None
-
-    @classmethod
-    def get_by_entity(cls, entity: Union[TypeChat, TypePeer, TypeUser, TypeUserFull,
-                                         TypeInputPeer],
-                      receiver_id: Optional[TelegramID] = None, create: bool = True
-                      ) -> Optional['Portal']:
-        entity_type = type(entity)
-        if entity_type in (Chat, ChatFull):
-            type_name = "chat"
-            entity_id = entity.id
-        elif entity_type in (PeerChat, InputPeerChat):
-            type_name = "chat"
-            entity_id = entity.chat_id
-        elif entity_type in (Channel, ChannelFull):
-            type_name = "channel"
-            entity_id = entity.id
-        elif entity_type in (PeerChannel, InputPeerChannel, InputChannel):
-            type_name = "channel"
-            entity_id = entity.channel_id
-        elif entity_type in (User, UserFull):
-            type_name = "user"
-            entity_id = entity.id
-        elif entity_type in (PeerUser, InputPeerUser, InputUser):
-            type_name = "user"
-            entity_id = entity.user_id
-        else:
-            raise ValueError(f"Unknown entity type {entity_type.__name__}")
-        return cls.get_by_tgid(TelegramID(entity_id),
-                               receiver_id if type_name == "user" else entity_id,
-                               type_name if create else None)
-
-    # endregion
-    # region Abstract methods (cross-called in matrix/metadata/telegram classes)
-
-    @abstractmethod
-    async def update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
-                                 direct: bool, puppet: p.Puppet = None,
-                                 levels: PowerLevelStateEventContent = None,
-                                 users: List[User] = None,
-                                 participants: List[TypeParticipant] = None) -> None:
-        pass
-
-    @abstractmethod
-    async def create_matrix_room(self, user: 'AbstractUser', entity: TypeChat = None,
-                                 invites: InviteList = None, update_if_exists: bool = True,
-                                 synchronous: bool = False) -> Optional[str]:
-        pass
-
-    @abstractmethod
-    async def _add_telegram_user(self, user_id: TelegramID, source: Optional['AbstractUser'] = None
-                                 ) -> None:
-        pass
-
-    @abstractmethod
-    async def _delete_telegram_user(self, user_id: TelegramID, sender: p.Puppet) -> None:
-        pass
-
-    @abstractmethod
-    async def _update_title(self, title: str, sender: Optional['p.Puppet'] = None,
-                            save: bool = False) -> bool:
-        pass
-
-    @abstractmethod
-    async def _update_avatar(self, user: 'AbstractUser', photo: Union[TypeChatPhoto],
-                             sender: Optional['p.Puppet'] = None, save: bool = False) -> bool:
-        pass
-
-    @abstractmethod
-    def _migrate_and_save_telegram(self, new_id: TelegramID) -> None:
-        pass
-
-    @abstractmethod
-    async def update_bridge_info(self) -> None:
-        pass
-
-    @abstractmethod
-    def handle_matrix_power_levels(self, sender: 'u.User', new_levels: Dict[UserID, int],
-                                   old_levels: Dict[UserID, int], event_id: Optional[EventID]
-                                   ) -> Awaitable[None]:
-        pass
-
-    @abstractmethod
-    def backfill(self, source: 'AbstractUser', is_initial: bool = False,
-                 limit: Optional[int] = None, last_id: Optional[int] = None) -> Awaitable[None]:
-        pass
-
-    @abstractmethod
-    async def _send_delivery_receipt(self, event_id: EventID, room_id: Optional[RoomID] = None
-                                     ) -> None:
-        pass
-
-    # endregion
-
-
-def init(context: Context) -> None:
-    global config
-    BasePortal.az, config, BasePortal.loop, BasePortal.bot = context.core
-    BasePortal.matrix = context.mx
-    BasePortal.bridge = context.bridge
-    BasePortal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
-    BasePortal.sync_channel_members = config["bridge.sync_channel_members"]
-    BasePortal.sync_matrix_state = config["bridge.sync_matrix_state"]
-    BasePortal.public_portals = config["bridge.public_portals"]
-    BasePortal.private_chat_portal_meta = config["bridge.private_chat_portal_meta"]
-    BasePortal.filter_mode = config["bridge.filter.mode"]
-    BasePortal.filter_list = config["bridge.filter.list"]
-    BasePortal.hs_domain = config["homeserver.domain"]
-    BasePortal.alias_template = SimpleTemplate(config["bridge.alias_template"], "groupname",
-                                               prefix="#", suffix=f":{BasePortal.hs_domain}")

mautrix_telegram/portal/deduplication.py

@@ -1,133 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Deque, Dict, Tuple, TYPE_CHECKING
-from collections import deque
-import hashlib
-
-from telethon.tl.patched import Message, MessageService
-from telethon.tl.types import (MessageMediaContact, MessageMediaDocument, MessageMediaGeo,
-                               MessageMediaPhoto, TypeMessage, TypeUpdates, UpdateNewMessage,
-                               UpdateNewChannelMessage)
-
-from mautrix.types import EventID
-
-from ..context import Context
-from ..types import TelegramID
-
-if TYPE_CHECKING:
-    from .base import BasePortal
-
-DedupMXID = Tuple[EventID, TelegramID]
-
-
-class PortalDedup:
-    pre_db_check: bool = False
-    cache_queue_length: int = 20
-
-    _dedup: Deque[str]
-    _dedup_mxid: Dict[str, DedupMXID]
-    _dedup_action: Deque[str]
-    _portal: 'BasePortal'
-
-    def __init__(self, portal: 'BasePortal') -> None:
-        self._dedup = deque()
-        self._dedup_mxid = {}
-        self._dedup_action = deque()
-        self._portal = portal
-
-    @property
-    def _always_force_hash(self) -> bool:
-        return self._portal.peer_type == 'chat'
-
-    @staticmethod
-    def _hash_event(event: TypeMessage) -> str:
-        # Non-channel messages are unique per-user (wtf telegram), so we have no other choice than
-        # to deduplicate based on a hash of the message content.
-
-        # The timestamp is only accurate to the second, so we can't rely solely on that either.
-        if isinstance(event, MessageService):
-            hash_content = [event.date.timestamp(), event.from_id, event.action]
-        else:
-            hash_content = [event.date.timestamp(), event.message]
-            if event.fwd_from:
-                hash_content += [event.fwd_from.from_id, event.fwd_from.channel_id]
-            elif isinstance(event, Message) and event.media:
-                try:
-                    hash_content += {
-                        MessageMediaContact: lambda media: [media.user_id],
-                        MessageMediaDocument: lambda media: [media.document.id],
-                        MessageMediaPhoto: lambda media: [media.photo.id if media.photo else 0],
-                        MessageMediaGeo: lambda media: [media.geo.long, media.geo.lat],
-                    }[type(event.media)](event.media)
-                except KeyError:
-                    pass
-        return hashlib.md5("-"
-                           .join(str(a) for a in hash_content)
-                           .encode("utf-8")
-                           ).hexdigest()
-
-    def check_action(self, event: TypeMessage) -> bool:
-        evt_hash = self._hash_event(event) if self._always_force_hash else event.id
-        if evt_hash in self._dedup_action:
-            return True
-
-        self._dedup_action.append(evt_hash)
-
-        if len(self._dedup_action) > self.cache_queue_length:
-            self._dedup_action.popleft()
-        return False
-
-    def update(self, event: TypeMessage, mxid: DedupMXID = None,
-               expected_mxid: Optional[DedupMXID] = None, force_hash: bool = False
-               ) -> Optional[DedupMXID]:
-        evt_hash = self._hash_event(event) if self._always_force_hash or force_hash else event.id
-        try:
-            found_mxid = self._dedup_mxid[evt_hash]
-        except KeyError:
-            return EventID("None"), TelegramID(0)
-
-        if found_mxid != expected_mxid:
-            return found_mxid
-        self._dedup_mxid[evt_hash] = mxid
-        return None
-
-    def check(self, event: TypeMessage, mxid: DedupMXID = None, force_hash: bool = False
-              ) -> Optional[DedupMXID]:
-        evt_hash = (self._hash_event(event)
-                    if self._always_force_hash or force_hash
-                    else event.id)
-        if evt_hash in self._dedup:
-            return self._dedup_mxid[evt_hash]
-
-        self._dedup_mxid[evt_hash] = mxid
-        self._dedup.append(evt_hash)
-
-        if len(self._dedup) > self.cache_queue_length:
-            del self._dedup_mxid[self._dedup.popleft()]
-        return None
-
-    def register_outgoing_actions(self, response: TypeUpdates) -> None:
-        for update in response.updates:
-            check_dedup = (isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage))
-                           and isinstance(update.message, MessageService))
-            if check_dedup:
-                self.check(update.message)
-
-
-def init(context: Context) -> None:
-    cfg = context.config
-    PortalDedup.dedup_pre_db_check = cfg["bridge.deduplication.pre_db_check"]
-    PortalDedup.dedup_cache_queue_length = cfg["bridge.deduplication.cache_queue_length"]

mautrix_telegram/portal/matrix.py

@@ -1,583 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, Optional, Union, Any, TYPE_CHECKING
-from html import escape as escape_html
-from string import Template
-from abc import ABC
-
-import magic
-
-from telethon.tl.functions.messages import (EditChatPhotoRequest, EditChatTitleRequest,
-                                            UpdatePinnedMessageRequest, SetTypingRequest,
-                                            EditChatAboutRequest)
-from telethon.tl.functions.channels import EditPhotoRequest, EditTitleRequest, JoinChannelRequest
-from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError,
-                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError,
-                             RPCError)
-from telethon.tl.patched import Message, MessageService
-from telethon.tl.types import (DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
-                               InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
-                               SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer,
-                               UpdateNewMessage, InputMediaUploadedDocument,
-                               InputMediaUploadedPhoto)
-
-from mautrix.types import (EventID, RoomID, UserID, ContentURI, MessageType, MessageEventContent,
-                           TextMessageEventContent, MediaMessageEventContent, Format,
-                           LocationMessageEventContent, ImageInfo, VideoInfo)
-
-from ..types import TelegramID
-from ..db import Message as DBMessage
-from ..util import sane_mimetypes, parallel_transfer_to_telegram
-from ..context import Context
-from .. import puppet as p, user as u, formatter, util
-from .base import BasePortal
-
-if TYPE_CHECKING:
-    from ..abstract_user import AbstractUser
-    from ..tgclient import MautrixTelegramClient
-    from ..config import Config
-
-try:
-    from mautrix.crypto.attachments import decrypt_attachment
-except ImportError:
-    decrypt_attachment = None
-
-TypeMessage = Union[Message, MessageService]
-
-config: Optional['Config'] = None
-
-
-class PortalMatrix(BasePortal, ABC):
-    async def _get_state_change_message(self, event: str, user: 'u.User', **kwargs: Any
-                                        ) -> Optional[str]:
-        tpl = self.get_config(f"state_event_formats.{event}")
-        if len(tpl) == 0:
-            # Empty format means they don't want the message
-            return None
-        displayname = await self.get_displayname(user)
-
-        tpl_args = {
-            "mxid": user.mxid,
-            "username": user.mxid_localpart,
-            "displayname": escape_html(displayname),
-            **kwargs,
-        }
-        return Template(tpl).safe_substitute(tpl_args)
-
-    async def _send_state_change_message(self, event: str, user: 'u.User', event_id: EventID,
-                                         **kwargs: Any) -> None:
-        if not self.has_bot:
-            return
-        elif self.peer_type == "user" and not config["bridge.relaybot.private_chat.state_changes"]:
-            return
-        async with self.send_lock(self.bot.tgid):
-            message = await self._get_state_change_message(event, user, **kwargs)
-            if not message:
-                return
-            message, entities = await formatter.matrix_to_telegram(self.bot.client, html=message)
-            response = await self.bot.client.send_message(self.peer, message,
-                                                          formatting_entities=entities)
-            space = self.tgid if self.peer_type == "channel" else self.bot.tgid
-            self.dedup.check(response, (event_id, space))
-
-    async def name_change_matrix(self, user: 'u.User', displayname: str, prev_displayname: str,
-                                 event_id: EventID) -> None:
-        await self._send_state_change_message("name_change", user, event_id,
-                                              displayname=displayname,
-                                              prev_displayname=prev_displayname)
-
-    async def get_displayname(self, user: 'u.User') -> str:
-        return await self.main_intent.get_room_displayname(self.mxid, user.mxid) or user.mxid
-
-    def set_typing(self, user: 'u.User', typing: bool = True,
-                   action: type = SendMessageTypingAction) -> Awaitable[bool]:
-        return user.client(SetTypingRequest(
-            self.peer, action() if typing else SendMessageCancelAction()))
-
-    async def mark_read(self, user: 'u.User', event_id: EventID) -> None:
-        if user.is_bot:
-            return
-        space = self.tgid if self.peer_type == "channel" else user.tgid
-        message = DBMessage.get_by_mxid(event_id, self.mxid, space)
-        if not message:
-            return
-        await user.client.send_read_acknowledge(self.peer, max_id=message.tgid,
-                                                clear_mentions=True)
-
-    async def _preproc_kick_ban(self, user: Union['u.User', 'p.Puppet'], source: 'u.User'
-                                ) -> Optional['AbstractUser']:
-        if user.tgid == source.tgid:
-            return None
-        if self.peer_type == "user" and user.tgid == self.tgid:
-            await self.delete()
-            return None
-        if isinstance(user, u.User) and await user.needs_relaybot(self):
-            if not self.bot:
-                return None
-            # TODO kick message
-            return None
-        if await source.needs_relaybot(self):
-            if not self.has_bot:
-                return None
-            return self.bot
-        return source
-
-    async def kick_matrix(self, user: Union['u.User', 'p.Puppet'], source: 'u.User') -> None:
-        source = await self._preproc_kick_ban(user, source)
-        if source is not None:
-            await source.client.kick_participant(self.peer, user.peer)
-
-    async def ban_matrix(self, user: Union['u.User', 'p.Puppet'], source: 'u.User'):
-        source = await self._preproc_kick_ban(user, source)
-        if source is not None:
-            await source.client.edit_permissions(self.peer, user.peer, view_messages=False)
-
-    async def leave_matrix(self, user: 'u.User', event_id: EventID) -> None:
-        if await user.needs_relaybot(self):
-            await self._send_state_change_message("leave", user, event_id)
-            return
-
-        if self.peer_type == "user":
-            await self.main_intent.leave_room(self.mxid)
-            await self.delete()
-            try:
-                del self.by_tgid[self.tgid_full]
-                del self.by_mxid[self.mxid]
-            except KeyError:
-                pass
-        else:
-            await user.client.delete_dialog(self.peer)
-
-    async def join_matrix(self, user: 'u.User', event_id: EventID) -> None:
-        if await user.needs_relaybot(self):
-            await self._send_state_change_message("join", user, event_id)
-            return
-
-        if self.peer_type == "channel" and not user.is_bot:
-            await user.client(JoinChannelRequest(channel=await self.get_input_entity(user)))
-        else:
-            # We'll just assume the user is already in the chat.
-            pass
-
-    async def _apply_msg_format(self, sender: 'u.User', content: MessageEventContent
-                                ) -> None:
-        if not isinstance(content, TextMessageEventContent) or content.format != Format.HTML:
-            content.format = Format.HTML
-            content.formatted_body = escape_html(content.body).replace("\n", "<br/>")
-
-        tpl = (self.get_config(f"message_formats.[{content.msgtype.value}]")
-               or "<b>$sender_displayname</b>: $message")
-        displayname = await self.get_displayname(sender)
-        tpl_args = dict(sender_mxid=sender.mxid,
-                        sender_username=sender.mxid_localpart,
-                        sender_displayname=escape_html(displayname),
-                        message=content.formatted_body,
-                        body=content.body, formatted_body=content.formatted_body)
-        content.formatted_body = Template(tpl).safe_substitute(tpl_args)
-
-    async def _apply_emote_format(self, sender: 'u.User',
-                                  content: TextMessageEventContent) -> None:
-        if content.format != Format.HTML:
-            content.format = Format.HTML
-            content.formatted_body = escape_html(content.body).replace("\n", "<br/>")
-
-        tpl = self.get_config("emote_format")
-        puppet = p.Puppet.get(sender.tgid)
-        content.formatted_body = Template(tpl).safe_substitute(
-            dict(sender_mxid=sender.mxid,
-                 sender_username=sender.mxid_localpart,
-                 sender_displayname=escape_html(await self.get_displayname(sender)),
-                 mention=f"<a href='https://matrix.to/#/{puppet.mxid}'>{puppet.displayname}</a>",
-                 username=sender.username,
-                 displayname=puppet.displayname,
-                 body=content.body,
-                 formatted_body=content.formatted_body))
-        content.msgtype = MessageType.TEXT
-
-    async def _pre_process_matrix_message(self, sender: 'u.User', use_relaybot: bool,
-                                          content: MessageEventContent) -> None:
-        if use_relaybot:
-            await self._apply_msg_format(sender, content)
-        elif content.msgtype == MessageType.EMOTE:
-            await self._apply_emote_format(sender, content)
-
-    async def _handle_matrix_text(self, sender_id: TelegramID, event_id: EventID,
-                                  space: TelegramID, client: 'MautrixTelegramClient',
-                                  content: TextMessageEventContent, reply_to: TelegramID) -> None:
-        message, entities = await formatter.matrix_to_telegram(client, text=content.body,
-                                                               html=content.formatted(Format.HTML))
-        async with self.send_lock(sender_id):
-            lp = self.get_config("telegram_link_preview")
-            if content.get_edit():
-                orig_msg = DBMessage.get_by_mxid(content.get_edit(), self.mxid, space)
-                if orig_msg:
-                    response = await client.edit_message(self.peer, orig_msg.tgid, message,
-                                                         formatting_entities=entities,
-                                                         link_preview=lp)
-                    self._add_telegram_message_to_db(event_id, space, -1, response)
-                    return
-            response = await client.send_message(self.peer, message, reply_to=reply_to,
-                                                 formatting_entities=entities,
-                                                 link_preview=lp)
-            self._add_telegram_message_to_db(event_id, space, 0, response)
-        await self._send_delivery_receipt(event_id)
-
-    async def _handle_matrix_file(self, sender_id: TelegramID, event_id: EventID,
-                                  space: TelegramID, client: 'MautrixTelegramClient',
-                                  content: MediaMessageEventContent, reply_to: TelegramID,
-                                  caption: TextMessageEventContent = None) -> None:
-        mime = content.info.mimetype
-        if isinstance(content.info, (ImageInfo, VideoInfo)):
-            w, h = content.info.width, content.info.height
-        else:
-            w = h = None
-        file_name = content["net.maunium.telegram.internal.filename"]
-        max_image_size = config["bridge.image_as_file_size"] * 1000 ** 2
-
-        if config["bridge.parallel_file_transfer"] and content.url:
-            file_handle, file_size = await parallel_transfer_to_telegram(client, self.main_intent,
-                                                                         content.url, sender_id)
-        else:
-            if content.file:
-                if not decrypt_attachment:
-                    self.log.warning(f"Can't bridge encrypted media event {event_id}:"
-                                     " matrix-nio not installed")
-                    return
-                file = await self.main_intent.download_media(content.file.url)
-                file = decrypt_attachment(file, content.file.key.key,
-                                          content.file.hashes.get("sha256"), content.file.iv)
-            else:
-                file = await self.main_intent.download_media(content.url)
-
-            if content.msgtype == MessageType.STICKER:
-                if mime != "image/gif":
-                    mime, file, w, h = util.convert_image(file, source_mime=mime,
-                                                          target_type="webp")
-                else:
-                    # Remove sticker description
-                    file_name = "sticker.gif"
-
-            file_handle = await client.upload_file(file)
-            file_size = len(file)
-
-        file_handle.name = file_name
-
-        attributes = [DocumentAttributeFilename(file_name=file_name)]
-        if w and h:
-            attributes.append(DocumentAttributeImageSize(w, h))
-
-        if (mime == "image/png" or mime == "image/jpeg") and file_size < max_image_size:
-            media = InputMediaUploadedPhoto(file_handle)
-        else:
-            media = InputMediaUploadedDocument(file=file_handle, attributes=attributes,
-                                               mime_type=mime or "application/octet-stream")
-
-        capt, entities = (await formatter.matrix_to_telegram(client, text=caption.body,
-                                                             html=caption.formatted(Format.HTML))
-                          if caption else (None, None))
-
-        async with self.send_lock(sender_id):
-            if await self._matrix_document_edit(client, content, space, capt, media, event_id):
-                return
-            try:
-                response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=capt, entities=entities)
-            except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
-                media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
-                                                   attributes=attributes)
-                response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=capt, entities=entities)
-            self._add_telegram_message_to_db(event_id, space, 0, response)
-        await self._send_delivery_receipt(event_id)
-
-    async def _matrix_document_edit(self, client: 'MautrixTelegramClient',
-                                    content: MessageEventContent, space: TelegramID,
-                                    caption: str, media: Any, event_id: EventID) -> bool:
-        if content.get_edit():
-            orig_msg = DBMessage.get_by_mxid(content.get_edit(), self.mxid, space)
-            if orig_msg:
-                response = await client.edit_message(self.peer, orig_msg.tgid,
-                                                     caption, file=media)
-                self._add_telegram_message_to_db(event_id, space, -1, response)
-                await self._send_delivery_receipt(event_id)
-                return True
-        return False
-
-    async def _handle_matrix_location(self, sender_id: TelegramID, event_id: EventID,
-                                      space: TelegramID, client: 'MautrixTelegramClient',
-                                      content: LocationMessageEventContent, reply_to: TelegramID
-                                      ) -> None:
-        try:
-            lat, long = content.geo_uri[len("geo:"):].split(",")
-            lat, long = float(lat), float(long)
-        except (KeyError, ValueError):
-            self.log.exception("Failed to parse location")
-            return None
-        caption, entities = await formatter.matrix_to_telegram(client, text=content.body)
-        media = MessageMediaGeo(geo=GeoPoint(lat, long, access_hash=0))
-
-        async with self.send_lock(sender_id):
-            if await self._matrix_document_edit(client, content, space, caption, media, event_id):
-                return
-            response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                               caption=caption, entities=entities)
-            self._add_telegram_message_to_db(event_id, space, 0, response)
-        await self._send_delivery_receipt(event_id)
-
-    def _add_telegram_message_to_db(self, event_id: EventID, space: TelegramID,
-                                    edit_index: int, response: TypeMessage) -> None:
-        self.log.trace("Handled Matrix message: %s", response)
-        self.dedup.check(response, (event_id, space), force_hash=edit_index != 0)
-        if edit_index < 0:
-            prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1)
-            edit_index = prev_edit.edit_index + 1
-        DBMessage(
-            tgid=TelegramID(response.id),
-            tg_space=space,
-            mx_room=self.mxid,
-            mxid=event_id,
-            edit_index=edit_index).insert()
-
-    async def _send_bridge_error(self, msg: str) -> None:
-        if config["bridge.delivery_error_reports"]:
-            await self._send_message(self.main_intent,
-                                     TextMessageEventContent(msgtype=MessageType.NOTICE, body=msg))
-
-    async def handle_matrix_message(self, sender: 'u.User', content: MessageEventContent,
-                                    event_id: EventID) -> None:
-        try:
-            await self._handle_matrix_message(sender, content, event_id)
-        except RPCError as e:
-            if config["bridge.delivery_error_reports"]:
-                await self._send_bridge_error(
-                    f"\u26a0 Your message may not have been bridged: {e}")
-            raise
-
-    async def _handle_matrix_message(self, sender: 'u.User', content: MessageEventContent,
-                                     event_id: EventID) -> None:
-        if not content.body or not content.msgtype:
-            self.log.debug(f"Ignoring message {event_id} in {self.mxid} without body or msgtype")
-            return
-
-        logged_in = not await sender.needs_relaybot(self)
-        client = sender.client if logged_in else self.bot.client
-        sender_id = sender.tgid if logged_in else self.bot.tgid
-        space = (self.tgid if self.peer_type == "channel"  # Channels have their own ID space
-                 else (sender.tgid if logged_in else self.bot.tgid))
-        reply_to = formatter.matrix_reply_to_telegram(content, space, room_id=self.mxid)
-
-        media = (MessageType.STICKER, MessageType.IMAGE, MessageType.FILE, MessageType.AUDIO,
-                 MessageType.VIDEO)
-
-        if content.msgtype == MessageType.NOTICE:
-            bridge_notices = self.get_config("bridge_notices.default")
-            excepted = sender.mxid in self.get_config("bridge_notices.exceptions")
-            if not bridge_notices and not excepted:
-                return
-
-        if content.msgtype in (MessageType.TEXT, MessageType.EMOTE, MessageType.NOTICE):
-            await self._pre_process_matrix_message(sender, not logged_in, content)
-            await self._handle_matrix_text(sender_id, event_id, space, client, content, reply_to)
-        elif content.msgtype == MessageType.LOCATION:
-            await self._pre_process_matrix_message(sender, not logged_in, content)
-            await self._handle_matrix_location(sender_id, event_id, space, client, content,
-                                               reply_to)
-        elif content.msgtype in media:
-            content["net.maunium.telegram.internal.filename"] = content.body
-            try:
-                caption_content: MessageEventContent = sender.command_status["caption"]
-                reply_to = reply_to or formatter.matrix_reply_to_telegram(caption_content, space,
-                                                                          room_id=self.mxid)
-                sender.command_status = None
-            except (KeyError, TypeError):
-                caption_content = None if logged_in else TextMessageEventContent(body=content.body)
-            if caption_content:
-                caption_content.msgtype = content.msgtype
-                await self._pre_process_matrix_message(sender, not logged_in, caption_content)
-            await self._handle_matrix_file(sender_id, event_id, space, client, content, reply_to,
-                                           caption_content)
-        else:
-            self.log.trace("Unhandled Matrix event: %s", content)
-
-    async def handle_matrix_pin(self, sender: 'u.User', pinned_message: Optional[EventID],
-                                pin_event_id: EventID) -> None:
-        if self.peer_type != "chat" and self.peer_type != "channel":
-            return
-        try:
-            if not pinned_message:
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=0))
-            else:
-                tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
-                message = DBMessage.get_by_mxid(pinned_message, self.mxid, tg_space)
-                if message is None:
-                    self.log.warning(f"Could not find pinned {pinned_message} in {self.mxid}")
-                    return
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=message.tgid))
-            await self._send_delivery_receipt(pin_event_id)
-        except ChatNotModifiedError:
-            pass
-
-    async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID,
-                                     redaction_event_id: EventID) -> None:
-        real_deleter = deleter if not await deleter.needs_relaybot(self) else self.bot
-        space = self.tgid if self.peer_type == "channel" else real_deleter.tgid
-        message = DBMessage.get_by_mxid(event_id, self.mxid, space)
-        if not message:
-            return
-        if message.edit_index == 0:
-            await real_deleter.client.delete_messages(self.peer, [message.tgid])
-            await self._send_delivery_receipt(redaction_event_id)
-        else:
-            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
-
-    async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
-                                           level: int) -> None:
-        moderator = level >= 50
-        admin = level >= 75
-        await sender.client.edit_admin(self.peer, user_id,
-                                       change_info=moderator, post_messages=moderator,
-                                       edit_messages=moderator, delete_messages=moderator,
-                                       ban_users=moderator, invite_users=moderator,
-                                       pin_messages=moderator, add_admins=admin)
-
-    async def handle_matrix_power_levels(self, sender: 'u.User', new_users: Dict[UserID, int],
-                                         old_users: Dict[UserID, int], event_id: Optional[EventID]
-                                         ) -> None:
-        # TODO handle all power level changes and bridge exact admin rights to supergroups/channels
-        for user, level in new_users.items():
-            if not user or user == self.main_intent.mxid or user == sender.mxid:
-                continue
-            user_id = p.Puppet.get_id_from_mxid(user)
-            if not user_id:
-                mx_user = u.User.get_by_mxid(user, create=False)
-                if not mx_user or not mx_user.tgid:
-                    continue
-                user_id = mx_user.tgid
-            if not user_id or user_id == sender.tgid:
-                continue
-            if user not in old_users or level != old_users[user]:
-                await self._update_telegram_power_level(sender, user_id, level)
-
-    async def handle_matrix_about(self, sender: 'u.User', about: str, event_id: EventID) -> None:
-        if self.peer_type not in ("chat", "channel"):
-            return
-        peer = await self.get_input_entity(sender)
-        await sender.client(EditChatAboutRequest(peer=peer, about=about))
-        self.about = about
-        await self.save()
-        await self._send_delivery_receipt(event_id)
-
-    async def handle_matrix_title(self, sender: 'u.User', title: str, event_id: EventID) -> None:
-        if self.peer_type not in ("chat", "channel"):
-            return
-
-        if self.peer_type == "chat":
-            response = await sender.client(EditChatTitleRequest(chat_id=self.tgid, title=title))
-        else:
-            channel = await self.get_input_entity(sender)
-            response = await sender.client(EditTitleRequest(channel=channel, title=title))
-        self.dedup.register_outgoing_actions(response)
-        self.title = title
-        await self.save()
-        await self._send_delivery_receipt(event_id)
-        await self.update_bridge_info()
-
-    async def handle_matrix_avatar(self, sender: 'u.User', url: ContentURI, event_id: EventID
-                                   ) -> None:
-        if self.peer_type not in ("chat", "channel"):
-            # Invalid peer type
-            return
-        elif self.avatar_url == url:
-            return
-
-        self.avatar_url = url
-        file = await self.main_intent.download_media(url)
-        mime = magic.from_buffer(file, mime=True)
-        ext = sane_mimetypes.guess_extension(mime)
-        uploaded = await sender.client.upload_file(file, file_name=f"avatar{ext}")
-        photo = InputChatUploadedPhoto(file=uploaded)
-
-        if self.peer_type == "chat":
-            response = await sender.client(EditChatPhotoRequest(chat_id=self.tgid, photo=photo))
-        else:
-            channel = await self.get_input_entity(sender)
-            response = await sender.client(EditPhotoRequest(channel=channel, photo=photo))
-        self.dedup.register_outgoing_actions(response)
-        for update in response.updates:
-            is_photo_update = (isinstance(update, UpdateNewMessage)
-                               and isinstance(update.message, MessageService)
-                               and isinstance(update.message.action, MessageActionChatEditPhoto))
-            if is_photo_update:
-                loc, size = self._get_largest_photo_size(update.message.action.photo)
-                self.photo_id = f"{size.location.volume_id}-{size.location.local_id}"
-                await self.save()
-                break
-        await self._send_delivery_receipt(event_id)
-        await self.update_bridge_info()
-
-    async def handle_matrix_upgrade(self, sender: UserID, new_room: RoomID, event_id: EventID
-                                    ) -> None:
-        _, server = self.main_intent.parse_user_id(sender)
-        old_room = self.mxid
-        self.migrate_and_save_matrix(new_room)
-        await self.main_intent.join_room(new_room, servers=[server])
-        entity: Optional[TypeInputPeer] = None
-        user: Optional[AbstractUser] = None
-        if self.bot and self.has_bot:
-            user = self.bot
-            entity = await self.get_input_entity(self.bot)
-        if not entity:
-            user_mxids = await self.main_intent.get_room_members(self.mxid)
-            for user_str in user_mxids:
-                user_id = UserID(user_str)
-                if user_id == self.az.bot_mxid:
-                    continue
-                user = u.User.get_by_mxid(user_id, create=False)
-                if user and user.tgid:
-                    entity = await self.get_input_entity(user)
-                    if entity:
-                        break
-        if not entity:
-            self.log.error("Failed to fully migrate to upgraded Matrix room: "
-                           "no Telegram user found.")
-            return
-        await self.update_matrix_room(user, entity, direct=self.peer_type == "user")
-        self.log.info(f"{sender} upgraded room from {old_room} to {self.mxid}")
-        await self._send_delivery_receipt(event_id, room_id=old_room)
-
-    def migrate_and_save_matrix(self, new_id: RoomID) -> None:
-        try:
-            del self.by_mxid[self.mxid]
-        except KeyError:
-            pass
-        self.mxid = new_id
-        self.db_instance.edit(mxid=self.mxid)
-        self.by_mxid[self.mxid] = self
-
-    async def enable_dm_encryption(self) -> bool:
-        ok = await super().enable_dm_encryption()
-        if ok:
-            try:
-                puppet = p.Puppet.get(self.tgid)
-                await self.main_intent.set_room_name(self.mxid, puppet.displayname)
-            except Exception:
-                self.log.warning(f"Failed to set room name", exc_info=True)
-        return ok
-
-
-def init(context: Context) -> None:
-    global config
-    config = context.config

mautrix_telegram/portal/metadata.py

@@ -1,850 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Optional, Iterable, Union, Dict, Any, TYPE_CHECKING
-from abc import ABC
-import asyncio
-
-from telethon.tl.functions.messages import (AddChatUserRequest, CreateChatRequest,
-                                            GetFullChatRequest, MigrateChatRequest)
-from telethon.tl.functions.channels import (CreateChannelRequest, GetParticipantsRequest,
-                                            InviteToChannelRequest, UpdateUsernameRequest)
-from telethon.errors import ChatAdminRequiredError
-from telethon.tl.types import (
-    Channel, ChatBannedRights, ChannelParticipantsRecent, ChannelParticipantsSearch, ChatPhoto,
-    PhotoEmpty, InputChannel, InputUser, ChatPhotoEmpty, PeerUser, Photo, TypeChat, TypeInputPeer,
-    TypeUser, User, InputPeerPhotoFileLocation, ChatParticipantAdmin, ChannelParticipantAdmin,
-    ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty)
-
-from mautrix.errors import MForbidden
-from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership,
-                           PowerLevelStateEventContent, RoomTopicStateEventContent,
-                           RoomNameStateEventContent, RoomAvatarStateEventContent,
-                           StateEventContent, EventID)
-
-from ..types import TelegramID
-from ..context import Context
-from .. import puppet as p, user as u, util
-from .base import BasePortal, InviteList, TypeParticipant, TypeChatPhoto
-
-if TYPE_CHECKING:
-    from ..abstract_user import AbstractUser
-    from ..config import Config
-
-config: Optional['Config'] = None
-
-StateBridge = EventType.find("m.bridge", EventType.Class.STATE)
-StateHalfShotBridge = EventType.find("uk.half-shot.bridge", EventType.Class.STATE)
-
-
-class PortalMetadata(BasePortal, ABC):
-    _room_create_lock: asyncio.Lock
-
-    def __init__(self, *args, **kwargs) -> None:
-        super().__init__(*args, **kwargs)
-        self._room_create_lock = asyncio.Lock()
-
-    # region Matrix -> Telegram
-
-    async def _get_telegram_users_in_matrix_room(self) -> List[Union[InputUser, PeerUser]]:
-        user_tgids = set()
-        user_mxids = await self.main_intent.get_room_members(self.mxid, (Membership.JOIN,
-                                                                         Membership.INVITE))
-        for user_str in user_mxids:
-            user = UserID(user_str)
-            if user == self.az.bot_mxid:
-                continue
-            mx_user = u.User.get_by_mxid(user, create=False)
-            if mx_user and mx_user.tgid:
-                user_tgids.add(mx_user.tgid)
-            puppet_id = p.Puppet.get_id_from_mxid(user)
-            if puppet_id:
-                user_tgids.add(puppet_id)
-        return [PeerUser(user_id) for user_id in user_tgids]
-
-    async def upgrade_telegram_chat(self, source: 'u.User') -> None:
-        if self.peer_type != "chat":
-            raise ValueError("Only normal group chats are upgradable to supergroups.")
-
-        response = await source.client(MigrateChatRequest(chat_id=self.tgid))
-        entity = None
-        for chat in response.chats:
-            if isinstance(chat, Channel):
-                entity = chat
-                break
-        if not entity:
-            raise ValueError("Upgrade may have failed: output channel not found.")
-        self.peer_type = "channel"
-        self._migrate_and_save_telegram(TelegramID(entity.id))
-        await self.update_info(source, entity)
-
-    def _migrate_and_save_telegram(self, new_id: TelegramID) -> None:
-        try:
-            del self.by_tgid[self.tgid_full]
-        except KeyError:
-            pass
-        try:
-            existing = self.by_tgid[(new_id, new_id)]
-            existing.delete_sync()
-        except KeyError:
-            pass
-        self.db_instance.edit(tgid=new_id, tg_receiver=new_id, peer_type=self.peer_type)
-        old_id = self.tgid
-        self.tgid = new_id
-        self.tg_receiver = new_id
-        self.by_tgid[self.tgid_full] = self
-        self.log = self.base_log.getChild(self.tgid_log)
-        self.log.info(f"Telegram chat upgraded from {old_id}")
-
-    async def set_telegram_username(self, source: 'u.User', username: str) -> None:
-        if self.peer_type != "channel":
-            raise ValueError("Only channels and supergroups have usernames.")
-        await source.client(
-            UpdateUsernameRequest(await self.get_input_entity(source), username))
-        if await self._update_username(username):
-            await self.save()
-
-    async def create_telegram_chat(self, source: 'u.User', supergroup: bool = False) -> None:
-        if not self.mxid:
-            raise ValueError("Can't create Telegram chat for portal without Matrix room.")
-        elif self.tgid:
-            raise ValueError("Can't create Telegram chat for portal with existing Telegram chat.")
-
-        invites = await self._get_telegram_users_in_matrix_room()
-        if len(invites) < 2:
-            if self.bot is not None:
-                info, mxid = await self.bot.get_me()
-                raise ValueError("Not enough Telegram users to create a chat. "
-                                 "Invite more Telegram ghost users to the room, such as the "
-                                 f"relaybot ([{info.first_name}](https://matrix.to/#/{mxid})).")
-            raise ValueError("Not enough Telegram users to create a chat. "
-                             "Invite more Telegram ghost users to the room.")
-        if self.peer_type == "chat":
-            response = await source.client(CreateChatRequest(title=self.title, users=invites))
-            entity = response.chats[0]
-        elif self.peer_type == "channel":
-            response = await source.client(CreateChannelRequest(title=self.title,
-                                                                about=self.about or "",
-                                                                megagroup=supergroup))
-            entity = response.chats[0]
-            await source.client(InviteToChannelRequest(
-                channel=await source.client.get_input_entity(entity),
-                users=invites))
-        else:
-            raise ValueError("Invalid peer type for Telegram chat creation")
-
-        self.tgid = entity.id
-        self.tg_receiver = self.tgid
-        self.by_tgid[self.tgid_full] = self
-        await self.update_info(source, entity)
-        self.db_instance.insert()
-        self.log = self.base_log.getChild(self.tgid_log)
-
-        if self.bot and self.bot.tgid in invites:
-            self.bot.add_chat(self.tgid, self.peer_type)
-
-        levels = await self.main_intent.get_power_levels(self.mxid)
-        if levels.get_user_level(self.main_intent.mxid) == 100:
-            levels = self._get_base_power_levels(levels, entity)
-            await self.main_intent.set_power_levels(self.mxid, levels)
-        await self.handle_matrix_power_levels(source, levels.users, {}, None)
-
-    async def invite_telegram(self, source: 'u.User',
-                              puppet: Union[p.Puppet, 'AbstractUser']) -> None:
-        if self.peer_type == "chat":
-            await source.client(
-                AddChatUserRequest(chat_id=self.tgid, user_id=puppet.tgid, fwd_limit=0))
-        elif self.peer_type == "channel":
-            await source.client(InviteToChannelRequest(channel=self.peer, users=[puppet.tgid]))
-        # We don't care if there are invites for private chat portals with the relaybot.
-        elif not self.bot or self.tg_receiver != self.bot.tgid:
-            raise ValueError("Invalid peer type for Telegram user invite")
-
-    # endregion
-    # region Telegram -> Matrix
-
-    async def invite_to_matrix(self, users: InviteList) -> None:
-        if isinstance(users, list):
-            for user in users:
-                await self.main_intent.invite_user(self.mxid, user, check_cache=True)
-        else:
-            await self.main_intent.invite_user(self.mxid, users, check_cache=True)
-
-    async def update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
-                                 direct: bool = None, puppet: p.Puppet = None,
-                                 levels: PowerLevelStateEventContent = None,
-                                 users: List[User] = None) -> None:
-        if direct is None:
-            direct = self.peer_type == "user"
-        try:
-            await self._update_matrix_room(user, entity, direct, puppet, levels, users)
-        except Exception:
-            self.log.exception("Fatal error updating Matrix room")
-
-    async def _update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
-                                  direct: bool, puppet: p.Puppet = None,
-                                  levels: PowerLevelStateEventContent = None,
-                                  users: List[User] = None) -> None:
-        if not direct:
-            await self.update_info(user, entity)
-            if not users:
-                users = await self._get_users(user, entity)
-            await self._sync_telegram_users(user, users)
-            await self.update_power_levels(users, levels)
-        else:
-            if not puppet:
-                puppet = p.Puppet.get(self.tgid)
-            await puppet.update_info(user, entity)
-            await puppet.intent_for(self).join_room(self.mxid)
-            if self.encrypted or self.private_chat_portal_meta:
-                # The bridge bot needs to join for e2ee, but that messes up the default name
-                # generation. If/when canonical DMs happen, this might not be necessary anymore.
-                changed = await self._update_title(puppet.displayname)
-                changed = await self._update_avatar(user, entity.photo) or changed
-                if changed:
-                    await self.save()
-                    await self.update_bridge_info()
-
-            puppet = await p.Puppet.get_by_custom_mxid(user.mxid)
-            if puppet:
-                try:
-                    did_join = await puppet.intent.ensure_joined(self.mxid)
-                    if isinstance(user, u.User) and did_join and self.peer_type == "user":
-                        await user.update_direct_chats({self.main_intent.mxid: [self.mxid]})
-                except Exception:
-                    self.log.exception("Failed to ensure %s is joined to portal", user.mxid)
-
-        if self.sync_matrix_state:
-            await self.main_intent.get_joined_members(self.mxid)
-
-    async def create_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User] = None,
-                                 invites: InviteList = None, update_if_exists: bool = True
-                                 ) -> Optional[RoomID]:
-        if self.mxid:
-            if update_if_exists:
-                if not entity:
-                    try:
-                        entity = await self.get_entity(user)
-                    except Exception:
-                        self.log.exception(f"Failed to get entity through {user.tgid} for update")
-                        return self.mxid
-                update = self.update_matrix_room(user, entity, self.peer_type == "user")
-                self.loop.create_task(update)
-                await self.invite_to_matrix(invites or [])
-            return self.mxid
-        async with self._room_create_lock:
-            try:
-                return await self._create_matrix_room(user, entity, invites)
-            except Exception:
-                self.log.exception("Fatal error creating Matrix room")
-
-    @property
-    def bridge_info_state_key(self) -> str:
-        return f"net.maunium.telegram://telegram/{self.tgid}"
-
-    @property
-    def bridge_info(self) -> Dict[str, Any]:
-        info = {
-            "bridgebot": self.az.bot_mxid,
-            "creator": self.main_intent.mxid,
-            "protocol": {
-                "id": "telegram",
-                "displayname": "Telegram",
-                "avatar_url": config["appservice.bot_avatar"],
-                "external_url": "https://telegram.org",
-            },
-            "channel": {
-                "id": str(self.tgid),
-                "displayname": self.title,
-                "avatar_url": self.avatar_url,
-            }
-        }
-        if self.username:
-            info["channel"]["external_url"] = f"https://t.me/{self.username}"
-        elif self.peer_type == "user":
-            puppet = p.Puppet.get(self.tgid)
-            if puppet and puppet.username:
-                info["channel"]["external_url"] = f"https://t.me/{puppet.username}"
-        return info
-
-    async def update_bridge_info(self) -> None:
-        if not self.mxid:
-            self.log.debug("Not updating bridge info: no Matrix room created")
-            return
-        try:
-            self.log.debug("Updating bridge info...")
-            await self.main_intent.send_state_event(self.mxid, StateBridge,
-                                                    self.bridge_info, self.bridge_info_state_key)
-            # TODO remove this once https://github.com/matrix-org/matrix-doc/pull/2346 is in spec
-            await self.main_intent.send_state_event(self.mxid, StateHalfShotBridge,
-                                                    self.bridge_info, self.bridge_info_state_key)
-        except Exception:
-            self.log.warning("Failed to update bridge info", exc_info=True)
-
-    async def _create_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
-                                  invites: InviteList) -> Optional[RoomID]:
-        if self.mxid:
-            return self.mxid
-        elif not self.allow_bridging:
-            return None
-
-        direct = self.peer_type == "user"
-        invites = invites or []
-
-        if not entity:
-            entity = await self.get_entity(user)
-            self.log.trace("Fetched data: %s", entity)
-
-        self.log.debug("Creating room")
-
-        try:
-            self.title = entity.title
-        except AttributeError:
-            self.title = None
-
-        if direct and self.tgid == user.tgid:
-            self.title = "Telegram Saved Messages"
-            self.about = "Your Telegram cloud storage chat"
-
-        puppet = p.Puppet.get(self.tgid) if direct else None
-        if puppet:
-            await puppet.update_info(user, entity)
-        self._main_intent = puppet.intent_for(self) if direct else self.az.intent
-
-        if self.peer_type == "channel":
-            self.megagroup = entity.megagroup
-
-        if self.peer_type == "channel" and entity.username:
-            preset = RoomCreatePreset.PUBLIC
-            self.username = entity.username
-            alias = self.alias_localpart
-        else:
-            preset = RoomCreatePreset.PRIVATE
-            # TODO invite link alias?
-            alias = None
-
-        if alias:
-            # TODO? properly handle existing room aliases
-            await self.main_intent.remove_room_alias(alias)
-
-        power_levels = self._get_base_power_levels(entity=entity)
-        users = None
-        if not direct:
-            users = await self._get_users(user, entity)
-            if self.has_bot:
-                extra_invites = config["bridge.relaybot.group_chat_invite"]
-                invites += extra_invites
-                for invite in extra_invites:
-                    power_levels.users.setdefault(invite, 100)
-            await self._participants_to_power_levels(users, power_levels)
-        elif self.bot and self.tg_receiver == self.bot.tgid:
-            invites = config["bridge.relaybot.private_chat.invite"]
-            for invite in invites:
-                power_levels.users.setdefault(invite, 100)
-            self.title = puppet.displayname
-
-        initial_state = [{
-            "type": EventType.ROOM_POWER_LEVELS.serialize(),
-            "content": power_levels.serialize(),
-        }, {
-            "type": str(StateBridge),
-            "state_key": self.bridge_info_state_key,
-            "content": self.bridge_info,
-        }, {
-            # TODO remove this once https://github.com/matrix-org/matrix-doc/pull/2346 is in spec
-            "type": str(StateHalfShotBridge),
-            "state_key": self.bridge_info_state_key,
-            "content": self.bridge_info,
-        }]
-        if config["bridge.encryption.default"] and self.matrix.e2ee:
-            self.encrypted = True
-            initial_state.append({
-                "type": "m.room.encryption",
-                "content": {"algorithm": "m.megolm.v1.aes-sha2"},
-            })
-            if direct:
-                invites.append(self.az.bot_mxid)
-        if direct and (self.encrypted or self.private_chat_portal_meta):
-            self.title = puppet.displayname
-        if config["appservice.community_id"]:
-            initial_state.append({
-                "type": "m.room.related_groups",
-                "content": {"groups": [config["appservice.community_id"]]},
-            })
-        creation_content = {}
-        if not config["bridge.federate_rooms"]:
-            creation_content["m.federate"] = False
-
-        with self.backfill_lock:
-            room_id = await self.main_intent.create_room(alias_localpart=alias, preset=preset,
-                                                         is_direct=direct, invitees=invites or [],
-                                                         name=self.title, topic=self.about,
-                                                         initial_state=initial_state,
-                                                         creation_content=creation_content)
-            if not room_id:
-                raise Exception(f"Failed to create room")
-
-            if self.encrypted and self.matrix.e2ee and direct:
-                try:
-                    await self.az.intent.ensure_joined(room_id)
-                except Exception:
-                    self.log.warning(f"Failed to add bridge bot to new private chat {room_id}")
-
-            self.mxid = room_id
-            self.by_mxid[self.mxid] = self
-            await self.save()
-            await self.az.state_store.set_power_levels(self.mxid, power_levels)
-            await user.register_portal(self)
-
-            update_room = self.loop.create_task(self.update_matrix_room(
-                user, entity, direct, puppet,
-                levels=power_levels, users=users))
-
-            if config["bridge.backfill.initial_limit"] > 0:
-                self.log.debug("Initial backfill is enabled. Waiting for room members to sync "
-                               "and then starting backfill")
-                await update_room
-
-                try:
-                    await self.backfill(user, is_initial=True)
-                except Exception:
-                    self.log.exception("Failed to backfill new portal")
-
-        return self.mxid
-
-    def _get_base_power_levels(self, levels: PowerLevelStateEventContent = None,
-                               entity: TypeChat = None) -> PowerLevelStateEventContent:
-        levels = levels or PowerLevelStateEventContent()
-        if self.peer_type == "user":
-            overrides = config["bridge.initial_power_level_overrides.user"]
-            levels.ban = overrides.get("ban", 100)
-            levels.kick = overrides.get("kick", 100)
-            levels.invite = overrides.get("invite", 100)
-            levels.redact = overrides.get("redact", 0)
-            levels.events[EventType.ROOM_NAME] = 0
-            levels.events[EventType.ROOM_AVATAR] = 0
-            levels.events[EventType.ROOM_TOPIC] = 0
-            levels.state_default = overrides.get("state_default", 0)
-            levels.users_default = overrides.get("users_default", 0)
-            levels.events_default = overrides.get("events_default", 0)
-        else:
-            overrides = config["bridge.initial_power_level_overrides.group"]
-            dbr = entity.default_banned_rights
-            if not dbr:
-                self.log.debug(f"default_banned_rights is None in {entity}")
-                dbr = ChatBannedRights(invite_users=True, change_info=True, pin_messages=True,
-                                       send_stickers=False, send_messages=False, until_date=None)
-            levels.ban = overrides.get("ban", 50)
-            levels.kick = overrides.get("kick", 50)
-            levels.redact = overrides.get("redact", 50)
-            levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
-            levels.events[EventType.ROOM_ENCRYPTION] = 99
-            levels.events[EventType.ROOM_TOMBSTONE] = 99
-            levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
-            levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0
-            levels.events[EventType.ROOM_TOPIC] = 50 if dbr.change_info else 0
-            levels.events[EventType.ROOM_PINNED_EVENTS] = 50 if dbr.pin_messages else 0
-            levels.events[EventType.ROOM_POWER_LEVELS] = 75
-            levels.events[EventType.ROOM_HISTORY_VISIBILITY] = 75
-            levels.events[EventType.STICKER] = 50 if dbr.send_stickers else levels.events_default
-            levels.state_default = overrides.get("state_default", 50)
-            levels.users_default = overrides.get("users_default", 0)
-            levels.events_default = (
-                overrides.get("events_default",
-                              50 if (self.peer_type == "channel" and not entity.megagroup
-                                     or entity.default_banned_rights.send_messages)
-                              else 0))
-        for evt_type, value in overrides.get("events", {}).items():
-            levels.events[EventType.find(evt_type)] = value
-        levels.users = overrides.get("users", {})
-        if self.main_intent.mxid not in levels.users:
-            levels.users[self.main_intent.mxid] = 100
-        return levels
-
-    @staticmethod
-    def _get_level_from_participant(participant: TypeParticipant) -> int:
-        # TODO use the power level requirements to get better precision in channels
-        if isinstance(participant, (ChatParticipantAdmin, ChannelParticipantAdmin)):
-            return 50
-        elif isinstance(participant, (ChatParticipantCreator, ChannelParticipantCreator)):
-            return 95
-        return 0
-
-    @staticmethod
-    def _participant_to_power_levels(levels: PowerLevelStateEventContent,
-                                     user: Union['u.User', p.Puppet], new_level: int,
-                                     bot_level: int) -> bool:
-        new_level = min(new_level, bot_level)
-        user_level = levels.get_user_level(user.mxid)
-        if user_level != new_level and user_level < bot_level:
-            levels.users[user.mxid] = new_level
-            return True
-        return False
-
-    async def _participants_to_power_levels(self, users: List[Union[TypeUser, TypeParticipant]],
-                                            levels: PowerLevelStateEventContent) -> bool:
-        bot_level = levels.get_user_level(self.main_intent.mxid)
-        if bot_level < levels.get_event_level(EventType.ROOM_POWER_LEVELS):
-            return False
-        changed = False
-        admin_power_level = min(75 if self.peer_type == "channel" else 50, bot_level)
-        if levels.get_event_level(EventType.ROOM_POWER_LEVELS) != admin_power_level:
-            changed = True
-            levels.events[EventType.ROOM_POWER_LEVELS] = admin_power_level
-
-        for user in users:
-            # The User objects we get from TelegramClient.get_participants have a custom
-            # participant property
-            participant = getattr(user, "participant", user)
-
-            puppet = p.Puppet.get(TelegramID(participant.user_id))
-            user = u.User.get_by_tgid(TelegramID(participant.user_id))
-            new_level = self._get_level_from_participant(participant)
-
-            if user:
-                await user.register_portal(self)
-                changed = self._participant_to_power_levels(levels, user, new_level,
-                                                            bot_level) or changed
-
-            if puppet:
-                changed = self._participant_to_power_levels(levels, puppet, new_level,
-                                                            bot_level) or changed
-        return changed
-
-    async def update_power_levels(self, users: List[Union[TypeUser, TypeParticipant]],
-                                  levels: PowerLevelStateEventContent = None) -> None:
-        if not levels:
-            levels = await self.main_intent.get_power_levels(self.mxid)
-        if await self._participants_to_power_levels(users, levels):
-            await self.main_intent.set_power_levels(self.mxid, levels)
-
-    async def _add_bot_chat(self, bot: User) -> None:
-        if self.bot and bot.id == self.bot.tgid:
-            self.bot.add_chat(self.tgid, self.peer_type)
-            return
-
-        user = u.User.get_by_tgid(TelegramID(bot.id))
-        if user and user.is_bot:
-            await user.register_portal(self)
-
-    async def _sync_telegram_users(self, source: 'AbstractUser', users: List[User]) -> None:
-        allowed_tgids = set()
-        skip_deleted = config["bridge.skip_deleted_members"]
-        for entity in users:
-            puppet = p.Puppet.get(TelegramID(entity.id))
-            if entity.bot:
-                await self._add_bot_chat(entity)
-            allowed_tgids.add(entity.id)
-
-            await puppet.update_info(source, entity)
-            if skip_deleted and entity.deleted:
-                continue
-
-            await puppet.intent_for(self).ensure_joined(self.mxid)
-
-            user = u.User.get_by_tgid(TelegramID(entity.id))
-            if user:
-                await self.invite_to_matrix(user.mxid)
-
-                puppet = await p.Puppet.get_by_custom_mxid(user.mxid)
-                if puppet:
-                    try:
-                        await puppet.intent.ensure_joined(self.mxid)
-                    except Exception:
-                        self.log.exception("Failed to ensure %s is joined to portal", user.mxid)
-
-        # We can't trust the member list if any of the following cases is true:
-        #  * There are close to 10 000 users, because Telegram might not be sending all members.
-        #  * The member sync count is limited, because then we might ignore some members.
-        #  * It's a channel, because non-admins don't have access to the member list.
-        trust_member_list = ((len(allowed_tgids) < 9900
-                              if self.max_initial_member_sync < 0
-                              else len(allowed_tgids) < self.max_initial_member_sync - 10)
-                             and (self.megagroup or self.peer_type != "channel"))
-        if not trust_member_list:
-            return
-
-        for user_mxid in await self.main_intent.get_room_members(self.mxid):
-            if user_mxid == self.az.bot_mxid:
-                continue
-
-            puppet_id = p.Puppet.get_id_from_mxid(user_mxid)
-            if puppet_id:
-                if puppet_id in allowed_tgids:
-                    continue
-                if self.bot and puppet_id == self.bot.tgid:
-                    self.bot.remove_chat(self.tgid)
-                try:
-                    await self.main_intent.kick_user(self.mxid, user_mxid,
-                                                     "User had left this Telegram chat.")
-                except MForbidden:
-                    pass
-                continue
-
-            mx_user = u.User.get_by_mxid(user_mxid, create=False)
-            if mx_user:
-                if mx_user.tgid in allowed_tgids:
-                    continue
-                if mx_user.is_bot:
-                    await mx_user.unregister_portal(*self.tgid_full)
-                if not self.has_bot:
-                    try:
-                        await self.main_intent.kick_user(self.mxid, mx_user.mxid,
-                                                         "You had left this Telegram chat.")
-                    except MForbidden:
-                        pass
-
-    async def _add_telegram_user(self, user_id: TelegramID, source: Optional['AbstractUser'] = None
-                                 ) -> None:
-        puppet = p.Puppet.get(user_id)
-        if source:
-            entity: User = await source.client.get_entity(PeerUser(user_id))
-            await puppet.update_info(source, entity)
-            await puppet.intent_for(self).ensure_joined(self.mxid)
-
-        user = u.User.get_by_tgid(user_id)
-        if user:
-            await user.register_portal(self)
-            await self.invite_to_matrix(user.mxid)
-
-    async def _delete_telegram_user(self, user_id: TelegramID, sender: p.Puppet) -> None:
-        puppet = p.Puppet.get(user_id)
-        user = u.User.get_by_tgid(user_id)
-        kick_message = (f"Kicked by {sender.displayname}"
-                        if sender and sender.tgid != puppet.tgid
-                        else "Left Telegram chat")
-        if sender.tgid != puppet.tgid:
-            try:
-                await sender.intent_for(self).kick_user(self.mxid, puppet.mxid)
-            except MForbidden:
-                await self.main_intent.kick_user(self.mxid, puppet.mxid, kick_message)
-        else:
-            await puppet.intent_for(self).leave_room(self.mxid)
-        if user:
-            await user.unregister_portal(*self.tgid_full)
-            if sender.tgid != puppet.tgid:
-                try:
-                    await sender.intent_for(self).kick_user(self.mxid, puppet.mxid)
-                    return
-                except MForbidden:
-                    pass
-            try:
-                await self.main_intent.kick_user(self.mxid, user.mxid, kick_message)
-            except MForbidden as e:
-                self.log.warning(f"Failed to kick {user.mxid}: {e}")
-
-    async def update_info(self, user: 'AbstractUser', entity: TypeChat = None) -> None:
-        if self.peer_type == "user":
-            self.log.warning("Called update_info() for direct chat portal")
-            return
-
-        changed = False
-        self.log.debug("Updating info")
-        try:
-            if not entity:
-                entity = await self.get_entity(user)
-                self.log.trace("Fetched data: %s", entity)
-
-            if self.peer_type == "channel":
-                changed = self.megagroup != entity.megagroup or changed
-                self.megagroup = entity.megagroup
-                changed = await self._update_username(entity.username) or changed
-
-            if hasattr(entity, "about"):
-                changed = self._update_about(entity.about) or changed
-
-            changed = await self._update_title(entity.title) or changed
-
-            if isinstance(entity.photo, ChatPhoto):
-                changed = await self._update_avatar(user, entity.photo) or changed
-        except Exception:
-            self.log.exception(f"Failed to update info from source {user.tgid}")
-
-        if changed:
-            await self.save()
-            await self.update_bridge_info()
-
-    async def _update_username(self, username: str, save: bool = False) -> bool:
-        if self.username == username:
-            return False
-
-        if self.username:
-            await self.main_intent.remove_room_alias(self.alias_localpart)
-        self.username = username or None
-        if self.username:
-            await self.main_intent.add_room_alias(self.mxid, self.alias_localpart, override=True)
-            if self.public_portals:
-                await self.main_intent.set_join_rule(self.mxid, "public")
-        else:
-            await self.main_intent.set_join_rule(self.mxid, "invite")
-
-        if save:
-            await self.save()
-        return True
-
-    async def _try_set_state(self, sender: Optional['p.Puppet'], evt_type: EventType,
-                             content: StateEventContent) -> None:
-        if sender:
-            try:
-                intent = sender.intent_for(self)
-                if sender.is_real_user:
-                    content[self.az.real_user_content_key] = True
-                await intent.send_state_event(self.mxid, evt_type, content)
-            except MForbidden:
-                await self.main_intent.send_state_event(self.mxid, evt_type, content)
-        else:
-            await self.main_intent.send_state_event(self.mxid, evt_type, content)
-
-    async def _update_about(self, about: str, sender: Optional['p.Puppet'] = None,
-                            save: bool = False) -> bool:
-        if self.about == about:
-            return False
-
-        self.about = about
-        await self._try_set_state(sender, EventType.ROOM_TOPIC,
-                                  RoomTopicStateEventContent(topic=self.about))
-        if save:
-            await self.save()
-        return True
-
-    async def _update_title(self, title: str, sender: Optional['p.Puppet'] = None,
-                            save: bool = False) -> bool:
-        if self.title == title:
-            return False
-
-        self.title = title
-        await self._try_set_state(sender, EventType.ROOM_NAME,
-                                  RoomNameStateEventContent(name=self.title))
-        if save:
-            await self.save()
-        return True
-
-    async def _update_avatar(self, user: 'AbstractUser', photo: TypeChatPhoto,
-                             sender: Optional['p.Puppet'] = None, save: bool = False) -> bool:
-        if isinstance(photo, (ChatPhoto, UserProfilePhoto)):
-            loc = InputPeerPhotoFileLocation(
-                peer=await self.get_input_entity(user),
-                local_id=photo.photo_big.local_id,
-                volume_id=photo.photo_big.volume_id,
-                big=True
-            )
-            photo_id = (f"{loc.volume_id}-{loc.local_id}" if isinstance(photo, ChatPhoto)
-                        else photo.photo_id)
-        elif isinstance(photo, Photo):
-            loc, largest = self._get_largest_photo_size(photo)
-            photo_id = f"{largest.location.volume_id}-{largest.location.local_id}"
-        elif isinstance(photo, (UserProfilePhotoEmpty, ChatPhotoEmpty, PhotoEmpty, type(None))):
-            photo_id = ""
-            loc = None
-        else:
-            raise ValueError(f"Unknown photo type {type(photo)}")
-        if self.peer_type == "user" and not photo_id and not config["bridge.allow_avatar_remove"]:
-            return False
-        if self.photo_id != photo_id:
-            if not photo_id:
-                await self._try_set_state(sender, EventType.ROOM_AVATAR,
-                                          RoomAvatarStateEventContent(url=None))
-                self.photo_id = ""
-                self.avatar_url = None
-                if save:
-                    await self.save()
-                return True
-            file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
-            if file:
-                await self._try_set_state(sender, EventType.ROOM_AVATAR,
-                                          RoomAvatarStateEventContent(url=file.mxc))
-                self.photo_id = photo_id
-                self.avatar_url = file.mxc
-                if save:
-                    await self.save()
-                return True
-        return False
-
-    @staticmethod
-    def _filter_participants(users: List[TypeUser], participants: List[TypeParticipant]
-                             ) -> Iterable[TypeUser]:
-        participant_map = {part.user_id: part for part in participants}
-        for user in users:
-            try:
-                user.participant = participant_map[user.id]
-            except KeyError:
-                pass
-            else:
-                yield user
-
-    async def _get_channel_users(self, user: 'AbstractUser', entity: InputChannel, limit: int
-                                 ) -> List[TypeUser]:
-        if 0 < limit <= 200:
-            response = await user.client(GetParticipantsRequest(
-                entity, ChannelParticipantsRecent(), offset=0, limit=limit, hash=0))
-            return list(self._filter_participants(response.users, response.participants))
-        elif limit > 200 or limit == -1:
-            users: List[TypeUser] = []
-            offset = 0
-            remaining_quota = limit if limit > 0 else 1000000
-            query = (ChannelParticipantsSearch("") if limit == -1
-                     else ChannelParticipantsRecent())
-            while True:
-                if remaining_quota <= 0:
-                    break
-                response = await user.client(GetParticipantsRequest(
-                    entity, query, offset=offset, limit=min(remaining_quota, 200), hash=0))
-                if not response.users:
-                    break
-                users += self._filter_participants(response.users, response.participants)
-                offset += len(response.participants)
-                remaining_quota -= len(response.participants)
-            return users
-
-    async def _get_users(self, user: 'AbstractUser',
-                         entity: Union[TypeInputPeer, InputUser, TypeChat, TypeUser, InputChannel]
-                         ) -> List[TypeUser]:
-        if self.peer_type == "chat":
-            chat = await user.client(GetFullChatRequest(chat_id=self.tgid))
-            return list(self._filter_participants(chat.users,
-                                                  chat.full_chat.participants.participants))
-        elif self.peer_type == "channel":
-            if not self.megagroup and not self.sync_channel_members:
-                return []
-
-            limit = self.max_initial_member_sync
-            if limit == 0:
-                return []
-
-            try:
-                return await self._get_channel_users(user, entity, limit)
-            except ChatAdminRequiredError:
-                return []
-        elif self.peer_type == "user":
-            return [entity]
-        else:
-            raise RuntimeError(f"Unexpected peer type {self.peer_type}")
-
-    # endregion
-
-    async def _send_delivery_receipt(self, event_id: EventID, room_id: Optional[RoomID] = None
-                                     ) -> None:
-        # TODO maybe check if the bot is in the room rather than assuming based on self.encrypted
-        if event_id and config["bridge.delivery_receipts"] and (self.encrypted
-                                                                or self.peer_type != "user"):
-            try:
-                await self.az.intent.mark_read(room_id or self.mxid, event_id)
-            except Exception:
-                self.log.exception("Failed to send delivery receipt for %s", event_id)
-
-
-def init(context: Context) -> None:
-    global config
-    config = context.config

mautrix_telegram/portal/send_lock.py

@@ -1,44 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict
-from asyncio import Lock
-
-from ..types import TelegramID
-
-
-class FakeLock:
-    async def __aenter__(self) -> None:
-        pass
-
-    async def __aexit__(self, exc_type, exc, tb) -> None:
-        pass
-
-
-class PortalSendLock:
-    _send_locks: Dict[int, Lock]
-    _noop_lock: Lock = FakeLock()
-
-    def __init__(self) -> None:
-        self._send_locks = {}
-
-    def __call__(self, user_id: TelegramID, required: bool = True) -> Lock:
-        if user_id is None and required:
-            raise ValueError("Required send lock for none id")
-        try:
-            return self._send_locks[user_id]
-        except KeyError:
-            return (self._send_locks.setdefault(user_id, Lock())
-                    if required else self._noop_lock)

mautrix_telegram/portal/telegram.py

@@ -1,718 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple, Union, NamedTuple, TYPE_CHECKING
-from abc import ABC
-import random
-import mimetypes
-import codecs
-import unicodedata
-import base64
-import asyncio
-
-from sqlalchemy.exc import IntegrityError
-
-from telethon.tl.patched import Message, MessageService
-from telethon.tl.types import (
-    Poll, DocumentAttributeFilename, DocumentAttributeSticker, DocumentAttributeVideo,
-    MessageMediaPoll, MessageActionChannelCreate, MessageActionChatAddUser,
-    MessageActionChatCreate, MessageActionChatDeletePhoto, MessageActionChatDeleteUser,
-    MessageActionChatEditPhoto, MessageActionChatEditTitle, MessageActionChatJoinedByLink,
-    MessageActionChatMigrateTo, MessageActionGameScore, MessageMediaDocument, MessageMediaGeo,
-    MessageMediaPhoto, MessageMediaDice, MessageMediaGame, MessageMediaUnsupported, PeerUser,
-    PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant, TypeDocumentAttribute,
-    TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping, UpdateUserTyping,
-    MessageEntityPre, ChatPhotoEmpty)
-
-from mautrix.appservice import IntentAPI
-from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType,
-                           EventType, MediaMessageEventContent, TextMessageEventContent,
-                           LocationMessageEventContent, Format)
-from mautrix.bridge import NotificationDisabler
-
-from ..types import TelegramID
-from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
-from ..util import sane_mimetypes
-from ..context import Context
-from ..tgclient import TelegramClient
-from .. import puppet as p, user as u, formatter, util
-from .base import BasePortal
-
-if TYPE_CHECKING:
-    from ..abstract_user import AbstractUser
-    from ..config import Config
-
-InviteList = Union[UserID, List[UserID]]
-TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
-DocAttrs = NamedTuple("DocAttrs", name=Optional[str], mime_type=Optional[str], is_sticker=bool,
-                      sticker_alt=Optional[str], width=int, height=int)
-
-config: Optional['Config'] = None
-
-
-class PortalTelegram(BasePortal, ABC):
-    async def handle_telegram_typing(self, user: p.Puppet,
-                                     _: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
-        await user.intent_for(self).set_typing(self.mxid, is_typing=True)
-
-    def _get_external_url(self, evt: Message) -> Optional[str]:
-        if self.peer_type == "channel" and self.username is not None:
-            return f"https://t.me/{self.username}/{evt.id}"
-        elif self.peer_type != "user":
-            return f"https://t.me/c/{self.tgid}/{evt.id}"
-        return None
-
-    async def _expire_telegram_photo(self, intent: IntentAPI, event_id: EventID, ttl: int) -> None:
-        try:
-            content = TextMessageEventContent(msgtype=MessageType.NOTICE, body="Photo has expired")
-            content.set_edit(event_id)
-            await asyncio.sleep(ttl)
-            await self._send_message(intent, content)
-        except Exception:
-            self.log.warning("Failed to expire Telegram photo %s", event_id, exc_info=True)
-
-    async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
-                                    relates_to: RelatesTo = None) -> Optional[EventID]:
-        media: MessageMediaPhoto = evt.media
-        if media.photo is None and media.ttl_seconds:
-            return await self._send_message(intent, TextMessageEventContent(
-                msgtype=MessageType.NOTICE, body="Photo has expired"))
-        loc, largest_size = self._get_largest_photo_size(media.photo)
-        if loc is None:
-            content = TextMessageEventContent(msgtype=MessageType.TEXT,
-                                              body="Failed to bridge image",
-                                              external_url=self._get_external_url(evt))
-            return await self._send_message(intent, content, timestamp=evt.date)
-        file = await util.transfer_file_to_matrix(source.client, intent, loc,
-                                                  encrypt=self.encrypted)
-        if not file:
-            return None
-        if self.get_config("inline_images") and (evt.message or evt.fwd_from or evt.reply_to):
-            content = await formatter.telegram_to_matrix(
-                evt, source, self.main_intent,
-                prefix_html=f"<img src='{file.mxc}' alt='Inline Telegram photo'/><br/>",
-                prefix_text="Inline image: ")
-            content.external_url = self._get_external_url(evt)
-            await intent.set_typing(self.mxid, is_typing=False)
-            return await self._send_message(intent, content, timestamp=evt.date)
-        info = ImageInfo(
-            height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type,
-            size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize))
-                  else largest_size.size))
-        ext = sane_mimetypes.guess_extension(file.mime_type)
-        name = f"disappearing_image{ext}" if media.ttl_seconds else f"image{ext}"
-        await intent.set_typing(self.mxid, is_typing=False)
-        content = MediaMessageEventContent(msgtype=MessageType.IMAGE, info=info,
-                                           body=name, relates_to=relates_to,
-                                           external_url=self._get_external_url(evt))
-        if file.decryption_info:
-            content.file = file.decryption_info
-        else:
-            content.url = file.mxc
-        result = await self._send_message(intent, content, timestamp=evt.date)
-        if media.ttl_seconds:
-            self.loop.create_task(self._expire_telegram_photo(intent, result,
-                                                              media.ttl_seconds))
-        if evt.message:
-            caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
-                                                                 no_reply_fallback=True)
-            caption_content.external_url = content.external_url
-            result = await self._send_message(intent, caption_content, timestamp=evt.date)
-        return result
-
-    @staticmethod
-    def _parse_telegram_document_attributes(attributes: List[TypeDocumentAttribute]) -> DocAttrs:
-        name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
-        for attr in attributes:
-            if isinstance(attr, DocumentAttributeFilename):
-                name = name or attr.file_name
-                mime_type, _ = mimetypes.guess_type(attr.file_name)
-            elif isinstance(attr, DocumentAttributeSticker):
-                is_sticker = True
-                sticker_alt = attr.alt
-            elif isinstance(attr, DocumentAttributeVideo):
-                width, height = attr.w, attr.h
-        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height)
-
-    @staticmethod
-    def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: DocAttrs,
-                                      thumb_size: TypePhotoSize) -> Tuple[ImageInfo, str]:
-        document = evt.media.document
-        name = attrs.name
-        if attrs.is_sticker:
-            alt = attrs.sticker_alt
-            if len(alt) > 0:
-                try:
-                    name = f"{alt} ({unicodedata.name(alt[0]).lower()})"
-                except ValueError:
-                    name = alt
-
-        generic_types = ("text/plain", "application/octet-stream")
-        if file.mime_type in generic_types and document.mime_type not in generic_types:
-            mime_type = document.mime_type or file.mime_type
-        elif file.mime_type == 'application/ogg':
-            mime_type = 'audio/ogg'
-        else:
-            mime_type = file.mime_type or document.mime_type
-        info = ImageInfo(size=file.size, mimetype=mime_type)
-
-        if attrs.mime_type and not file.was_converted:
-            file.mime_type = attrs.mime_type or file.mime_type
-        if file.width and file.height:
-            info.width, info.height = file.width, file.height
-        elif attrs.width and attrs.height:
-            info.width, info.height = attrs.width, attrs.height
-
-        if file.thumbnail:
-            if file.thumbnail.decryption_info:
-                info.thumbnail_file = file.thumbnail.decryption_info
-            else:
-                info.thumbnail_url = file.thumbnail.mxc
-            info.thumbnail_info = ThumbnailInfo(mimetype=file.thumbnail.mime_type,
-                                                height=file.thumbnail.height or thumb_size.h,
-                                                width=file.thumbnail.width or thumb_size.w,
-                                                size=file.thumbnail.size)
-        else:
-            # This is a hack for bad clients like Riot iOS that require a thumbnail
-            if file.decryption_info:
-                info.thumbnail_file = file.decryption_info
-            else:
-                info.thumbnail_url = file.mxc
-            info.thumbnail_info = ImageInfo.deserialize(info.serialize())
-
-        return info, name
-
-    async def handle_telegram_document(self, source: 'AbstractUser', intent: IntentAPI,
-                                       evt: Message, relates_to: RelatesTo = None
-                                       ) -> Optional[EventID]:
-        document = evt.media.document
-
-        attrs = self._parse_telegram_document_attributes(document.attributes)
-
-        if document.size > config["bridge.max_document_size"] * 1000 ** 2:
-            name = attrs.name or ""
-            caption = f"\n{evt.message}" if evt.message else ""
-            # TODO encrypt
-            return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
-
-        thumb_loc, thumb_size = self._get_largest_photo_size(document)
-        if thumb_size and not isinstance(thumb_size, (PhotoSize, PhotoCachedSize)):
-            self.log.debug(f"Unsupported thumbnail type {type(thumb_size)}")
-            thumb_loc = None
-            thumb_size = None
-        parallel_id = source.tgid if config["bridge.parallel_file_transfer"] else None
-        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
-                                                  is_sticker=attrs.is_sticker,
-                                                  tgs_convert=config["bridge.animated_sticker"],
-                                                  filename=attrs.name, parallel_id=parallel_id,
-                                                  encrypt=self.encrypted)
-        if not file:
-            return None
-
-        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb_size)
-
-        await intent.set_typing(self.mxid, is_typing=False)
-
-        event_type = EventType.ROOM_MESSAGE
-        # Riot only supports images as stickers, so send animated webm stickers as m.video
-        if attrs.is_sticker and file.mime_type.startswith("image/"):
-            event_type = EventType.STICKER
-        content = MediaMessageEventContent(
-            body=name or "unnamed file", info=info, relates_to=relates_to,
-            external_url=self._get_external_url(evt),
-            msgtype={
-                "video/": MessageType.VIDEO,
-                "audio/": MessageType.AUDIO,
-                "image/": MessageType.IMAGE,
-            }.get(info.mimetype[:6], MessageType.FILE))
-        if file.decryption_info:
-            content.file = file.decryption_info
-        else:
-            content.url = file.mxc
-        res = await self._send_message(intent, content, event_type=event_type, timestamp=evt.date)
-        if evt.message:
-            caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
-                                                                 no_reply_fallback=True)
-            caption_content.external_url = content.external_url
-            res = await self._send_message(intent, caption_content, timestamp=evt.date)
-        return res
-
-    def handle_telegram_location(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
-                                 relates_to: RelatesTo = None) -> Awaitable[EventID]:
-        long = evt.media.geo.long
-        lat = evt.media.geo.lat
-        long_char = "E" if long > 0 else "W"
-        lat_char = "N" if lat > 0 else "S"
-        geo = f"{round(lat, 6)},{round(long, 6)}"
-
-        body = f"{round(abs(lat), 4)}° {lat_char}, {round(abs(long), 4)}° {long_char}"
-        url = f"https://maps.google.com/?q={geo}"
-
-        content = LocationMessageEventContent(
-            msgtype=MessageType.LOCATION, geo_uri=f"geo:{geo}",
-            body=f"Location: {body}\n{url}",
-            relates_to=relates_to, external_url=self._get_external_url(evt))
-        content["format"] = str(Format.HTML)
-        content["formatted_body"] = f"Location: <a href='{url}'>{body}</a>"
-
-        return self._send_message(intent, content, timestamp=evt.date)
-
-    async def handle_telegram_text(self, source: 'AbstractUser', intent: IntentAPI, is_bot: bool,
-                                   evt: Message) -> EventID:
-        self.log.trace(f"Sending {evt.message} to {self.mxid} by {intent.mxid}")
-        content = await formatter.telegram_to_matrix(evt, source, self.main_intent)
-        content.external_url = self._get_external_url(evt)
-        if is_bot and self.get_config("bot_messages_as_notices"):
-            content.msgtype = MessageType.NOTICE
-        await intent.set_typing(self.mxid, is_typing=False)
-        return await self._send_message(intent, content, timestamp=evt.date)
-
-    async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
-                                          evt: Message, relates_to: RelatesTo = None) -> EventID:
-        override_text = ("This message is not supported on your version of Mautrix-Telegram. "
-                         "Please check https://github.com/tulir/mautrix-telegram or ask your "
-                         "bridge administrator about possible updates.")
-        content = await formatter.telegram_to_matrix(
-            evt, source, self.main_intent, override_text=override_text)
-        content.msgtype = MessageType.NOTICE
-        content.external_url = self._get_external_url(evt)
-        content["net.maunium.telegram.unsupported"] = True
-        await intent.set_typing(self.mxid, is_typing=False)
-        return await self._send_message(intent, content, timestamp=evt.date)
-
-    async def handle_telegram_poll(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
-                                   relates_to: RelatesTo) -> EventID:
-        poll: Poll = evt.media.poll
-        poll_id = self._encode_msgid(source, evt)
-
-        _n = 0
-
-        def n() -> int:
-            nonlocal _n
-            _n += 1
-            return _n
-
-        text_answers = "\n".join(f"{n()}. {answer.text}" for answer in poll.answers)
-        html_answers = "\n".join(f"<li>{answer.text}</li>" for answer in poll.answers)
-        content = TextMessageEventContent(
-            msgtype=MessageType.TEXT, format=Format.HTML,
-            body=f"Poll: {poll.question}\n{text_answers}\n"
-                 f"Vote with !tg vote {poll_id} <choice number>",
-            formatted_body=f"<strong>Poll</strong>: {poll.question}<br/>\n"
-                           f"<ol>{html_answers}</ol>\n"
-                           f"Vote with <code>!tg vote {poll_id} &lt;choice number&gt;</code>",
-            relates_to=relates_to, external_url=self._get_external_url(evt))
-
-        await intent.set_typing(self.mxid, is_typing=False)
-        return await self._send_message(intent, content, timestamp=evt.date)
-
-    async def handle_telegram_dice(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
-                                   relates_to: RelatesTo) -> EventID:
-        emoji_text = {
-            "\U0001F3AF": " Dart throw",
-            "\U0001F3B2": " Dice roll",
-            "\U0001F3C0": " Basketball throw",
-            "\u26BD": " Football kick"
-        }
-        roll: MessageMediaDice = evt.media
-        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {roll.value}"
-        content = TextMessageEventContent(msgtype=MessageType.TEXT, format=Format.HTML, body=text,
-                                          formatted_body=f"<h4>{text}</h4>", relates_to=relates_to,
-                                          external_url=self._get_external_url(evt))
-        content["net.maunium.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
-        await intent.set_typing(self.mxid, is_typing=False)
-        return await self._send_message(intent, content, timestamp=evt.date)
-
-    @staticmethod
-    def _int_to_bytes(i: int) -> bytes:
-        hex_value = "{0:010x}".format(i).encode("utf-8")
-        return codecs.decode(hex_value, "hex_codec")
-
-    def _encode_msgid(self, source: 'AbstractUser', evt: Message) -> str:
-        if self.peer_type == "channel":
-            play_id = (b"c"
-                       + self._int_to_bytes(self.tgid)
-                       + self._int_to_bytes(evt.id))
-        elif self.peer_type == "chat":
-            play_id = (b"g"
-                       + self._int_to_bytes(self.tgid)
-                       + self._int_to_bytes(evt.id)
-                       + self._int_to_bytes(source.tgid))
-        elif self.peer_type == "user":
-            play_id = (b"u"
-                       + self._int_to_bytes(self.tgid)
-                       + self._int_to_bytes(evt.id))
-        else:
-            raise ValueError("Portal has invalid peer type")
-        return base64.b64encode(play_id).decode("utf-8").rstrip("=")
-
-    async def handle_telegram_game(self, source: 'AbstractUser', intent: IntentAPI,
-                                   evt: Message, relates_to: RelatesTo = None) -> EventID:
-        game = evt.media.game
-        play_id = self._encode_msgid(source, evt)
-        command = f"!tg play {play_id}"
-        override_text = f"Run {command} in your bridge management room to play {game.title}"
-        override_entities = [
-            MessageEntityPre(offset=len("Run "), length=len(command), language="")]
-
-        content = await formatter.telegram_to_matrix(
-            evt, source, self.main_intent,
-            override_text=override_text, override_entities=override_entities)
-        content.msgtype = MessageType.NOTICE
-        content.external_url = self._get_external_url(evt)
-        content["net.maunium.telegram.game"] = play_id
-
-        await intent.set_typing(self.mxid, is_typing=False)
-        return await self._send_message(intent, content, timestamp=evt.date)
-
-    async def handle_telegram_edit(self, source: 'AbstractUser', sender: p.Puppet, evt: Message
-                                   ) -> None:
-        if not self.mxid:
-            self.log.trace("Ignoring edit to %d as chat has no Matrix room", evt.id)
-            return
-        elif hasattr(evt, "media") and isinstance(evt.media, MessageMediaGame):
-            self.log.debug("Ignoring game message edit event")
-            return
-
-        async with self.send_lock(sender.tgid if sender else None, required=False):
-            tg_space = self.tgid if self.peer_type == "channel" else source.tgid
-
-            temporary_identifier = EventID(
-                f"${random.randint(1000000000000, 9999999999999)}TGBRIDGEDITEMP")
-            duplicate_found = self.dedup.check(evt, (temporary_identifier, tg_space),
-                                               force_hash=True)
-            if duplicate_found:
-                mxid, other_tg_space = duplicate_found
-                if tg_space != other_tg_space:
-                    prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1)
-                    if not prev_edit_msg:
-                        return
-                    DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space,
-                              tgid=TelegramID(evt.id), edit_index=prev_edit_msg.edit_index + 1
-                              ).insert()
-                return
-
-        content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
-                                                     no_reply_fallback=True)
-        editing_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
-        if not editing_msg:
-            self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) "
-                          "in database.")
-            return
-
-        content.msgtype = (MessageType.NOTICE if (sender and sender.is_bot
-                                                  and self.get_config("bot_messages_as_notices"))
-                           else MessageType.TEXT)
-        content.external_url = self._get_external_url(evt)
-        content.set_edit(editing_msg.mxid)
-
-        intent = sender.intent_for(self) if sender else self.main_intent
-        await intent.set_typing(self.mxid, is_typing=False)
-        event_id = await self._send_message(intent, content)
-
-        prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
-        DBMessage(mxid=event_id, mx_room=self.mxid, tg_space=tg_space, tgid=TelegramID(evt.id),
-                  edit_index=prev_edit_msg.edit_index + 1).insert()
-        DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=event_id)
-
-    @property
-    def _takeout_options(self) -> Dict[str, Union[bool, int]]:
-        return {
-            "files": True,
-            "megagroups": self.megagroup,
-            "chats": self.peer_type == "chat",
-            "users": self.peer_type == "user",
-            "channels": (self.peer_type == "channel" and not self.megagroup),
-            "max_file_size": min(config["bridge.max_document_size"], 2000) * 1024 * 1024
-        }
-
-    async def backfill(self, source: 'u.User', is_initial: bool = False,
-                       limit: Optional[int] = None, last_id: Optional[int] = None) -> None:
-        async with self.backfill_method_lock:
-            await self._locked_backfill(source, is_initial, limit, last_id)
-
-    async def _locked_backfill(self, source: 'u.User', is_initial: bool = False,
-                               limit: Optional[int] = None, last_id: Optional[int] = None) -> None:
-        limit = limit or (config["bridge.backfill.initial_limit"] if is_initial
-                          else config["bridge.backfill.missed_limit"])
-        if limit == 0:
-            return
-        if not config["bridge.backfill.normal_groups"] and self.peer_type == "chat":
-            return
-        last = DBMessage.find_last(self.mxid, (source.tgid if self.peer_type != "channel"
-                                               else self.tgid))
-        min_id = last.tgid if last else 0
-        if last_id is None:
-            messages = await source.client.get_messages(self.peer, limit=1)
-            if not messages:
-                # The chat seems empty
-                return
-            last_id = messages[0].id
-        if last_id <= min_id:
-            # Nothing to backfill
-            return
-        if limit < 0:
-            limit = last_id - min_id
-            self.log.debug(f"Backfilling approximately {last_id - min_id} messages "
-                           f"through {source.mxid}")
-        elif self.peer_type == "channel":
-            # This is a channel or supergroup, so we'll backfill messages based on the ID.
-            # There are some cases, such as deleted messages, where this may backfill less
-            # messages than the limit.
-            min_id = max(last_id - limit, min_id)
-            self.log.debug(f"Backfilling messages after ID {min_id} (last message: {last_id}) "
-                           f"through {source.mxid}")
-        else:
-            # Private chats and normal groups don't have their own message ID namespace,
-            # which means we'll have to fetch messages a different way.
-            # The _backfill_messages method will detect min_id=None and not use reverse=True
-            min_id = None
-            self.log.debug(f"Backfilling up to {limit} messages through {source.mxid}")
-        with self.backfill_lock:
-            await self._backfill(source, min_id, limit)
-
-    async def _backfill(self, source: 'u.User', min_id: Optional[int], limit: int) -> None:
-        self.backfill_leave = set()
-        if ((self.peer_type == "user" and self.tgid != source.tgid
-             and config["bridge.backfill.invite_own_puppet"])):
-            self.log.debug("Adding %s's default puppet to room for backfilling", source.mxid)
-            sender = p.Puppet.get(source.tgid)
-            await self.main_intent.invite_user(self.mxid, sender.default_mxid)
-            await sender.default_mxid_intent.join_room_by_id(self.mxid)
-            self.backfill_leave.add(sender.default_mxid_intent)
-
-        client = source.client
-        async with NotificationDisabler(self.mxid, source):
-            if limit > config["bridge.backfill.takeout_limit"]:
-                self.log.debug(f"Opening takeout client for {source.tgid}")
-                async with client.takeout(**self._takeout_options) as takeout:
-                    count = await self._backfill_messages(source, min_id, limit, takeout)
-            else:
-                count = await self._backfill_messages(source, min_id, limit, client)
-
-        for intent in self.backfill_leave:
-            self.log.trace("Leaving room with %s post-backfill", intent.mxid)
-            await intent.leave_room(self.mxid)
-        self.backfill_leave = None
-        self.log.info("Backfilled %d messages through %s", count, source.mxid)
-
-    async def _backfill_messages(self, source: 'AbstractUser', min_id: Optional[int], limit: int,
-                                 client: TelegramClient) -> int:
-        count = 0
-        entity = await self.get_input_entity(source)
-        if min_id is not None:
-            self.log.debug(f"Iterating all messages starting with {min_id} (approx: {limit})")
-            messages = client.iter_messages(entity, reverse=True, min_id=min_id)
-            async for message in messages:
-                sender = (p.Puppet.get(message.from_id.user_id)
-                          if isinstance(message.from_id, PeerUser) else None)
-                # TODO handle service messages?
-                await self.handle_telegram_message(source, sender, message)
-                count += 1
-        else:
-            self.log.debug(f"Fetching up to {limit} most recent messages")
-            messages = await client.get_messages(entity, limit=limit)
-            for message in reversed(messages):
-                sender = (p.Puppet.get(TelegramID(message.from_id.user_id))
-                          if isinstance(message.from_id, PeerUser) else None)
-                await self.handle_telegram_message(source, sender, message)
-                count += 1
-        return count
-
-    async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
-                                      evt: Message) -> None:
-        if not self.mxid:
-            self.log.trace("Got telegram message %d, but no room exists, creating...", evt.id)
-            await self.create_matrix_room(source, invites=[source.mxid], update_if_exists=False)
-
-        if (self.peer_type == "user" and sender and sender.tgid == self.tg_receiver
-            and not sender.is_real_user and not await self.az.state_store.is_joined(self.mxid,
-                                                                                    sender.mxid)):
-            self.log.debug(f"Ignoring private chat message {evt.id}@{source.tgid} as receiver does"
-                           " not have matrix puppeting and their default puppet isn't in the room")
-            return
-
-        async with self.send_lock(sender.tgid if sender else None, required=False):
-            tg_space = self.tgid if self.peer_type == "channel" else source.tgid
-
-            temporary_identifier = EventID(
-                f"${random.randint(1000000000000, 9999999999999)}TGBRIDGETEMP")
-            duplicate_found = self.dedup.check(evt, (temporary_identifier, tg_space))
-            if duplicate_found:
-                mxid, other_tg_space = duplicate_found
-                self.log.debug(f"Ignoring message {evt.id}@{tg_space} (src {source.tgid}) "
-                               f"as it was already handled (in space {other_tg_space})")
-                if tg_space != other_tg_space:
-                    DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                              tg_space=tg_space, edit_index=0).insert()
-                return
-
-        if self.backfill_lock.locked or (self.dedup.pre_db_check and self.peer_type == "channel"):
-            msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
-            if msg:
-                self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already "
-                               f"handled into {msg.mxid}. This duplicate was catched in the db "
-                               "check. If you get this message often, consider increasing "
-                               "bridge.deduplication.cache_queue_length in the config.")
-                return
-
-        self.log.trace("Handling Telegram message %s", evt)
-
-        if sender and not sender.displayname:
-            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a "
-                           "displayname, updating info...")
-            entity = await source.client.get_entity(PeerUser(sender.tgid))
-            await sender.update_info(source, entity)
-
-        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
-                         MessageMediaGame, MessageMediaDice, MessageMediaPoll,
-                         MessageMediaUnsupported)
-        media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
-                                                                  allowed_media) else None
-        if sender:
-            intent = sender.intent_for(self)
-            if ((self.backfill_lock.locked and intent != sender.default_mxid_intent
-                 and config["bridge.backfill.invite_own_puppet"])):
-                intent = sender.default_mxid_intent
-                self.backfill_leave.add(intent)
-        else:
-            intent = self.main_intent
-        if not media and evt.message:
-            is_bot = sender.is_bot if sender else False
-            event_id = await self.handle_telegram_text(source, intent, is_bot, evt)
-        elif media:
-            event_id = await {
-                MessageMediaPhoto: self.handle_telegram_photo,
-                MessageMediaDocument: self.handle_telegram_document,
-                MessageMediaGeo: self.handle_telegram_location,
-                MessageMediaPoll: self.handle_telegram_poll,
-                MessageMediaDice: self.handle_telegram_dice,
-                MessageMediaUnsupported: self.handle_telegram_unsupported,
-                MessageMediaGame: self.handle_telegram_game,
-            }[type(media)](source, intent, evt,
-                           relates_to=formatter.telegram_reply_to_matrix(evt, source))
-        else:
-            self.log.debug("Unhandled Telegram message %d", evt.id)
-            return
-
-        if not event_id:
-            return
-
-        prev_id = self.dedup.update(evt, (event_id, tg_space), (temporary_identifier, tg_space))
-        if prev_id:
-            self.log.debug(f"Sent message {evt.id}@{tg_space} to Matrix as {event_id}. "
-                           f"Temporary dedup identifier was {temporary_identifier}, "
-                           f"but dedup map contained {prev_id[1]} instead! -- "
-                           "This was probably a race condition caused by Telegram sending updates"
-                           "to other clients before responding to the sender. I'll just redact "
-                           "the likely duplicate message now.")
-            await intent.redact(self.mxid, event_id)
-            return
-
-        self.log.debug("Handled telegram message %d -> %s", evt.id, event_id)
-        try:
-            DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=event_id,
-                      tg_space=tg_space, edit_index=0).insert()
-            DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=event_id)
-        except IntegrityError as e:
-            self.log.exception(f"{e.__class__.__name__} while saving message mapping. "
-                               "This might mean that an update was handled after it left the "
-                               "dedup cache queue. You can try enabling bridge.deduplication."
-                               "pre_db_check in the config.")
-            await intent.redact(self.mxid, event_id)
-        await self._send_delivery_receipt(event_id)
-
-    async def _create_room_on_action(self, source: 'AbstractUser',
-                                     action: TypeMessageAction) -> bool:
-        if source.is_relaybot and config["bridge.ignore_unbridged_group_chat"]:
-            return False
-        create_and_exit = (MessageActionChatCreate, MessageActionChannelCreate)
-        create_and_continue = (MessageActionChatAddUser, MessageActionChatJoinedByLink)
-        if isinstance(action, create_and_exit) or isinstance(action, create_and_continue):
-            await self.create_matrix_room(source, invites=[source.mxid],
-                                          update_if_exists=isinstance(action, create_and_exit))
-        if not isinstance(action, create_and_continue):
-            return False
-        return True
-
-    async def handle_telegram_action(self, source: 'AbstractUser', sender: p.Puppet,
-                                     update: MessageService) -> None:
-        action = update.action
-        should_ignore = ((not self.mxid and not await self._create_room_on_action(source, action))
-                         or self.dedup.check_action(update))
-        if should_ignore or not self.mxid:
-            return
-        if isinstance(action, MessageActionChatEditTitle):
-            await self._update_title(action.title, sender=sender, save=True)
-            await self.update_bridge_info()
-        elif isinstance(action, MessageActionChatEditPhoto):
-            await self._update_avatar(source, action.photo, sender=sender, save=True)
-            await self.update_bridge_info()
-        elif isinstance(action, MessageActionChatDeletePhoto):
-            await self._update_avatar(source, ChatPhotoEmpty(), sender=sender, save=True)
-            await self.update_bridge_info()
-        elif isinstance(action, MessageActionChatAddUser):
-            for user_id in action.users:
-                await self._add_telegram_user(TelegramID(user_id), source)
-        elif isinstance(action, MessageActionChatJoinedByLink):
-            await self._add_telegram_user(sender.id, source)
-        elif isinstance(action, MessageActionChatDeleteUser):
-            await self._delete_telegram_user(TelegramID(action.user_id), sender)
-        elif isinstance(action, MessageActionChatMigrateTo):
-            self.peer_type = "channel"
-            self._migrate_and_save_telegram(TelegramID(action.channel_id))
-            # TODO encrypt
-            await sender.intent_for(self).send_emote(self.mxid,
-                                                     "upgraded this group to a supergroup.")
-            await self.update_bridge_info()
-        elif isinstance(action, MessageActionGameScore):
-            # TODO handle game score
-            pass
-        else:
-            self.log.trace("Unhandled Telegram action in %s: %s", self.title, action)
-
-    async def set_telegram_admin(self, user_id: TelegramID) -> None:
-        puppet = p.Puppet.get(user_id)
-        user = u.User.get_by_tgid(user_id)
-
-        levels = await self.main_intent.get_power_levels(self.mxid)
-        if user:
-            levels.users[user.mxid] = 50
-        if puppet:
-            levels.users[puppet.mxid] = 50
-        await self.main_intent.set_power_levels(self.mxid, levels)
-
-    async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
-        tg_space = receiver if self.peer_type != "channel" else self.tgid
-        message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
-        if message:
-            await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
-        else:
-            await self.main_intent.set_pinned_messages(self.mxid, [])
-
-    async def set_telegram_admins_enabled(self, enabled: bool) -> None:
-        level = 50 if enabled else 10
-        levels = await self.main_intent.get_power_levels(self.mxid)
-        levels.invite = level
-        levels.events[EventType.ROOM_NAME] = level
-        levels.events[EventType.ROOM_AVATAR] = level
-        await self.main_intent.set_power_levels(self.mxid, levels)
-
-
-def init(context: Context) -> None:
-    global config
-    config = context.config
-    NotificationDisabler.puppet_cls = p.Puppet
-    NotificationDisabler.config_enabled = config["bridge.backfill.disable_notifications"]

mautrix_telegram/puppet.py

@@ -1,5 +1,6 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,25 +14,21 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Any, Dict, Iterable, Optional, Union, TYPE_CHECKING
+from typing import Awaitable, Any, Dict, List, Iterable, Optional, Pattern, Union, TYPE_CHECKING
 from difflib import SequenceMatcher
-import unicodedata
+from enum import Enum
+from aiohttp import ServerDisconnectedError
 import asyncio
 import logging
+import re
 
 from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
-                               InputPeerPhotoFileLocation, UserProfilePhotoEmpty, TypeInputUser)
-from yarl import URL
+                               InputPeerPhotoFileLocation, UserProfilePhotoEmpty)
+from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
-from mautrix.appservice import AppService, IntentAPI
-from mautrix.errors import MatrixRequestError
-from mautrix.bridge import BasePuppet
-from mautrix.types import UserID, SyncToken, RoomID
-from mautrix.util.simple_template import SimpleTemplate
-
-from .types import TelegramID
+from .types import MatrixUserID, TelegramID
 from .db import Puppet as DBPuppet
-from . import util, portal as p
+from . import util
 
 if TYPE_CHECKING:
     from .matrix import MatrixHandler
@@ -39,49 +36,26 @@ if TYPE_CHECKING:
     from .context import Context
     from .abstract_user import AbstractUser
 
-config: Optional['Config'] = None
+PuppetError = Enum('PuppetError', 'Success OnlyLoginSelf InvalidAccessToken')
+
+config = None  # type: Config
 
 
-class Puppet(BasePuppet):
-    log: logging.Logger = logging.getLogger("mau.puppet")
-    az: AppService
-    mx: 'MatrixHandler'
-    loop: asyncio.AbstractEventLoop
-    hs_domain: str
-    mxid_template: SimpleTemplate[TelegramID]
-    displayname_template: SimpleTemplate[str]
-
-    cache: Dict[TelegramID, 'Puppet'] = {}
-    by_custom_mxid: Dict[UserID, 'Puppet'] = {}
-
-    id: TelegramID
-    access_token: Optional[str]
-    custom_mxid: Optional[UserID]
-    _next_batch: Optional[SyncToken]
-    base_url: Optional[URL]
-    default_mxid: UserID
-
-    username: Optional[str]
-    displayname: Optional[str]
-    displayname_source: Optional[TelegramID]
-    photo_id: Optional[str]
-    is_bot: bool
-    is_registered: bool
-    disable_updates: bool
-
-    default_mxid_intent: IntentAPI
-    intent: IntentAPI
-
-    sync_task: Optional[asyncio.Future]
-
-    _db_instance: Optional[DBPuppet]
+class Puppet:
+    log = logging.getLogger("mau.puppet")  # type: logging.Logger
+    az = None  # type: AppService
+    mx = None  # type: MatrixHandler
+    loop = None  # type: asyncio.AbstractEventLoop
+    mxid_regex = None  # type: Pattern
+    username_template = None  # type: str
+    hs_domain = None  # type: str
+    cache = {}  # type: Dict[TelegramID, Puppet]
+    by_custom_mxid = {}  # type: Dict[str, Puppet]
 
     def __init__(self,
                  id: TelegramID,
                  access_token: Optional[str] = None,
-                 custom_mxid: Optional[UserID] = None,
-                 next_batch: Optional[SyncToken] = None,
-                 base_url: Optional[str] = None,
+                 custom_mxid: Optional[MatrixUserID] = None,
                  username: Optional[str] = None,
                  displayname: Optional[str] = None,
                  displayname_source: Optional[TelegramID] = None,
@@ -90,48 +64,40 @@ class Puppet(BasePuppet):
                  is_registered: bool = False,
                  disable_updates: bool = False,
                  db_instance: Optional[DBPuppet] = None) -> None:
-        self.id = id
-        self.access_token = access_token
-        self.custom_mxid = custom_mxid
-        self._next_batch = next_batch
-        self.base_url = URL(base_url) if base_url else None
-        self.default_mxid = self.get_mxid_from_id(self.id)
+        self.id = id  # type: TelegramID
+        self.access_token = access_token  # type: Optional[str]
+        self.custom_mxid = custom_mxid  # type: Optional[MatrixUserID]
+        self.default_mxid = self.get_mxid_from_id(self.id)  # type: MatrixUserID
 
-        self.username = username
-        self.displayname = displayname
-        self.displayname_source = displayname_source
-        self.photo_id = photo_id
-        self.is_bot = is_bot
-        self.is_registered = is_registered
-        self.disable_updates = disable_updates
-        self._db_instance = db_instance
+        self.username = username  # type: Optional[str]
+        self.displayname = displayname  # type: Optional[str]
+        self.displayname_source = displayname_source  # type: Optional[TelegramID]
+        self.photo_id = photo_id  # type: Optional[str]
+        self.is_bot = is_bot  # type: bool
+        self.is_registered = is_registered  # type: bool
+        self.disable_updates = disable_updates  # type: bool
+        self._db_instance = db_instance  # type: Optional[DBPuppet]
 
         self.default_mxid_intent = self.az.intent.user(self.default_mxid)
-        self.intent = self._fresh_intent()
-        self.sync_task = None
+        self.intent = self._fresh_intent()  # type: IntentAPI
+        self.sync_task = None  # type: Optional[asyncio.Future]
 
         self.cache[id] = self
         if self.custom_mxid:
             self.by_custom_mxid[self.custom_mxid] = self
 
-        self.log = self.log.getChild(str(self.id))
+    @property
+    def mxid(self) -> MatrixUserID:
+        return self.custom_mxid or self.default_mxid
 
     @property
     def tgid(self) -> TelegramID:
         return self.id
 
     @property
-    def peer(self) -> PeerUser:
-        return PeerUser(user_id=self.tgid)
-
-    @property
-    def next_batch(self) -> SyncToken:
-        return self._next_batch
-
-    @next_batch.setter
-    def next_batch(self, value: SyncToken) -> None:
-        self._next_batch = value
-        self.db_instance.edit(next_batch=self._next_batch)
+    def is_real_user(self) -> bool:
+        """ Is True when the puppet is a real Matrix user. """
+        return bool(self.custom_mxid and self.access_token)
 
     @staticmethod
     async def is_logged_in() -> bool:
@@ -140,17 +106,175 @@ class Puppet(BasePuppet):
 
     @property
     def plain_displayname(self) -> str:
-        return self.displayname_template.parse(self.displayname) or self.displayname
+        tpl = config["bridge.displayname_template"]
+        if tpl == "{displayname}":
+            # Template has no extra stuff, no need to parse.
+            return self.displayname
+        regex = re.compile("^" + re.escape(tpl).replace(re.escape("{displayname}"), "(.+?)") + "$")
+        match = regex.match(self.displayname)
+        return match.group(1) or self.displayname
 
-    def get_input_entity(self, user: 'AbstractUser'
-                         ) -> Awaitable[Union[TypeInputPeer, TypeInputUser]]:
-        return user.client.get_input_entity(self.peer)
+    def get_input_entity(self, user: 'AbstractUser') -> Awaitable[TypeInputPeer]:
+        return user.client.get_input_entity(PeerUser(user_id=self.tgid))
 
-    def intent_for(self, portal: 'p.Portal') -> IntentAPI:
-        if portal.tgid == self.tgid:
-            return self.default_mxid_intent
-        return self.intent
+    # region Custom puppet management
+    def _fresh_intent(self) -> IntentAPI:
+        return (self.az.intent.user(self.custom_mxid, self.access_token)
+                if self.is_real_user else self.default_mxid_intent)
 
+    async def switch_mxid(self, access_token: Optional[str],
+                          mxid: Optional[MatrixUserID]) -> PuppetError:
+        prev_mxid = self.custom_mxid
+        self.custom_mxid = mxid
+        self.access_token = access_token
+        self.intent = self._fresh_intent()
+
+        err = await self.init_custom_mxid()
+        if err != PuppetError.Success:
+            return err
+
+        try:
+            del self.by_custom_mxid[prev_mxid]  # type: ignore
+        except KeyError:
+            pass
+        if self.mxid != self.default_mxid:
+            self.by_custom_mxid[self.mxid] = self
+            await self.leave_rooms_with_default_user()
+        self.save()
+        return PuppetError.Success
+
+    async def init_custom_mxid(self) -> PuppetError:
+        if not self.is_real_user:
+            return PuppetError.Success
+
+        mxid = await self.intent.whoami()
+        if not mxid or mxid != self.custom_mxid:
+            self.custom_mxid = None
+            self.access_token = None
+            self.intent = self._fresh_intent()
+            if mxid != self.custom_mxid:
+                return PuppetError.OnlyLoginSelf
+            return PuppetError.InvalidAccessToken
+        if config["bridge.sync_with_custom_puppets"]:
+            self.sync_task = asyncio.ensure_future(self.sync(), loop=self.loop)
+        return PuppetError.Success
+
+    async def leave_rooms_with_default_user(self) -> None:
+        for room_id in await self.default_mxid_intent.get_joined_rooms():
+            try:
+                await self.default_mxid_intent.leave_room(room_id)
+                await self.intent.ensure_joined(room_id)
+            except (IntentError, MatrixRequestError):
+                pass
+
+    def create_sync_filter(self) -> Awaitable[str]:
+        return self.intent.client.create_filter(self.custom_mxid, {
+            "room": {
+                "include_leave": False,
+                "state": {
+                    "types": []
+                },
+                "timeline": {
+                    "types": [],
+                },
+                "ephemeral": {
+                    "types": ["m.typing", "m.receipt"],
+                },
+                "account_data": {
+                    "types": []
+                }
+            },
+            "account_data": {
+                "types": [],
+            },
+            "presence": {
+                "types": ["m.presence"],
+                "senders": [self.custom_mxid],
+            },
+        })
+
+    def filter_events(self, events: List[Dict]) -> List:
+        new_events = []
+        for event in events:
+            evt_type = event.get("type", None)
+            event.setdefault("content", {})
+            if evt_type == "m.typing":
+                is_typing = self.custom_mxid in event["content"].get("user_ids", [])
+                event["content"]["user_ids"] = [self.custom_mxid] if is_typing else []
+            elif evt_type == "m.receipt":
+                val = None
+                evt = None
+                for event_id in event["content"]:
+                    try:
+                        val = event["content"][event_id]["m.read"][self.custom_mxid]
+                        evt = event_id
+                        break
+                    except KeyError:
+                        pass
+                if val and evt:
+                    event["content"] = {evt: {"m.read": {
+                        self.custom_mxid: val
+                    }}}
+                else:
+                    continue
+            new_events.append(event)
+        return new_events
+
+    def handle_sync(self, presence: List, ephemeral: Dict) -> None:
+        presence_events = [self.mx.try_handle_ephemeral_event(event) for event in presence]
+
+        for room_id, events in ephemeral.items():
+            for event in events:
+                event["room_id"] = room_id
+
+        ephemeral_events = [self.mx.try_handle_ephemeral_event(event)
+                            for events in ephemeral.values()
+                            for event in self.filter_events(events)]
+
+        events = ephemeral_events + presence_events  # List[Callable[[int], Awaitable[None]]]
+        coro = asyncio.gather(*events, loop=self.loop)
+        asyncio.ensure_future(coro, loop=self.loop)
+
+    async def sync(self) -> None:
+        try:
+            await self._sync()
+        except asyncio.CancelledError:
+            self.log.info("Syncing cancelled")
+        except Exception:
+            self.log.exception("Fatal error syncing")
+
+    async def _sync(self) -> None:
+        if not self.is_real_user:
+            self.log.warning("Called sync() for non-custom puppet.")
+            return
+        custom_mxid = self.custom_mxid
+        access_token_at_start = self.access_token
+        errors = 0
+        next_batch = None
+        filter_id = await self.create_sync_filter()
+        self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.")
+        while access_token_at_start == self.access_token:
+            try:
+                sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch,
+                                                          set_presence="offline")  # type: Dict
+                errors = 0
+                if next_batch is not None:
+                    presence = sync_resp.get("presence", {}).get("events", [])  # type: List
+                    ephemeral = {room: data.get("ephemeral", {}).get("events", [])
+                                 for room, data
+                                 in sync_resp.get("rooms", {}).get("join", {}).items()
+                                 }  # type: Dict
+                    self.handle_sync(presence, ephemeral)
+                next_batch = sync_resp.get("next_batch", None)
+            except (MatrixRequestError, ServerDisconnectedError) as e:
+                wait = min(errors, 11) ** 2
+                self.log.warning(f"Syncer for {custom_mxid} errored: {e}. "
+                                 f"Waiting for {wait} seconds...")
+                errors += 1
+                await asyncio.sleep(wait)
+        self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.")
+
+    # endregion
     # region DB conversion
 
     @property
@@ -159,27 +283,26 @@ class Puppet(BasePuppet):
             self._db_instance = self.new_db_instance()
         return self._db_instance
 
-    @property
-    def _fields(self) -> Dict[str, Any]:
-        return dict(access_token=self.access_token, next_batch=self._next_batch,
-                    custom_mxid=self.custom_mxid, username=self.username, is_bot=self.is_bot,
-                    displayname=self.displayname, displayname_source=self.displayname_source,
-                    photo_id=self.photo_id, matrix_registered=self.is_registered,
-                    disable_updates=self.disable_updates, base_url=self.base_url)
-
     def new_db_instance(self) -> DBPuppet:
-        return DBPuppet(id=self.id, **self._fields)
-
-    async def save(self) -> None:
-        self.db_instance.edit(**self._fields)
+        return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
+                        username=self.username, displayname=self.displayname,
+                        displayname_source=self.displayname_source, photo_id=self.photo_id,
+                        is_bot=self.is_bot, matrix_registered=self.is_registered,
+                        disable_updates=self.disable_updates)
 
     @classmethod
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
-                      db_puppet.next_batch, db_puppet.base_url, db_puppet.username,
-                      db_puppet.displayname, db_puppet.displayname_source, db_puppet.photo_id,
-                      db_puppet.is_bot, db_puppet.matrix_registered, db_puppet.disable_updates,
-                      db_instance=db_puppet)
+                      db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
+                      db_puppet.disable_updates, db_instance=db_puppet)
+
+    def save(self) -> None:
+        self.db_instance.update(access_token=self.access_token, custom_mxid=self.custom_mxid,
+                                username=self.username, displayname=self.displayname,
+                                displayname_source=self.displayname_source, photo_id=self.photo_id,
+                                is_bot=self.is_bot, matrix_registered=self.is_registered,
+                                disable_updates=self.disable_updates)
 
     # endregion
     # region Info updating
@@ -193,26 +316,14 @@ class Puppet(BasePuppet):
         return int(round(similarity * 100))
 
     @staticmethod
-    def _filter_name(name: str) -> str:
-        if not name:
-            return ""
-        whitespace = ("\t\n\r\v\f \u00a0\u034f\u180e\u2063\u202f\u205f\u2800\u3000\u3164\ufeff"
-                      "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b"
-                      "\u200c\u200d\u200e\u200f\ufe0f")
-        name = "".join(c for c in name.strip(whitespace) if unicodedata.category(c) != 'Cf')
-        return name
-
-    @classmethod
-    def get_displayname(cls, info: User, enable_format: bool = True) -> str:
-        fn = cls._filter_name(info.first_name)
-        ln = cls._filter_name(info.last_name)
+    def get_displayname(info: User, enable_format: bool = True) -> str:
         data = {
             "phone number": info.phone if hasattr(info, "phone") else None,
             "username": info.username,
-            "full name": " ".join([fn, ln]).strip(),
-            "full name reversed": " ".join([ln, fn]).strip(),
-            "first name": fn,
-            "last name": ln,
+            "full name": " ".join([info.first_name or "", info.last_name or ""]).strip(),
+            "full name reversed": " ".join([info.first_name or "", info.last_name or ""]).strip(),
+            "first name": info.first_name,
+            "last name": info.last_name,
         }
         preferences = config["bridge.displayname_preference"]
         name = None
@@ -224,63 +335,51 @@ class Puppet(BasePuppet):
         if isinstance(info, User) and info.deleted:
             name = f"Deleted account {info.id}"
         elif not name:
-            name = str(info.id)
+            name = info.id
 
         if not enable_format:
             return name
-        return cls.displayname_template.format_full(name)
-
-    async def try_update_info(self, source: 'AbstractUser', info: User) -> None:
-        try:
-            await self.update_info(source, info)
-        except Exception:
-            source.log.exception(f"Failed to update info of {self.tgid}")
+        return config["bridge.displayname_template"].format(
+            displayname=name)
 
     async def update_info(self, source: 'AbstractUser', info: User) -> None:
+        if self.disable_updates:
+            return
         changed = False
         if self.username != info.username:
             self.username = info.username
             changed = True
 
-        if not self.disable_updates:
-            try:
-                changed = await self.update_displayname(source, info) or changed
-                changed = await self.update_avatar(source, info.photo) or changed
-            except Exception:
-                self.log.exception(f"Failed to update info from source {source.tgid}")
+        changed = await self.update_displayname(source, info) or changed
+        if isinstance(info.photo, UserProfilePhoto):
+            changed = await self.update_avatar(source, info.photo) or changed
 
         self.is_bot = info.bot
 
         if changed:
-            await self.save()
+            self.save()
 
     async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
                                  ) -> bool:
         if self.disable_updates:
             return False
-        if source.is_relaybot or source.is_bot:
-            allow_because = "user is bot"
-        elif self.displayname_source == source.tgid:
-            allow_because = "user is the primary source"
-        elif not isinstance(info, UpdateUserName) and not info.contact:
-            allow_because = "user is not a contact"
-        elif self.displayname_source is None:
-            allow_because = "no primary source set"
-        else:
+        allow_source = (source.is_relaybot
+                        or self.displayname_source == source.tgid
+                        # No displayname source, so just trust anything
+                        or self.displayname_source is None
+                        # No phone -> not in contact list -> can't set custom name
+                        or (isinstance(info, User) and info.phone is None))
+        if not allow_source:
             return False
-
-        if isinstance(info, UpdateUserName):
+        elif isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
-            self.log.debug(f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
-                           f"because {allow_because}) from {self.displayname} to {displayname}")
             self.displayname = displayname
             self.displayname_source = source.tgid
             try:
-                await self.default_mxid_intent.set_displayname(
-                    displayname[:config["bridge.displayname_max_length"]])
+                await self.default_mxid_intent.set_display_name(displayname[:100])
             except MatrixRequestError:
                 self.log.exception("Failed to set displayname")
                 self.displayname = ""
@@ -296,20 +395,15 @@ class Puppet(BasePuppet):
         if self.disable_updates:
             return False
 
-        if photo is None or isinstance(photo, UserProfilePhotoEmpty):
+        if isinstance(photo, UserProfilePhotoEmpty):
             photo_id = ""
-        elif isinstance(photo, UserProfilePhoto):
-            photo_id = str(photo.photo_id)
         else:
-            self.log.warning(f"Unknown user profile photo type: {type(photo)}")
-            return False
-        if not photo_id and not config["bridge.allow_avatar_remove"]:
-            return False
+            photo_id = str(photo.photo_id)
         if self.photo_id != photo_id:
             if not photo_id:
                 self.photo_id = ""
                 try:
-                    await self.default_mxid_intent.set_avatar_url("")
+                    await self.default_mxid_intent.set_avatar("")
                 except MatrixRequestError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
@@ -325,17 +419,13 @@ class Puppet(BasePuppet):
             if file:
                 self.photo_id = photo_id
                 try:
-                    await self.default_mxid_intent.set_avatar_url(file.mxc)
+                    await self.default_mxid_intent.set_avatar(file.mxc)
                 except MatrixRequestError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
                 return True
         return False
 
-    def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
-        portal: p.Portal = p.Portal.get_by_mxid(room_id)
-        return portal and not portal.backfill_lock.locked and portal.peer_type != "user"
-
     # endregion
     # region Getters
 
@@ -358,7 +448,7 @@ class Puppet(BasePuppet):
         return None
 
     @classmethod
-    def deprecated_sync_get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['Puppet']:
+    def get_by_mxid(cls, mxid: MatrixUserID, create: bool = True) -> Optional['Puppet']:
         tgid = cls.get_id_from_mxid(mxid)
         if tgid:
             return cls.get(tgid, create)
@@ -366,11 +456,7 @@ class Puppet(BasePuppet):
         return None
 
     @classmethod
-    async def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['Puppet']:
-        return cls.deprecated_sync_get_by_mxid(mxid, create)
-
-    @classmethod
-    def deprecated_sync_get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
+    def get_by_custom_mxid(cls, mxid: MatrixUserID) -> Optional['Puppet']:
         if not mxid:
             raise ValueError("Matrix ID can't be empty")
 
@@ -386,34 +472,31 @@ class Puppet(BasePuppet):
 
         return None
 
-    @classmethod
-    async def get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
-        return cls.deprecated_sync_get_by_custom_mxid(mxid)
-
     @classmethod
     def all_with_custom_mxid(cls) -> Iterable['Puppet']:
-        return (cls.by_custom_mxid[puppet.custom_mxid]
+        return (cls.by_custom_mxid[puppet.mxid]
                 if puppet.custom_mxid in cls.by_custom_mxid
                 else cls.from_db(puppet)
                 for puppet in DBPuppet.all_with_custom_mxid())
 
     @classmethod
-    def get_id_from_mxid(cls, mxid: UserID) -> Optional[TelegramID]:
-        return cls.mxid_template.parse(mxid)
+    def get_id_from_mxid(cls, mxid: MatrixUserID) -> Optional[TelegramID]:
+        match = cls.mxid_regex.match(mxid)
+        if match:
+            return TelegramID(int(match.group(1)))
+        return None
 
     @classmethod
-    def get_mxid_from_id(cls, tgid: TelegramID) -> UserID:
-        return UserID(cls.mxid_template.format_full(tgid))
+    def get_mxid_from_id(cls, tgid: TelegramID) -> MatrixUserID:
+        return MatrixUserID(f"@{cls.username_template.format(userid=tgid)}:{cls.hs_domain}")
 
     @classmethod
     def find_by_username(cls, username: str) -> Optional['Puppet']:
         if not username:
             return None
 
-        username = username.lower()
-
         for _, puppet in cls.cache.items():
-            if puppet.username and puppet.username.lower() == username:
+            if puppet.username and puppet.username.lower() == username.lower():
                 return puppet
 
         dbpuppet = DBPuppet.get_by_username(username)
@@ -439,23 +522,12 @@ class Puppet(BasePuppet):
     # endregion
 
 
-def init(context: 'Context') -> Iterable[Awaitable[Any]]:
+def init(context: 'Context') -> List[Awaitable[Any]]:  # [None, None, PuppetError]
     global config
     Puppet.az, config, Puppet.loop, _ = context.core
     Puppet.mx = context.mx
+    Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
     Puppet.hs_domain = config["homeserver"]["domain"]
-
-    Puppet.mxid_template = SimpleTemplate(config["bridge.username_template"], "userid",
-                                          prefix="@", suffix=f":{Puppet.hs_domain}", type=int)
-    Puppet.displayname_template = SimpleTemplate(config["bridge.displayname_template"],
-                                                 "displayname")
-
-    Puppet.sync_with_custom_puppets = config["bridge.sync_with_custom_puppets"]
-    Puppet.homeserver_url_map = {server: URL(url) for server, url
-                                 in config["bridge.double_puppet_server_map"].items()}
-    Puppet.allow_discover_url = config["bridge.double_puppet_allow_discovery"]
-    Puppet.login_shared_secret_map = {server: secret.encode("utf-8") for server, secret
-                                      in config["bridge.login_shared_secret_map"].items()}
-    Puppet.login_device_name = "Telegram Bridge"
-
-    return (puppet.try_start() for puppet in Puppet.all_with_custom_mxid())
+    Puppet.mxid_regex = re.compile(
+        f"@{Puppet.username_template.format(userid='([0-9]+)')}:{Puppet.hs_domain}")
+    return [puppet.init_custom_mxid() for puppet in Puppet.all_with_custom_mxid()]

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -1,9 +1,7 @@
-from typing import Union
 import argparse
-
+import sqlalchemy as sql
 from sqlalchemy import orm
 from sqlalchemy.ext.declarative import declarative_base
-import sqlalchemy as sql
 
 from alchemysession import AlchemySessionContainer
 
@@ -24,20 +22,16 @@ def log(message, end="\n"):
 
 
 def connect(to):
-    from mautrix.util.db import Base
-    from mautrix.client.state_store.sqlalchemy import RoomState, UserProfile
-    from mautrix_telegram.db import (Portal, Message, UserPortal, User, Contact, Puppet, BotChat,
-                                     TelegramFile)
-
+    import mautrix_telegram.db.base as base
+    base.Base = declarative_base(cls=base.BaseBase)
+    from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile,
+                                     Contact, Puppet, BotChat, TelegramFile)
     db_engine = sql.create_engine(to)
     db_factory = orm.sessionmaker(bind=db_engine)
-    db_session: Union[orm.Session, orm.scoped_session] = orm.scoped_session(db_factory)
-    Base.metadata.bind = db_engine
-
-    new_base = declarative_base()
-    new_base.metadata.bind = db_engine
+    db_session = orm.scoped_session(db_factory)  # type: orm.Session
+    base.Base.metadata.bind = db_engine
     session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
-                                                table_base=new_base, table_prefix="telethon_",
+                                                table_base=base.Base, table_prefix="telethon_",
                                                 manage_tables=False)
 
     return db_session, {
@@ -58,7 +52,6 @@ def connect(to):
         "TelegramFile": TelegramFile,
     }
 
-
 log("Connecting to old database")
 session, tables = connect(args.from_url)
 

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -14,14 +15,11 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Dict
-import argparse
-
 from sqlalchemy import orm
 import sqlalchemy as sql
+import argparse
 
-from mautrix.util.db import Base
-
-from mautrix_telegram.db import Portal, Message, Puppet, BotChat
+from mautrix_telegram.db import Base, Portal, Message, Puppet, BotChat
 from mautrix_telegram.config import Config
 
 from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as TelematrixBase
@@ -40,7 +38,8 @@ args = parser.parse_args()
 config = Config(args.config, None, None)
 config.load()
 
-mxtg_db_engine = sql.create_engine(config["appservice.database"])
+mxtg_db_engine = sql.create_engine(
+    config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
 mxtg = orm.sessionmaker(bind=mxtg_db_engine)()
 Base.metadata.bind = mxtg_db_engine
 
@@ -56,18 +55,18 @@ tm_messages = telematrix.query(TMMessage).all()
 telematrix.close()
 telematrix_db_engine.dispose()
 
-portals_by_tgid: Dict[int, Portal] = {}
-portals_by_mxid: Dict[str, Portal] = {}
-chats: Dict[int, BotChat] = {}
-messages: Dict[str, Message] = {}
-puppets: Dict[int, Puppet] = {}
+portals_by_tgid = {}  # type: Dict[int, Portal]
+portals_by_mxid = {}  # type: Dict[str, Portal]
+chats = {}  # type: Dict[int, BotChat]
+messages = {}  # type: Dict[str, Message]
+puppets = {}  # type: Dict[int, Puppet]
 
 for chat_link in chat_links:
     if type(chat_link.tg_room) is str:
-        print(f"Expected tg_room to be a number, got a string. Ignoring {chat_link.tg_room}")
+        print("Expected tg_room to be a number, got a string. Ignoring %s" % chat_link.tg_room)
         continue
     if chat_link.tg_room >= 0:
-        print(f"Unexpected unprefixed telegram chat ID: {chat_link.tg_room}, ignoring...")
+        print("Unexpected unprefixed telegram chat ID: %s, ignoring..." % chat_link.tg_room)
         continue
     tgid = str(chat_link.tg_room)
     if tgid.startswith("-100"):

mautrix_telegram/sqlstatestore.py

@@ -0,0 +1,119 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Tuple
+
+from mautrix_appservice import StateStore
+
+from .types import MatrixUserID, MatrixRoomID
+from . import puppet as pu
+from .db import RoomState, UserProfile
+
+
+class SQLStateStore(StateStore):
+    def __init__(self) -> None:
+        super().__init__()
+        self.profile_cache = {}  # type: Dict[Tuple[str, str], UserProfile]
+        self.room_state_cache = {}  # type: Dict[str, RoomState]
+
+    @staticmethod
+    def is_registered(user: MatrixUserID) -> bool:
+        puppet = pu.Puppet.get_by_mxid(user)
+        return puppet.is_registered if puppet else False
+
+    @staticmethod
+    def registered(user: MatrixUserID) -> None:
+        puppet = pu.Puppet.get_by_mxid(user)
+        if puppet:
+            puppet.is_registered = True
+            puppet.save()
+
+    def update_state(self, event: Dict) -> None:
+        event_type = event["type"]
+        if event_type == "m.room.power_levels":
+            self.set_power_levels(event["room_id"], event["content"])
+        elif event_type == "m.room.member":
+            self.set_member(event["room_id"], event["state_key"], event["content"])
+
+    def _get_user_profile(self, room_id: MatrixRoomID, user_id: MatrixUserID, create: bool = True
+                          ) -> UserProfile:
+        key = (room_id, user_id)
+        try:
+            return self.profile_cache[key]
+        except KeyError:
+            pass
+
+        profile = UserProfile.get(*key)
+        if profile:
+            self.profile_cache[key] = profile
+        elif create:
+            profile = UserProfile(room_id=room_id, user_id=user_id, membership="leave")
+            profile.insert()
+            self.profile_cache[key] = profile
+        return profile
+
+    def get_member(self, room: MatrixRoomID, user: MatrixUserID) -> Dict:
+        return self._get_user_profile(room, user).dict()
+
+    def set_member(self, room: MatrixRoomID, user: MatrixUserID, member: Dict) -> None:
+        profile = self._get_user_profile(room, user)
+        profile.membership = member.get("membership", profile.membership or "leave")
+        profile.displayname = member.get("displayname", profile.displayname)
+        profile.avatar_url = member.get("avatar_url", profile.avatar_url)
+        profile.update()
+
+    def set_membership(self, room: MatrixRoomID, user: MatrixUserID, membership: str) -> None:
+        self.set_member(room, user, {
+            "membership": membership,
+        })
+
+    def _get_room_state(self, room_id: MatrixRoomID, create: bool = True) -> RoomState:
+        try:
+            return self.room_state_cache[room_id]
+        except KeyError:
+            pass
+
+        room = RoomState.get(room_id)
+        if room:
+            self.room_state_cache[room_id] = room
+        elif create:
+            room = RoomState(room_id=room_id)
+            room.insert()
+            self.room_state_cache[room_id] = room
+        return room
+
+    def has_power_levels(self, room: MatrixRoomID) -> bool:
+        return bool(self._get_room_state(room).power_levels)
+
+    def get_power_levels(self, room: MatrixRoomID) -> Dict:
+        return self._get_room_state(room).power_levels
+
+    def set_power_level(self, room: MatrixRoomID, user: MatrixUserID, level: int) -> None:
+        room_state = self._get_room_state(room)
+        power_levels = room_state.power_levels
+        if not power_levels:
+            power_levels = {
+                "users": {},
+                "events": {},
+            }
+        power_levels[room]["users"][user] = level
+        room_state.power_levels = power_levels
+        room_state.update()
+
+    def set_power_levels(self, room: MatrixRoomID, content: Dict) -> None:
+        state = self._get_room_state(room)
+        state.power_levels = content
+        state.update()

mautrix_telegram/tgclient.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -17,21 +18,18 @@ from typing import List, Union, Optional
 
 from telethon import TelegramClient, utils
 from telethon.tl.functions.messages import SendMediaRequest
-from telethon.tl.types import (InputMediaUploadedDocument, InputMediaUploadedPhoto,
-                               TypeDocumentAttribute, TypeInputMedia, TypeInputPeer,
-                               TypeMessageEntity, TypeMessageMedia, TypePeer)
+from telethon.tl.types import (
+    InputMediaUploadedDocument, InputMediaUploadedPhoto, TypeDocumentAttribute, TypeInputMedia,
+    TypeInputPeer, TypeMessageEntity, TypeMessageMedia, TypePeer)
 from telethon.tl.patched import Message
-from telethon.sessions.abstract import Session
 
 
 class MautrixTelegramClient(TelegramClient):
-    session: Session
-
     async def upload_file_direct(self, file: bytes, mime_type: str = None,
                                  attributes: List[TypeDocumentAttribute] = None,
                                  file_name: str = None, max_image_size: float = 10 * 1000 ** 2,
                                  ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
-        file_handle = await super().upload_file(file, file_name=file_name)
+        file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)
 
         if (mime_type == "image/png" or mime_type == "image/jpeg") and len(file) < max_image_size:
             return InputMediaUploadedPhoto(file_handle)

mautrix_telegram/types.py

@@ -1,3 +1,9 @@
-from typing import NewType
+from typing import Dict, NewType
+
+MatrixUserID = NewType('MatrixUserID', str)
+MatrixRoomID = NewType('MatrixRoomID', str)
+MatrixEventID = NewType('MatrixEventID', str)
+
+MatrixEvent = NewType('MatrixEvent', Dict)
 
 TelegramID = NewType('TelegramID', int)

mautrix_telegram/user.py

@@ -1,5 +1,6 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,29 +14,21 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import (Awaitable, Dict, List, Iterable, NamedTuple, Optional, Tuple, Any, cast,
-                    TYPE_CHECKING)
-from collections import defaultdict
+from typing import Awaitable, Dict, List, Iterable, Match, NewType, Optional, Tuple, TYPE_CHECKING
 import logging
 import asyncio
+import re
 
-from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
-                               UpdateShortChatMessage, UpdateShortMessage, User as TLUser, Chat,
-                               ChatForbidden)
-from telethon.tl.custom import Dialog
+from telethon.tl.types import (
+    TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
+    UpdateShortChatMessage, UpdateShortMessage, User as TLUser)
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.account import UpdateStatusRequest
+from mautrix_appservice import MatrixRequestError
 
-from mautrix.client import Client
-from mautrix.errors import MatrixRequestError
-from mautrix.types import UserID, RoomID
-from mautrix.bridge import BaseUser
-from mautrix.util.logging import TraceLogger
-from mautrix.util.opt_prometheus import Gauge
-
-from .types import TelegramID
-from .db import User as DBUser, Portal as DBPortal
+from .types import MatrixUserID, TelegramID
+from .db import User as DBUser
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
@@ -43,53 +36,36 @@ if TYPE_CHECKING:
     from .config import Config
     from .context import Context
 
-config: Optional['Config'] = None
+config = None  # type: Config
 
-SearchResult = NamedTuple('SearchResult', puppet='pu.Puppet', similarity=int)
-
-METRIC_LOGGED_IN = Gauge('bridge_logged_in', 'Users logged into bridge')
-METRIC_CONNECTED = Gauge('bridge_connected', 'Users connected to Telegram')
+SearchResult = NewType('SearchResult', Tuple['pu.Puppet', int])
 
 
-class User(AbstractUser, BaseUser):
-    log: TraceLogger = logging.getLogger("mau.user")
-    by_mxid: Dict[str, 'User'] = {}
-    by_tgid: Dict[int, 'User'] = {}
+class User(AbstractUser):
+    log = logging.getLogger("mau.user")  # type: logging.Logger
+    by_mxid = {}  # type: Dict[str, User]
+    by_tgid = {}  # type: Dict[int, User]
 
-    phone: Optional[str]
-    contacts: List['pu.Puppet']
-    saved_contacts: int
-    portals: Dict[Tuple[TelegramID, TelegramID], 'po.Portal']
-    command_status: Optional[Dict[str, Any]]
-
-    _db_instance: Optional[DBUser]
-    _ensure_started_lock: asyncio.Lock
-    _track_connection_task: Optional[asyncio.Task]
-
-    def __init__(self, mxid: UserID, tgid: Optional[TelegramID] = None,
+    def __init__(self, mxid: MatrixUserID, tgid: Optional[TelegramID] = None,
                  username: Optional[str] = None, phone: Optional[str] = None,
                  db_contacts: Optional[Iterable[TelegramID]] = None,
                  saved_contacts: int = 0, is_bot: bool = False,
                  db_portals: Optional[Iterable[Tuple[TelegramID, TelegramID]]] = None,
                  db_instance: Optional[DBUser] = None) -> None:
         super().__init__()
-        self.mxid = mxid
-        self.tgid = tgid
-        self.is_bot = is_bot
-        self.username = username
-        self.phone = phone
-        self.contacts = []
-        self.saved_contacts = saved_contacts
+        self.mxid = mxid  # type: MatrixUserID
+        self.tgid = tgid  # type: TelegramID
+        self.is_bot = is_bot  # type: bool
+        self.username = username  # type: str
+        self.phone = phone  # type: str
+        self.contacts = []  # type: List[pu.Puppet]
+        self.saved_contacts = saved_contacts  # type: int
         self.db_contacts = db_contacts
-        self.portals = {}
+        self.portals = {}  # type: Dict[Tuple[TelegramID, TelegramID], po.Portal]
         self.db_portals = db_portals or []
-        self._db_instance = db_instance
-        self._ensure_started_lock = asyncio.Lock()
-        self.dm_update_lock = asyncio.Lock()
-        self._metric_value = defaultdict(lambda: False)
-        self._track_connection_task = None
+        self._db_instance = db_instance  # type: Optional[DBUser]
 
-        self.command_status = None
+        self.command_status = None  # type: Optional[Dict]
 
         (self.relaybot_whitelisted,
          self.whitelisted,
@@ -102,16 +78,14 @@ class User(AbstractUser, BaseUser):
         if tgid:
             self.by_tgid[tgid] = self
 
-        self.log = self.log.getChild(self.mxid)
-
     @property
     def name(self) -> str:
         return self.mxid
 
     @property
     def mxid_localpart(self) -> str:
-        localpart, server = Client.parse_user_id(self.mxid)
-        return localpart
+        match = re.compile("@(.+):(.+)").match(self.mxid)  # type: Match
+        return match.group(1)
 
     @property
     def human_tg_id(self) -> str:
@@ -161,9 +135,9 @@ class User(AbstractUser, BaseUser):
         return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
                       saved_contacts=self.saved_contacts, portals=self.db_portals)
 
-    async def save(self, contacts: bool = False, portals: bool = False) -> None:
-        self.db_instance.edit(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
-                              saved_contacts=self.saved_contacts)
+    def save(self, contacts: bool = False, portals: bool = False) -> None:
+        self.db_instance.update(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
+                                saved_contacts=self.saved_contacts)
         if contacts:
             self.db_instance.contacts = self.db_contacts
         if portals:
@@ -193,83 +167,52 @@ class User(AbstractUser, BaseUser):
         except Exception:
             self.log.exception("Exception in ensure_started")
 
-    async def ensure_started(self, even_if_no_session=False) -> 'User':
-        if not self.puppet_whitelisted or self.connected:
-            return self
-        async with self._ensure_started_lock:
-            return cast(User, await super().ensure_started(even_if_no_session))
+    def ensure_started(self, even_if_no_session=False) -> Awaitable['User']:
+        return super().ensure_started(even_if_no_session)
 
     async def start(self, delete_unless_authenticated: bool = False) -> 'User':
         await super().start()
         if await self.is_logged_in():
             self.log.debug(f"Ensuring post_login() for {self.name}")
-            self.loop.create_task(self.post_login())
+            asyncio.ensure_future(self.post_login(), loop=self.loop)
         elif delete_unless_authenticated:
             self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
             await self.client.disconnect()
             self.client.session.delete()
         return self
 
-    async def _track_connection(self) -> None:
-        self.log.debug("Starting loop to track connection state")
-        while True:
-            await asyncio.sleep(3)
-            connected = bool(self.client._sender._transport_connected
-                             if self.client and self.client._sender else False)
-            self._track_metric(METRIC_CONNECTED, connected)
-
-    async def stop(self) -> None:
-        await super().stop()
-        if self._track_connection_task:
-            self._track_connection_task.cancel()
-            self._track_connection_task = None
-        self._track_metric(METRIC_CONNECTED, False)
-
-    async def post_login(self, info: TLUser = None, first_login: bool = False) -> None:
-        if config["metrics.enabled"] and not self._track_connection_task:
-            self._track_connection_task = self.loop.create_task(self._track_connection())
-
+    async def post_login(self, info: TLUser = None) -> None:
         try:
             await self.update_info(info)
-        except Exception:
-            self.log.exception("Failed to update telegram account info")
-            return
-
-        self._track_metric(METRIC_LOGGED_IN, True)
-
-        try:
-            puppet = pu.Puppet.get(self.tgid)
-            if puppet.custom_mxid != self.mxid and puppet.can_auto_login(self.mxid):
-                self.log.info(f"Automatically enabling custom puppet")
-                await puppet.switch_mxid(access_token="auto", mxid=self.mxid)
-        except Exception:
-            self.log.exception("Failed to automatically enable custom puppet")
-
-        if not self.is_bot and config["bridge.startup_sync"]:
-            try:
+            if not self.is_bot and config["bridge.startup_sync"]:
                 await self.sync_dialogs()
                 await self.sync_contacts()
-            except Exception:
-                self.log.exception("Failed to run post-login sync")
+            if config["bridge.catch_up"]:
+                await self.client.catch_up()
+        except Exception:
+            self.log.exception("Failed to run post-login functions for %s", self.mxid)
 
     async def update(self, update: TypeUpdate) -> bool:
         if not self.is_bot:
             return False
 
         if isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
-            portal = po.Portal.get_by_entity(update.message.peer_id, receiver_id=self.tgid)
+            message = update.message
+            if isinstance(message.to_id, PeerUser) and not message.out:
+                portal = po.Portal.get_by_tgid(message.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
+            else:
+                portal = po.Portal.get_by_entity(message.to_id, receiver_id=self.tgid)
         elif isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
         elif isinstance(update, UpdateShortMessage):
             portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
         else:
             return False
 
         if portal:
-            await self.register_portal(portal)
-            return False
+            self.register_portal(portal)
 
-        # Don't bother handling the update
         return True
 
     # endregion
@@ -292,10 +235,10 @@ class User(AbstractUser, BaseUser):
             self.phone = info.phone
             changed = True
         if self.tgid != info.id:
-            self.tgid = TelegramID(info.id)
+            self.tgid = info.id
             self.by_tgid[self.tgid] = self
         if changed:
-            await self.save()
+            self.save()
 
     async def log_out(self) -> bool:
         puppet = pu.Puppet.get(self.tgid)
@@ -304,39 +247,33 @@ class User(AbstractUser, BaseUser):
         for _, portal in self.portals.items():
             if not portal or portal.deleted or not portal.mxid or portal.has_bot:
                 continue
-            if portal.peer_type == "user":
-                await portal.cleanup_portal("Logged out of Telegram")
-            else:
-                try:
-                    await portal.main_intent.kick_user(portal.mxid, self.mxid,
-                                                       "Logged out of Telegram.")
-                except MatrixRequestError:
-                    pass
+            try:
+                await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
+            except MatrixRequestError:
+                pass
         self.portals = {}
         self.contacts = []
-        await self.save(portals=True, contacts=True)
+        self.save(portals=True, contacts=True)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
             except KeyError:
                 pass
             self.tgid = None
-            await self.save()
+            self.save()
         ok = await self.client.log_out()
         if not ok:
             return False
         self.delete()
-        await self.stop()
-        self._track_metric(METRIC_LOGGED_IN, False)
         return True
 
     def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
                       ) -> List[SearchResult]:
-        results: List[SearchResult] = []
+        results = []  # type: List[SearchResult]
         for contact in self.contacts:
             similarity = contact.similarity(query)
             if similarity >= min_similarity:
-                results.append(SearchResult(contact, similarity))
+                results.append(SearchResult((contact, similarity)))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
@@ -344,11 +281,11 @@ class User(AbstractUser, BaseUser):
         if len(query) < 5:
             return []
         server_results = await self.client(SearchRequest(q=query, limit=max_results))
-        results: List[SearchResult] = []
+        results = []  # type: List[SearchResult]
         for user in server_results.users:
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
-            results.append(SearchResult(puppet, puppet.similarity(query)))
+            results.append(SearchResult((puppet, puppet.similarity(query))))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
@@ -363,81 +300,36 @@ class User(AbstractUser, BaseUser):
 
         return await self._search_remote(query), True
 
-    async def _catch(self, action: str, task: asyncio.Task) -> None:
-        try:
-            await task
-        except Exception:
-            self.log.exception(f"Error while {action}")
-
-    async def get_direct_chats(self) -> Dict[UserID, List[RoomID]]:
-        return {
-            pu.Puppet.get_mxid_from_id(portal.tgid): [portal.mxid]
-            for portal in DBPortal.find_private_chats(self.tgid)
-            if portal.mxid
-        }
-
-    async def sync_dialogs(self) -> None:
-        if self.is_bot:
-            return
+    async def sync_dialogs(self, synchronous_create: bool = False) -> None:
         creators = []
-        update_limit = config["bridge.sync_update_limit"] or None
-        create_limit = config["bridge.sync_create_limit"]
-        index = 0
-        self.log.debug(f"Syncing dialogs (update_limit={update_limit}, "
-                       f"create_limit={create_limit})")
-        dialog: Dialog
-        async for dialog in self.client.iter_dialogs(limit=update_limit, ignore_migrated=True,
-                                                     archived=False):
-            entity = dialog.entity
-            if isinstance(entity, ChatForbidden):
-                self.log.warning(f"Ignoring forbidden chat {entity} while syncing")
-                continue
-            elif isinstance(entity, Chat) and (entity.deactivated or entity.left):
-                self.log.warning(f"Ignoring deactivated or left chat {entity} while syncing")
-                continue
-            elif isinstance(entity, TLUser) and not config["bridge.sync_direct_chats"]:
-                self.log.trace(f"Ignoring user {entity.id} while syncing")
-                continue
-            portal = po.Portal.get_by_entity(entity, receiver_id=self.tgid)
+        for entity in await self.get_dialogs(limit=config["bridge.sync_dialog_limit"] or None):
+            portal = po.Portal.get_by_entity(entity)
             self.portals[portal.tgid_full] = portal
-            if portal.mxid:
-                update_task = portal.update_matrix_room(self, entity)
-                backfill_task = portal.backfill(self, last_id=dialog.message.id)
-                creators.append(self._catch(f"updating {portal.tgid_log}",
-                                            self.loop.create_task(update_task)))
-                creators.append(self._catch(f"backfilling {portal.tgid_log}",
-                                            self.loop.create_task(backfill_task)))
-            elif not create_limit or index < create_limit:
-                create_task = portal.create_matrix_room(self, entity, invites=[self.mxid])
-                creators.append(self._catch(f"creating {portal.tgid_log}",
-                                            self.loop.create_task(create_task)))
-            index += 1
-        await self.save(portals=True)
-        await asyncio.gather(*creators)
-        await self.update_direct_chats()
-        self.log.debug("Dialog syncing complete")
+            creators.append(
+                portal.create_matrix_room(self, entity, invites=[self.mxid],
+                                          synchronous=synchronous_create))
+        self.save(portals=True)
+        await asyncio.gather(*creators, loop=self.loop)
 
-    async def register_portal(self, portal: po.Portal) -> None:
-        self.log.trace(f"Registering portal {portal.tgid_full}")
+    def register_portal(self, portal: po.Portal) -> None:
         try:
             if self.portals[portal.tgid_full] == portal:
                 return
         except KeyError:
             pass
         self.portals[portal.tgid_full] = portal
-        await self.save(portals=True)
+        self.save(portals=True)
 
-    async def unregister_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
-        self.log.trace(f"Unregistering portal {(tgid, tg_receiver)}")
+    def unregister_portal(self, portal: po.Portal) -> None:
         try:
-            del self.portals[(tgid, tg_receiver)]
-            await self.save(portals=True)
+            del self.portals[portal.tgid_full]
+            self.save(portals=True)
         except KeyError:
             pass
 
     async def needs_relaybot(self, portal: po.Portal) -> bool:
         return not await self.is_logged_in() or (
-            (portal.has_bot or self.is_bot) and portal.tgid_full not in self.portals)
+            (portal.has_bot or self.bot) and portal.tgid_full not in self.portals)
 
     def _hash_contacts(self) -> int:
         acc = 0
@@ -456,14 +348,13 @@ class User(AbstractUser, BaseUser):
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
             self.contacts.append(puppet)
-        await self.save(contacts=True)
+        self.save(contacts=True)
 
     # endregion
     # region Class instance lookup
 
     @classmethod
-    def get_by_mxid(cls, mxid: UserID, create: bool = True, check_db: bool = True
-                    ) -> Optional['User']:
+    def get_by_mxid(cls, mxid: MatrixUserID, create: bool = True) -> Optional['User']:
         if not mxid:
             raise ValueError("Matrix ID can't be empty")
 
@@ -472,11 +363,10 @@ class User(AbstractUser, BaseUser):
         except KeyError:
             pass
 
-        if check_db:
-            user = DBUser.get_by_mxid(mxid)
-            if user:
-                user = cls.from_db(user)
-                return user
+        user = DBUser.get_by_mxid(mxid)
+        if user:
+            user = cls.from_db(user)
+            return user
 
         if create:
             user = cls(mxid)
@@ -504,10 +394,8 @@ class User(AbstractUser, BaseUser):
         if not username:
             return None
 
-        username = username.lower()
-
         for _, user in cls.by_tgid.items():
-            if user.username and user.username.lower() == username:
+            if user.username and user.username.lower() == username.lower():
                 return user
 
         puppet = DBUser.get_by_username(username)
@@ -518,10 +406,9 @@ class User(AbstractUser, BaseUser):
     # endregion
 
 
-def init(context: 'Context') -> Iterable[Awaitable['User']]:
+def init(context: 'Context') -> List[Awaitable[None]]:
     global config
     config = context.config
-    User.bridge = context.bridge
 
-    return (User.from_db(db_user).try_ensure_started()
-            for db_user in DBUser.all_with_tgid())
+    users = [User.from_db(user) for user in DBUser.all()]
+    return [user.try_ensure_started() for user in users if user.tgid]

mautrix_telegram/util/__init__.py

@@ -1,5 +1,7 @@
 from .file_transfer import transfer_file_to_matrix, convert_image
-from .parallel_file_transfer import parallel_transfer_to_telegram
 from .format_duration import format_duration
+from .signed_token import sign_token, verify_token
 from .recursive_dict import recursive_del, recursive_set, recursive_get
-from .color_log import ColorFormatter
+
+def ignore_coro(coro):
+    pass

mautrix_telegram/util/color_log.py

@@ -1,30 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.util.logging.color import (ColorFormatter as BaseColorFormatter,
-                                        PREFIX, MXID_COLOR, RESET)
-
-TELETHON_COLOR = PREFIX + "35;1m"  # magenta
-TELETHON_MODULE_COLOR = PREFIX + "35m"
-
-
-class ColorFormatter(BaseColorFormatter):
-    def _color_name(self, module: str) -> str:
-        if module.startswith("telethon"):
-            prefix, user_id, module = module.split(".", 2)
-            return (f"{TELETHON_COLOR}{prefix}{RESET}."
-                    f"{MXID_COLOR}{user_id}{RESET}."
-                    f"{TELETHON_MODULE_COLOR}{module}{RESET}")
-        return super()._color_name(module)

mautrix_telegram/util/file_transfer.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #
@@ -18,7 +19,6 @@ from io import BytesIO
 import time
 import logging
 import asyncio
-import tempfile
 
 import magic
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
@@ -28,32 +28,26 @@ from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLoc
                                InputPeerPhotoFileLocation)
 from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
                              SecurityError, FileIdInvalidError)
-
-from mautrix.appservice import IntentAPI
-from mautrix.util.network_retry import call_with_net_retry
+from mautrix_appservice import IntentAPI
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
 from ..util import sane_mimetypes
-from .parallel_file_transfer import parallel_transfer_to_matrix
-from .tgs_converter import convert_tgs_to
 
 try:
     from PIL import Image
 except ImportError:
     Image = None
-
 try:
     from moviepy.editor import VideoFileClip
+    import random
+    import string
+    import os
+    import mimetypes
 except ImportError:
-    VideoFileClip = None
+    VideoFileClip = random = string = os = mimetypes = None
 
-try:
-    from mautrix.crypto.attachments import encrypt_attachment
-except ImportError:
-    encrypt_attachment = None
-
-log: logging.Logger = logging.getLogger("mau.util")
+log = logging.getLogger("mau.util")  # type: logging.Logger
 
 TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
                      InputFileLocation, InputPhotoFileLocation]
@@ -65,7 +59,7 @@ def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str
     if not Image:
         return source_mime, file, None, None
     try:
-        image: Image.Image = Image.open(BytesIO(file)).convert("RGBA")
+        image = Image.open(BytesIO(file)).convert("RGBA")  # type: Image.Image
         if thumbnail_to:
             image.thumbnail(thumbnail_to, Image.ANTIALIAS)
         new_file = BytesIO()
@@ -77,41 +71,46 @@ def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str
         return source_mime, file, None, None
 
 
+def _temp_file_name(ext: str) -> str:
+    return ("/tmp/mxtg-video-"
+            + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
+            + ext)
+
+
 def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str = "png",
                           max_size: Tuple[int, int] = (1024, 720)) -> Tuple[bytes, int, int]:
-    with tempfile.NamedTemporaryFile(prefix="mxtg_video_", suffix=f".{video_ext}") as file:
-        # We don't have any way to read the video from memory, so save it to disk.
+    # We don't have any way to read the video from memory, so save it to disk.
+    temp_file = _temp_file_name(video_ext)
+    with open(temp_file, "wb") as file:
         file.write(data)
 
-        # Read temp file and get frame
-        frame = VideoFileClip(file.name).get_frame(0)
+    # Read temp file and get frame
+    clip = VideoFileClip(temp_file)
+    frame = clip.get_frame(0)
 
     # Convert to png and save to BytesIO
     image = Image.fromarray(frame).convert("RGBA")
-
     thumbnail_file = BytesIO()
     if max_size:
         image.thumbnail(max_size, Image.ANTIALIAS)
     image.save(thumbnail_file, frame_ext)
 
+    os.remove(temp_file)
+
     w, h = image.size
     return thumbnail_file.getvalue(), w, h
 
 
 def _location_to_id(location: TypeLocation) -> str:
-    if isinstance(location, Document):
+    if isinstance(location, (Document, InputDocumentFileLocation, InputPhotoFileLocation)):
         return f"{location.id}-{location.access_hash}"
-    elif isinstance(location, (InputDocumentFileLocation, InputPhotoFileLocation)):
-        return f"{location.id}-{location.access_hash}-{location.thumb_size}"
     elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
         return f"{location.volume_id}-{location.local_id}"
 
 
 async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                       thumbnail_loc: TypeLocation, mime_type: str, encrypt: bool,
-                                       video: Optional[bytes], custom_data: Optional[bytes] = None,
-                                       width: Optional[int] = None, height: [int] = None
-                                       ) -> Optional[DBTelegramFile]:
+                                       thumbnail_loc: TypeLocation, video: bytes,
+                                       mime: str) -> Optional[DBTelegramFile]:
     if not Image or not VideoFileClip:
         return None
 
@@ -119,17 +118,12 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if not loc_id:
         return None
 
-    if custom_data:
-        loc_id += "-mau_custom_thumbnail"
-
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
-    video_ext = sane_mimetypes.guess_extension(mime_type)
-    if custom_data:
-        file = custom_data
-    elif VideoFileClip and video_ext and video:
+    video_ext = sane_mimetypes.guess_extension(mime)
+    if VideoFileClip and video_ext:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
         except OSError:
@@ -140,19 +134,11 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
         width, height = None, None
         mime_type = magic.from_buffer(file, mime=True)
 
-    decryption_info = None
-    upload_mime_type = mime_type
-    if encrypt:
-        file, decryption_info = encrypt_attachment(file)
-        upload_mime_type = "application/octet-stream"
-    content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
-                                            _action="upload media")
-    if decryption_info:
-        decryption_info.url = content_uri
+    content_uri = await intent.upload_file(file, mime_type)
 
     db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
                              was_converted=False, timestamp=int(time.time()), size=len(file),
-                             width=width, height=height, decryption_info=decryption_info)
+                             width=width, height=height)
     try:
         db_file.insert()
     except (IntegrityError, InvalidRequestError) as e:
@@ -162,16 +148,14 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     return db_file
 
 
-transfer_locks: Dict[str, asyncio.Lock] = {}
+transfer_locks = {}  # type: Dict[str, asyncio.Lock]
 
 TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
 
 
 async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: TypeThumbnail = None, *,
-                                  is_sticker: bool = False, tgs_convert: Optional[dict] = None,
-                                  filename: Optional[str] = None, encrypt: bool = False,
-                                  parallel_id: Optional[int] = None) -> Optional[DBTelegramFile]:
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
+                                  is_sticker: bool = False) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
         return None
@@ -187,89 +171,48 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA
         transfer_locks[location_id] = lock
     async with lock:
         return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
-                                                       thumbnail, is_sticker, tgs_convert,
-                                                       filename, encrypt, parallel_id)
+                                                       thumbnail, is_sticker)
 
 
 async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                             loc_id: str, location: TypeLocation,
-                                            thumbnail: TypeThumbnail, is_sticker: bool,
-                                            tgs_convert: Optional[dict], filename: Optional[str],
-                                            encrypt: bool, parallel_id: Optional[int]
+                                            thumbnail: TypeThumbnail, is_sticker: bool
                                             ) -> Optional[DBTelegramFile]:
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
-    converted_anim = None
+    try:
+        file = await client.download_file(location)
+    except (LocationInvalidError, FileIdInvalidError):
+        return None
+    except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
+        log.exception(f"{e.__class__.__name__} while downloading a file.")
+        return None
 
-    if parallel_id and isinstance(location, Document) and (not is_sticker or not tgs_convert):
-        db_file = await parallel_transfer_to_matrix(client, intent, loc_id, location, filename,
-                                                    encrypt, parallel_id)
-        mime_type = location.mime_type
-        file = None
-    else:
-        try:
-            file = await client.download_file(location)
-        except (LocationInvalidError, FileIdInvalidError):
-            return None
-        except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
-            log.exception(f"{e.__class__.__name__} while downloading a file.")
-            return None
+    width, height = None, None
+    mime_type = magic.from_buffer(file, mime=True)
 
-        width, height = None, None
-        mime_type = magic.from_buffer(file, mime=True)
+    image_converted = False
+    if mime_type == "image/webp":
+        new_mime_type, file, width, height = convert_image(
+            file, source_mime="image/webp", target_type="png",
+            thumbnail_to=(256, 256) if is_sticker else None)
+        image_converted = new_mime_type != mime_type
+        mime_type = new_mime_type
+        thumbnail = None
 
-        image_converted = False
-        # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
-        is_tgs = (mime_type == "application/gzip" or (mime_type == "application/octet-stream"
-                                                      and magic.from_buffer(file).startswith(
-                "gzip")))
-        if is_sticker and tgs_convert and is_tgs:
-            converted_anim = await convert_tgs_to(file, tgs_convert["target"],
-                                                  **tgs_convert["args"])
-            mime_type = converted_anim.mime
-            file = converted_anim.data
-            width, height = converted_anim.width, converted_anim.height
-            image_converted = mime_type != "application/gzip"
-            thumbnail = None
+    content_uri = await intent.upload_file(file, mime_type)
 
-        if mime_type == "image/webp":
-            new_mime_type, file, width, height = convert_image(
-                file, source_mime="image/webp", target_type="png",
-                thumbnail_to=(256, 256) if is_sticker else None)
-            image_converted = new_mime_type != mime_type
-            mime_type = new_mime_type
-            thumbnail = None
-
-        decryption_info = None
-        upload_mime_type = mime_type
-        if encrypt and encrypt_attachment:
-            file, decryption_info = encrypt_attachment(file)
-            upload_mime_type = "application/octet-stream"
-        content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
-                                                _action="upload media")
-        if decryption_info:
-            decryption_info.url = content_uri
-
-        db_file = DBTelegramFile(id=loc_id, mxc=content_uri, decryption_info=decryption_info,
-                                 mime_type=mime_type, was_converted=image_converted,
-                                 timestamp=int(time.time()), size=len(file),
-                                 width=width, height=height)
+    db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
+                             mime_type=mime_type, was_converted=image_converted,
+                             timestamp=int(time.time()), size=len(file),
+                             width=width, height=height)
     if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
         if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
             thumbnail = thumbnail.location
-        try:
-            db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail,
-                                                                   video=file, mime_type=mime_type,
-                                                                   encrypt=encrypt)
-        except FileIdInvalidError:
-            log.warning(f"Failed to transfer thumbnail for {thumbnail!s}", exc_info=True)
-    elif converted_anim and converted_anim.thumbnail_data:
-        db_file.thumbnail = await transfer_thumbnail_to_matrix(
-            client, intent, location, video=None, encrypt=encrypt,
-            custom_data=converted_anim.thumbnail_data, mime_type=converted_anim.thumbnail_mime,
-            width=converted_anim.width, height=converted_anim.height)
+        db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
+                                                               mime_type)
 
     try:
         db_file.insert()

mautrix_telegram/util/format_duration.py

@@ -1,3 +1,4 @@
+# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
 # Copyright (C) 2019 Tulir Asokan
 #

mautrix_telegram/util/parallel_file_transfer.py

@@ -1,320 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, List, AsyncGenerator, Union, Awaitable, DefaultDict, Tuple, cast
-from collections import defaultdict
-import hashlib
-import asyncio
-import logging
-import time
-import math
-
-from aiohttp import ClientResponse
-
-from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLocation,
-                               InputPhotoFileLocation, InputPeerPhotoFileLocation, TypeInputFile,
-                               InputFileBig, InputFile)
-from telethon.tl.functions.auth import ExportAuthorizationRequest, ImportAuthorizationRequest
-from telethon.tl.functions.upload import (GetFileRequest, SaveFilePartRequest,
-                                          SaveBigFilePartRequest)
-from telethon.network import MTProtoSender
-from telethon.crypto import AuthKey
-from telethon import utils, helpers
-
-from mautrix.appservice import IntentAPI
-from mautrix.types import ContentURI, EncryptedFile
-from mautrix.util.logging import TraceLogger
-
-from ..tgclient import MautrixTelegramClient
-from ..db import TelegramFile as DBTelegramFile
-
-try:
-    from mautrix.crypto.attachments import async_encrypt_attachment
-except ImportError:
-    async_encrypt_attachment = None
-
-log: TraceLogger = cast(TraceLogger, logging.getLogger("mau.util"))
-
-TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
-                     InputFileLocation, InputPhotoFileLocation]
-
-
-class DownloadSender:
-    sender: MTProtoSender
-    request: GetFileRequest
-    remaining: int
-    stride: int
-
-    def __init__(self, sender: MTProtoSender, file: TypeLocation, offset: int, limit: int,
-                 stride: int, count: int) -> None:
-        self.sender = sender
-        self.request = GetFileRequest(file, offset=offset, limit=limit)
-        self.stride = stride
-        self.remaining = count
-
-    async def next(self) -> Optional[bytes]:
-        if not self.remaining:
-            return None
-        result = await self.sender.send(self.request)
-        self.remaining -= 1
-        self.request.offset += self.stride
-        return result.bytes
-
-    def disconnect(self) -> Awaitable[None]:
-        return self.sender.disconnect()
-
-
-class UploadSender:
-    sender: MTProtoSender
-    request: Union[SaveFilePartRequest, SaveBigFilePartRequest]
-    part_count: int
-    stride: int
-    previous: Optional[asyncio.Task]
-    loop: asyncio.AbstractEventLoop
-
-    def __init__(self, sender: MTProtoSender, file_id: int, part_count: int, big: bool, index: int,
-                 stride: int, loop: asyncio.AbstractEventLoop) -> None:
-        self.sender = sender
-        self.part_count = part_count
-        if big:
-            self.request = SaveBigFilePartRequest(file_id, index, part_count, b"")
-        else:
-            self.request = SaveFilePartRequest(file_id, index, b"")
-        self.stride = stride
-        self.previous = None
-        self.loop = loop
-
-    async def next(self, data: bytes) -> None:
-        if self.previous:
-            await self.previous
-        self.previous = self.loop.create_task(self._next(data))
-
-    async def _next(self, data: bytes) -> None:
-        self.request.bytes = data
-        log.trace(f"Sending file part {self.request.file_part}/{self.part_count}"
-                  f" with {len(data)} bytes")
-        await self.sender.send(self.request)
-        self.request.file_part += self.stride
-
-    async def disconnect(self) -> None:
-        if self.previous:
-            await self.previous
-        return await self.sender.disconnect()
-
-
-class ParallelTransferrer:
-    client: MautrixTelegramClient
-    loop: asyncio.AbstractEventLoop
-    dc_id: int
-    senders: Optional[List[Union[DownloadSender, UploadSender]]]
-    auth_key: AuthKey
-    upload_ticker: int
-
-    def __init__(self, client: MautrixTelegramClient, dc_id: Optional[int] = None) -> None:
-        self.client = client
-        self.loop = self.client.loop
-        self.dc_id = dc_id or self.client.session.dc_id
-        self.auth_key = (None if dc_id and self.client.session.dc_id != dc_id
-                         else self.client.session.auth_key)
-        self.senders = None
-        self.upload_ticker = 0
-
-    async def _cleanup(self) -> None:
-        await asyncio.gather(*[sender.disconnect() for sender in self.senders])
-        self.senders = None
-
-    @staticmethod
-    def _get_connection_count(file_size: int, max_count: int = 20,
-                              full_size: int = 100 * 1024 * 1024) -> int:
-        if file_size > full_size:
-            return max_count
-        return math.ceil((file_size / full_size) * max_count)
-
-    async def _init_download(self, connections: int, file: TypeLocation, part_count: int,
-                             part_size: int) -> None:
-        minimum, remainder = divmod(part_count, connections)
-
-        def get_part_count() -> int:
-            nonlocal remainder
-            if remainder > 0:
-                remainder -= 1
-                return minimum + 1
-            return minimum
-
-        # The first cross-DC sender will export+import the authorization, so we always create it
-        # before creating any other senders.
-        self.senders = [
-            await self._create_download_sender(file, 0, part_size, connections * part_size,
-                                               get_part_count()),
-            *await asyncio.gather(
-                *[self._create_download_sender(file, i, part_size, connections * part_size,
-                                               get_part_count())
-                  for i in range(1, connections)])
-        ]
-
-    async def _create_download_sender(self, file: TypeLocation, index: int, part_size: int,
-                                      stride: int,
-                                      part_count: int) -> DownloadSender:
-        return DownloadSender(await self._create_sender(), file, index * part_size, part_size,
-                              stride, part_count)
-
-    async def _init_upload(self, connections: int, file_id: int, part_count: int, big: bool
-                           ) -> None:
-        self.senders = [
-            await self._create_upload_sender(file_id, part_count, big, 0, connections),
-            *await asyncio.gather(
-                *[self._create_upload_sender(file_id, part_count, big, i, connections)
-                  for i in range(1, connections)])
-        ]
-
-    async def _create_upload_sender(self, file_id: int, part_count: int, big: bool, index: int,
-                                    stride: int) -> UploadSender:
-        return UploadSender(await self._create_sender(), file_id, part_count, big, index, stride,
-                            loop=self.loop)
-
-    async def _create_sender(self) -> MTProtoSender:
-        dc = await self.client._get_dc(self.dc_id)
-        sender = MTProtoSender(self.auth_key, loggers=self.client._log)
-        await sender.connect(self.client._connection(dc.ip_address, dc.port, dc.id,
-                                                     loggers=self.client._log,
-                                                     proxy=self.client._proxy))
-        if not self.auth_key:
-            log.debug(f"Exporting auth to DC {self.dc_id}")
-            auth = await self.client(ExportAuthorizationRequest(self.dc_id))
-            req = self.client._init_with(ImportAuthorizationRequest(
-                id=auth.id, bytes=auth.bytes
-            ))
-            await sender.send(req)
-            self.auth_key = sender.auth_key
-        return sender
-
-    async def init_upload(self, file_id: int, file_size: int, part_size_kb: Optional[float] = None,
-                          connection_count: Optional[int] = None) -> Tuple[int, int, bool]:
-        connection_count = connection_count or self._get_connection_count(file_size)
-        part_size = (part_size_kb or utils.get_appropriated_part_size(file_size)) * 1024
-        part_count = (file_size + part_size - 1) // part_size
-        is_large = file_size > 10 * 1024 * 1024
-        await self._init_upload(connection_count, file_id, part_count, is_large)
-        return part_size, part_count, is_large
-
-    async def upload(self, part: bytes) -> None:
-        await self.senders[self.upload_ticker].next(part)
-        self.upload_ticker = (self.upload_ticker + 1) % len(self.senders)
-
-    async def finish_upload(self) -> None:
-        await self._cleanup()
-
-    async def download(self, file: TypeLocation, file_size: int,
-                       part_size_kb: Optional[float] = None,
-                       connection_count: Optional[int] = None) -> AsyncGenerator[bytes, None]:
-        connection_count = connection_count or self._get_connection_count(file_size)
-        part_size = (part_size_kb or utils.get_appropriated_part_size(file_size)) * 1024
-        part_count = math.ceil(file_size / part_size)
-        log.debug("Starting parallel download: "
-                  f"{connection_count} {part_size} {part_count} {file!s}")
-        await self._init_download(connection_count, file, part_count, part_size)
-
-        part = 0
-        while part < part_count:
-            tasks = []
-            for sender in self.senders:
-                tasks.append(self.loop.create_task(sender.next()))
-            for task in tasks:
-                data = await task
-                if not data:
-                    break
-                yield data
-                part += 1
-                log.trace(f"Part {part} downloaded")
-
-        log.debug("Parallel download finished, cleaning up connections")
-        await self._cleanup()
-
-
-parallel_transfer_locks: DefaultDict[int, asyncio.Lock] = defaultdict(lambda: asyncio.Lock())
-
-
-async def parallel_transfer_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                      loc_id: str, location: TypeLocation, filename: str,
-                                      encrypt: bool, parallel_id: int) -> DBTelegramFile:
-    size = location.size
-    mime_type = location.mime_type
-    dc_id, location = utils.get_input_location(location)
-    # We lock the transfers because telegram has connection count limits
-    async with parallel_transfer_locks[parallel_id]:
-        downloader = ParallelTransferrer(client, dc_id)
-        data = downloader.download(location, size)
-        decryption_info = None
-        up_mime_type = mime_type
-        if encrypt and async_encrypt_attachment:
-            async def encrypted(stream):
-                nonlocal decryption_info
-                async for chunk in async_encrypt_attachment(stream):
-                    if isinstance(chunk, EncryptedFile):
-                        decryption_info = chunk
-                    else:
-                        yield chunk
-
-            data = encrypted(data)
-            up_mime_type = "application/octet-stream"
-        content_uri = await intent.upload_media(data, mime_type=up_mime_type, filename=filename,
-                                                size=size if not encrypt else None)
-        if decryption_info:
-            decryption_info.url = content_uri
-    return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
-                          was_converted=False, timestamp=int(time.time()), size=size,
-                          width=None, height=None, decryption_info=decryption_info)
-
-
-async def _internal_transfer_to_telegram(client: MautrixTelegramClient, response: ClientResponse
-                                         ) -> Tuple[TypeInputFile, int]:
-    file_id = helpers.generate_random_long()
-    file_size = response.content_length
-
-    hash_md5 = hashlib.md5()
-    uploader = ParallelTransferrer(client)
-    part_size, part_count, is_large = await uploader.init_upload(file_id, file_size)
-    buffer = bytearray()
-    async for data in response.content:
-        if not is_large:
-            hash_md5.update(data)
-        if len(buffer) == 0 and len(data) == part_size:
-            await uploader.upload(data)
-            continue
-        new_len = len(buffer) + len(data)
-        if new_len >= part_size:
-            cutoff = part_size - len(buffer)
-            buffer.extend(data[:cutoff])
-            await uploader.upload(bytes(buffer))
-            buffer.clear()
-            buffer.extend(data[cutoff:])
-        else:
-            buffer.extend(data)
-    if len(buffer) > 0:
-        await uploader.upload(bytes(buffer))
-    await uploader.finish_upload()
-    if is_large:
-        return InputFileBig(file_id, part_count, "upload"), file_size
-    else:
-        return InputFile(file_id, part_count, "upload", hash_md5.hexdigest()), file_size
-
-
-async def parallel_transfer_to_telegram(client: MautrixTelegramClient, intent: IntentAPI,
-                                        uri: ContentURI, parallel_id: int
-                                        ) -> Tuple[TypeInputFile, int]:
-    url = intent.api.get_download_url(uri)
-    async with parallel_transfer_locks[parallel_id]:
-        async with intent.api.session.get(url) as response:
-            return await _internal_transfer_to_telegram(client, response)