Bump version to 0.9.0rc3

If it just kicks the user, logging in again later would cause the
bridge to think there's a portal, but fail to invite the user again.

Fixes #397

.codeclimate.yml

@@ -0,0 +1,8 @@
+engines:
+  sonar-python:
+    enabled: true
+    checks:
+      python:S107:
+        enabled: false
+exclude_patterns:
+- "alembic/"

.dockerignore

@@ -0,0 +1,10 @@
+.editorconfig
+.codeclimate.yml
+*.png
+*.md
+logs
+.venv
+start
+config.yaml
+registration.yaml
+*.db

.editorconfig

@@ -13,3 +13,6 @@ max_line_length = 99
 
 [*.{yaml,yml,py}]
 indent_style = space
+
+[.gitlab-ci.yml]
+indent_size = 2

.gitignore

@@ -1,12 +1,18 @@
-.idea/
+/.idea/
 
-.venv
+/.venv
+/env/
 pip-selfcheck.json
 *.pyc
 __pycache__
+/build
+/dist
+/*.egg-info
+/.eggs
 
-config.yaml
-registration.yaml
+/config.yaml
+/registration.yaml
+*.log*
 *.db
-*.session
-*.json
+*.pickle
+*.bak

.gitlab-ci.yml

@@ -0,0 +1,48 @@
+image: docker:stable
+
+stages:
+- build
+- manifest
+
+default:
+  before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+
+build amd64:
+  stage: build
+  tags:
+  - amd64
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  after_script:
+  - |
+    if [ "$CI_COMMIT_BRANCH" = "master" ]; then
+      apk add --update curl
+      rm -rf /var/cache/apk/*
+      curl "$NOVA_ADMIN_API_URL" -H "Content-Type: application/json" -d '{"password":"'"$NOVA_ADMIN_NIGHTLY_PASS"'","bridge":"'$NOVA_BRIDGE_TYPE'","image":"'$CI_REGISTRY_IMAGE':'$CI_COMMIT_SHA'-amd64"}'
+    fi
+
+build arm64:
+  stage: build
+  tags:
+  - arm64
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+
+manifest:
+  stage: manifest
+  before_script:
+  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
+  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64

.isort.cfg

@@ -0,0 +1,16 @@
+[settings]
+line_length=99
+indent=4
+
+multi_line_output=5
+
+sections=FUTURE,STDLIB,THIRDPARTY,TELETHON,MAUTRIX,FIRSTPARTY,LOCALFOLDER
+no_lines_before=LOCALFOLDER
+default_section=FIRSTPARTY
+
+known_thirdparty=aiohttp,sqlalchemy,alembic,commonmark,ruamel.yaml,PIL,moviepy,prometheus_client,yarl,mako,pkg_resources
+known_telethon=telethon,alchemysession,cryptg
+known_mautrix=mautrix
+
+balanced_wrapping=True
+length_sort=True

Dockerfile

@@ -1,30 +1,73 @@
-FROM docker.io/alpine:3.7
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.12
 
-ENV UID=1337 \
-    GID=1337
+ARG TARGETARCH=amd64
+
+RUN echo $'\
+@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\n\
+@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing\n\
+@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
 
-COPY . /opt/mautrixtelegram
 RUN apk add --no-cache \
-      python3-dev \
+      python3 py3-pip py3-setuptools py3-wheel \
       py3-virtualenv \
-      build-base \
-      zlib-dev \
-      jpeg-dev \
-      libxslt-dev \
-      libxml2-dev \
-      libmagic \
+      py3-pillow \
+      py3-aiohttp \
+      py3-magic \
+      py3-sqlalchemy \
+      py3-telethon-session-sqlalchemy@edge \
+      py3-alembic@edge \
+      py3-psycopg2 \
+      py3-ruamel.yaml \
+      py3-commonmark@edge \
+      # Indirect dependencies
+      py3-idna \
+      #moviepy
+        py3-decorator \
+        py3-tqdm \
+        py3-requests \
+        #imageio
+          py3-numpy \
+      #py3-telethon@edge \ (outdated)
+        # Optional for socks proxies
+        py3-pysocks \
+        # cryptg
+          py3-cffi \
+	  py3-qrcode@edge \
+      py3-brotli \
+      # Other dependencies
       ffmpeg \
-      bash \
       ca-certificates \
       su-exec \
-      s6 \
- && cd /opt/mautrixtelegram \
- && cp -r docker/root/* / \
- && rm docker -rf \
- && virtualenv -p /usr/bin/python3 .venv \
- && source .venv/bin/activate \
- && pip install -r requirements.txt -r optional-requirements.txt
+      netcat-openbsd \
+      # encryption
+      olm-dev \
+      py3-pycryptodome \
+      py3-unpaddedbase64 \
+      py3-future \
+      bash \
+      curl \
+      jq && \
+  curl -sLo yq https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${TARGETARCH} && \
+  chmod +x yq && mv yq /usr/bin/yq
+
+COPY requirements.txt /opt/mautrix-telegram/requirements.txt
+COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
+WORKDIR /opt/mautrix-telegram
+RUN apk add --virtual .build-deps \
+      python3-dev \
+      libffi-dev \
+      build-base \
+ && sed -Ei 's/psycopg2-binary.+//' optional-requirements.txt \
+ && pip3 install -r requirements.txt -r optional-requirements.txt \
+ && apk del .build-deps
+
+COPY . /opt/mautrix-telegram
+RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics,e2be] && apk del git \
+  # This doesn't make the image smaller, but it's needed so that the `version` command works properly
+  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
 
 VOLUME /data
+ENV UID=1337 GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
 
-CMD ["/bin/s6-svscan", "/etc/s6.d"]
+CMD ["/opt/mautrix-telegram/docker-run.sh"]

MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include LICENSE
+include requirements.txt
+include optional-requirements.txt

README.md

@@ -1,9 +1,28 @@
 # mautrix-telegram
+![Languages](https://img.shields.io/github/languages/top/tulir/mautrix-telegram.svg)
+[![License](https://img.shields.io/github/license/tulir/mautrix-telegram.svg)](LICENSE)
+[![Release](https://img.shields.io/github/release/tulir/mautrix-telegram/all.svg)](https://github.com/tulir/mautrix-telegram/releases)
+[![GitLab CI](https://mau.dev/tulir/mautrix-telegram/badges/master/pipeline.svg)](https://mau.dev/tulir/mautrix-telegram/container_registry)
+[![Maintainability](https://img.shields.io/codeclimate/maintainability/tulir/mautrix-telegram.svg)](https://codeclimate.com/github/tulir/mautrix-telegram)
+
 A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
-### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)
+## Sponsors
+* [Joel Lehtonen / Zouppen](https://github.com/zouppen)
 
-### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+### Wiki
+All setup and usage instructions are located in the GitHub
+[wiki](https://github.com/tulir/mautrix-telegram/wiki). Some quick links:
+
+* [Bridge setup](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup)
+  (or [with Docker](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup-with-Docker))
+* Basic usage: [Authentication](https://github.com/tulir/mautrix-telegram/wiki/Authentication),
+  [Creating chats](https://github.com/tulir/mautrix-telegram/wiki/Creating-and-managing-chats),
+  [Relaybot setup](https://github.com/tulir/mautrix-telegram/wiki/Relay-bot)
+
+### Features & Roadmap
+[ROADMAP.md](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+contains a general overview of what is supported by the bridge.
 
 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)
@@ -11,4 +30,4 @@ Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net
 Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
 
 ## Preview
-![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)
+![Preview](preview.png)

ROADMAP.md

@@ -3,10 +3,11 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message edits
   * [ ] ‡ Message history
-  * [ ] † Presence
-  * [ ] † Typing notifications
-  * [ ] † Read receipts
+  * [x] Presence
+  * [x] Typing notifications
+  * [x] Read receipts
   * [x] Pinning messages
   * [x] Power level
     * [x] Normal chats
@@ -21,9 +22,16 @@
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
+  * [ ] Advanced message content/media
+    * [x] Polls
+	* [x] Games
+	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
-  * [ ] Message history
+  * [x] Message history
+    * [x] Manually (`!tg backfill`)
+    * [x] Automatically when creating portal
+    * [x] Automatically for missed messages
   * [x] Avatars
   * [x] Presence
   * [x] Typing notifications
@@ -45,9 +53,11 @@
     * [x] At startup
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [x] Option to use bot to relay messages for unauthenticated Matrix users
-  * [ ] Option to use own Matrix account for messages sent from other Telegram clients
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
+  * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
+  * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
 
-† Information not automatically sent from source, i.e. implementation may not be possible  
+† Information not automatically sent from source, i.e. implementation may not be possible
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic.ini

@@ -35,9 +35,6 @@ script_location = alembic
 # are written from script.py.mako
 # output_encoding = utf-8
 
-sqlalchemy.url = sqlite:///mautrix-telegram.db
-
-
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -1,19 +1,28 @@
-from __future__ import with_statement
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig
 
 import sys
 from os.path import abspath, dirname
+
 sys.path.insert(0, dirname(dirname(abspath(__file__))))
 
-from mautrix_telegram.base import Base
+from mautrix.util.db import Base
 import mautrix_telegram.db
+from mautrix_telegram.config import Config
+from alchemysession import AlchemySessionContainer
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
 
+mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+mxtg_config = Config(mxtg_config_path, None, None)
+mxtg_config.load()
+config.set_main_option("sqlalchemy.url", mxtg_config["appservice.database"].replace("%", "%%"))
+
+AlchemySessionContainer.create_table_classes(None, "telethon_", Base)
+
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
@@ -24,6 +33,7 @@ fileConfig(config.config_file_name)
 # target_metadata = mymodel.Base.metadata
 target_metadata = Base.metadata
 
+
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
@@ -44,7 +54,8 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True)
+        url=url, target_metadata=target_metadata, literal_binds=True,
+        render_as_batch=True)
 
     with context.begin_transaction():
         context.run_migrations()
@@ -65,12 +76,14 @@ def run_migrations_online():
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
-            target_metadata=target_metadata
+            target_metadata=target_metadata,
+            render_as_batch=True
         )
 
         with context.begin_transaction():
             context.run_migrations()
 
+
 if context.is_offline_mode():
     run_migrations_offline()
 else:

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -0,0 +1,27 @@
+"""Add disable_updates field for puppets
+
+Revision ID: 17574c57f3f8
+Revises: a9119be92164
+Create Date: 2019-05-15 00:24:46.967529
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '17574c57f3f8'
+down_revision = 'a9119be92164'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("disable_updates")

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -17,8 +17,10 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
 
 
 def downgrade():
-    op.drop_column('puppet', 'is_bot')
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('is_bot')

alembic/versions/2228d49c383f_add_cascade_rules_to_userportal.py

@@ -0,0 +1,41 @@
+"""Add cascade rules to UserPortal
+
+Revision ID: 2228d49c383f
+Revises: bcfefa1f1299
+Create Date: 2018-05-31 11:11:59.482112
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '2228d49c383f'
+down_revision = 'bcfefa1f1299'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+            batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+            batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+            batch_op.create_foreign_key("user_portal_user_fkey", "user", ["user"], ["tgid"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+            batch_op.create_foreign_key("user_portal_portal_fkey", "portal",
+                                        ["portal", "portal_receiver"], ["tgid", "tg_receiver"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+    except ValueError:
+        return
+
+
+def downgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+                batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+                batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+                batch_op.create_foreign_key("user_portal_user_fkey", "portal",
+                                            ["portal", "portal_receiver"], ["tgid", "tg_receiver"])
+                batch_op.create_foreign_key("user_portal_portal_fkey", "user", ["user"], ["tgid"])
+    except ValueError:
+        return

alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py

@@ -0,0 +1,27 @@
+"""Add encrypted field for portals
+
+Revision ID: 24f31fc8a72b
+Revises: a7c04a56041b
+Create Date: 2020-03-28 20:14:29.046699
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "24f31fc8a72b"
+down_revision = "a7c04a56041b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column("encrypted", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column("encrypted")

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -16,8 +16,10 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
 
 
 def downgrade():
-    op.drop_column('portal', 'megagroup')
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column('megagroup')

alembic/versions/3e3745baa458_store_matrix_avatar_url_in_database.py

@@ -0,0 +1,32 @@
+"""Store Matrix avatar URL in database
+
+Revision ID: 3e3745baa458
+Revises: dff56c93da8d
+Create Date: 2020-06-15 14:32:10.454033
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3e3745baa458'
+down_revision = 'dff56c93da8d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('portal', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('avatar_url', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('portal', schema=None) as batch_op:
+        batch_op.drop_column('avatar_url')
+
+    # ### end Alembic commands ###

alembic/versions/4f7d7ed5792a_switch_mx_user_profile_to_native_enum.py

@@ -0,0 +1,26 @@
+"""Switch mx_user_profile to native enum
+
+Revision ID: 4f7d7ed5792a
+Revises: 9e9c89b0b877
+Create Date: 2019-08-04 17:47:36.568120
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = '4f7d7ed5792a'
+down_revision = '9e9c89b0b877'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    conn = op.get_bind()
+    conn.execute("UPDATE mx_user_profile SET membership=UPPER(membership)")
+    conn.execute("UPDATE mx_user_profile SET membership='LEAVE' WHERE membership='LEFT'")
+
+
+def downgrade():
+    conn = op.get_bind()
+    conn.execute("UPDATE mx_user_profile SET membership=LOWER(membership)")

alembic/versions/501dad2868bc_move_sessions_to_main_database.py

@@ -19,31 +19,31 @@ depends_on = None
 
 def upgrade():
     Session = op.create_table('telethon_sessions',
-                              sa.Column('session_id', sa.VARCHAR(), nullable=False),
-                              sa.Column('dc_id', sa.INTEGER(), nullable=False),
-                              sa.Column('server_address', sa.VARCHAR(), nullable=True),
-                              sa.Column('port', sa.INTEGER(), nullable=True),
-                              sa.Column('auth_key', sa.BLOB(), nullable=True),
+                              sa.Column('session_id', sa.String, nullable=False),
+                              sa.Column('dc_id', sa.Integer, nullable=False),
+                              sa.Column('server_address', sa.String, nullable=True),
+                              sa.Column('port', sa.Integer, nullable=True),
+                              sa.Column('auth_key', sa.LargeBinary, nullable=True),
                               sa.PrimaryKeyConstraint('session_id', 'dc_id'))
     SentFile = op.create_table('telethon_sent_files',
-                               sa.Column('session_id', sa.VARCHAR(), nullable=False),
-                               sa.Column('md5_digest', sa.BLOB(), nullable=False),
-                               sa.Column('file_size', sa.INTEGER(), nullable=False),
-                               sa.Column('type', sa.INTEGER(), nullable=False),
-                               sa.Column('id', sa.INTEGER(), nullable=True),
-                               sa.Column('hash', sa.INTEGER(), nullable=True),
+                               sa.Column('session_id', sa.String, nullable=False),
+                               sa.Column('md5_digest', sa.LargeBinary, nullable=False),
+                               sa.Column('file_size', sa.Integer, nullable=False),
+                               sa.Column('type', sa.Integer, nullable=False),
+                               sa.Column('id', sa.BigInteger, nullable=True),
+                               sa.Column('hash', sa.BigInteger, nullable=True),
                                sa.PrimaryKeyConstraint('session_id', 'md5_digest', 'file_size',
                                                        'type'))
     Entity = op.create_table('telethon_entities',
-                             sa.Column('session_id', sa.VARCHAR(), nullable=False),
-                             sa.Column('id', sa.INTEGER(), nullable=False),
-                             sa.Column('hash', sa.INTEGER(), nullable=False),
-                             sa.Column('username', sa.VARCHAR(), nullable=True),
-                             sa.Column('phone', sa.INTEGER(), nullable=True),
-                             sa.Column('name', sa.VARCHAR(), nullable=True),
+                             sa.Column('session_id', sa.String, nullable=False),
+                             sa.Column('id', sa.Integer, nullable=False),
+                             sa.Column('hash', sa.Integer, nullable=False),
+                             sa.Column('username', sa.String, nullable=True),
+                             sa.Column('phone', sa.Integer, nullable=True),
+                             sa.Column('name', sa.String, nullable=True),
                              sa.PrimaryKeyConstraint('session_id', 'id'))
     Version = op.create_table('telethon_version',
-                              sa.Column('version', sa.INTEGER(), nullable=False),
+                              sa.Column('version', sa.Integer, nullable=False),
                               sa.PrimaryKeyConstraint('version'))
     conn = op.get_bind()
     sessions = [os.path.basename(f) for f in os.listdir(".") if f.endswith(".session")]

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -0,0 +1,136 @@
+"""Move state store to main database
+
+Revision ID: 6ca3d74d51e4
+Revises: 2228d49c383f
+Create Date: 2018-06-26 21:31:26.911307
+
+"""
+import json
+import re
+
+from alembic import context, op
+import sqlalchemy.orm as orm
+import sqlalchemy as sa
+
+from mautrix.util.db import Base
+
+from mautrix_telegram.config import Config
+
+# revision identifiers, used by Alembic.
+revision = "6ca3d74d51e4"
+down_revision = "2228d49c383f"
+branch_labels = None
+depends_on = None
+
+
+class RoomState(Base):
+    __tablename__ = "mx_room_state"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    power_levels = sa.Column("power_levels", sa.Text, nullable=True)
+
+
+class UserProfile(Base):
+    __tablename__ = "mx_user_profile"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    user_id = sa.Column(sa.String, primary_key=True)
+    membership = sa.Column(sa.String, nullable=False, default="leave")
+    displayname = sa.Column(sa.String, nullable=True)
+    avatar_url = sa.Column(sa.String, nullable=True)
+
+
+class Puppet(Base):
+    __tablename__ = "puppet"
+    __table_args__ = {"extend_existing": True}
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    displayname = sa.Column(sa.String, nullable=True)
+    displayname_source = sa.Column(sa.Integer, nullable=True)
+    username = sa.Column(sa.String, nullable=True)
+    photo_id = sa.Column(sa.String, nullable=True)
+    is_bot = sa.Column(sa.Boolean, nullable=True)
+    matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False)
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+    op.create_table("mx_room_state",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("power_levels", sa.Text(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id"))
+    op.create_table("mx_user_profile",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("user_id", sa.String(), nullable=False),
+                    sa.Column("membership", sa.String(), nullable=False,
+                              default="leave"),
+                    sa.Column("displayname", sa.String(), nullable=True),
+                    sa.Column("avatar_url", sa.String(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id", "user_id"))
+
+    try:
+        migrate_state_store()
+    except Exception as e:
+        print("Failed to migrate state store:", e)
+        print("Migrating the state store isn't required, but you can retry by alembic downgrading "
+              "to revision 2228d49c383f and upgrading again.")
+
+
+def migrate_state_store():
+    conn = op.get_bind()
+    session: orm.Session = orm.sessionmaker(bind=conn)()
+
+    try:
+        with open("mx-state.json") as file:
+            data = json.load(file)
+    except FileNotFoundError:
+        return
+    if not data:
+        return
+    registrations = data.get("registrations", [])
+
+    mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+    mxtg_config = Config(mxtg_config_path, None, None)
+    mxtg_config.load()
+
+    username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}")
+    hs_domain = mxtg_config["homeserver.domain"]
+    localpart = username_template.format(userid="(.+)")
+    mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain))
+    for user in registrations:
+        match = mxid_regex.match(user)
+        if not match:
+            continue
+
+        puppet = session.query(Puppet).get(match.group(1))
+        if not puppet:
+            continue
+
+        puppet.matrix_registered = True
+        session.merge(puppet)
+    session.commit()
+
+    user_profiles = [UserProfile(room_id=room, user_id=user,
+                                 membership=member.get("membership", "leave"),
+                                 displayname=member.get("displayname", None),
+                                 avatar_url=member.get("avatar_url", None))
+                     for room, members in data.get("members", {}).items()
+                     for user, member in members.items()]
+    session.add_all(user_profiles)
+    session.commit()
+
+    room_state = [RoomState(room_id=room, power_levels=json.dumps(levels))
+                  for room, levels in data.get("power_levels", {}).items()]
+    session.add_all(room_state)
+    session.commit()
+
+
+def downgrade():
+    op.drop_table("mx_user_profile")
+    op.drop_table("mx_room_state")
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("matrix_registered")

alembic/versions/7d47d84380b6_add_timestamp_to_telegramfile.py

@@ -17,9 +17,10 @@ depends_on = None
 
 def upgrade():
     op.add_column('telegram_file',
-                  sa.Column('timestamp', sa.BigInteger(), nullable=False, default=0,
-                            server_default="true"))
+                  sa.Column('timestamp', sa.BigInteger(), nullable=True, default=0,
+                            server_default="0"))
 
 
 def downgrade():
-    op.drop_column('telegram_file', 'timestamp')
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column('timestamp')

alembic/versions/888275d58e57_add_double_puppet_base_url_to_puppet_.py

@@ -0,0 +1,30 @@
+"""Add double puppet base URL to puppet table
+
+Revision ID: 888275d58e57
+Revises: a328bf4f0932
+Create Date: 2020-10-14 18:52:00.730666
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '888275d58e57'
+down_revision = 'a328bf4f0932'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('base_url', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('base_url')
+    # ### end Alembic commands ###

alembic/versions/97d2a942bcf8_initial_revision.py

@@ -29,7 +29,7 @@ def upgrade():
                     sa.UniqueConstraint('mxid'))
     op.create_table('user',
                     sa.Column('mxid', sa.String),
-                    sa.Column('tgid', sa.Integer, nullable=True),
+                    sa.Column('tgid', sa.Integer, nullable=True, unique=True),
                     sa.Column('tg_username', sa.String, nullable=True),
                     sa.Column('saved_contacts', sa.Integer, nullable=False, default=0),
                     sa.PrimaryKeyConstraint('mxid'))
@@ -50,9 +50,13 @@ def upgrade():
                     sa.Column('portal', sa.Integer),
                     sa.Column('portal_receiver', sa.Integer),
                     sa.PrimaryKeyConstraint('user', 'portal', 'portal_receiver'),
-                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",),
+                                            name="user_portal_user_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"),
                     sa.ForeignKeyConstraint(("portal", "portal_receiver"),
-                                            ("portal.tgid", "portal.tg_receiver")))
+                                            ("portal.tgid", "portal.tg_receiver"),
+                                            name="user_portal_portal_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"))
     op.create_table('message',
                     sa.Column('mxid', sa.String),
                     sa.Column('mx_room', sa.String),

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,48 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

alembic/versions/a328bf4f0932_store_encryption_state_event_in_db.py

@@ -0,0 +1,38 @@
+"""Store encryption state event in db
+
+Revision ID: a328bf4f0932
+Revises: ccbaff858240
+Create Date: 2020-07-11 21:31:27.059813
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from mautrix.client.state_store.sqlalchemy import SerializableType
+from mautrix.types import RoomEncryptionStateEventContent
+
+# revision identifiers, used by Alembic.
+revision = 'a328bf4f0932'
+down_revision = 'ccbaff858240'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('encryption',
+                                      SerializableType(RoomEncryptionStateEventContent),
+                                      nullable=True))
+        batch_op.add_column(sa.Column('has_full_member_list', sa.Boolean(), nullable=True))
+        batch_op.add_column(sa.Column('is_encrypted', sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
+        batch_op.drop_column('is_encrypted')
+        batch_op.drop_column('has_full_member_list')
+        batch_op.drop_column('encryption')
+    # ### end Alembic commands ###

alembic/versions/a7c04a56041b_store_custom_puppet_next_batch_in_.py

@@ -0,0 +1,26 @@
+"""Store custom puppet next_batch in database
+
+Revision ID: a7c04a56041b
+Revises: 4f7d7ed5792a
+Create Date: 2019-08-06 23:08:51.087651
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "a7c04a56041b"
+down_revision = "4f7d7ed5792a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("next_batch", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("next_batch")

alembic/versions/a9119be92164_add_phone_number_field_to_users.py

@@ -0,0 +1,25 @@
+"""Add phone number field to users
+
+Revision ID: a9119be92164
+Revises: b54929c22c86
+Create Date: 2018-09-28 02:38:40.626282
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "a9119be92164"
+down_revision = "b54929c22c86"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("user", sa.Column("tg_phone", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("user") as batch_op:
+        batch_op.drop_column("tg_phone")

alembic/versions/b54929c22c86_add_portal_specific_config.py

@@ -0,0 +1,25 @@
+"""Add portal-specific config
+
+Revision ID: b54929c22c86
+Revises: d5f7b8b4b456
+Create Date: 2018-09-24 23:40:33.528710
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "b54929c22c86"
+down_revision = "d5f7b8b4b456"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("portal", sa.Column("config", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column("config")

alembic/versions/bcfefa1f1299_add_displayname_source_fields_for_.py

@@ -20,4 +20,5 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_column('puppet', 'displayname_source')
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('displayname_source')

alembic/versions/ccbaff858240_switch_to_mautrix_python_crypto.py

@@ -0,0 +1,71 @@
+"""Switch to mautrix-python crypto
+
+Revision ID: ccbaff858240
+Revises: 3e3745baa458
+Create Date: 2020-07-08 19:06:12.588047
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'ccbaff858240'
+down_revision = '3e3745baa458'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('nio_account')
+    op.drop_table('nio_device_key')
+    op.drop_table('nio_outgoing_key_request')
+    op.drop_table('nio_olm_session')
+    op.drop_table('nio_megolm_inbound_session')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('nio_megolm_inbound_session',
+    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('fp_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.Column('forwarded_chains', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('session_id', name='nio_megolm_inbound_session_pkey')
+    )
+    op.create_table('nio_olm_session',
+    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.Column('last_used', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('session_id', name='nio_olm_session_pkey')
+    )
+    op.create_table('nio_outgoing_key_request',
+    sa.Column('request_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('algorithm', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('request_id', name='nio_outgoing_key_request_pkey')
+    )
+    op.create_table('nio_device_key',
+    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('display_name', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('deleted', sa.BOOLEAN(), autoincrement=False, nullable=False),
+    sa.Column('keys', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_device_key_pkey')
+    )
+    op.create_table('nio_account',
+    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('shared', sa.BOOLEAN(), autoincrement=False, nullable=False),
+    sa.Column('sync_token', sa.TEXT(), autoincrement=False, nullable=False),
+    sa.Column('account', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_account_pkey')
+    )
+    # ### end Alembic commands ###

alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py

@@ -0,0 +1,26 @@
+"""Add decryption info field for reuploaded telegram files
+
+Revision ID: d3c922a6acd2
+Revises: 24f31fc8a72b
+Create Date: 2020-03-30 20:07:17.340346
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd3c922a6acd2'
+down_revision = '24f31fc8a72b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.add_column(sa.Column("decryption_info", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column("decryption_info")

alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py

@@ -0,0 +1,26 @@
+"""Add access_token and custom_mxid fields for puppets
+
+Revision ID: d5f7b8b4b456
+Revises: 6ca3d74d51e4
+Create Date: 2018-07-20 12:09:30.277960
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d5f7b8b4b456"
+down_revision = "6ca3d74d51e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
+    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("custom_mxid")
+        batch_op.drop_column("access_token")

alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py

@@ -0,0 +1,71 @@
+"""Add matrix-nio state store to main db
+
+Revision ID: dff56c93da8d
+Revises: d3c922a6acd2
+Create Date: 2020-03-31 22:04:04.014048
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'dff56c93da8d'
+down_revision = 'd3c922a6acd2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('nio_account',
+    sa.Column('user_id', sa.String(length=255), nullable=False),
+    sa.Column('device_id', sa.String(length=255), nullable=False),
+    sa.Column('shared', sa.Boolean(), nullable=False),
+    sa.Column('sync_token', sa.Text(), nullable=False),
+    sa.Column('account', sa.LargeBinary(), nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id')
+    )
+    op.create_table('nio_device_key',
+    sa.Column('user_id', sa.String(length=255), nullable=False),
+    sa.Column('device_id', sa.String(length=255), nullable=False),
+    sa.Column('display_name', sa.String(length=255), nullable=False),
+    sa.Column('deleted', sa.Boolean(), nullable=False),
+    sa.Column('keys', sa.PickleType(), nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id')
+    )
+    op.create_table('nio_megolm_inbound_session',
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('sender_key', sa.String(length=255), nullable=False),
+    sa.Column('fp_key', sa.String(length=255), nullable=False),
+    sa.Column('room_id', sa.String(length=255), nullable=False),
+    sa.Column('session', sa.LargeBinary(), nullable=False),
+    sa.Column('forwarded_chains', sa.PickleType(), nullable=False),
+    sa.PrimaryKeyConstraint('session_id')
+    )
+    op.create_table('nio_olm_session',
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('sender_key', sa.String(length=255), nullable=False),
+    sa.Column('session', sa.LargeBinary(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('last_used', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('session_id')
+    )
+    op.create_table('nio_outgoing_key_request',
+    sa.Column('request_id', sa.String(length=255), nullable=False),
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('room_id', sa.String(length=255), nullable=False),
+    sa.Column('algorithm', sa.String(length=255), nullable=False),
+    sa.PrimaryKeyConstraint('request_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('nio_outgoing_key_request')
+    op.drop_table('nio_olm_session')
+    op.drop_table('nio_megolm_inbound_session')
+    op.drop_table('nio_device_key')
+    op.drop_table('nio_account')
+    # ### end Alembic commands ###

docker-run.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# Define functions.
+function fixperms {
+	chown -R $UID:$GID /data /opt/mautrix-telegram
+}
+
+cd /opt/mautrix-telegram
+
+# Replace database path in config.
+sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
+
+if [ -f /data/mx-state.json ]; then
+	ln -s /data/mx-state.json
+fi
+
+if [ ! -f /data/config.yaml ]; then
+	cp example-config.yaml /data/config.yaml
+	echo "Didn't find a config file."
+	echo "Copied default config file to /data/config.yaml"
+	echo "Modify that config file to your liking."
+	echo "Start the container again after that to generate the registration file."
+	fixperms
+	exit
+fi
+
+if [ ! -f /data/registration.yaml ]; then
+	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
+	echo "Didn't find a registration file."
+	echo "Generated one for you."
+	echo "Copy that over to synapses app service directory."
+	fixperms
+	exit
+fi
+
+# Check that database is in the right state
+alembic -x config=/data/config.yaml upgrade head
+
+fixperms
+exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

docker/root/etc/s6.d/.s6-svscan/finish

@@ -1 +0,0 @@
-#!/bin/sh

docker/root/etc/s6.d/mautrix-telegram/finish

@@ -1,2 +0,0 @@
-#!/bin/bash
-s6-svscanctl -t /etc/s6.d

docker/root/etc/s6.d/mautrix-telegram/run

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Define functions
-function fixperms {
-	chown -R ${UID}:${GID} /data /opt/mautrixtelegram
-}
-
-
-# Go into env
-cd /opt/mautrixtelegram
-source .venv/bin/activate
-
-# Replace database path in alembic.ini
-sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" alembic.ini
-sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
-
-# Check that database is in the right state
-alembic upgrade head
-
-if [[ ! -f /data/config.yaml ]]; then
-	cp example-config.yaml /data/config.yaml
-	echo "Didn't find a config file."
-	echo "Copied default config file to /data/config.yaml"
-	echo "Modify that config file to your liking."
-	echo "Start the container again after that to generate the registration file."
-	fixperms
-	exit
-fi
-
-if [[ ! -f /data/registration.yaml ]]; then
-	python -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
-	echo "Didn't find a registration file."
-	echo "Generated ode for you."
-	echo "Copy that over to synapses app service directory."
-	fixperms
-	exit
-fi
-
-export FFMPEG_BINARY=/usr/bin/ffmpeg
-
-fixperms
-exec su-exec ${UID}:${GID} python -m mautrix_telegram -c /data/config.yaml

example-config.yaml

@@ -1,162 +0,0 @@
-# Homeserver details
-homeserver:
-    # The address that this appservice can use to connect to the homeserver.
-    address: https://matrix.org
-    # The domain of the homeserver (for MXIDs, etc).
-    domain: matrix.org
-    # Whether or not to verify the SSL certificate of the homeserver.
-    # Only applies if address starts with https://
-    verify_ssl: true
-
-# Application service host/registration related details
-# Changing these values requires regeneration of the registration.
-appservice:
-    # The protocol the homeserver should use when connecting to this appservice.
-    # Usually "http" or "https".
-    protocol: http
-
-    # The hostname and port where the homeserver can find this appservice.
-    hostname: localhost
-    port: 8080
-
-    # The full URI to the database.
-    database: sqlite:///mautrix-telegram.db
-
-    # Public part of web server for out-of-Matrix interaction with the bridge.
-    # Used for things like login if the user wants to make sure the 2FA password isn't stored in
-    # the HS database.
-    public:
-        # Whether or not the public-facing endpoints should be enabled.
-        enabled: true
-        # The prefix to use in the public-facing endpoints.
-        prefix: /public
-        # The base URL where the public-facing endpoints are available. The prefix is not added
-        # implicitly.
-        external: https://example.com/public
-
-    # Whether or not to enable debug messages in the console.
-    debug: true
-
-    # The unique ID of this appservice.
-    id: telegram
-    # Username of the appservice bot.
-    bot_username: telegrambot
-    bot_displayname: Telegram bridge bot
-
-    # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
-    as_token: "This value is generated when generating the registration"
-    hs_token: "This value is generated when generating the registration"
-
-# Bridge config
-bridge:
-    # Localpart template of MXIDs for Telegram users.
-    # {userid} is replaced with the user ID of the Telegram user.
-    username_template: "telegram_{userid}"
-    # Localpart template of room aliases for Telegram portal rooms.
-    # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} )
-    alias_template: "telegram_{groupname}"
-    # Displayname template for Telegram users.
-    # {displayname} is replaced with the display name of the Telegram user.
-    displayname_template: "{displayname} (Telegram)"
-
-    # Set the preferred order of user identifiers which to use in the Matrix puppet display name.
-    # In the (hopefully unlikely) scenario that none of the given keys are found, the numeric user
-    # ID is used.
-    #
-    # If the bridge is working properly, a phone number or an username should always be known, but
-    # the other one can very well be empty.
-    #
-    # Valid keys:
-    #   "full name"          (First and/or last name)
-    #   "full name reversed" (Last and/or first name)
-    #   "first name"
-    #   "last name"
-    #   "username"
-    #   "phone number"
-    displayname_preference:
-    - full name
-    - username
-    - phone number
-
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: false
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
-    # Whether or not Matrix bot messages (type m.notice) should be bridged.
-    bridge_notices: true
-    # Whether to bridge Telegram bot messages as m.notices or m.texts.
-    bot_messages_as_notices: true
-    # Maximum number of members to sync per portal when starting up. Other members will be
-    # synced when they send messages. The maximum is 10000, after which the Telegram server
-    # will not send any more members.
-    # Defaults to no local limit (-> limited to 10000 by server)
-    max_initial_member_sync: -1
-    # The maximum number of simultaneous Telegram deletions to handle.
-    # A large number of simultaneous redactions could put strain on your homeserver.
-    max_telegram_delete: 10
-    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
-    # login website (see appservice.public config section)
-    allow_matrix_login: true
-    # Use inline images instead of m.image to make rich captions possible.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
-    inline_images: false
-    # Whether or not to bridge plaintext highlights.
-    # Only enable this if your displayname_template has some static part that the bridge can use to
-    # reliably identify what is a plaintext highlight.
-    plaintext_highlights: false
-    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
-    public_portals: true
-    # Whether to send stickers as the new native m.sticker type or normal m.images.
-    # Old versions of Riot don't support the new type at all.
-    # Remember that proper sticker support always requires Pillow to convert webp into png.
-    native_stickers: true
-    # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
-    # WARNING: Probably buggy, might get stuck in infinite loop.
-    catch_up: false
-
-    filter:
-        # Filter mode to use. Either "blacklist" or "whitelist".
-        # If the mode is "blacklist", the listed chats will never be bridged. An empty blacklist disables the filter.
-        # If the mode is "whitelist", only the listed chats can be bridged.
-        # Direct chats are not affected.
-        mode: blacklist
-        # The list of group/channel IDs to filter.
-        list: []
-
-    # The prefix for commands. Only required in non-management rooms.
-    command_prefix: "!tg"
-
-    # Permissions for using the bridge.
-    # Permitted values:
-    #   relaybot - Only use the bridge via the relaybot, no access to commands.
-    #       full - Full access to use the bridge via relaybot or logging in with Telegram account.
-    #      admin - Full access to use the bridge and some extra administration commands.
-    # Permitted keys:
-    #        * - All Matrix users
-    #   domain - All users on that homeserver
-    #     mxid - Specific user
-    permissions:
-        "*": "relaybot"
-        "example.com": "full"
-        "public.example.com": "full"
-        "@admin:example.com": "admin"
-
-    # Options related to the message relay Telegram bot.
-    relaybot:
-        # Whether or not to allow creating portals from Telegram.
-        authless_portals: true
-        # Whether or not to allow Telegram group admins to use the bot commands.
-        whitelist_group_admins: true
-        # List of usernames/user IDs who are also allowed to use the bot commands.
-        whitelist:
-        - myusername
-        - 12345678
-
-# Telegram config
-telegram:
-    # Get your own API keys at https://my.telegram.org/apps
-    api_id: 12345
-    api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
-    # (Optional) Create your own bot at https://t.me/BotFather
-    bot_token: disabled

helm/mautrix-telegram/.editorconfig

@@ -0,0 +1,2 @@
+[*.{yaml,yml}]
+indent_size = 2

helm/mautrix-telegram/.gitignore

@@ -0,0 +1 @@
+charts/*

helm/mautrix-telegram/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm/mautrix-telegram/Chart.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+name: mautrix-telegram
+version: 0.1.0
+appVersion: "0.7.0"
+description: A Matrix-Telegram hybrid puppeting/relaybot bridge.
+keywords:
+  - matrix
+  - bridge
+  - telegram
+maintainers:
+  - name: Tulir Asokan
+    email: tulir@maunium.net
+sources:
+  - https://github.com/tulir/mautrix-telegram

helm/mautrix-telegram/requirements.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: postgresql
+  repository: https://kubernetes-charts.storage.googleapis.com/
+  version: 6.5.0
+digest: sha256:85139e9d4207e49c11c5f84d7920d0135cffd3d427f3f3638d4e51258990de2a
+generated: "2019-10-23T22:11:37.005827507+03:00"

helm/mautrix-telegram/requirements.yaml

@@ -0,0 +1,5 @@
+dependencies:
+  - name: postgresql
+    version: 6.5.0
+    repository: https://kubernetes-charts.storage.googleapis.com/
+    condition: postgresql.enabled

helm/mautrix-telegram/templates/NOTES.txt

@@ -0,0 +1,21 @@
+Your registration file is below. Save it into a YAML file and give the path to that file to synapse:
+
+id: {{ .Values.appservice.id }}
+as_token: {{ .Values.appservice.asToken }}
+hs_token: {{ .Values.appservice.hsToken }}
+namespaces:
+    users:
+        - exclusive: true
+          regex: "@{{ .Values.bridge.username_template | replace "{userid}" ".+"}}:{{ .Values.homeserver.domain }}"
+          {{- if .Values.appservice.communityID }}
+          group_id: {{ .Values.appservice.communityID }}
+          {{- end }}
+    aliases:
+        - exclusive: true
+          regex: "@{{ .Values.bridge.alias_template | replace "{groupname}" ".+"}}:{{ .Values.homeserver.domain }}"
+          {{- if .Values.appservice.communityID }}
+          group_id: {{ .Values.appservice.communityID }}
+          {{- end }}
+url: {{ .Values.appservice.address }}
+sender_localpart: {{ .Values.appservice.botUsername }}
+rate_limited: false

helm/mautrix-telegram/templates/_helpers.tpl

@@ -0,0 +1,55 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mautrix-telegram.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mautrix-telegram.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mautrix-telegram.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "mautrix-telegram.labels" -}}
+app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
+helm.sh/chart: {{ include "mautrix-telegram.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mautrix-telegram.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "mautrix-telegram.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}

helm/mautrix-telegram/templates/configmap.yaml

@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "mautrix-telegram.fullname" . }}
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    app.kubernetes.io/name: {{ template "mautrix-telegram.name" . }}
+data:
+  config.yaml: |
+    homeserver:
+      address: {{ .Values.homeserver.address }}
+      domain: {{ .Values.homeserver.domain }}
+      verify_ssl: {{ .Values.homeserver.verifySSL }}
+
+    appservice:
+      address: http://{{ include "mautrix-telegram.fullname" . }}:{{ .Values.service.port }}
+
+      hostname: 0.0.0.0
+      port: {{ .Values.service.port }}
+      max_body_size: {{ .Values.appservice.maxBodySize }}
+
+      {{- if .Values.postgresql.enabled }}
+      database: "postgres://postgres:{{ .Values.postgresql.postgresqlPassword }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.postgresqlDatabase }}"
+      {{- else }}
+      database: {{ .Values.appservice.database | quote }}
+      {{- end }}
+
+      public:
+        {{- toYaml .Values.appservice.public | nindent 8 }}
+
+      provisioning:
+        {{- toYaml .Values.appservice.provisioning | nindent 8 }}
+
+      id: {{ .Values.appservice.id }}
+      bot_username: {{ .Values.appservice.botUsername }}
+      bot_displayname: {{ .Values.appservice.botDisplayname }}
+      bot_avatar:  {{ .Values.appservice.botAvatar }}
+
+      community_id: {{ .Values.appservice.communityID }}
+
+      as_token: {{ .Values.appservice.asToken }}
+      hs_token: {{ .Values.appservice.hsToken }}
+
+    metrics:
+      {{- toYaml .Values.metrics | nindent 6 }}
+
+    bridge:
+      {{- toYaml .Values.bridge | nindent 6 }}
+
+    telegram:
+      {{- toYaml .Values.telegram | nindent 6 }}
+
+    logging:
+      {{- toYaml .Values.logging | nindent 6 }}
+  registration.yaml: ""

helm/mautrix-telegram/templates/deployment.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "mautrix-telegram.fullname" . }}
+  labels:
+    {{- include "mautrix-telegram.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    {{- if .Values.podAnnotations }}
+    annotations:
+      {{- toYaml .Values.podAnnotations | nindent 6 }}
+    {{- end }}
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      serviceAccountName: {{ template "mautrix-telegram.serviceAccountName" . }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          volumeMounts:
+            - mountPath: /data
+              name: config-volume
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /_matrix/mau/live
+              port: http
+            initialDelaySeconds: 60
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /_matrix/mau/ready
+              port: http
+            initialDelaySeconds: 60
+            periodSeconds: 5
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ template "mautrix-telegram.fullname" . }}
+
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

helm/mautrix-telegram/templates/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "mautrix-telegram.fullname" . }}
+  labels:
+{{ include "mautrix-telegram.labels" . | indent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}

helm/mautrix-telegram/templates/serviceaccount.yaml

@@ -0,0 +1,8 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "mautrix-telegram.serviceAccountName" . }}
+  labels:
+{{ include "mautrix-telegram.labels" . | indent 4 }}
+{{- end -}}

helm/mautrix-telegram/values.yaml

@@ -0,0 +1,141 @@
+image:
+  repository: dock.mau.dev/tulir/mautrix-telegram
+  tag: latest
+  pullPolicy: IfNotPresent
+
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:
+
+service:
+  type: ClusterIP
+  port: 29317
+
+resources: {}
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+# Postgres pod configs
+postgresql:
+  enabled: true
+  postgresqlDatabase: mxtg
+  postgresqlPassword: SET TO RANDOM STRING
+  persistence:
+    size: 2Gi
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 100m
+
+# Homeserver details
+homeserver:
+  # The address that this appservice can use to connect to the homeserver.
+  address: https://example.com
+  # The domain of the homeserver (for MXIDs, etc).
+  domain: example.com
+  # Whether or not to verify the SSL certificate of the homeserver.
+  # Only applies if address starts with https://
+  verifySSL: true
+
+# Application service host/registration related details
+# Changing these values requires regeneration of the registration.
+appservice:
+  # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+  # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+  maxBodySize: 1
+
+  # Public part of web server for out-of-Matrix interaction with the bridge.
+  # Used for things like login if the user wants to make sure the 2FA password isn't stored in
+  # the HS database.
+  public:
+    # Whether or not the public-facing endpoints should be enabled.
+    enabled: true
+    # The prefix to use in the public-facing endpoints.
+    prefix: /public
+    # The base URL where the public-facing endpoints are available. The prefix is not added
+    # implicitly.
+    external: https://example.com/public
+
+  # Provisioning API part of the web server for automated portal creation and fetching information.
+  # Used by things like Dimension (https://dimension.t2bot.io/).
+  provisioning:
+    # Whether or not the provisioning API should be enabled.
+    enabled: true
+    # The prefix to use in the provisioning API endpoints.
+    prefix: /_matrix/provision/v1
+    # The shared secret to authorize users of the API.
+    shared_secret: SET TO RANDOM STRING
+
+  id: telegram
+  botUsername: telegrambot
+  # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+  # to leave display name/avatar as-is.
+  botDisplayname: Telegram bridge bot
+  botAvatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
+
+  # Community ID for bridged users (changes registration file) and rooms.
+  # Must be created manually.
+  communityID: false
+
+  # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
+  asToken: SET TO RANDOM STRING
+  hsToken: SET TO RANDOM STRING
+
+# The keys below can be used to override the configs in the base config:
+# https://github.com/tulir/mautrix-telegram/blob/master/example-config.yaml
+# Note that the "appservice" and "homeserver" sections are above and slightly different than the base.
+
+# Bridge config
+bridge:
+  # Localpart template of MXIDs for Telegram users.
+  # {userid} is replaced with the user ID of the Telegram user.
+  username_template: "telegram_{userid}"
+  # Localpart template of room aliases for Telegram portal rooms.
+  # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} )
+  alias_template: "telegram_{groupname}"
+
+  # Permissions for using the bridge.
+  # Permitted values:
+  #   relaybot - Only use the bridge via the relaybot, no access to commands.
+  #       user - Relaybot level + access to commands to create bridges.
+  #  puppeting - User level + logging in with a Telegram account.
+  #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
+  #      admin - Full access to use the bridge and some extra administration commands.
+  # Permitted keys:
+  #        * - All Matrix users
+  #   domain - All users on that homeserver
+  #     mxid - Specific user
+  permissions:
+    "*": "relaybot"
+    "public.example.com": "user"
+    "example.com": "full"
+    "@admin:example.com": "admin"
+
+# Prometheus telemetry config.
+metrics:
+  enabled: false
+  listen_port: 8000
+
+# Telegram config
+telegram:
+  # Get your own API keys at https://my.telegram.org/apps
+  api_id: 12345
+  api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
+  # (Optional) Create your own bot at https://t.me/BotFather
+  # bot_token: 123456789:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.2.0rc2"
+__version__ = "0.9.0rc3"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,107 +13,121 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import argparse
-import sys
-import logging
-import asyncio
-
-import sqlalchemy as sql
-from sqlalchemy import orm
+from typing import Optional
 
 from alchemysession import AlchemySessionContainer
-from mautrix_appservice import AppService
 
-from .base import Base
-from .config import Config, DictWithRecursion
-from .matrix import MatrixHandler
+from mautrix.types import UserID, RoomID
+from mautrix.bridge import Bridge
+from mautrix.util.db import Base
 
-from .db import init as init_db
+from .web.provisioning import ProvisioningAPI
+from .web.public import PublicBridgeWebsite
+from .commands.manhole import ManholeState
 from .abstract_user import init as init_abstract_user
-from .user import init as init_user, User
-from .bot import init as init_bot
-from .portal import init as init_portal
-from .puppet import init as init_puppet
-from .formatter import init as init_formatter
-from .public import PublicBridgeWebsite
+from .bot import Bot, init as init_bot
+from .config import Config
 from .context import Context
+from .db import init as init_db
+from .formatter import init as init_formatter
+from .matrix import MatrixHandler
+from .portal import Portal, init as init_portal
+from .puppet import Puppet, init as init_puppet
+from .user import User, init as init_user
+from .version import version, linkified_version
 
-log = logging.getLogger("mau")
-time_formatter = logging.Formatter("[%(asctime)s] [%(levelname)s@%(name)s] %(message)s")
-handler = logging.StreamHandler()
-handler.setFormatter(time_formatter)
-log.addHandler(handler)
+try:
+    import prometheus_client as prometheus
+except ImportError:
+    prometheus = None
 
-parser = argparse.ArgumentParser(
-    description="A Matrix-Telegram puppeting bridge.",
-    prog="python -m mautrix-telegram")
-parser.add_argument("-c", "--config", type=str, default="config.yaml",
-                    metavar="<path>", help="the path to your config file")
-parser.add_argument("-b", "--base-config", type=str, default="example-config.yaml",
-                    metavar="<path>", help="the path to the example config "
-                                           "(for automatic config updates)")
-parser.add_argument("-g", "--generate-registration", action="store_true",
-                    help="generate registration and quit")
-parser.add_argument("-r", "--registration", type=str, default="registration.yaml",
-                    metavar="<path>", help="the path to save the generated registration to")
-args = parser.parse_args()
 
-config = Config(args.config, args.registration, args.base_config)
-config.load()
-config.update()
+class TelegramBridge(Bridge):
+    module = "mautrix_telegram"
+    name = "mautrix-telegram"
+    command = "python -m mautrix-telegram"
+    description = "A Matrix-Telegram puppeting bridge."
+    repo_url = "https://github.com/tulir/mautrix-telegram"
+    real_user_content_key = "net.maunium.telegram.puppet"
+    version = version
+    markdown_version = linkified_version
+    config_class = Config
+    matrix_class = MatrixHandler
 
-if args.generate_registration:
-    config.generate_registration()
-    config.save()
-    print(f"Registration generated and saved to {config.registration_path}")
-    sys.exit(0)
+    config: Config
+    session_container: AlchemySessionContainer
+    bot: Bot
+    manhole: Optional[ManholeState]
 
-if config["appservice.debug"]:
-    telethon_log = logging.getLogger("telethon")
-    telethon_log.addHandler(handler)
-    telethon_log.setLevel(logging.DEBUG)
-    log.setLevel(logging.DEBUG)
-    log.debug("Debug messages enabled.")
+    def prepare_db(self) -> None:
+        super().prepare_db()
+        init_db(self.db)
+        self.session_container = AlchemySessionContainer(
+            engine=self.db, table_base=Base, session=False,
+            table_prefix="telethon_", manage_tables=False)
 
-db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
-db_factory = orm.sessionmaker(bind=db_engine)
-db_session = orm.scoping.scoped_session(db_factory)
-Base.metadata.bind = db_engine
+    def _prepare_website(self, context: Context) -> None:
+        if self.config["appservice.public.enabled"]:
+            public_website = PublicBridgeWebsite(self.loop)
+            self.az.app.add_subapp(self.config["appservice.public.prefix"], public_website.app)
+            context.public_website = public_website
 
-telethon_session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
-                                                     table_base=Base, table_prefix="telethon_",
-                                                     manage_tables=False)
+        if self.config["appservice.provisioning.enabled"]:
+            provisioning_api = ProvisioningAPI(context)
+            self.az.app.add_subapp(self.config["appservice.provisioning.prefix"],
+                                   provisioning_api.app)
+            context.provisioning_api = provisioning_api
 
-loop = asyncio.get_event_loop()
+    def prepare_bridge(self) -> None:
+        self.bot = init_bot(self.config)
+        context = Context(self.az, self.config, self.loop, self.session_container, self, self.bot)
+        self._prepare_website(context)
+        self.matrix = context.mx = MatrixHandler(context)
+        self.manhole = None
 
-appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
-                     config["appservice.as_token"], config["appservice.hs_token"],
-                     config["appservice.bot_username"], log="mau.as", loop=loop,
-                     verify_ssl=config["homeserver.verify_ssl"])
+        init_abstract_user(context)
+        init_formatter(context)
+        init_portal(context)
+        self.add_startup_actions(init_puppet(context))
+        self.add_startup_actions(init_user(context))
+        if self.bot:
+            self.add_startup_actions(self.bot.start())
+        if self.config["bridge.resend_bridge_info"]:
+            self.add_startup_actions(self.resend_bridge_info())
 
-context = Context(appserv, db_session, config, loop, None, None, telethon_session_container)
+    async def resend_bridge_info(self) -> None:
+        self.config["bridge.resend_bridge_info"] = False
+        self.config.save()
+        self.log.info("Re-sending bridge info state event to all portals")
+        for portal in Portal.all():
+            await portal.update_bridge_info()
+        self.log.info("Finished re-sending bridge info state events")
 
-if config["appservice.public.enabled"]:
-    public = PublicBridgeWebsite(loop)
-    appserv.app.add_subapp(config.get("appservice.public.prefix", "/public"), public.app)
+    def prepare_stop(self) -> None:
+        for puppet in Puppet.by_custom_mxid.values():
+            puppet.stop()
+        self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
+        if self.manhole:
+            self.manhole.close()
+            self.manhole = None
 
-with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
-    init_db(db_session)
-    init_abstract_user(context)
-    context.bot = init_bot(context)
-    context.mx = MatrixHandler(context)
-    init_formatter(context)
-    init_portal(context)
-    init_puppet(context)
-    startup_actions = init_user(context) + [start, context.mx.init_as_bot()]
+    async def get_user(self, user_id: UserID, create: bool = True) -> User:
+        user = User.get_by_mxid(user_id, create=create)
+        if user:
+            await user.ensure_started()
+        return user
 
-    if context.bot:
-        startup_actions.append(context.bot.start())
+    async def get_portal(self, room_id: RoomID) -> Portal:
+        return Portal.get_by_mxid(room_id)
 
-    try:
-        loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
-        loop.run_forever()
-    except KeyboardInterrupt:
-        for user in User.by_tgid.values():
-            user.stop()
-        sys.exit(0)
+    async def get_puppet(self, user_id: UserID, create: bool = False) -> Puppet:
+        return await Puppet.get_by_mxid(user_id, create=create)
+
+    async def get_double_puppet(self, user_id: UserID) -> Puppet:
+        return await Puppet.get_by_custom_mxid(user_id)
+
+    def is_bridge_ghost(self, user_id: UserID) -> bool:
+        return bool(Puppet.get_id_from_mxid(user_id))
+
+
+TelegramBridge().run()

mautrix_telegram/abstract_user.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,103 +13,230 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Optional, Union, Dict, Type, Any, TYPE_CHECKING
+from abc import ABC, abstractmethod
+import asyncio
+import logging
 import platform
-import os
+import time
 
-from telethon.tl.types import *
-from mautrix_appservice import MatrixRequestError
+from telethon.sessions import Session
+from telethon.network import (ConnectionTcpMTProxyRandomizedIntermediate, ConnectionTcpFull,
+                              Connection)
+from telethon.tl.patched import MessageService, Message
+from telethon.tl.types import (
+    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdateChatPinnedMessage,
+    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
+    UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
+    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
+    UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
+    UpdateUserTyping, User, UserStatusOffline, UserStatusOnline, UpdateReadHistoryInbox,
+    UpdateReadChannelInbox, MessageEmpty)
+
+from mautrix.types import UserID, PresenceState
+from mautrix.errors import MatrixError
+from mautrix.appservice import AppService
+from mautrix.util.logging import TraceLogger
+from mautrix.util.opt_prometheus import Histogram, Counter
+from alchemysession import AlchemySessionContainer
 
-from .tgclient import MautrixTelegramClient
-from .db import Message as DBMessage
 from . import portal as po, puppet as pu, __version__
+from .db import Message as DBMessage
+from .types import TelegramID
+from .tgclient import MautrixTelegramClient
 
-config = None
+if TYPE_CHECKING:
+    from .context import Context
+    from .config import Config
+    from .bot import Bot
+
+config: Optional['Config'] = None
 # Value updated from config in init()
-MAX_DELETIONS = 10
+MAX_DELETIONS: int = 10
+
+UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                      UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
+UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
+
+UPDATE_TIME = Histogram("bridge_telegram_update", "Time spent processing Telegram updates",
+                        ("update_type",))
+UPDATE_ERRORS = Counter("bridge_telegram_update_error",
+                        "Number of fatal errors while handling Telegram updates", ("update_type",))
 
 
-class AbstractUser:
-    session_container = None
-    loop = None
-    log = None
-    db = None
-    az = None
+class AbstractUser(ABC):
+    session_container: AlchemySessionContainer = None
+    loop: asyncio.AbstractEventLoop = None
+    log: TraceLogger
+    az: AppService
+    relaybot: Optional['Bot']
+    ignore_incoming_bot_events: bool = True
 
-    def __init__(self):
-        self.connected = False
+    client: Optional[MautrixTelegramClient]
+    mxid: Optional[UserID]
+
+    tgid: Optional[TelegramID]
+    username: Optional['str']
+    is_bot: bool
+
+    is_relaybot: bool
+
+    puppet_whitelisted: bool
+    whitelisted: bool
+    relaybot_whitelisted: bool
+    matrix_puppet_whitelisted: bool
+    is_admin: bool
+
+    def __init__(self) -> None:
+        self.is_admin = False
+        self.matrix_puppet_whitelisted = False
+        self.puppet_whitelisted = False
         self.whitelisted = False
+        self.relaybot_whitelisted = False
         self.client = None
-        self.tgid = None
-        self.mxid = None
         self.is_relaybot = False
+        self.is_bot = False
 
-    async def _init_client(self):
+    @property
+    def connected(self) -> bool:
+        return self.client and self.client.is_connected()
+
+    @property
+    def _proxy_settings(self) -> Tuple[Type[Connection], Optional[Tuple[Any, ...]]]:
+        proxy_type = config["telegram.proxy.type"].lower()
+        connection = ConnectionTcpFull
+        connection_data = (config["telegram.proxy.address"],
+                           config["telegram.proxy.port"],
+                           config["telegram.proxy.rdns"],
+                           config["telegram.proxy.username"],
+                           config["telegram.proxy.password"])
+        if proxy_type == "disabled":
+            connection_data = None
+        elif proxy_type == "socks4":
+            connection_data = (1,) + connection_data
+        elif proxy_type == "socks5":
+            connection_data = (2,) + connection_data
+        elif proxy_type == "http":
+            connection_data = (3,) + connection_data
+        elif proxy_type == "mtproxy":
+            connection = ConnectionTcpMTProxyRandomizedIntermediate
+            connection_data = (connection_data[0], connection_data[1], connection_data[4])
+
+        return connection, connection_data
+
+    def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
-        self.session = self.session_container.new_session(self.name)
-        self.client = MautrixTelegramClient(session=self.session,
-                                            api_id=config["telegram.api_id"],
-                                            api_hash=config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device,
-                                            report_errors=False)
-        await self.client.add_event_handler(self._update_catch)
 
-    async def update(self, update):
+        self.session = self.session_container.new_session(self.name)
+        if config["telegram.server.enabled"]:
+            self.session.set_dc(config["telegram.server.dc"],
+                                config["telegram.server.ip"],
+                                config["telegram.server.port"])
+
+        if self.is_relaybot:
+            base_logger = logging.getLogger("telethon.relaybot")
+        else:
+            base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
+
+        device = config["telegram.device_info.device_model"]
+        sysversion = config["telegram.device_info.system_version"]
+        appversion = config["telegram.device_info.app_version"]
+        connection, proxy = self._proxy_settings
+
+        assert isinstance(self.session, Session)
+
+        self.client = MautrixTelegramClient(
+            session=self.session,
+
+            api_id=config["telegram.api_id"],
+            api_hash=config["telegram.api_hash"],
+
+            app_version=__version__ if appversion == "auto" else appversion,
+            system_version=(MautrixTelegramClient.__version__
+                            if sysversion == "auto" else sysversion),
+            device_model=(f"{platform.system()} {platform.release()}"
+                          if device == "auto" else device),
+
+            timeout=config["telegram.connection.timeout"],
+            connection_retries=config["telegram.connection.retries"],
+            retry_delay=config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
+            request_retries=config["telegram.connection.request_retries"],
+            connection=connection,
+            proxy=proxy,
+            raise_last_call_error=True,
+
+            loop=self.loop,
+            base_logger=base_logger
+        )
+        self.client.add_event_handler(self._update_catch)
+
+    @abstractmethod
+    async def update(self, update: TypeUpdate) -> bool:
         return False
 
-    async def post_login(self):
+    @abstractmethod
+    async def post_login(self) -> None:
         raise NotImplementedError()
 
-    async def _update_catch(self, update):
+    @abstractmethod
+    async def register_portal(self, portal: po.Portal) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
+        raise NotImplementedError()
+
+    async def _update_catch(self, update: TypeUpdate) -> None:
+        start_time = time.time()
+        update_type = type(update).__name__
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
-            self.log.exception("Failed to handle Telegram update")
-
-    async def _get_dialogs(self, limit=None):
-        dialogs = await self.client.get_dialogs(limit=limit)
-        return [dialog.entity for dialog in dialogs if (
-            not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden))
-            and not (isinstance(dialog.entity, Chat)
-                     and (dialog.entity.deactivated or dialog.entity.left)))]
+            self.log.exception(f"Failed to handle Telegram update {update}")
+            UPDATE_ERRORS.labels(update_type=update_type).inc()
+        UPDATE_TIME.labels(update_type=update_type).observe(time.time() - start_time)
 
     @property
-    def name(self):
+    @abstractmethod
+    def name(self) -> str:
         raise NotImplementedError()
 
-    @property
-    def logged_in(self):
-        return self.client and self.client.is_user_authorized()
+    async def is_logged_in(self) -> bool:
+        return (self.client and self.client.is_connected()
+                and await self.client.is_user_authorized())
 
-    @property
-    def has_full_access(self):
-        return self.logged_in and self.whitelisted
+    async def has_full_access(self, allow_bot: bool = False) -> bool:
+        return (self.puppet_whitelisted
+                and (not self.is_bot or allow_bot)
+                and await self.is_logged_in())
 
-    async def start(self):
+    async def start(self, delete_unless_authenticated: bool = False) -> 'AbstractUser':
         if not self.client:
-            await self._init_client()
-        self.connected = await self.client.connect()
-
-    async def ensure_started(self, even_if_no_session=False):
-        if not self.whitelisted:
-            return self
-        elif not self.connected and (even_if_no_session or os.path.exists(f"{self.name}.session")):
-            return await self.start()
+            self._init_client()
+        await self.client.connect()
+        self.log.debug(f"{'Bot' if self.is_relaybot else self.mxid} connected: {self.connected}")
         return self
 
-    def stop(self):
-        self.client.disconnect()
+    async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
+        if self.connected:
+            return self
+        if even_if_no_session or self.session_container.has_session(self.mxid):
+            self.log.debug("Starting client due to ensure_started"
+                           f"(even_if_no_session={even_if_no_session})")
+            await self.start(delete_unless_authenticated=not even_if_no_session)
+        return self
+
+    async def stop(self) -> None:
+        await self.client.disconnect()
         self.client = None
-        self.connected = False
 
     # region Telegram update handling
 
-    async def _update(self, update):
+    async def _update(self, update: TypeUpdate) -> None:
+        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+                              loop=self.loop)
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -122,177 +248,239 @@ class AbstractUser:
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
-        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
+        elif isinstance(update, UpdateChatParticipantAdmin):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, UpdateChannelPinnedMessage):
+        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
         elif isinstance(update, UpdateReadHistoryOutbox):
             await self.update_read_receipt(update)
+        elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
+            await self.update_own_read_receipt(update)
         else:
-            self.log.debug("Unhandled update: %s", update)
+            self.log.trace("Unhandled update: %s", update)
 
-    async def update_pinned_messages(self, update):
-        portal = po.Portal.get_by_tgid(update.channel_id)
+    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
+                                                         UpdateChatPinnedMessage]) -> None:
+        if isinstance(update, UpdateChatPinnedMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        else:
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id)
+            await portal.receive_telegram_pin_id(update.id, self.tgid)
 
-    async def update_participants(self, update):
-        portal = po.Portal.get_by_tgid(update.participants.chat_id)
+    @staticmethod
+    async def update_participants(update: UpdateChatParticipants) -> None:
+        portal = po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
         if portal and portal.mxid:
-            await portal.update_telegram_participants(update.participants.participants)
+            await portal.update_power_levels(update.participants.participants)
 
-    async def update_read_receipt(self, update):
+    async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
         if not isinstance(update.peer, PeerUser):
             self.log.debug("Unexpected read receipt peer: %s", update.peer)
             return
 
-        portal = po.Portal.get_by_tgid(update.peer.user_id, self.tgid)
+        portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
         if not portal or not portal.mxid:
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.query.get((update.max_id, self.tgid))
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
         if not message:
             return
 
-        puppet = pu.Puppet.get(update.peer.user_id)
+        puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
-    async def update_admin(self, update):
-        # TODO duplication not checked
-        portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        if isinstance(update, UpdateChatAdmins):
-            await portal.set_telegram_admins_enabled(update.enabled)
-        elif isinstance(update, UpdateChatParticipantAdmin):
-            await portal.set_telegram_admin(update.user_id)
-        else:
-            self.log.warning("Unexpected admin status update: %s", update)
+    async def update_own_read_receipt(self, update: Union[UpdateReadHistoryInbox,
+                                                          UpdateReadChannelInbox]) -> None:
+        puppet = pu.Puppet.get(self.tgid)
+        if not puppet.is_real_user:
+            return
 
-    async def update_typing(self, update):
-        if isinstance(update, UpdateUserTyping):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+        if isinstance(update, UpdateReadChannelInbox):
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update.peer, PeerChat):
+            portal = po.Portal.get_by_tgid(TelegramID(update.peer.chat_id))
+        elif isinstance(update.peer, PeerUser):
+            portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
         else:
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        sender = pu.Puppet.get(update.user_id)
+            self.log.debug("Unexpected own read receipt peer: %s", update.peer)
+            return
+
+        if not portal or not portal.mxid:
+            return
+
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), tg_space, edit_index=-1)
+        if not message:
+            return
+
+        await puppet.intent.mark_read(portal.mxid, message.mxid)
+
+    async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
+        # TODO duplication not checked
+        portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        if not portal or not portal.mxid:
+            return
+
+        await portal.set_telegram_admin(TelegramID(update.user_id))
+
+    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+        if isinstance(update, UpdateUserTyping):
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
+        else:
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+
+        if not portal or not portal.mxid:
+            return
+
+        sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
-    async def update_others_info(self, update):
-        # TODO duplication not checked
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update, UpdateUserName):
-            if await puppet.update_displayname(self, update):
-                puppet.save()
-        elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
-                puppet.save()
-        else:
-            self.log.warning("Unexpected other user info update: %s", update)
+    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
+                                     ) -> None:
+        try:
+            users = (entity for entity in entities.values() if isinstance(entity, User))
+            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
+            await asyncio.gather(*[puppet.try_update_info(self, info)
+                                   for puppet, info in puppets if puppet])
+        except Exception:
+            self.log.exception("Failed to handle entity updates")
 
-    async def update_status(self, update):
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update.status, UserStatusOnline):
-            await puppet.intent.set_presence("online")
-        elif isinstance(update.status, UserStatusOffline):
-            await puppet.intent.set_presence("offline")
+    async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
+        # TODO duplication not checked
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
+        if isinstance(update, UpdateUserName):
+            puppet.username = update.username
+            if await puppet.update_displayname(self, update):
+                await puppet.save()
+        elif isinstance(update, UpdateUserPhoto):
+            if await puppet.update_avatar(self, update.photo):
+                await puppet.save()
         else:
-            self.log.warning("Unexpected user status update: %s", update)
+            self.log.warning(f"Unexpected other user info update: {type(update)}")
+
+    async def update_status(self, update: UpdateUserStatus) -> None:
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
+        if isinstance(update.status, UserStatusOnline):
+            await puppet.default_mxid_intent.set_presence(PresenceState.ONLINE)
+        elif isinstance(update.status, UserStatusOffline):
+            await puppet.default_mxid_intent.set_presence(PresenceState.OFFLINE)
+        else:
+            self.log.warning(f"Unexpected user status update: type({update})")
         return
 
-    def get_message_details(self, update):
+    def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
+                                                                  Optional[pu.Puppet],
+                                                                  Optional[po.Portal]]:
         if isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-            sender = pu.Puppet.get(update.from_id)
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+            if not portal:
+                self.log.warning(f"Received message in chat with unknown type {update.chat_id}")
+            sender = pu.Puppet.get(TelegramID(update.from_id))
         elif isinstance(update, UpdateShortMessage):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
             sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
         elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
                                  UpdateEditMessage, UpdateEditChannelMessage)):
             update = update.message
-            if isinstance(update.to_id, PeerUser) and not update.out:
-                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
+            if isinstance(update, MessageEmpty):
+                return update, None, None
+            portal = po.Portal.get_by_entity(update.peer_id, receiver_id=self.tgid)
+            if update.out:
+                sender = pu.Puppet.get(self.tgid)
+            elif isinstance(update.from_id, PeerUser):
+                sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
             else:
-                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
-            sender = pu.Puppet.get(update.from_id) if update.from_id else None
+                sender = None
         else:
-            self.log.warning(
-                f"Unexpected message type in User#get_message_details: {type(update)}")
+            self.log.warning("Unexpected message type in User#get_message_details: "
+                             f"{type(update)}")
             return update, None, None
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal, message):
+    async def _try_redact(message: DBMessage) -> None:
+        portal = po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
             await portal.main_intent.redact(message.mx_room, message.mxid)
-        except MatrixRequestError:
+        except MatrixError:
             pass
 
-    async def delete_message(self, update):
+    async def delete_message(self, update: UpdateDeleteMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        for message in update.messages:
-            message = DBMessage.query.get((message, self.tgid))
-            if not message:
-                continue
-            self.db.delete(message)
-            number_left = DBMessage.query.filter(DBMessage.mxid == message.mxid,
-                                                 DBMessage.mx_room == message.mx_room).count()
-            if number_left == 0:
-                portal = po.Portal.get_by_mxid(message.mx_room)
-                await self._try_redact(portal, message)
-        self.db.commit()
+        for message_id in update.messages:
+            for message in DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    await self._try_redact(message)
 
-    async def delete_channel_message(self, update):
+    async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        portal = po.Portal.get_by_tgid(update.channel_id)
+        channel_id = TelegramID(update.channel_id)
+
+        for message_id in update.messages:
+            for message in DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
+                message.delete()
+                await self._try_redact(message)
+
+    async def update_message(self, original_update: UpdateMessage) -> None:
+        update, sender, portal = self.get_message_details(original_update)
         if not portal:
             return
+        elif portal and not portal.allow_bridging:
+            self.log.debug(f"Ignoring message in portal {portal.tgid_log} (bridging disallowed)")
+            return
 
-        for message in update.messages:
-            message = DBMessage.query.get((message, portal.tgid))
-            if not message:
-                continue
-            self.db.delete(message)
-            await self._try_redact(portal, message)
-        self.db.commit()
+        if self.is_relaybot:
+            if update.is_private:
+                if not config["bridge.relaybot.private_chat.invite"]:
+                    self.log.debug(f"Ignoring private message to bot from {sender.id}")
+                    return
+            elif not portal.mxid and config["bridge.relaybot.ignore_unbridged_group_chat"]:
+                self.log.debug("Ignoring message received by bot"
+                               f" in unbridged chat {portal.tgid_log}")
+                return
 
-    async def update_message(self, original_update):
-        update, sender, portal = self.get_message_details(original_update)
+        if ((self.ignore_incoming_bot_events and self.relaybot
+             and sender and sender.id == self.relaybot.tgid)):
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
+            return
+
+        await portal.backfill_lock.wait(update.id)
 
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.debug(f"Ignoring action %s to %s by %d", update.action,
-                               portal.tgid_log,
-                               sender.id)
+                self.log.trace(f"Received %s in %s by %d, unregistering portal...",
+                               update.action, portal.tgid_log, sender.id)
+                await self.unregister_portal(update.action.chat_id, update.action.chat_id)
+                await self.register_portal(portal)
                 return
-            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
+            self.log.trace("Handling action %s to %s by %d", update.action, portal.tgid_log,
                            sender.id)
             return await portal.handle_telegram_action(self, sender, update)
 
-        user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            if config["bridge.edits_as_replies"]:
-                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-                return await portal.handle_telegram_edit(self, sender, update)
-            return
-
-        self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
+            return await portal.handle_telegram_edit(self, sender, update)
         return await portal.handle_telegram_message(self, sender, update)
 
     # endregion
 
 
-def init(context):
+def init(context: 'Context') -> None:
     global config, MAX_DELETIONS
-    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, _ = context
-    AbstractUser.session_container = context.telethon_session_container
+    AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
+    AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
+    AbstractUser.session_container = context.session_container
     MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/base.py

@@ -1,2 +0,0 @@
-from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()

mautrix_telegram/bot.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,108 +13,129 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable
+from typing import Awaitable, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING
 import logging
-import re
 
-from telethon.tl.types import *
+from telethon.tl.patched import Message, MessageService
+from telethon.tl.types import (
+    ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
+    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser, PeerUser,
+    MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
 
+from mautrix.types import UserID
+
 from .abstract_user import AbstractUser
 from .db import BotChat
+from .types import TelegramID
 from . import puppet as pu, portal as po, user as u
 
-config = None
+if TYPE_CHECKING:
+    from .config import Config
+
+config: Optional['Config'] = None
 
 ReplyFunc = Callable[[str], Awaitable[Message]]
 
 
 class Bot(AbstractUser):
-    log = logging.getLogger("mau.bot")
-    mxid_regex = re.compile("@.+:.+")
+    log: logging.Logger = logging.getLogger("mau.user.bot")
 
-    def __init__(self, token: str):
+    token: str
+    chats: Dict[int, str]
+    tg_whitelist: List[int]
+    whitelist_group_admins: bool
+    _me_info: Optional[User]
+    _me_mxid: Optional[UserID]
+
+    def __init__(self, token: str) -> None:
         super().__init__()
         self.token = token
+        self.tgid = None
+        self.mxid = None
+        self.puppet_whitelisted = True
         self.whitelisted = True
+        self.relaybot_whitelisted = True
         self.username = None
         self.is_relaybot = True
-        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}
+        self.is_bot = True
+        self.chats = {}
         self.tg_whitelist = []
-        self.whitelist_group_admins = config["bridge.relaybot.whitelist_group_admins"] or False
+        self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
+                                       or False)
+        self._me_info = None
+        self._me_mxid = None
 
-    async def init_permissions(self):
+    async def get_me(self, use_cache: bool = True) -> Tuple[User, UserID]:
+        if not use_cache or not self._me_mxid:
+            self._me_info = await self.client.get_me()
+            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
+        return self._me_info, self._me_mxid
+
+    async def init_permissions(self) -> None:
         whitelist = config["bridge.relaybot.whitelist"] or []
-        for id in whitelist:
-            if isinstance(id, str):
-                entity = await self.client.get_input_entity(id)
+        for user_id in whitelist:
+            if isinstance(user_id, str):
+                entity = await self.client.get_input_entity(user_id)
                 if isinstance(entity, InputUser):
-                    id = entity.user_id
+                    user_id = entity.user_id
                 else:
-                    id = None
-            if isinstance(id, int):
-                self.tg_whitelist.append(id)
+                    user_id = None
+            if isinstance(user_id, int):
+                self.tg_whitelist.append(user_id)
 
-    async def start(self):
-        await super().start()
-        if not self.logged_in:
+    async def start(self, delete_unless_authenticated: bool = False) -> 'Bot':
+        self.chats = {chat.id: chat.type for chat in BotChat.all()}
+        await super().start(delete_unless_authenticated)
+        if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
         await self.post_login()
         return self
 
-    async def post_login(self):
+    async def post_login(self) -> None:
         await self.init_permissions()
         info = await self.client.get_me()
-        self.tgid = info.id
+        self.tgid = TelegramID(info.id)
         self.username = info.username
         self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
 
-        chat_ids = [id for id, type in self.chats.items() if type == "chat"]
+        chat_ids = [chat_id for chat_id, chat_type in self.chats.items() if chat_type == "chat"]
         response = await self.client(GetChatsRequest(chat_ids))
         for chat in response.chats:
             if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
-                self.remove_chat(chat.id)
+                self.remove_chat(TelegramID(chat.id))
 
-        channel_ids = [InputChannel(id, 0)
-                       for id, type in self.chats.items()
-                       if type == "channel"]
-        for id in channel_ids:
+        channel_ids = [InputChannel(chat_id, 0)
+                       for chat_id, chat_type in self.chats.items()
+                       if chat_type == "channel"]
+        for channel_id in channel_ids:
             try:
-                await self.client(GetChannelsRequest([id]))
+                await self.client(GetChannelsRequest([channel_id]))
             except (ChannelPrivateError, ChannelInvalidError):
-                self.remove_chat(id.channel_id)
+                self.remove_chat(TelegramID(channel_id.channel_id))
 
-        if config["bridge.catch_up"]:
-            try:
-                await self.client.catch_up()
-            except Exception:
-                self.log.exception("Failed to run catch_up() for bot")
-
-    def register_portal(self, portal: po.Portal):
+    async def register_portal(self, portal: po.Portal) -> None:
         self.add_chat(portal.tgid, portal.peer_type)
 
-    def unregister_portal(self, portal: po.Portal):
-        self.remove_chat(portal.tgid)
+    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
+        self.remove_chat(tgid)
 
-    def add_chat(self, id: int, type: str):
-        if id not in self.chats:
-            self.chats[id] = type
-            self.db.add(BotChat(id=id, type=type))
-            self.db.commit()
+    def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
+        if chat_id not in self.chats:
+            self.chats[chat_id] = chat_type
+            BotChat(id=TelegramID(chat_id), type=chat_type).insert()
 
-    def remove_chat(self, id: int):
+    def remove_chat(self, chat_id: TelegramID) -> None:
         try:
-            del self.chats[id]
+            del self.chats[chat_id]
         except KeyError:
             pass
-        existing_chat = BotChat.query.get(id)
-        if existing_chat:
-            self.db.delete(existing_chat)
-            self.db.commit()
+        BotChat.delete_by_id(chat_id)
 
-    async def _can_use_commands(self, chat, tgid):
+    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
@@ -127,25 +147,26 @@ class Bot(AbstractUser):
         if self.whitelist_group_admins:
             if isinstance(chat, PeerChannel):
                 p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
+                return isinstance(p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin))
             elif isinstance(chat, PeerChat):
                 chat = await self.client(GetFullChatRequest(chat.chat_id))
                 participants = chat.full_chat.participants.participants
                 for p in participants:
                     if p.user_id == tgid:
                         return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+        return False
 
-    async def check_can_use_commands(self, event: Message, reply: ReplyFunc):
-        if not await self._can_use_commands(event.to_id, event.from_id):
+    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
+        if not await self._can_use_commands(event.to_id, TelegramID(event.from_id)):
             await reply("You do not have the permission to use that command.")
             return False
         return True
 
-    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc):
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> Message:
         if not config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
-        if not portal.allow_bridging():
+        if not portal.allow_bridging:
             return await reply("This bridge doesn't allow bridging this chat.")
 
         await portal.create_matrix_room(self)
@@ -157,31 +178,38 @@ class Bot(AbstractUser):
                 return await reply(
                     "Portal is not public. Use `/invite <mxid>` to get an invite.")
 
-    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc, mxid: str):
-        if len(mxid) == 0:
+    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc,
+                                    mxid_input: UserID) -> Message:
+        if len(mxid_input) == 0:
             return await reply("Usage: `/invite <mxid>`")
         elif not portal.mxid:
             return await reply("Portal does not have Matrix room. "
                                "Create one with /portal first.")
-        if not self.mxid_regex.match(mxid):
+        if mxid_input[0] != '@' or mxid_input.find(':') < 2:
             return await reply("That doesn't look like a Matrix ID.")
-        user = await u.User.get_by_mxid(mxid).ensure_started()
-        if not user.whitelisted:
+        user = await u.User.get_by_mxid(mxid_input).ensure_started()
+        if not user.relaybot_whitelisted:
             return await reply("That user is not whitelisted to use the bridge.")
-        elif user.logged_in:
+        elif await user.is_logged_in():
             displayname = f"@{user.username}" if user.username else user.displayname
             return await reply("That user seems to be logged in. "
                                f"Just invite [{displayname}](tg://user?id={user.tgid})")
         else:
-            await portal.main_intent.invite(portal.mxid, user.mxid)
+            await portal.main_intent.invite_user(portal.mxid, user.mxid)
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
-    def handle_command_id(self, message: Message, reply: ReplyFunc):
+    @staticmethod
+    def handle_command_id(message: Message, reply: ReplyFunc) -> Awaitable[Message]:
         # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
         # chat is a normal group or a supergroup/channel when using the ID.
         if isinstance(message.to_id, PeerChannel):
             return reply(f"-100{message.to_id.channel_id}")
-        return reply(str(-message.to_id.chat_id))
+        elif isinstance(message.to_id, PeerChat):
+            return reply(str(-message.to_id.chat_id))
+        elif isinstance(message.to_id, PeerUser):
+            return reply(f"Your user ID is {message.from_id}.")
+        else:
+            return reply("Failed to find chat ID.")
 
     def match_command(self, text: str, command: str) -> bool:
         text = text.lower()
@@ -198,59 +226,73 @@ class Bot(AbstractUser):
 
         return False
 
-    async def handle_command(self, message: Message):
-        def reply(reply_text):
-            return self.client.send_message(message.to_id, reply_text, markdown=True,
-                                            reply_to=message.id)
+    async def handle_command(self, message: Message) -> None:
+        def reply(reply_text: str) -> Awaitable[Message]:
+            return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)
 
         text = message.message
 
-        if self.match_command(text, "id"):
-            return await self.handle_command_id(message, reply)
+        if self.match_command(text, "start"):
+            pcm = config["bridge.relaybot.private_chat.message"]
+            if pcm:
+                await reply(pcm)
+            return
+        elif self.match_command(text, "id"):
+            await self.handle_command_id(message, reply)
+            return
+        elif message.is_private:
+            return
 
         portal = po.Portal.get_by_entity(message.to_id)
 
-        if self.match_command(text, "portal"):
+        is_portal_cmd = self.match_command(text, "portal")
+        is_invite_cmd = self.match_command(text, "invite")
+        if is_portal_cmd or is_invite_cmd:
             if not await self.check_can_use_commands(message, reply):
                 return
-            await self.handle_command_portal(portal, reply)
-        elif self.match_command(text, "invite"):
-            if not await self.check_can_use_commands(message, reply):
-                return
-            try:
-                mxid = text[text.index(" ") + 1:]
-            except ValueError:
-                mxid = ""
-            await self.handle_command_invite(portal, reply, mxid=mxid)
+            if is_portal_cmd:
+                await self.handle_command_portal(portal, reply)
+            elif is_invite_cmd:
+                try:
+                    mxid = text[text.index(" ") + 1:]
+                except ValueError:
+                    mxid = ""
+                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
 
-    def handle_service_message(self, message: MessageService):
-        to_id = message.to_id
-        if isinstance(to_id, PeerChannel):
-            to_id = to_id.channel_id
-            type = "channel"
-        elif isinstance(to_id, PeerChat):
-            to_id = to_id.chat_id
-            type = "chat"
+    def handle_service_message(self, message: MessageService) -> None:
+        to_peer = message.to_id
+        if isinstance(to_peer, PeerChannel):
+            to_id = TelegramID(to_peer.channel_id)
+            chat_type = "channel"
+        elif isinstance(to_peer, PeerChat):
+            to_id = TelegramID(to_peer.chat_id)
+            chat_type = "chat"
         else:
             return
 
         action = message.action
         if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
-            self.add_chat(to_id, type)
+            self.add_chat(to_id, chat_type)
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
             self.remove_chat(to_id)
+        elif isinstance(action, MessageActionChatMigrateTo):
+            self.remove_chat(to_id)
+            self.add_chat(TelegramID(action.channel_id), "channel")
 
-    async def update(self, update):
+    async def update(self, update) -> bool:
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
-            return
+            return False
         if isinstance(update.message, MessageService):
-            return self.handle_service_message(update.message)
+            self.handle_service_message(update.message)
+            return False
 
         is_command = (isinstance(update.message, Message)
                       and update.message.entities and len(update.message.entities) > 0
-                      and isinstance(update.message.entities[0], MessageEntityBotCommand))
+                      and isinstance(update.message.entities[0], MessageEntityBotCommand)
+                      and update.message.entities[0].offset == 0)
         if is_command:
-            return await self.handle_command(update.message)
+            await self.handle_command(update.message)
+        return False
 
     def is_in_chat(self, peer_id) -> bool:
         return peer_id in self.chats
@@ -260,9 +302,9 @@ class Bot(AbstractUser):
         return "bot"
 
 
-def init(context):
+def init(cfg: 'Config') -> Optional[Bot]:
     global config
-    config = context.config
+    config = cfg
     token = config["telegram.bot_token"]
     if token and not token.lower().startswith("disable"):
         return Bot(token)

mautrix_telegram/commands/__init__.py

@@ -1,2 +1,8 @@
-from .handler import command_handler, CommandHandler, CommandEvent
-from . import clean_rooms, auth, meta, telegram, portal
+from .handler import (command_handler, CommandHandler, CommandProcessor, CommandEvent,
+                      SECTION_AUTH, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT,
+                      SECTION_MISC, SECTION_ADMIN)
+from . import portal, telegram, matrix_auth, manhole
+
+__all__ = ["command_handler", "CommandHandler", "CommandProcessor", "CommandEvent",
+           "SECTION_AUTH", "SECTION_MISC", "SECTION_ADMIN", "SECTION_CREATING_PORTALS",
+           "SECTION_PORTAL_MANAGEMENT"]

mautrix_telegram/commands/auth.py

@@ -1,232 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import asyncio
-
-from telethon.errors import *
-
-from . import command_handler
-from .. import puppet as pu
-from ..util import format_duration
-
-
-@command_handler(needs_auth=False)
-async def ping(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
-    me = await evt.sender.client.get_me()
-    if me:
-        return await evt.reply(f"You're logged in as @{me.username}")
-    else:
-        return await evt.reply("You're not logged in.")
-
-
-@command_handler()
-async def ping_bot(evt):
-    if not evt.tgbot:
-        return await evt.reply("Telegram message relay bot not configured.")
-    bot_info = await evt.tgbot.client.get_me()
-    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
-    displayname = bot_info.first_name
-    return await evt.reply("Telegram message relay bot is active: "
-                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
-                           "To use the bot, simply invite it to a portal room.")
-
-
-@command_handler(needs_auth=False, management_only=True)
-def register(evt):
-    return evt.reply("Not yet implemented.")
-
-
-@command_handler(needs_auth=False, management_only=True)
-async def register(evt):
-    if evt.sender.logged_in:
-        return await evt.reply("You are already logged in.")
-    elif len(evt.args) < 1:
-        return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
-
-    phone_number = evt.args[0]
-    if len(evt.args) == 2:
-        full_name = evt.args[1], ""
-    else:
-        full_name = " ".join(evt.args[1:-1]), evt.args[-1]
-
-    await request_code(evt, phone_number, {
-        "next": enter_code_register,
-        "action": "Register",
-        "full_name": full_name,
-    })
-
-
-async def enter_code_register(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
-    try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        first_name, last_name = evt.sender.command_status["full_name"]
-        user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully registered to Telegram.")
-    except PhoneNumberOccupiedError:
-        return await evt.reply("That phone number has already been registered. "
-                               "You can log in with `$cmdprefix+sp login`.")
-    except FirstNameInvalidError:
-        return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
-    except PhoneCodeExpiredError:
-        return await evt.reply(
-            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`.")
-    except PhoneCodeInvalidError:
-        return await evt.reply("Invalid phone code.")
-    except Exception:
-        evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code. "
-                               "Check console for more details.")
-
-
-@command_handler(needs_auth=False, management_only=True)
-async def login(evt):
-    if evt.sender.logged_in:
-        return await evt.reply("You are already logged in.")
-
-    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
-    if allow_matrix_login:
-        evt.sender.command_status = {
-            "next": enter_phone,
-            "action": "Login",
-        }
-
-    if evt.config["appservice.public.enabled"]:
-        prefix = evt.config["appservice.public.external"]
-        url = f"{prefix}/login?mxid={evt.sender.mxid}"
-        if evt.config.get("bridge.allow_matrix_login", True):
-            return await evt.reply("\n\n".join((
-                "This bridge instance allows you to log in inside or outside Matrix.",
-                "If you would like to log in within Matrix, please send your phone number here.",
-                f"If you would like to log in outside of Matrix, [click here]({url}).")))
-        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
-                               f"Please visit [the login page]({url}) to log in.")
-    elif allow_matrix_login:
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number here to start the login process.")
-    return await evt.reply("This bridge instance has been configured to not allow logging in.")
-
-
-async def request_code(evt, phone_number, next_status):
-    ok = False
-    try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        await evt.sender.client.sign_in(phone_number)
-        ok = True
-        return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
-    except PhoneNumberAppSignupForbiddenError:
-        return await evt.reply(
-            "Your phone number does not allow 3rd party apps to sign in.")
-    except PhoneNumberFloodError:
-        return await evt.reply(
-            "Your phone number has been temporarily blocked for flooding. "
-            "The ban is usually applied for around a day.")
-    except FloodWaitError as e:
-        return await evt.reply(
-            "Your phone number has been temporarily blocked for flooding. "
-            f"Please wait for {format_duration(e.seconds)} before trying again.")
-    except PhoneNumberBannedError:
-        return await  evt.reply("Your phone number has been banned from Telegram.")
-    except PhoneNumberUnoccupiedError:
-        return await  evt.reply("That phone number has not been registered. "
-                                "Please register with `$cmdprefix+sp register <phone>`.")
-    except Exception:
-        evt.log.exception("Error requesting phone code")
-        return await evt.reply("Unhandled exception while requesting code. "
-                               "Check console for more details.")
-    finally:
-        evt.sender.command_status = next_status if ok else None
-
-
-@command_handler(needs_auth=False)
-async def enter_phone(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone <phone>`")
-    elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
-
-    phone_number = evt.args[0]
-    await request_code(evt, phone_number, {
-        "next": enter_code,
-        "action": "Login",
-    })
-
-
-@command_handler(needs_auth=False)
-async def enter_code(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
-    elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
-    try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        user = await evt.sender.client.sign_in(code=evt.args[0])
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PhoneCodeExpiredError:
-        return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
-    except PhoneCodeInvalidError:
-        return await evt.reply("Invalid phone code.")
-    except SessionPasswordNeededError:
-        evt.sender.command_status = {
-            "next": enter_password,
-            "action": "Login (password entry)",
-        }
-        return await evt.reply("Your account has two-factor authentication. "
-                               "Please send your password here.")
-    except Exception:
-        evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code. "
-                               "Check console for more details.")
-
-
-@command_handler(needs_auth=False)
-async def enter_password(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
-    elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
-    try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        user = await evt.sender.client.sign_in(password=" ".join(evt.args))
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PasswordHashInvalidError:
-        return await evt.reply("Incorrect password.")
-    except Exception:
-        evt.log.exception("Error sending password")
-        return await evt.reply("Unhandled exception while sending password. "
-                               "Check console for more details.")
-
-
-@command_handler(needs_auth=False)
-async def logout(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
-    if await evt.sender.log_out():
-        return await evt.reply("Logged out successfully.")
-    return await evt.reply("Failed to log out.")

mautrix_telegram/commands/clean_rooms.py

@@ -1,170 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix_appservice import MatrixRequestError
-
-from . import command_handler
-from .. import puppet as pu, portal as po
-
-
-async def _find_rooms(intent):
-    management_rooms = []
-    unidentified_rooms = []
-    portals = []
-    empty_portals = []
-
-    rooms = await intent.get_joined_rooms()
-    for room in rooms:
-        portal = po.Portal.get_by_mxid(room)
-        if not portal:
-            try:
-                members = await intent.get_room_members(room)
-            except MatrixRequestError:
-                members = []
-            if len(members) == 2:
-                other_member = members[0] if members[0] != intent.mxid else members[1]
-                if pu.Puppet.get_id_from_mxid(other_member):
-                    unidentified_rooms.append(room)
-                else:
-                    management_rooms.append((room, other_member))
-            else:
-                unidentified_rooms.append(room)
-        else:
-            members = await portal.get_authenticated_matrix_users()
-            if len(members) == 0:
-                empty_portals.append(portal)
-            else:
-                portals.append(portal)
-
-    return management_rooms, unidentified_rooms, portals, empty_portals
-
-
-@command_handler(needs_admin=True, needs_auth=False, name="clean-rooms")
-async def clean_rooms(evt):
-    if not evt.is_management:
-        return await evt.reply("`clean-rooms` is a particularly spammy command. Please don't "
-                               "run it in non-management rooms.")
-
-    management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
-
-    reply = ["#### Management rooms (M)"]
-    reply += ([f"{n+1}. [M{n+1}](https://matrix.to/#/{room}) (with {other_member}"
-               for n, (room, other_member) in enumerate(management_rooms)]
-              or ["No management rooms found."])
-    reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [P{n+1}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(portals)]
-              or ["No active portal rooms found."])
-    reply.append("#### Unidentified rooms (U)")
-    reply += ([f"{n+1}. [U{n+1}](https://matrix.to/#/{room})"
-               for n, room in enumerate(unidentified_rooms)]
-              or ["No unidentified rooms found."])
-    reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [E{n}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(empty_portals)]
-              or ["No inactive portal rooms found."])
-
-    reply += ["#### Usage",
-              ("To clean the recommended set of rooms (unidentified & inactive portals), "
-               "type `$cmdprefix+sp clean-recommended`"),
-              "",
-              ("To clean other groups of rooms, type `$cmdprefix+sp clean-groups <letters>` "
-               "where `letters` are the first letters of the group names (M, A, U, I)"),
-              "",
-              ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
-               "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name."),
-              "",
-              ("Please note that you will have to re-run `$cmdprefix+sp cleanrooms` "
-               "between each use of the commands above.")]
-
-    evt.sender.command_status = {
-        "next": lambda clean_evt: set_rooms_to_clean(clean_evt, management_rooms,
-                                                     unidentified_rooms, portals, empty_portals),
-        "action": "Room cleaning",
-    }
-
-    return await evt.reply("\n".join(reply))
-
-
-async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals, empty_portals):
-    command = evt.args[0]
-    rooms_to_clean = []
-    if command == "clean-recommended":
-        rooms_to_clean = empty_portals + unidentified_rooms
-    elif command == "clean-groups":
-        if len(evt.args) < 2:
-            return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1]
-        if "M" in groups_to_clean:
-            rooms_to_clean += management_rooms
-        if "A" in groups_to_clean:
-            rooms_to_clean += portals
-        if "U" in groups_to_clean:
-            rooms_to_clean += unidentified_rooms
-        if "I" in groups_to_clean:
-            rooms_to_clean += empty_portals
-    elif command == "clean-range":
-        try:
-            range = evt.args[1]
-            group, range = range[0], range[1:]
-            start, end = range.split("-")
-            start, end = int(start), int(end)
-            if group == "M":
-                group = management_rooms
-            elif group == "A":
-                group = portals
-            elif group == "U":
-                group = unidentified_rooms
-            elif group == "I":
-                group = empty_portals
-            else:
-                raise ValueError("Unknown group")
-            rooms_to_clean = group[start - 1:end]
-        except (KeyError, ValueError):
-            return await evt.reply(
-                "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
-    else:
-        return await evt.reply(f"Unknown room cleaning action `{command}`. "
-                               "Use `$cmdprefix+sp cancel` to cancel room "
-                               "cleaning.")
-
-    evt.sender.command_status = {
-        "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
-        "action": "Room cleaning",
-    }
-    await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type"
-                    "`$cmdprefix+sp confirm-clean`.")
-
-
-async def execute_room_cleanup(evt, rooms_to_clean):
-    if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
-        await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
-                        "This might take a while.")
-        cleaned = 0
-        for room in rooms_to_clean:
-            if isinstance(room, po.Portal):
-                await room.cleanup_and_delete()
-                cleaned += 1
-            elif isinstance(room, str):
-                await po.Portal.cleanup_room(evt.az.intent, room, message="Room deleted")
-                cleaned += 1
-        evt.sender.command_status = None
-        await evt.reply(f"{cleaned} rooms cleaned up successfully.")
-    else:
-        await evt.reply("Room cleaning cancelled.")

mautrix_telegram/commands/handler.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,90 +13,111 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import markdown
-import logging
+"""This module contains classes handling commands issued by Matrix users."""
+from typing import Awaitable, Callable, List, Optional, NamedTuple, Any
 
 from telethon.errors import FloodWaitError
 
+from mautrix.types import RoomID, EventID, MessageEventContent
+from mautrix.bridge.commands import (HelpSection, CommandEvent as BaseCommandEvent,
+                                     CommandHandler as BaseCommandHandler,
+                                     CommandProcessor as BaseCommandProcessor,
+                                     CommandHandlerFunc, command_handler as base_command_handler)
+
 from ..util import format_duration
-
-command_handlers = {}
+from .. import user as u, context as c, portal as po
 
 
-def command_handler(needs_auth=True, management_only=False, needs_admin=False, name=None):
-    def decorator(func):
-        def wrapper(evt):
-            if management_only and not evt.is_management:
-                return evt.reply(f"`{evt.command}` is a restricted command:"
-                                 "you may only run it in management rooms.")
-            elif needs_auth and not evt.sender.logged_in:
-                return evt.reply("This command requires you to be logged in.")
-            elif needs_admin and not evt.sender.is_admin:
-                return evt.reply("This is command requires administrator privileges.")
-            return func(evt)
-
-        command_handlers[name or func.__name__.replace("_", "-")] = wrapper
-        return wrapper
-
-    return decorator
+class HelpCacheKey(NamedTuple):
+    is_management: bool
+    is_portal: bool
+    puppet_whitelisted: bool
+    matrix_puppet_whitelisted: bool
+    is_admin: bool
+    is_logged_in: bool
 
 
-class CommandEvent:
-    def __init__(self, handler, room, sender, command, args, is_management, is_portal):
-        self.az = handler.az
-        self.log = handler.log
-        self.loop = handler.loop
-        self.tgbot = handler.tgbot
-        self.config = handler.config
-        self.command_prefix = handler.command_prefix
-        self.room_id = room
-        self.sender = sender
-        self.command = command
-        self.args = args
-        self.is_management = is_management
-        self.is_portal = is_portal
-
-    def reply(self, message, allow_html=False, render_markdown=True):
-        message = message.replace("$cmdprefix+sp ",
-                                  "" if self.is_management else f"{self.command_prefix} ")
-        message = message.replace("$cmdprefix", self.command_prefix)
-        html = None
-        if render_markdown:
-            html = markdown.markdown(message, safe_mode="escape" if allow_html else False)
-        elif allow_html:
-            html = message
-        return self.az.intent.send_notice(self.room_id, message, html=html)
+SECTION_AUTH = HelpSection("Authentication", 10, "")
+SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
+SECTION_PORTAL_MANAGEMENT = HelpSection("Portal management", 30, "")
+SECTION_MISC = HelpSection("Miscellaneous", 40, "")
+SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 
-class CommandHandler:
-    log = logging.getLogger("mau.commands")
+class CommandEvent(BaseCommandEvent):
+    sender: u.User
+    portal: po.Portal
 
-    def __init__(self, context):
-        self.az, self.db, self.config, self.loop, self.tgbot = context
-        self.command_prefix = self.config["bridge.command_prefix"]
+    def __init__(self, processor: 'CommandProcessor', room_id: RoomID, event_id: EventID,
+                 sender: u.User, command: str, args: List[str], content: MessageEventContent,
+                 portal: Optional['po.Portal'], is_management: bool, has_bridge_bot: bool) -> None:
+        super().__init__(processor, room_id, event_id, sender, command, args, content,
+                         portal, is_management, has_bridge_bot)
+        self.bridge = processor.bridge
+        self.tgbot = processor.tgbot
+        self.config = processor.config
+        self.public_website = processor.public_website
 
-    # region Utility functions for handling commands
+    @property
+    def print_error_traceback(self) -> bool:
+        return self.sender.is_admin
 
-    async def handle(self, room, sender, command, args, is_management, is_portal):
-        evt = CommandEvent(self, room, sender, command, args,
-                           is_management, is_portal)
-        orig_command = command
-        command = command.lower()
+    async def get_help_key(self) -> HelpCacheKey:
+        return HelpCacheKey(self.is_management, self.portal is not None,
+                            self.sender.puppet_whitelisted, self.sender.matrix_puppet_whitelisted,
+                            self.sender.is_admin, await self.sender.is_logged_in())
+
+
+class CommandHandler(BaseCommandHandler):
+    name: str
+
+    needs_puppeting: bool
+    needs_matrix_puppeting: bool
+
+    def __init__(self, handler: Callable[[CommandEvent], Awaitable[EventID]],
+                 management_only: bool, name: str, help_text: str, help_args: str,
+                 help_section: HelpSection, needs_auth: bool, needs_puppeting: bool,
+                 needs_matrix_puppeting: bool, needs_admin: bool) -> None:
+        super().__init__(handler, management_only, name, help_text, help_args, help_section,
+                         needs_auth=needs_auth, needs_puppeting=needs_puppeting,
+                         needs_matrix_puppeting=needs_matrix_puppeting, needs_admin=needs_admin)
+
+    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        if self.needs_puppeting and not evt.sender.puppet_whitelisted:
+            return "This command requires puppeting privileges."
+        elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
+            return "This command requires Matrix puppeting privileges."
+        return await super().get_permission_error(evt)
+
+    def has_permission(self, key: HelpCacheKey) -> bool:
+        return (super().has_permission(key) and
+                (not self.needs_puppeting or key.puppet_whitelisted) and
+                (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted))
+
+
+def command_handler(_func: Optional[CommandHandlerFunc] = None, *, needs_auth: bool = True,
+                    needs_puppeting: bool = True, needs_matrix_puppeting: bool = False,
+                    needs_admin: bool = False, management_only: bool = False,
+                    name: Optional[str] = None, help_text: str = "", help_args: str = "",
+                    help_section: HelpSection = None) -> Callable[[CommandHandlerFunc],
+                                                                  CommandHandler]:
+    return base_command_handler(
+        _func, _handler_class=CommandHandler, name=name, help_text=help_text, help_args=help_args,
+        help_section=help_section, management_only=management_only, needs_auth=needs_auth,
+        needs_admin=needs_admin, needs_puppeting=needs_puppeting,
+        needs_matrix_puppeting=needs_matrix_puppeting)
+
+
+class CommandProcessor(BaseCommandProcessor):
+    def __init__(self, context: c.Context) -> None:
+        super().__init__(event_class=CommandEvent, bridge=context.bridge)
+        self.tgbot = context.bot
+        self.public_website = context.public_website
+
+    @staticmethod
+    async def _run_handler(handler: Callable[[CommandEvent], Awaitable[Any]], evt: CommandEvent
+                           ) -> Any:
         try:
-            command = command_handlers[command]
-        except KeyError:
-            if sender.command_status and "next" in sender.command_status:
-                args.insert(0, orig_command)
-                evt.command = ""
-                command = sender.command_status["next"]
-            else:
-                command = command_handlers["unknown-command"]
-        try:
-            await command(evt)
+            return await handler(evt)
         except FloodWaitError as e:
             return await evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
-        except Exception:
-            self.log.exception("Fatal error handling command "
-                               f"{evt.command} {' '.join(args)} from {sender.mxid}")
-            return await evt.reply("Fatal error while handling command. "
-                                   "Check logs for more details.")

mautrix_telegram/commands/manhole.py

@@ -0,0 +1,128 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Set, Callable
+import asyncio
+import sys
+import os
+
+from attr import dataclass
+
+from telethon import __version__ as __telethon_version__
+
+from mautrix import __version__ as __mautrix_version__
+from mautrix.types import UserID
+from mautrix.errors import MatrixConnectionError
+from mautrix.util.manhole import start_manhole
+
+from .. import __version__
+from . import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@dataclass
+class ManholeState:
+    server: asyncio.AbstractServer
+    opened_by: UserID
+    close: Callable[[], None]
+    whitelist: Set[int]
+
+
+@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
+                 help_text="Open a manhole into the bridge.", help_args="<_uid..._>")
+async def open_manhole(evt: CommandEvent) -> None:
+    if not evt.config["manhole.enabled"]:
+        await evt.reply("The manhole has been disabled in the config.")
+        return
+    elif len(evt.args) == 0:
+        await evt.reply("**Usage:** `$cmdprefix+sp open-manhole <uid...>`")
+        return
+
+    whitelist = set()
+    whitelist_whitelist = evt.config["manhole.whitelist"]
+    for arg in evt.args:
+        try:
+            uid = int(arg)
+        except ValueError:
+            await evt.reply(f"{arg} is not an integer.")
+            return
+        if whitelist_whitelist and uid not in whitelist_whitelist:
+            await evt.reply(f"{uid} is not in the list of allowed UIDs.")
+            return
+        whitelist.add(uid)
+
+    if evt.bridge.manhole:
+        added = [uid for uid in whitelist
+                 if uid not in evt.bridge.manhole.whitelist]
+        evt.bridge.manhole.whitelist |= set(added)
+        if len(added) == 0:
+            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
+                            " and all the given UIDs are already whitelisted.")
+        else:
+            added_str = (f"{', '.join(str(uid) for uid in added[:-1])} and {added[-1]}"
+                         if len(added) > 1 else added[0])
+            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
+                            f". Added {added_str} to the whitelist.")
+            evt.log.info(f"{evt.sender.mxid} added {added_str} to the manhole whitelist.")
+        return
+
+    from ..portal import Portal
+    from ..puppet import Puppet
+    from ..user import User
+    namespace = {
+        "bridge": evt.bridge,
+        "User": User,
+        "Portal": Portal,
+        "Puppet": Puppet,
+    }
+    banner = (f"Python {sys.version} on {sys.platform}\n"
+              f"mautrix-telegram {__version__} with mautrix-python {__mautrix_version__} "
+              f"and Telethon {__telethon_version__}\n\nManhole opened by {evt.sender.mxid}\n")
+    path = evt.config["manhole.path"]
+
+    wl_list = list(whitelist)
+    whitelist_str = (f"{', '.join(str(uid) for uid in wl_list[:-1])} and {wl_list[-1]}"
+                     if len(wl_list) > 1 else wl_list[0])
+    evt.log.info(f"{evt.sender.mxid} opened a manhole with {whitelist_str} whitelisted.")
+    server, close = await start_manhole(path=path, banner=banner, namespace=namespace,
+                                        loop=evt.loop, whitelist=whitelist)
+    evt.bridge.manhole = ManholeState(server=server, opened_by=evt.sender.mxid, close=close,
+                                      whitelist=whitelist)
+    plrl = "s" if len(whitelist) != 1 else ""
+    await evt.reply(f"Opened manhole at unix://{path} with UID{plrl} {whitelist_str} whitelisted")
+    await server.wait_closed()
+    evt.bridge.manhole = None
+    try:
+        os.unlink(path)
+    except FileNotFoundError:
+        pass
+    evt.log.info(f"{evt.sender.mxid}'s manhole was closed.")
+    try:
+        await evt.reply("Your manhole was closed.")
+    except (AttributeError, MatrixConnectionError) as e:
+        evt.log.warning(f"Failed to send manhole close notification: {e}")
+
+
+@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
+                 help_text="Close an open manhole.")
+async def close_manhole(evt: CommandEvent) -> None:
+    if not evt.bridge.manhole:
+        await evt.reply("There is no open manhole.")
+        return
+
+    opened_by = evt.bridge.manhole.opened_by
+    evt.bridge.manhole.close()
+    evt.bridge.manhole = None
+    if opened_by != evt.sender.mxid:
+        await evt.reply(f"Closed manhole opened by {opened_by}")

mautrix_telegram/commands/matrix_auth.py

@@ -0,0 +1,113 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.types import EventID
+from mautrix.bridge import InvalidAccessToken, OnlyLoginSelf
+
+from . import command_handler, CommandEvent, SECTION_AUTH
+from .. import puppet as pu
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH, help_text="Revert your Telegram account's Matrix "
+                                                      "puppet to use the default Matrix account.")
+async def logout_matrix(evt: CommandEvent) -> EventID:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
+
+
+@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account.")
+async def login_matrix(evt: CommandEvent) -> EventID:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        token = evt.public_website.make_token(evt.sender.mxid, "/matrix-login")
+        url = f"{prefix}/matrix-login?token={token}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Pings the server with the stored matrix authentication.")
+async def ping_matrix(evt: CommandEvent) -> EventID:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    try:
+        await puppet.start()
+    except InvalidAccessToken:
+        return await evt.reply("Your access token is invalid.")
+    return await evt.reply("Your Matrix login is working.")
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True, help_section=SECTION_AUTH,
+                 help_text="Clear the Matrix sync token stored for your custom puppet.")
+async def clear_cache_matrix(evt: CommandEvent) -> EventID:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    try:
+        puppet.stop()
+        puppet.next_batch = None
+        await puppet.start()
+    except InvalidAccessToken:
+        return await evt.reply("Your access token is invalid.")
+    return await evt.reply("Cleared cache successfully.")
+
+
+async def enter_matrix_token(evt: CommandEvent) -> EventID:
+    evt.sender.command_status = None
+
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    try:
+        await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    except OnlyLoginSelf:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    except InvalidAccessToken:
+        return await evt.reply("Failed to verify access token.")
+    return await evt.reply("Replaced your Telegram account's Matrix puppet "
+                           f"with {puppet.custom_mxid}.")

mautrix_telegram/commands/meta.py

@@ -1,88 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from . import command_handler
-
-
-@command_handler(needs_auth=False)
-def cancel(evt):
-    if evt.sender.command_status:
-        action = evt.sender.command_status["action"]
-        evt.sender.command_status = None
-        return evt.reply(f"{action} cancelled.")
-    else:
-        return evt.reply("No ongoing command.")
-
-
-@command_handler(needs_auth=False)
-def unknown_command(evt):
-    return evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
-
-
-@command_handler(needs_auth=False)
-def help(evt):
-    if evt.is_management:
-        management_status = ("This is a management room: prefixing commands "
-                             "with `$cmdprefix` is not required.\n")
-    elif evt.is_portal:
-        management_status = ("**This is a portal room**: you must always "
-                             "prefix commands with `$cmdprefix`.\n"
-                             "Management commands will not be sent to Telegram.")
-    else:
-        management_status = ("**This is not a management room**: you must "
-                             "prefix commands with `$cmdprefix`.\n")
-    help = """\n
-#### Generic bridge commands
-**help**   - Show this help message.  
-**cancel** - Cancel an ongoing action (such as login).
-
-#### Authentication
-**login**  - Request an authentication code.  
-**logout** - Log out from Telegram.  
-**ping**   - Check if you're logged into Telegram.
-
-#### Miscellaneous things
-**search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users.  
-**sync** [`chats`|`contacts`|`me`]   - Synchronize your chat portals, contacts and/or own info.  
-**ping-bot**                         - Get info of the message relay Telegram bot.  
-**set-pl** <_level_> [_mxid_]        - Set a temporary power level without affecting Telegram.
-
-#### Initiating chats
-**pm** <_identifier_> - Open a private chat with the given Telegram user. The identifier is either
-                        the internal user ID, the username or the phone number.  
-**join** <_link_>     - Join a chat with an invite link.  
-**create** [_type_]   - Create a Telegram chat of the given type for the current Matrix room. The
-                        type is either `group`, `supergroup` or `channel` (defaults to `group`).
-
-#### Portal management  
-**upgrade**                 - Upgrade a normal Telegram group to a supergroup.  
-**invite-link**             - Get a Telegram invite link to the current chat.  
-**delete-portal**           - Remove all users from the current portal room and forget the portal.
-                              Only works for group chats; to delete a private chat portal, simply
-                              leave the room.  
-**unbridge**                - Remove puppets from the current portal room and forget the portal.  
-**bridge** [_id_]           - Bridge the current Matrix room to the Telegram chat with the given
-                              ID. The ID must be the prefixed version that you get with the `/id`
-                              command of the Telegram-side bot.  
-**group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash
-                             (`-`) as the name.  
-**clean-rooms**             - Clean up unused portal/management rooms.
-
-**filter** <`whitelist`|`blacklist`> <_chat ID_> - Allow or disallow bridging a specific chat.  
-**filter-mode** <`whitelist`|`blacklist`>      - Change whether the bridge will allow or disallow
-                                                 bridging rooms by default.
-"""
-    return evt.reply(management_status + help)

mautrix_telegram/commands/portal.py

@@ -1,426 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import asyncio
-
-from telethon.errors import *
-from mautrix_appservice import MatrixRequestError
-
-from .. import portal as po
-from . import command_handler, CommandEvent
-
-
-@command_handler(needs_admin=True, needs_auth=False, name="set-pl")
-async def set_power_level(evt: CommandEvent):
-    try:
-        level = int(evt.args[0])
-    except KeyError:
-        return await evt.reply("**Usage:** `$cmdprefix+sp set-power <level> [mxid]`")
-    except ValueError:
-        return await evt.reply("The level must be an integer.")
-    levels = await evt.az.intent.get_power_levels(evt.room_id)
-    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
-    levels["users"][mxid] = level
-    try:
-        await evt.az.intent.set_power_levels(evt.room_id, levels)
-    except MatrixRequestError:
-        evt.log.exception("Failed to set power level.")
-        return await evt.reply("Failed to set power level.")
-
-
-@command_handler()
-async def invite_link(evt: CommandEvent):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
-        return await evt.reply("You can't invite users to private chats.")
-
-    try:
-        link = await portal.get_invite_link(evt.sender)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to create an invite link.")
-
-
-async def _has_access_to(room, intent, sender, event, default=50):
-    if sender.is_admin:
-        return True
-    # Make sure the state store contains the power levels.
-    try:
-        await intent.get_power_levels(room)
-    except MatrixRequestError:
-        return False
-    return intent.state_store.has_power_level(room, sender.mxid,
-                                              event=f"net.maunium.telegram.{event}",
-                                              default=default)
-
-
-async def _get_portal_and_check_permission(evt, permission, action=None):
-    room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if not portal:
-        that_this = "This" if room_id == evt.room_id else "That"
-        return await evt.reply(f"{that_this} is not a portal room."), False
-
-    if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, permission):
-        action = action or f"{permission.replace('_', ' ')}s"
-        return await evt.reply(f"You do not have the permissions to {action} that portal."), False
-    return portal, True
-
-
-def _get_portal_murder_function(action, room_id, function, command, completed_message):
-    async def post_confirm(confirm):
-        confirm.sender.command_status = None
-        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
-            await function()
-            if confirm.room_id != room_id:
-                return await confirm.reply(completed_message)
-        else:
-            return await confirm.reply(f"{action} cancelled.")
-
-    return {
-        "next": post_confirm,
-        "action": action,
-    }
-
-
-@command_handler()
-async def delete_portal(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "delete_portal")
-    if not ok:
-        return
-
-    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
-                                                            portal.cleanup_and_delete, "delete",
-                                                            "Portal successfully deleted.")
-    return await evt.reply("Please confirm deletion of portal "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           f"to Telegram chat \"{portal.title}\" "
-                           "by typing `$cmdprefix+sp confirm-delete`")
-
-
-@command_handler()
-async def unbridge(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "unbridge_room")
-    if not ok:
-        return
-
-    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
-                                                            portal.unbridge, "unbridge",
-                                                            "Room successfully unbridged.")
-    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           "by typing `$cmdprefix+sp confirm-unbridge`")
-
-
-@command_handler()
-async def bridge(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** "
-                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
-    room_id = evt.args[1] if len(evt.args) > 1 else evt.room_id
-    that_this = "This" if room_id == evt.room_id else "That"
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if portal:
-        return await evt.reply(f"{that_this} room is already a portal room.")
-
-    if not await _has_access_to(room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply("You do not have the permissions to bridge that room.")
-
-    # The /id bot command provides the prefixed ID, so we assume
-    tgid = evt.args[0]
-    if tgid.startswith("-100"):
-        tgid = int(tgid[4:])
-        peer_type = "channel"
-    elif tgid.startswith("-"):
-        tgid = -int(tgid)
-        peer_type = "chat"
-    else:
-        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
-                               "If you did not get the ID using the `/id` bot command, please "
-                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
-                               "Bridging private chats to existing rooms is not allowed.")
-
-    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
-    if not portal.allow_bridging():
-        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
-                               "If you're the bridge admin, try"
-                               "`$cmdprefix+sp whitelist <Telegram chat ID>` first.")
-    if portal.mxid:
-        has_portal_message = (
-            "That Telegram chat already has a portal at "
-            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
-        if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-            return await evt.reply(f"{has_portal_message}"
-                                   "Additionally, you do not have the permissions to unbridge "
-                                   "that room.")
-        evt.sender.command_status = {
-            "next": confirm_bridge,
-            "action": "Room bridging",
-            "mxid": portal.mxid,
-            "bridge_to_mxid": room_id,
-            "tgid": portal.tgid,
-            "peer_type": portal.peer_type,
-        }
-        return await evt.reply(f"{has_portal_message}"
-                               "However, you have the permissions to unbridge that room.\n\n"
-                               "To delete that portal completely and continue bridging, use "
-                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
-                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
-                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
-    evt.sender.command_status = {
-        "next": confirm_bridge,
-        "action": "Room bridging",
-        "bridge_to_mxid": room_id,
-        "tgid": portal.tgid,
-        "peer_type": portal.peer_type,
-    }
-    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
-                           "chat to this room, use `$cmdprefix+sp continue`")
-
-
-async def cleanup_old_portal_while_bridging(evt, portal):
-    if not portal.mxid:
-        await evt.reply("The portal seems to have lost its Matrix room between you"
-                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                        "Continuing without touching previous Matrix room...")
-        return True, None
-    elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Portal deleted (moving to another room)")
-    elif evt.args[0] == "unbridge-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Room unbridged (portal moving to another room)",
-                                         puppets_only=True)
-    else:
-        await evt.reply(
-            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
-            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
-            "continue` to either delete or unbridge the existing room (respectively) and "
-            "continue with the bridging.\n\n"
-            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
-        return False, None
-
-
-async def confirm_bridge(evt: CommandEvent):
-    status = evt.sender.command_status
-    try:
-        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
-        bridge_to_mxid = status["bridge_to_mxid"]
-    except KeyError:
-        evt.sender.command_status = None
-        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
-                               "This shouldn't happen unless you're messing with the command "
-                               "handler code.")
-    if "mxid" in status:
-        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
-        if not ok:
-            return
-        elif coro:
-            asyncio.ensure_future(coro, loop=evt.loop)
-            await evt.reply("Cleaning up previous portal room...")
-    elif portal.mxid:
-        evt.sender.command_status = None
-        return await evt.reply("The portal seems to have created a Matrix room between you "
-                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                               "Please start over by calling the bridge command again.")
-    elif evt.args[0] != "continue":
-        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
-                               "`$cmdprefix+sp cancel` to cancel.")
-
-    portal.mxid = bridge_to_mxid
-    portal.title, portal.about, levels = await _get_initial_state(evt)
-    portal.photo_id = ""
-    portal.save()
-
-    entity = await evt.sender.client.get_entity(portal.peer)
-    direct = False
-    asyncio.ensure_future(portal.update_matrix_room(evt.sender, entity, direct, levels=levels),
-                          loop=evt.loop)
-
-    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")
-
-
-async def _get_initial_state(evt: CommandEvent):
-    state = await evt.az.intent.get_room_state(evt.room_id)
-    title = None
-    about = None
-    levels = None
-    for event in state:
-        if event["type"] == "m.room.name":
-            title = event["content"]["name"]
-        elif event["type"] == "m.room.topic":
-            about = event["content"]["topic"]
-        elif event["type"] == "m.room.power_levels":
-            levels = event["content"]
-        elif event["type"] == "m.room.canonical_alias":
-            title = title or event["content"]["alias"]
-    return title, about, levels
-
-
-@command_handler()
-async def create(evt: CommandEvent):
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
-
-    if po.Portal.get_by_mxid(evt.room_id):
-        return await evt.reply("This is already a portal room.")
-
-    title, about, levels = await _get_initial_state(evt)
-    if not title:
-        return await evt.reply("Please set a title before creating a Telegram chat.")
-
-    supergroup = type == "supergroup"
-    type = {
-        "supergroup": "channel",
-        "channel": "channel",
-        "chat": "chat",
-        "group": "chat",
-    }[type]
-
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
-    try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
-    except ValueError as e:
-        portal.delete()
-        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
-
-
-@command_handler()
-async def upgrade(evt: CommandEvent):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type == "channel":
-        return await evt.reply("This is already a supergroup or a channel.")
-    elif portal.peer_type == "user":
-        return await evt.reply("You can't upgrade private chats.")
-
-    try:
-        await portal.upgrade_telegram_chat(evt.sender)
-        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to upgrade this group.")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-
-
-@command_handler()
-async def group_name(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type != "channel":
-        return await evt.reply("Only channels and supergroups have usernames.")
-
-    try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
-        if portal.username:
-            return await evt.reply(f"Username of channel changed to {portal.username}.")
-        else:
-            return await evt.reply(f"Channel is now private.")
-    except ChatAdminRequiredError:
-        return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
-    except UsernameNotModifiedError:
-        if portal.username:
-            return await evt.reply("That is already the username of this channel.")
-        else:
-            return await evt.reply("This channel is already private")
-    except UsernameOccupiedError:
-        return await evt.reply("That username is already in use.")
-    except UsernameInvalidError:
-        return await evt.reply("Invalid username")
-
-
-@command_handler(needs_admin=True)
-async def filter_mode(evt: CommandEvent):
-    try:
-        mode = evt.args[0]
-        if mode not in ("whitelist", "blacklist"):
-            raise ValueError()
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
-
-    evt.config["bridge.filter.mode"] = mode
-    evt.config.save()
-    po.Portal.filter_mode = mode
-    if mode == "whitelist":
-        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
-                               "To allow bridging a specific chat, use"
-                               "`!filter whitelist <chat ID>`.")
-    else:
-        return await evt.reply("The bridge will now allow bridging chats by default.\n"
-                               "To disallow bridging a specific chat, use"
-                               "`!filter blacklist <chat ID>`.")
-
-
-@command_handler(needs_admin=True)
-async def filter(evt: CommandEvent):
-    try:
-        action = evt.args[0]
-        if action not in ("whitelist", "blacklist", "add", "remove"):
-            raise ValueError()
-
-        id = evt.args[1]
-        if id.startswith("-100"):
-            id = int(id[4:])
-        elif id.startswith("-"):
-            id = int(id[1:])
-        else:
-            id = int(id)
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
-
-    mode = evt.config["bridge.filter.mode"]
-    if mode not in ("blacklist", "whitelist"):
-        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
-
-    list = evt.config["bridge.filter.list"]
-
-    if action in ("blacklist", "whitelist"):
-        action = "add" if mode == action else "remove"
-
-    def save():
-        evt.config["bridge.filter.list"] = list
-        evt.config.save()
-        po.Portal.filter_list = list
-
-    if action == "add":
-        if id in list:
-            return await evt.reply(f"That chat is already {mode}ed.")
-        list.append(id)
-        save()
-        return await evt.reply(f"Chat ID added to {mode}.")
-    elif action == "remove":
-        if id not in list:
-            return await evt.reply(f"That chat is not {mode}ed.")
-        list.remove(id)
-        save()
-        return await evt.reply(f"Chat ID removed from {mode}.")

mautrix_telegram/commands/portal/__init__.py

@@ -0,0 +1 @@
+from . import admin, bridge, config, create_chat, filter, misc, unbridge

mautrix_telegram/commands/portal/admin.py

@@ -0,0 +1,76 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import asyncio
+
+from mautrix.types import EventID
+
+from ... import portal as po, puppet as pu, user as u
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`portal`|`puppet`|`user`>",
+                 help_text="Clear internal bridge caches")
+async def clear_db_cache(evt: CommandEvent) -> EventID:
+    try:
+        section = evt.args[0].lower()
+    except IndexError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+    if section == "portal":
+        po.Portal.by_tgid = {}
+        po.Portal.by_mxid = {}
+        await evt.reply("Cleared portal cache")
+    elif section == "puppet":
+        pu.Puppet.cache = {}
+        for puppet in pu.Puppet.by_custom_mxid.values():
+            puppet.sync_task.cancel()
+        pu.Puppet.by_custom_mxid = {}
+        await asyncio.gather(*[puppet.try_start() for puppet in pu.Puppet.all_with_custom_mxid()],
+                             loop=evt.loop)
+        await evt.reply("Cleared puppet cache and restarted custom puppet syncers")
+    elif section == "user":
+        u.User.by_mxid = {
+            user.mxid: user
+            for user in u.User.by_tgid.values()
+        }
+        await evt.reply("Cleared non-logged-in user cache")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="[_mxid_]",
+                 help_text="Reload and reconnect a user")
+async def reload_user(evt: CommandEvent) -> EventID:
+    if len(evt.args) > 0:
+        mxid = evt.args[0]
+    else:
+        mxid = evt.sender.mxid
+    user = u.User.get_by_mxid(mxid, create=False)
+    if not user:
+        return await evt.reply("User not found")
+    puppet = await pu.Puppet.get_by_custom_mxid(mxid)
+    if puppet:
+        puppet.sync_task.cancel()
+    await user.stop()
+    user.delete(delete_db=False)
+    user = u.User.get_by_mxid(mxid)
+    await user.ensure_started()
+    if puppet:
+        await puppet.start()
+    return await evt.reply(f"Reloaded and reconnected {user.mxid} (telegram: {user.human_tg_id})")

mautrix_telegram/commands/portal/bridge.py

@@ -0,0 +1,193 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Tuple, Awaitable
+import asyncio
+
+from telethon.tl.types import ChatForbidden, ChannelForbidden
+
+from mautrix.types import EventID, RoomID
+
+from ...types import TelegramID
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_id_]",
+                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
+                           "ID. The ID must be the prefixed version that you get with the `/id` "
+                           "command of the Telegram-side bot.")
+async def bridge(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** "
+                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    force_use_bot = False
+    if evt.args[0] == "--usebot" and evt.sender.is_admin:
+        force_use_bot = True
+        evt.args = evt.args[1:]
+    room_id = RoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
+    that_this = "This" if room_id == evt.room_id else "That"
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if portal:
+        return await evt.reply(f"{that_this} room is already a portal room.")
+
+    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
+
+    # The /id bot command provides the prefixed ID, so we assume
+    tgid_str = evt.args[0]
+    if tgid_str.startswith("-100"):
+        tgid = TelegramID(int(tgid_str[4:]))
+        peer_type = "channel"
+    elif tgid_str.startswith("-"):
+        tgid = TelegramID(-int(tgid_str))
+        peer_type = "chat"
+    else:
+        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
+                               "If you did not get the ID using the `/id` bot command, please "
+                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+                               "Bridging private chats to existing rooms is not allowed.")
+
+    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    if not portal.allow_bridging:
+        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
+                               "If you're the bridge admin, try "
+                               "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
+    if portal.mxid:
+        has_portal_message = (
+            "That Telegram chat already has a portal at "
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
+        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+            return await evt.reply(f"{has_portal_message}"
+                                   "Additionally, you do not have the permissions to unbridge "
+                                   "that room.")
+        evt.sender.command_status = {
+            "next": confirm_bridge,
+            "action": "Room bridging",
+            "mxid": portal.mxid,
+            "bridge_to_mxid": room_id,
+            "tgid": portal.tgid,
+            "peer_type": portal.peer_type,
+            "force_use_bot": force_use_bot,
+        }
+        return await evt.reply(f"{has_portal_message}"
+                               "However, you have the permissions to unbridge that room.\n\n"
+                               "To delete that portal completely and continue bridging, use "
+                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
+    evt.sender.command_status = {
+        "next": confirm_bridge,
+        "action": "Room bridging",
+        "bridge_to_mxid": room_id,
+        "tgid": portal.tgid,
+        "peer_type": portal.peer_type,
+        "force_use_bot": force_use_bot,
+    }
+    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
+                           "chat to this room, use `$cmdprefix+sp continue`")
+
+
+async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
+                                            ) -> Tuple[bool, Optional[Awaitable[None]]]:
+    if not portal.mxid:
+        await evt.reply("The portal seems to have lost its Matrix room between you"
+                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                        "Continuing without touching previous Matrix room...")
+        return True, None
+    elif evt.args[0] == "delete-and-continue":
+        return True, portal.cleanup_portal("Portal deleted (moving to another room)", delete=False)
+    elif evt.args[0] == "unbridge-and-continue":
+        return True, portal.cleanup_portal("Room unbridged (portal moving to another room)",
+                                           puppets_only=True, delete=False)
+    else:
+        await evt.reply(
+            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
+            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
+            "continue` to either delete or unbridge the existing room (respectively) and "
+            "continue with the bridging.\n\n"
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
+        return False, None
+
+
+async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
+    status = evt.sender.command_status
+    try:
+        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        bridge_to_mxid = status["bridge_to_mxid"]
+    except KeyError:
+        evt.sender.command_status = None
+        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
+                               "This shouldn't happen unless you're messing with the command "
+                               "handler code.")
+
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+
+    if "mxid" in status:
+        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
+        if not ok:
+            return None
+        elif coro:
+            asyncio.ensure_future(coro, loop=evt.loop)
+            await evt.reply("Cleaning up previous portal room...")
+    elif portal.mxid:
+        evt.sender.command_status = None
+        return await evt.reply("The portal seems to have created a Matrix room between you "
+                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                               "Please start over by calling the bridge command again.")
+    elif evt.args[0] != "continue":
+        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
+                               "`$cmdprefix+sp cancel` to cancel.")
+
+    evt.sender.command_status = None
+    async with portal._room_create_lock:
+        await _locked_confirm_bridge(evt, portal=portal, room_id=bridge_to_mxid,
+                                     is_logged_in=is_logged_in)
+
+
+async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id: RoomID,
+                                 is_logged_in: bool) -> Optional[EventID]:
+    user = evt.sender if is_logged_in else evt.tgbot
+    try:
+        entity = await user.client.get_entity(portal.peer)
+    except Exception:
+        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+        if is_logged_in:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You are logged in, are you in that chat?")
+        else:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You're not logged in, is the relay bot in the chat?")
+    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
+        if is_logged_in:
+            return await evt.reply("You don't seem to be in that chat.")
+        else:
+            return await evt.reply("The bot doesn't seem to be in that chat.")
+
+    portal.mxid = room_id
+    portal.by_mxid[portal.mxid] = portal
+    (portal.title, portal.about, levels,
+     portal.encrypted) = await get_initial_state(evt.az.intent, evt.room_id)
+    portal.photo_id = ""
+    await portal.save()
+
+    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
+                          loop=evt.loop)
+
+    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -0,0 +1,151 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Any
+from io import StringIO
+
+from ruamel.yaml import YAMLError
+
+from mautrix.util.config import yaml
+from mautrix.types import EventID
+
+from ... import portal as po, util
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+
+
+@command_handler(needs_auth=False, help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="View or change per-portal settings.",
+                 help_args="<`help`|_subcommand_> [...]")
+async def config(evt: CommandEvent) -> None:
+    cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
+    if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
+        await config_help(evt)
+        return
+    elif cmd == "defaults":
+        await config_defaults(evt)
+        return
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        await evt.reply("This is not a portal room.")
+        return
+    elif cmd == "view":
+        await config_view(evt, portal)
+        return
+
+    if not await portal.can_user_perform(evt.sender, "config"):
+        await evt.reply("You do not have the permissions to configure this room.")
+        return
+
+    key = evt.args[1] if len(evt.args) > 1 else None
+    try:
+        value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    except YAMLError as e:
+        await evt.reply(f"Invalid value provided. Values must be valid YAML.\n{e}")
+        return
+    if cmd == "set":
+        await config_set(evt, portal, key, value)
+    elif cmd == "unset":
+        await config_unset(evt, portal, key)
+    elif cmd == "add" or cmd == "del":
+        await config_add_del(evt, portal, key, value, cmd)
+    else:
+        return
+    await portal.save()
+
+
+def config_help(evt: CommandEvent) -> Awaitable[EventID]:
+    return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
+
+* **help** - View this help text.
+* **view** - View the current config data.
+* **defaults** - View the default config values.
+* **set** <_key_> <_value_> - Set a config value.
+* **unset** <_key_> - Remove a config value.
+* **add** <_key_> <_value_> - Add a value to an array.
+* **del** <_key_> <_value_> - Remove a value from an array.
+""")
+
+
+def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[EventID]:
+    return evt.reply(f"Room-specific config:\n{_str_value(portal.local_config).rstrip()}")
+
+
+def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
+    value = _str_value({
+        "bridge_notices": {
+            "default": evt.config["bridge.bridge_notices.default"],
+            "exceptions": evt.config["bridge.bridge_notices.exceptions"],
+        },
+        "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
+        "inline_images": evt.config["bridge.inline_images"],
+        "message_formats": evt.config["bridge.message_formats"],
+        "emote_format": evt.config["bridge.emote_format"],
+        "state_event_formats": evt.config["bridge.state_event_formats"],
+        "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
+    })
+    return evt.reply(f"Bridge instance wide config:\n{value.rstrip()}")
+
+
+def _str_value(value: Any) -> str:
+    stream = StringIO()
+    yaml.dump(value, stream)
+    value_str = stream.getvalue()
+    if "\n" in value_str:
+        return f"\n```yaml\n{value_str}\n```\n"
+    else:
+        return f"`{value_str}`"
+
+
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: Any) -> Awaitable[EventID]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
+    elif util.recursive_set(portal.local_config, key, value):
+        return evt.reply(f"Successfully set the value of `{key}` to {_str_value(value)}".rstrip())
+    else:
+        return evt.reply(f"Failed to set value of `{key}`. "
+                         "Does the path contain non-map types?")
+
+
+def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[EventID]:
+    if not key:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config unset <key>`")
+    elif util.recursive_del(portal.local_config, key):
+        return evt.reply(f"Successfully deleted `{key}` from config.")
+    else:
+        return evt.reply(f"`{key}` not found in config.")
+
+
+def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
+                   ) -> Awaitable[EventID]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
+
+    arr = util.recursive_get(portal.local_config, key)
+    if not arr:
+        return evt.reply(f"`{key}` not found in config. "
+                         f"Maybe do `$cmdprefix+sp config set {key} []` first?")
+    elif not isinstance(arr, list):
+        return evt.reply("`{key}` does not seem to be an array.")
+    elif cmd == "add":
+        if value in arr:
+            return evt.reply(f"The array at `{key}` already contains {_str_value(value)}".rstrip())
+        arr.append(value)
+        return evt.reply(f"Successfully added {_str_value(value)} to the array at `{key}`")
+    else:
+        if value not in arr:
+            return evt.reply(f"The array at `{key}` does not contain {_str_value(value)}")
+        arr.remove(value)
+        return evt.reply(f"Successfully removed {_str_value(value)} from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -0,0 +1,60 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.types import EventID
+
+from ... import portal as po
+from ...types import TelegramID
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_type_]",
+                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
+                           "The type is either `group`, `supergroup` or `channel` (defaults to "
+                           "`supergroup`).")
+async def create(evt: CommandEvent) -> EventID:
+    type = evt.args[0] if len(evt.args) > 0 else "supergroup"
+    if type not in ("chat", "group", "supergroup", "channel"):
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
+
+    if po.Portal.get_by_mxid(evt.room_id):
+        return await evt.reply("This is already a portal room.")
+
+    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply("You do not have the permissions to bridge this room.")
+
+    title, about, levels, encrypted = await get_initial_state(evt.az.intent, evt.room_id)
+    if not title:
+        return await evt.reply("Please set a title before creating a Telegram chat.")
+
+    supergroup = type == "supergroup"
+    type = {
+        "supergroup": "channel",
+        "channel": "channel",
+        "chat": "chat",
+        "group": "chat",
+    }[type]
+
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type, mxid=evt.room_id,
+                       title=title, about=about, encrypted=encrypted)
+    try:
+        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+    except ValueError as e:
+        await portal.delete()
+        return await evt.reply(e.args[0])
+    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/filter.py

@@ -0,0 +1,95 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.types import EventID
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`>",
+                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
+                           "default.")
+async def filter_mode(evt: CommandEvent) -> EventID:
+    try:
+        mode = evt.args[0]
+        if mode not in ("whitelist", "blacklist"):
+            raise ValueError()
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
+
+    evt.config["bridge.filter.mode"] = mode
+    evt.config.save()
+    po.Portal.filter_mode = mode
+    if mode == "whitelist":
+        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
+                               "To allow bridging a specific chat, use"
+                               "`!filter whitelist <chat ID>`.")
+    else:
+        return await evt.reply("The bridge will now allow bridging chats by default.\n"
+                               "To disallow bridging a specific chat, use"
+                               "`!filter blacklist <chat ID>`.")
+
+
+@command_handler(name="filter", needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+                 help_text="Allow or disallow bridging a specific chat.")
+async def edit_filter(evt: CommandEvent) -> EventID:
+    try:
+        action = evt.args[0]
+        if action not in ("whitelist", "blacklist", "add", "remove"):
+            raise ValueError()
+
+        id_str = evt.args[1]
+        if id_str.startswith("-100"):
+            filter_id = int(id_str[4:])
+        elif id_str.startswith("-"):
+            filter_id = int(id_str[1:])
+        else:
+            filter_id = int(id_str)
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
+
+    mode = evt.config["bridge.filter.mode"]
+    if mode not in ("blacklist", "whitelist"):
+        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
+
+    filter_id_list = evt.config["bridge.filter.list"]
+
+    if action in ("blacklist", "whitelist"):
+        action = "add" if mode == action else "remove"
+
+    def save() -> None:
+        evt.config["bridge.filter.list"] = filter_id_list
+        evt.config.save()
+        po.Portal.filter_list = filter_id_list
+
+    if action == "add":
+        if filter_id in filter_id_list:
+            return await evt.reply(f"That chat is already {mode}ed.")
+        filter_id_list.append(filter_id)
+        save()
+        return await evt.reply(f"Chat ID added to {mode}.")
+    elif action == "remove":
+        if filter_id not in filter_id_list:
+            return await evt.reply(f"That chat is not {mode}ed.")
+        filter_id_list.remove(filter_id)
+        save()
+        return await evt.reply(f"Chat ID removed from {mode}.")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")

mautrix_telegram/commands/portal/misc.py

@@ -0,0 +1,154 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from telethon.tl.functions.channels import GetFullChannelRequest
+from telethon.tl.functions.messages import GetFullChatRequest
+from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
+                             UsernameNotModifiedError, UsernameOccupiedError, RPCError)
+
+from mautrix.types import EventID
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT, SECTION_MISC
+from .util import user_has_power_level
+
+
+@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.")
+async def sync_state(evt: CommandEvent) -> EventID:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to synchronize this room.")
+
+    await portal.main_intent.get_joined_members(portal.mxid)
+    await evt.reply("Synchronization complete")
+
+
+@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC)
+async def sync_full(evt: CommandEvent) -> EventID:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if len(evt.args) > 0 and evt.args[0] == "--usebot" and evt.sender.is_admin:
+        src = evt.tgbot
+    else:
+        src = evt.tgbot if await evt.sender.needs_relaybot(portal) else evt.sender
+
+    try:
+        if portal.peer_type == "channel":
+            res = await src.client(GetFullChannelRequest(portal.peer))
+        elif portal.peer_type == "chat":
+            res = await src.client(GetFullChatRequest(portal.tgid))
+        else:
+            return await evt.reply("This is not a channel or chat portal.")
+    except (ValueError, RPCError):
+        return await evt.reply("Failed to get portal info from Telegram.")
+
+    await portal.update_matrix_room(src, res.full_chat)
+    return await evt.reply("Portal synced successfully.")
+
+
+@command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Get the ID of the Telegram chat where this room is bridged.")
+async def get_id(evt: CommandEvent) -> EventID:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    tgid = portal.tgid
+    if portal.peer_type == "chat":
+        tgid = -tgid
+    elif portal.peer_type == "channel":
+        tgid = f"-100{tgid}"
+    await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Get a Telegram invite link to the current chat.")
+async def invite_link(evt: CommandEvent) -> EventID:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if portal.peer_type == "user":
+        return await evt.reply("You can't invite users to private chats.")
+
+    try:
+        link = await portal.get_invite_link(evt.sender)
+        return await evt.reply(f"Invite link to {portal.title}: {link}")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to create an invite link.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Upgrade a normal Telegram group to a supergroup.")
+async def upgrade(evt: CommandEvent) -> EventID:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type == "channel":
+        return await evt.reply("This is already a supergroup or a channel.")
+    elif portal.peer_type == "user":
+        return await evt.reply("You can't upgrade private chats.")
+
+    try:
+        await portal.upgrade_telegram_chat(evt.sender)
+        return await evt.reply(f"Group upgraded to supergroup. New ID: -100{portal.tgid}")
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to upgrade this group.")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="<_name_|`-`>",
+                 help_text="Change the username of a supergroup/channel. "
+                           "To disable, use a dash (`-`) as the name.")
+async def group_name(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type != "channel":
+        return await evt.reply("Only channels and supergroups have usernames.")
+
+    try:
+        await portal.set_telegram_username(evt.sender,
+                                           evt.args[0] if evt.args[0] != "-" else "")
+        if portal.username:
+            return await evt.reply(f"Username of channel changed to {portal.username}.")
+        else:
+            return await evt.reply(f"Channel is now private.")
+    except ChatAdminRequiredError:
+        return await evt.reply(
+            "You don't have the permission to set the username of this channel.")
+    except UsernameNotModifiedError:
+        if portal.username:
+            return await evt.reply("That is already the username of this channel.")
+        else:
+            return await evt.reply("This channel is already private")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username")

mautrix_telegram/commands/portal/unbridge.py

@@ -0,0 +1,100 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Callable, Optional
+
+from mautrix.types import RoomID, EventID
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+from .util import user_has_power_level
+
+
+async def _get_portal_and_check_permission(evt: CommandEvent) -> Optional[po.Portal]:
+    room_id = RoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if not portal:
+        that_this = "This" if room_id == evt.room_id else "That"
+        await evt.reply(f"{that_this} is not a portal room.")
+        return None
+
+    if portal.peer_type == "user":
+        if portal.tg_receiver != evt.sender.tgid:
+            await evt.reply("You do not have the permissions to unbridge that portal.")
+            return None
+        return portal
+
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+        await evt.reply("You do not have the permissions to unbridge that portal.")
+        return None
+    return portal
+
+
+def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
+                                completed_message: str) -> Dict:
+    async def post_confirm(confirm) -> Optional[EventID]:
+        confirm.sender.command_status = None
+        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
+            await function()
+            if confirm.room_id != room_id:
+                return await confirm.reply(completed_message)
+        else:
+            return await confirm.reply(f"{action} cancelled.")
+        return None
+
+    return {
+        "next": post_confirm,
+        "action": action,
+    }
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove all users from the current portal room and forget the portal. "
+                           "Only works for group chats; to delete a private chat portal, simply "
+                           "leave the room.")
+async def delete_portal(evt: CommandEvent) -> Optional[EventID]:
+    portal = await _get_portal_and_check_permission(evt)
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
+                                                            portal.cleanup_and_delete, "delete",
+                                                            "Portal successfully deleted.")
+    return await evt.reply("Please confirm deletion of portal "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           f"to Telegram chat \"{portal.title}\" "
+                           "by typing `$cmdprefix+sp confirm-delete`"
+                           "\n\n"
+                           "**WARNING:** If the bridge bot has the power level to do so, **this "
+                           "will kick ALL users** in the room. If you just want to remove the "
+                           "bridge, use `$cmdprefix+sp unbridge` instead.")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove puppets from the current portal room and forget the portal.")
+async def unbridge(evt: CommandEvent) -> Optional[EventID]:
+    portal = await _get_portal_and_check_permission(evt)
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
+                                                            portal.unbridge, "unbridge",
+                                                            "Room successfully unbridged.")
+    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           "by typing `$cmdprefix+sp confirm-unbridge`")

mautrix_telegram/commands/portal/util.py

@@ -0,0 +1,62 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Optional
+
+from mautrix.errors import MatrixRequestError
+from mautrix.appservice import IntentAPI
+from mautrix.types import RoomID, EventType, PowerLevelStateEventContent
+
+from ... import user as u
+
+OptStr = Optional[str]
+
+
+async def get_initial_state(intent: IntentAPI, room_id: RoomID
+                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent], bool]:
+    state = await intent.get_state(room_id)
+    title: OptStr = None
+    about: OptStr = None
+    levels: Optional[PowerLevelStateEventContent] = None
+    encrypted: bool = False
+    for event in state:
+        try:
+            if event.type == EventType.ROOM_NAME:
+                title = event.content.name
+            elif event.type == EventType.ROOM_TOPIC:
+                about = event.content.topic
+            elif event.type == EventType.ROOM_POWER_LEVELS:
+                levels = event.content
+            elif event.type == EventType.ROOM_CANONICAL_ALIAS:
+                title = title or event.content.canonical_alias
+            elif event.type == EventType.ROOM_ENCRYPTION:
+                encrypted = True
+        except KeyError:
+            # Some state event probably has empty content
+            pass
+    return title, about, levels, encrypted
+
+
+async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.User,
+                               event: str) -> bool:
+    if sender.is_admin:
+        return True
+    # Make sure the state store contains the power levels.
+    try:
+        await intent.get_power_levels(room_id)
+    except MatrixRequestError:
+        return False
+    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
+    return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)

mautrix_telegram/commands/telegram.py

@@ -1,142 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from telethon.errors import *
-from telethon.tl.types import User as TLUser
-from telethon.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest
-from telethon.tl.functions.channels import JoinChannelRequest
-
-from .. import puppet as pu, portal as po
-from . import command_handler
-
-
-@command_handler()
-async def search(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
-
-    force_remote = False
-    if evt.args[0] in {"-r", "--remote"}:
-        force_remote = True
-        evt.args.pop(0)
-
-    query = " ".join(evt.args)
-    if force_remote and len(query) < 5:
-        return await evt.reply("Minimum length of query for remote search is 5 characters.")
-
-    results, remote = await evt.sender.search(query, force_remote)
-
-    if not results:
-        if len(query) < 5 and remote:
-            return await evt.reply("No local results. "
-                                   "Minimum length of remote query is 5 characters.")
-        return await evt.reply("No results 3:")
-
-    reply = []
-    if remote:
-        reply += ["**Results from Telegram server:**", ""]
-    else:
-        reply += ["**Results in contacts:**", ""]
-    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
-               f"{puppet.id} ({similarity}% match)")
-              for puppet, similarity in results]
-
-    # TODO somehow show remote channel results when joining by alias is possible?
-
-    return await evt.reply("\n".join(reply))
-
-
-@command_handler(name="pm")
-async def private_message(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
-
-    try:
-        user = await evt.sender.client.get_entity(evt.args[0])
-    except ValueError:
-        return await evt.reply("Invalid user identifier or user not found.")
-
-    if not user:
-        return await evt.reply("User not found.")
-    elif not isinstance(user, TLUser):
-        return await evt.reply("That doesn't seem to be a user.")
-    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
-    await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply("Created private chat room with "
-                           f"{pu.Puppet.get_displayname(user, False)}")
-
-
-async def _join(evt, arg):
-    if arg.startswith("joinchat/"):
-        invite_hash = arg[len("joinchat/"):]
-        try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
-        except InviteHashInvalidError:
-            return None, await evt.reply("Invalid invite link.")
-        except InviteHashExpiredError:
-            return None, await evt.reply("Invite link expired.")
-        try:
-            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
-        except UserAlreadyParticipantError:
-            return None, await evt.reply("You are already in that chat.")
-    else:
-        channel = await evt.sender.client.get_entity(arg)
-        if not channel:
-            return None, await evt.reply("Channel/supergroup not found.")
-        return await evt.sender.client(JoinChannelRequest(channel)), None
-
-
-@command_handler()
-async def join(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
-
-    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
-    arg = regex.match(evt.args[0])
-    if not arg:
-        return await evt.reply("That doesn't look like a Telegram invite link.")
-
-    updates, _ = await _join(evt, arg.group(1))
-    if not updates:
-        return
-
-    for chat in updates.chats:
-        portal = po.Portal.get_by_entity(chat)
-        if portal.mxid:
-            await portal.invite_to_matrix([evt.sender.mxid])
-            return await evt.reply(f"Invited you to portal of {portal.title}")
-        else:
-            await evt.reply(f"Creating room for {chat.title}... This might take a while.")
-            await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
-            return await evt.reply(f"Created room for {portal.title}")
-
-
-@command_handler()
-async def sync(evt):
-    if len(evt.args) > 0:
-        sync_only = evt.args[0]
-        if sync_only not in ("chats", "contacts", "me"):
-            return await evt.reply("**Usage:** `$cmdprefix+sp sync [chats|contacts|me]`")
-    else:
-        sync_only = None
-
-    if not sync_only or sync_only == "chats":
-        await evt.sender.sync_dialogs(synchronous_create=True)
-    if not sync_only or sync_only == "contacts":
-        await evt.sender.sync_contacts()
-    if not sync_only or sync_only == "me":
-        await evt.sender.update_info()
-    return await evt.reply("Synchronization complete.")

mautrix_telegram/commands/telegram/__init__.py

@@ -0,0 +1 @@
+from . import account, auth, misc

mautrix_telegram/commands/telegram/account.py

@@ -0,0 +1,125 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional
+
+from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
+from telethon.tl.types import Authorization
+from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
+                                           ResetAuthorizationRequest, UpdateProfileRequest)
+
+from mautrix.types import EventID
+
+from .. import command_handler, CommandEvent, SECTION_AUTH
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new username_>",
+                 help_text="Change your Telegram username.")
+async def username(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp username <new username>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own username.")
+    new_name = evt.args[0]
+    if new_name == "-":
+        new_name = ""
+    try:
+        await evt.sender.client(UpdateUsernameRequest(username=new_name))
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username. Usernames must be between 5 and 30 alphanumeric "
+                               "characters.")
+    except UsernameNotModifiedError:
+        return await evt.reply("That is your current username.")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    await evt.sender.update_info()
+    if not evt.sender.username:
+        await evt.reply("Username removed")
+    else:
+        await evt.reply(f"Username changed to {evt.sender.username}")
+
+
+@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
+                 help_text="Change your Telegram displayname.")
+async def displayname(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own displayname.")
+
+    first_name, last_name = ((evt.args[0], "")
+                             if len(evt.args) == 1
+                             else (" ".join(evt.args[:-1]), evt.args[-1]))
+    try:
+        await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid first name")
+    await evt.sender.update_info()
+    return await evt.reply("Displayname updated")
+
+
+def _format_session(sess: Authorization) -> str:
+    return (f"**{sess.app_name} {sess.app_version}**  \n"
+            f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
+            f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
+            f"  **From:** {sess.ip} - {sess.region}, {sess.country}")
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<`list`|`terminate`> [_hash_]",
+                 help_text="View or delete other Telegram sessions.")
+async def session(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't manage their sessions")
+    cmd = evt.args[0].lower()
+    if cmd == "list":
+        res = await evt.sender.client(GetAuthorizationsRequest())
+        session_list = res.authorizations
+        current = [s for s in session_list if s.current][0]
+        current_text = _format_session(current)
+        other_text = "\n".join(f"* {_format_session(sess)}  \n"
+                               f"  **Hash:** {sess.hash}"
+                               for sess in session_list if not sess.current)
+        return await evt.reply(f"### Current session\n"
+                               f"{current_text}\n"
+                               f"\n"
+                               f"### Other active sessions\n"
+                               f"{other_text}")
+    elif cmd == "terminate" and len(evt.args) > 1:
+        try:
+            session_hash = int(evt.args[1])
+        except ValueError:
+            return await evt.reply("Hash must be an integer")
+        try:
+            ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
+        except HashInvalidError:
+            return await evt.reply("Invalid session hash.")
+        except AuthKeyError as e:
+            if e.message == "FRESH_RESET_AUTHORISATION_FORBIDDEN":
+                return await evt.reply("New sessions can't terminate other sessions. "
+                                       "Please wait a while.")
+            raise
+        if ok:
+            return await evt.reply("Session terminated successfully.")
+        else:
+            return await evt.reply("Session not found.")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")

mautrix_telegram/commands/telegram/auth.py

@@ -0,0 +1,355 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Any, Dict, Optional
+import asyncio
+import io
+
+from telethon.errors import (  # isort: skip
+    AccessTokenExpiredError, AccessTokenInvalidError, FirstNameInvalidError, FloodWaitError,
+    PasswordHashInvalidError, PhoneCodeExpiredError, PhoneCodeInvalidError,
+    PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
+    PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError,
+    PhoneNumberInvalidError)
+from telethon.tl.types import User
+
+from mautrix.types import (EventID, UserID, MediaMessageEventContent, ImageInfo, MessageType,
+                           TextMessageEventContent)
+
+from ... import user as u
+from ...types import TelegramID
+from ...commands import command_handler, CommandEvent, SECTION_AUTH
+from ...util import format_duration as fmt_duration
+
+try:
+    import qrcode
+    import PIL as _
+    from telethon.tl.custom import QRLogin
+except ImportError:
+    qrcode = None
+    QRLogin = None
+
+
+@command_handler(needs_auth=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Check if you're logged into Telegram.")
+async def ping(evt: CommandEvent) -> EventID:
+    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
+    if me:
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
+        return await evt.reply(f"You're logged in as {human_tg_id}")
+    else:
+        return await evt.reply("You're not logged in.")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Get the info of the message relay Telegram bot.")
+async def ping_bot(evt: CommandEvent) -> EventID:
+    if not evt.tgbot:
+        return await evt.reply("Telegram message relay bot not configured.")
+    info, mxid = await evt.tgbot.get_me(use_cache=False)
+    return await evt.reply("Telegram message relay bot is active: "
+                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
+                           "To use the bot, simply invite it to a portal room.")
+
+
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_phone_> <_full name_>",
+                 help_text="Register to Telegram")
+async def register(evt: CommandEvent) -> EventID:
+    if await evt.sender.is_logged_in():
+        return await evt.reply("You are already logged in.")
+    elif len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
+
+    phone_number = evt.args[0]
+    if len(evt.args) == 2:
+        full_name = evt.args[1], ""
+    else:
+        full_name = " ".join(evt.args[1:-1]), evt.args[-1]
+
+    await _request_code(evt, phone_number, {
+        "next": enter_code_register,
+        "action": "Register",
+        "full_name": full_name,
+    })
+    return await evt.reply("By signing up for Telegram, you agree to "
+                           "the terms of service: https://telegram.org/tos")
+
+
+async def enter_code_register(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        first_name, last_name = evt.sender.command_status["full_name"]
+        user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
+        asyncio.ensure_future(evt.sender.post_login(user, first_login=True), loop=evt.loop)
+        evt.sender.command_status = None
+        return await evt.reply(f"Successfully registered to Telegram.")
+    except PhoneNumberOccupiedError:
+        return await evt.reply("That phone number has already been registered. "
+                               "You can log in with `$cmdprefix+sp login`.")
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
+    except PhoneCodeExpiredError:
+        return await evt.reply(
+            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`.")
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
+    except Exception:
+        evt.log.exception("Error sending phone code")
+        return await evt.reply("Unhandled exception while sending code. "
+                               "Check console for more details.")
+
+
+@command_handler(needs_auth=False, management_only=True, help_section=SECTION_AUTH,
+                 help_text="Log in by scanning a QR code.")
+async def login_qr(evt: CommandEvent) -> EventID:
+    login_as = evt.sender
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        login_as = u.User.get_by_mxid(UserID(evt.args[0]))
+    if not qrcode or not QRLogin:
+        return await evt.reply("This bridge instance does not support logging in with a QR code.")
+    if await login_as.is_logged_in():
+        return await evt.reply(f"You are already logged in as {login_as.human_tg_id}.")
+
+    await login_as.ensure_started(even_if_no_session=True)
+    qr_login = QRLogin(login_as.client, ignored_ids=[])
+    qr_event_id: Optional[EventID] = None
+
+    async def upload_qr() -> None:
+        nonlocal qr_event_id
+        buffer = io.BytesIO()
+        image = qrcode.make(qr_login.url)
+        size = image.pixel_size
+        image.save(buffer, "PNG")
+        qr = buffer.getvalue()
+        mxc = await evt.az.intent.upload_media(qr, "image/png", "login-qr.png", len(qr))
+        content = MediaMessageEventContent(body=qr_login.url, url=mxc, msgtype=MessageType.IMAGE,
+                                           info=ImageInfo(mimetype="image/png", size=len(qr),
+                                                          width=size, height=size))
+        if qr_event_id:
+            content.set_edit(qr_event_id)
+            await evt.az.intent.send_message(evt.room_id, content)
+        else:
+            content.set_reply(evt.event_id)
+            qr_event_id = await evt.az.intent.send_message(evt.room_id, content)
+
+    retries = 4
+    while retries > 0:
+        await qr_login.recreate()
+        await upload_qr()
+        try:
+            user = await qr_login.wait()
+            break
+        except asyncio.TimeoutError:
+            retries -= 1
+        except SessionPasswordNeededError:
+            evt.sender.command_status = {
+                "next": enter_password,
+                "login_as": login_as if login_as != evt.sender else None,
+                "action": "Login (password entry)",
+            }
+            return await evt.reply("Your account has two-factor authentication. "
+                                   "Please send your password here.")
+    else:
+        timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
+        timeout.set_edit(qr_event_id)
+        return await evt.az.intent.send_message(evt.room_id, timeout)
+
+    return await _finish_sign_in(evt, user, login_as=login_as)
+
+
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Get instructions on how to log in.")
+async def login(evt: CommandEvent) -> EventID:
+    override_sender = False
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        evt.sender = await u.User.get_by_mxid(UserID(evt.args[0])).ensure_started()
+        override_sender = True
+    if await evt.sender.is_logged_in():
+        return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
+
+    allow_matrix_login = evt.config["bridge.allow_matrix_login"]
+    if allow_matrix_login and not override_sender:
+        evt.sender.command_status = {
+            "next": enter_phone_or_token,
+            "action": "Login",
+        }
+
+    nb = "**N.B. Logging in grants the bridge full access to your Telegram account.**"
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
+        if override_sender:
+            return await evt.reply(f"[Click here to log in]({url}) as "
+                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+        elif allow_matrix_login:
+            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
+                                   f" number (or bot auth token) here to log in.\n\n{nb}")
+        return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
+    elif allow_matrix_login:
+        if override_sender:
+            return await evt.reply(
+                "This bridge instance does not allow you to log in outside of Matrix. "
+                "Logging in as another user inside Matrix is not currently possible.")
+        return await evt.reply("Please send your phone number (or bot auth token) here to start "
+                               f"the login process.\n\n{nb}")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
+                        ) -> EventID:
+    ok = False
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        await evt.sender.client.sign_in(phone_number)
+        ok = True
+        return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
+    except PhoneNumberAppSignupForbiddenError:
+        return await evt.reply("Your phone number does not allow 3rd party apps to sign in.")
+    except PhoneNumberFloodError:
+        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
+                               "The ban is usually applied for around a day.")
+    except FloodWaitError as e:
+        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
+                               f"Please wait for {fmt_duration(e.seconds)} before trying again.")
+    except PhoneNumberBannedError:
+        return await evt.reply("Your phone number has been banned from Telegram.")
+    except PhoneNumberUnoccupiedError:
+        return await evt.reply("That phone number has not been registered. "
+                               "Please register with `$cmdprefix+sp register <phone>`.")
+    except PhoneNumberInvalidError:
+        return await evt.reply("That phone number is not valid.")
+    except Exception:
+        evt.log.exception("Error requesting phone code")
+        return await evt.reply("Unhandled exception while requesting code. "
+                               "Check console for more details.")
+    finally:
+        evt.sender.command_status = next_status if ok else None
+
+
+@command_handler(needs_auth=False)
+async def enter_phone_or_token(evt: CommandEvent) -> Optional[EventID]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+
+    # phone numbers don't contain colons but telegram bot auth tokens do
+    if evt.args[0].find(":") > 0:
+        try:
+            await _sign_in(evt, bot_token=evt.args[0])
+        except Exception:
+            evt.log.exception("Error sending auth token")
+            return await evt.reply("Unhandled exception while sending auth token. "
+                                   "Check console for more details.")
+    else:
+        await _request_code(evt, evt.args[0], {
+            "next": enter_code,
+            "action": "Login",
+        })
+    return None
+
+
+@command_handler(needs_auth=False)
+async def enter_code(evt: CommandEvent) -> Optional[EventID]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+    try:
+        await _sign_in(evt, code=evt.args[0])
+    except Exception:
+        evt.log.exception("Error sending phone code")
+        return await evt.reply("Unhandled exception while sending code. "
+                               "Check console for more details.")
+    return None
+
+
+@command_handler(needs_auth=False)
+async def enter_password(evt: CommandEvent) -> Optional[EventID]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+    try:
+        await _sign_in(evt, login_as=evt.sender.command_status.get("login_as", None),
+                       password=" ".join(evt.args))
+    except AccessTokenInvalidError:
+        return await evt.reply("That bot token is not valid.")
+    except AccessTokenExpiredError:
+        return await evt.reply("That bot token has expired.")
+    except Exception:
+        evt.log.exception("Error sending password")
+        return await evt.reply("Unhandled exception while sending password. "
+                               "Check console for more details.")
+    return None
+
+
+async def _sign_in(evt: CommandEvent, login_as: 'u.User' = None, **sign_in_info) -> EventID:
+    login_as = login_as or evt.sender
+    try:
+        await login_as.ensure_started(even_if_no_session=True)
+        user = await login_as.client.sign_in(**sign_in_info)
+        await _finish_sign_in(evt, user)
+    except PhoneCodeExpiredError:
+        return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
+    except PasswordHashInvalidError:
+        return await evt.reply("Incorrect password.")
+    except SessionPasswordNeededError:
+        evt.sender.command_status = {
+            "next": enter_password,
+            "action": "Login (password entry)",
+        }
+        return await evt.reply("Your account has two-factor authentication. "
+                               "Please send your password here.")
+
+
+async def _finish_sign_in(evt: CommandEvent, user: User, login_as: 'u.User' = None) -> EventID:
+    login_as = login_as or evt.sender
+    existing_user = u.User.get_by_tgid(TelegramID(user.id))
+    if existing_user and existing_user != login_as:
+        await existing_user.log_out()
+        await evt.reply(f"[{existing_user.displayname}]"
+                        f"(https://matrix.to/#/{existing_user.mxid})"
+                        " was logged out from the account.")
+    asyncio.ensure_future(login_as.post_login(user, first_login=True), loop=evt.loop)
+    evt.sender.command_status = None
+    name = f"@{user.username}" if user.username else f"+{user.phone}"
+    if login_as != evt.sender:
+        msg = (f"Successfully logged in [{login_as.mxid}](https://matrix.to/#/{login_as.mxid})"
+               f" as {name}")
+    else:
+        msg = f"Successfully logged in as {name}"
+    return await evt.reply(msg)
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Log out from Telegram.")
+async def logout(evt: CommandEvent) -> EventID:
+    if await evt.sender.log_out():
+        return await evt.reply("Logged out successfully.")
+    return await evt.reply("Failed to log out.")

mautrix_telegram/commands/telegram/misc.py

@@ -0,0 +1,361 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2020 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Optional, Tuple, cast
+import logging
+import codecs
+import base64
+import re
+
+from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
+                             UserAlreadyParticipantError, ChatIdInvalidError,
+                             TakeoutInitDelayError, EmoticonInvalidError)
+from telethon.tl.patched import Message
+from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
+                               TypeInputPeer, InputMediaDice)
+from telethon.tl.types.messages import BotCallbackAnswer
+from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
+                                            GetBotCallbackAnswerRequest, SendVoteRequest)
+from telethon.tl.functions.channels import JoinChannelRequest
+
+from mautrix.types import EventID, Format
+
+from ... import puppet as pu, portal as po
+from ...abstract_user import AbstractUser
+from ...db import Message as DBMessage
+from ...types import TelegramID
+from ...commands import (command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS,
+                         SECTION_PORTAL_MANAGEMENT)
+
+
+@command_handler(needs_auth=False,
+                 help_section=SECTION_MISC, help_args="<_caption_>",
+                 help_text="Set a caption for the next image you send")
+async def caption(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp caption <caption>`")
+
+    prefix = f"{evt.command_prefix} caption "
+    if evt.content.format == Format.HTML:
+        evt.content.formatted_body = evt.content.formatted_body.replace(prefix, "", 1)
+    evt.content.body = evt.content.body.replace(prefix, "", 1)
+    evt.sender.command_status = {"caption": evt.content, "action": "Caption"}
+    return await evt.reply("Your next image or file will be sent with that caption. "
+                           "Use `$cmdprefix+sp cancel` to cancel the caption.")
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[_-r|--remote_] <_query_>",
+                 help_text="Search your contacts or the Telegram servers for users.")
+async def search(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
+
+    force_remote = False
+    if evt.args[0] in {"-r", "--remote"}:
+        force_remote = True
+        evt.args.pop(0)
+
+    query = " ".join(evt.args)
+    if force_remote and len(query) < 5:
+        return await evt.reply("Minimum length of query for remote search is 5 characters.")
+
+    results, remote = await evt.sender.search(query, force_remote)
+
+    if not results:
+        if len(query) < 5 and remote:
+            return await evt.reply("No local results. "
+                                   "Minimum length of remote query is 5 characters.")
+        return await evt.reply("No results 3:")
+
+    reply: List[str] = []
+    if remote:
+        reply += ["**Results from Telegram server:**", ""]
+    else:
+        reply += ["**Results in contacts:**", ""]
+    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
+               f"{puppet.id} ({similarity}% match)")
+              for puppet, similarity in results]
+
+    # TODO somehow show remote channel results when joining by alias is possible?
+
+    return await evt.reply("\n".join(reply))
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS, help_args="<_identifier_>",
+                 help_text="Open a private chat with the given Telegram user. The identifier is "
+                           "either the internal user ID, the username or the phone number. "
+                           "**N.B.** The phone numbers you start chats with must already be in "
+                           "your contacts.")
+async def pm(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
+
+    try:
+        id = "".join(evt.args).translate({ord(c): None for c in "+()- "})
+        user = await evt.sender.client.get_entity(id)
+    except ValueError:
+        return await evt.reply("Invalid user identifier or user not found.")
+
+    if not user:
+        return await evt.reply("User not found.")
+    elif not isinstance(user, TLUser):
+        return await evt.reply("That doesn't seem to be a user.")
+    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
+    await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
+    return await evt.reply("Created private chat room with "
+                           f"{pu.Puppet.get_displayname(user, False)}")
+
+
+async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
+    if arg.startswith("joinchat/"):
+        invite_hash = arg[len("joinchat/"):]
+        try:
+            await evt.sender.client(CheckChatInviteRequest(invite_hash))
+        except InviteHashInvalidError:
+            return None, await evt.reply("Invalid invite link.")
+        except InviteHashExpiredError:
+            return None, await evt.reply("Invite link expired.")
+        try:
+            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
+        except UserAlreadyParticipantError:
+            return None, await evt.reply("You are already in that chat.")
+    else:
+        channel = await evt.sender.client.get_entity(arg)
+        if not channel:
+            return None, await evt.reply("Channel/supergroup not found.")
+        return await evt.sender.client(JoinChannelRequest(channel)), None
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_link_>",
+                 help_text="Join a chat with an invite link.")
+async def join(evt: CommandEvent) -> Optional[EventID]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
+
+    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
+    arg = regex.match(evt.args[0])
+    if not arg:
+        return await evt.reply("That doesn't look like a Telegram invite link.")
+
+    updates, _ = await _join(evt, arg.group(1))
+    if not updates:
+        return None
+
+    for chat in updates.chats:
+        portal = po.Portal.get_by_entity(chat)
+        if portal.mxid:
+            await portal.invite_to_matrix([evt.sender.mxid])
+            return await evt.reply(f"Invited you to portal of {portal.title}")
+        else:
+            await evt.reply(f"Creating room for {chat.title}... This might take a while.")
+            try:
+                await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
+            except ChatIdInvalidError as e:
+                logging.getLogger("mau.commands").trace("ChatIdInvalidError while creating portal "
+                                                        "from !tg join command: %s",
+                                                        updates.stringify())
+                raise e
+            return await evt.reply(f"Created room for {portal.title}")
+    return None
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[`chats`|`contacts`|`me`]",
+                 help_text="Synchronize your chat portals, contacts and/or own info.")
+async def sync(evt: CommandEvent) -> EventID:
+    if len(evt.args) > 0:
+        sync_only = evt.args[0]
+        if sync_only not in ("chats", "contacts", "me"):
+            return await evt.reply("**Usage:** `$cmdprefix+sp sync [chats|contacts|me]`")
+    else:
+        sync_only = None
+
+    if not sync_only or sync_only == "chats":
+        await evt.reply("Synchronizing chats...")
+        await evt.sender.sync_dialogs()
+    if not sync_only or sync_only == "contacts":
+        await evt.reply("Synchronizing contacts...")
+        await evt.sender.sync_contacts()
+    if not sync_only or sync_only == "me":
+        await evt.sender.update_info()
+    return await evt.reply("Synchronization complete.")
+
+
+PEER_TYPE_CHAT = b"g"
+
+
+class MessageIDError(ValueError):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+
+async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
+                               ) -> Tuple[TypeInputPeer, Message]:
+    try:
+        enc_id += (4 - len(enc_id) % 4) * "="
+        enc_id = base64.b64decode(enc_id)
+        peer_type, enc_id = bytes([enc_id[0]]), enc_id[1:]
+        tgid = TelegramID(int(codecs.encode(enc_id[0:5], "hex_codec"), 16))
+        msg_id = TelegramID(int(codecs.encode(enc_id[5:10], "hex_codec"), 16))
+        space = None
+        if peer_type == PEER_TYPE_CHAT:
+            space = TelegramID(int(codecs.encode(enc_id[10:15], "hex_codec"), 16))
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (format)") from e
+
+    if peer_type == PEER_TYPE_CHAT:
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
+        if not orig_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
+        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
+        if not new_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (your copy of message not found in db)")
+        msg_id = new_msg.tgid
+    try:
+        peer = await user.client.get_input_entity(tgid)
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (chat not found)") from e
+
+    msg = await user.client.get_messages(entity=peer, ids=msg_id)
+    if not msg:
+        raise MessageIDError(f"Invalid {type_name} ID (message not found)")
+    return peer, cast(Message, msg)
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_play ID_>",
+                 help_text="Play a Telegram game.")
+async def play(evt: CommandEvent) -> EventID:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp play <play ID>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to play games.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't play games :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="play")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaGame):
+        return await evt.reply("Invalid play ID (message doesn't look like a game)")
+
+    game = await evt.sender.client(
+        GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
+    if not isinstance(game, BotCallbackAnswer):
+        return await evt.reply("Game request response invalid")
+
+    return await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
+                           f"{msg.media.game.description}")
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_poll ID_> <_choice number_>",
+                 help_text="Vote in a Telegram poll.")
+async def vote(evt: CommandEvent) -> EventID:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to vote in polls.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't vote in polls :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="poll")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaPoll):
+        return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
+
+    options = []
+    for option in evt.args[1:]:
+        try:
+            if len(option) > 10:
+                raise ValueError("option index too long")
+            option_index = int(option) - 1
+        except ValueError:
+            option_index = None
+        if option_index is None:
+            return await evt.reply(f"Invalid option number \"{option}\"",
+                                   render_markdown=False, allow_html=False)
+        elif option_index < 0:
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"Option numbers must be positive.")
+        elif option_index >= len(msg.media.poll.answers):
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"The poll only has {len(msg.media.poll.answers)} options.")
+        options.append(msg.media.poll.answers[option_index].option)
+    options = [msg.media.poll.answers[int(option) - 1].option
+               for option in evt.args[1:]]
+    try:
+        resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
+    except OptionsTooMuchError:
+        return await evt.reply("You passed too many options.")
+    # TODO use response
+    return await evt.mark_read()
+
+
+@command_handler(help_section=SECTION_MISC, help_args="<_emoji_>",
+                 help_text="Roll a dice (\U0001F3B2), kick a football (\u26BD\uFE0F) or throw a "
+                           "dart (\U0001F3AF) or basketball (\U0001F3C0) on the Telegram servers.")
+async def random(evt: CommandEvent) -> EventID:
+    if not evt.is_portal:
+        return await evt.reply("You can only randomize values in portal rooms")
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    arg = evt.args[0] if len(evt.args) > 0 else "dice"
+    emoticon = {
+        "dart": "\U0001F3AF",
+        "dice": "\U0001F3B2",
+        "ball": "\U0001F3C0",
+        "basketball": "\U0001F3C0",
+        "football": "\u26BD",
+        "soccer": "\u26BD",
+    }.get(arg, arg)
+    try:
+        await evt.sender.client.send_media(await portal.get_input_entity(evt.sender),
+                                           InputMediaDice(emoticon))
+    except EmoticonInvalidError:
+        return await evt.reply("Invalid emoji for randomization")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT, help_args="[_limit_]",
+                 help_text="Backfill messages from Telegram history.")
+async def backfill(evt: CommandEvent) -> None:
+    if not evt.is_portal:
+        await evt.reply("You can only use backfill in portal rooms")
+        return
+    try:
+        limit = int(evt.args[0])
+    except (ValueError, IndexError):
+        limit = -1
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
+        await evt.reply("Backfilling normal groups is disabled in the bridge config")
+        return
+    try:
+        await portal.backfill(evt.sender, limit=limit)
+    except TakeoutInitDelayError:
+        msg = ("Please accept the data export request from a mobile device, "
+               "then re-run the backfill command.")
+        if portal.peer_type == "user":
+            from mautrix.appservice import IntentAPI
+            await portal.main_intent.send_notice(evt.room_id, msg)
+        else:
+            await evt.reply(msg)

mautrix_telegram/config.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,172 +13,154 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from ruamel.yaml import YAML
+from typing import Any, List, NamedTuple
 from ruamel.yaml.comments import CommentedMap
-import random
-import string
+import os
 
-yaml = YAML()
-yaml.indent(4)
+from mautrix.types import UserID
+from mautrix.client import Client
+from mautrix.bridge.config import BaseBridgeConfig
+from mautrix.util.config import ForbiddenKey, ForbiddenDefault, ConfigUpdateHelper
+
+Permissions = NamedTuple("Permissions", relaybot=bool, user=bool, puppeting=bool,
+                         matrix_puppeting=bool, admin=bool, level=str)
 
 
-class DictWithRecursion:
-    def __init__(self, data=None):
-        self._data = data or CommentedMap()
-
-    def _recursive_get(self, data, key, default_value):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
-            next_data = data.get(key, CommentedMap())
-            return self._recursive_get(next_data, next_key, default_value)
-        return data.get(key, default_value)
-
-    def get(self, key, default_value, allow_recursion=True):
-        if allow_recursion and '.' in key:
-            return self._recursive_get(self._data, key, default_value)
-        return self._data.get(key, default_value)
-
-    def __getitem__(self, key):
-        return self.get(key, None)
-
-    def __contains__(self, key):
-        return self[key] is not None
-
-    def _recursive_set(self, data, key, value):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
-            if key not in data:
-                data[key] = CommentedMap()
-            next_data = data.get(key, CommentedMap())
-            self._recursive_set(next_data, next_key, value)
-            return
-        data[key] = value
-
-    def set(self, key, value, allow_recursion=True):
-        if allow_recursion and '.' in key:
-            self._recursive_set(self._data, key, value)
-            return
-        self._data[key] = value
-
-    def __setitem__(self, key, value):
-        self.set(key, value)
-
-    def _recursive_del(self, data, key):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
-            if key not in data:
-                return
-            next_data = data[key]
-            self._recursive_del(next_data, next_key)
-            return
+class Config(BaseBridgeConfig):
+    def __getitem__(self, key: str) -> Any:
         try:
-            del data[key]
-            del data.ca.items[key]
+            return os.environ[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
         except KeyError:
-            pass
+            return super().__getitem__(key)
 
-    def delete(self, key, allow_recursion=True):
-        if allow_recursion and '.' in key:
-            self._recursive_del(self._data, key)
-            return
-        try:
-            del self._data[key]
-            del self._data.ca.items[key]
-        except KeyError:
-            pass
+    @property
+    def forbidden_defaults(self) -> List[ForbiddenDefault]:
+        return [
+            *super().forbidden_defaults,
+            ForbiddenDefault("appservice.public.external", "https://example.com/public",
+                             condition="appservice.public.enabled"),
+            ForbiddenDefault("bridge.permissions", ForbiddenKey("example.com")),
+            ForbiddenDefault("telegram.api_id", 12345),
+            ForbiddenDefault("telegram.api_hash", "tjyd5yge35lbodk1xwzw2jstp90k55qz"),
+        ]
 
-    def __delitem__(self, key):
-        self.delete(key)
+    def do_update(self, helper: ConfigUpdateHelper) -> None:
+        super().do_update(helper)
+        copy, copy_dict, base = helper
 
+        copy("homeserver.asmux")
 
-class Config(DictWithRecursion):
-    def __init__(self, path, registration_path, base_path):
-        super().__init__()
-        self.path = path
-        self.registration_path = registration_path
-        self.base_path = base_path
-        self._registration = None
-
-    def load(self):
-        with open(self.path, 'r') as stream:
-            self._data = yaml.load(stream)
-
-    def load_base(self):
-        try:
-            with open(self.base_path, 'r') as stream:
-                return DictWithRecursion(yaml.load(stream))
-        except OSError:
-            pass
-        return None
-
-    def save(self):
-        with open(self.path, 'w') as stream:
-            yaml.dump(self._data, stream)
-        if self._registration and self.registration_path:
-            with open(self.registration_path, 'w') as stream:
-                yaml.dump(self._registration, stream)
-
-    @staticmethod
-    def _new_token():
-        return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
-
-    def update(self):
-        base = self.load_base()
-        if not base:
-            return
-
-        def copy(from_path, to_path=None):
-            if from_path in self:
-                base[to_path or from_path] = self[from_path]
-
-        def copy_dict(from_path, to_path=None):
-            if from_path in self:
-                to_path = to_path or from_path
-                base[to_path] = CommentedMap()
-                for key, value in self[from_path].items():
-                    base[to_path][key] = value
-
-        copy("homeserver.address")
-        copy("homeserver.verify_ssl")
-        copy("homeserver.domain")
-
-        copy("appservice.protocol")
-        copy("appservice.hostname")
-        copy("appservice.port")
-
-        copy("appservice.database")
+        if "appservice.protocol" in self and "appservice.address" not in self:
+            protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
+                                        self["appservice.port"])
+            base["appservice.address"] = f"{protocol}://{hostname}:{port}"
+        if "appservice.debug" in self and "logging" not in self:
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
+            base["logging.root.level"] = level
+            base["logging.loggers.mau.level"] = level
+            base["logging.loggers.telethon.level"] = level
 
         copy("appservice.public.enabled")
         copy("appservice.public.prefix")
         copy("appservice.public.external")
 
-        copy("appservice.debug")
+        copy("appservice.provisioning.enabled")
+        copy("appservice.provisioning.prefix")
+        copy("appservice.provisioning.shared_secret")
+        if base["appservice.provisioning.shared_secret"] == "generate":
+            base["appservice.provisioning.shared_secret"] = self._new_token()
 
-        copy("appservice.id")
-        copy("appservice.bot_username")
-        copy("appservice.bot_displayname")
+        copy("appservice.community_id")
 
-        copy("appservice.as_token")
-        copy("appservice.hs_token")
+        copy("metrics.enabled")
+        copy("metrics.listen_port")
+
+        copy("manhole.enabled")
+        copy("manhole.path")
+        copy("manhole.whitelist")
 
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
 
         copy("bridge.displayname_preference")
+        copy("bridge.displayname_max_length")
+        copy("bridge.allow_avatar_remove")
 
-        copy("bridge.edits_as_replies")
-        copy("bridge.highlight_edits")
-        copy("bridge.bridge_notices")
-        copy("bridge.bot_messages_as_notices")
         copy("bridge.max_initial_member_sync")
+        copy("bridge.sync_channel_members")
+        copy("bridge.skip_deleted_members")
+        copy("bridge.startup_sync")
+        if "bridge.sync_dialog_limit" in self:
+            base["bridge.sync_create_limit"] = self["bridge.sync_dialog_limit"]
+            base["bridge.sync_update_limit"] = self["bridge.sync_dialog_limit"]
+        else:
+            copy("bridge.sync_update_limit")
+            copy("bridge.sync_create_limit")
+        copy("bridge.sync_direct_chats")
         copy("bridge.max_telegram_delete")
+        copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
-        copy("bridge.inline_images")
         copy("bridge.plaintext_highlights")
         copy("bridge.public_portals")
-        copy("bridge.native_stickers")
-        copy("bridge.catch_up")
+        copy("bridge.sync_with_custom_puppets")
+        copy("bridge.sync_direct_chat_list")
+        copy("bridge.double_puppet_server_map")
+        copy("bridge.double_puppet_allow_discovery")
+        if "bridge.login_shared_secret" in self:
+            base["bridge.login_shared_secret_map"] = {
+                base["homeserver.domain"]: self["bridge.login_shared_secret"]
+            }
+        else:
+            copy("bridge.login_shared_secret_map")
+        copy("bridge.telegram_link_preview")
+        copy("bridge.inline_images")
+        copy("bridge.image_as_file_size")
+        copy("bridge.max_document_size")
+        copy("bridge.parallel_file_transfer")
+        copy("bridge.federate_rooms")
+        copy("bridge.animated_sticker.target")
+        copy("bridge.animated_sticker.args")
+        copy("bridge.encryption.allow")
+        copy("bridge.encryption.default")
+        copy("bridge.encryption.database")
+        copy("bridge.encryption.key_sharing.allow")
+        copy("bridge.encryption.key_sharing.require_cross_signing")
+        copy("bridge.encryption.key_sharing.require_verification")
+        copy("bridge.private_chat_portal_meta")
+        copy("bridge.delivery_receipts")
+        copy("bridge.delivery_error_reports")
+        copy("bridge.resend_bridge_info")
+        copy("bridge.backfill.invite_own_puppet")
+        copy("bridge.backfill.takeout_limit")
+        copy("bridge.backfill.initial_limit")
+        copy("bridge.backfill.missed_limit")
+        copy("bridge.backfill.disable_notifications")
+        copy("bridge.backfill.normal_groups")
+
+        copy("bridge.initial_power_level_overrides.group")
+        copy("bridge.initial_power_level_overrides.user")
+
+        copy("bridge.bot_messages_as_notices")
+        if isinstance(self["bridge.bridge_notices"], bool):
+            base["bridge.bridge_notices"] = {
+                "default": self["bridge.bridge_notices"],
+                "exceptions": ["@importantbot:example.com"],
+            }
+        else:
+            copy("bridge.bridge_notices")
+
+        copy("bridge.deduplication.pre_db_check")
+        copy("bridge.deduplication.cache_queue_length")
+
+        if "bridge.message_formats.m_text" in self:
+            del self["bridge.message_formats"]
+        copy_dict("bridge.message_formats", override_existing_map=False)
+        copy("bridge.emote_format")
+
+        copy("bridge.state_event_formats.join")
+        copy("bridge.state_event_formats.leave")
+        copy("bridge.state_event_formats.name_change")
 
         copy("bridge.filter.mode")
         copy("bridge.filter.list")
@@ -202,63 +183,60 @@ class Config(DictWithRecursion):
         if "bridge.relaybot" not in self:
             copy("bridge.authless_relaybot_portals", "bridge.relaybot.authless_portals")
         else:
+            copy("bridge.relaybot.private_chat.invite")
+            copy("bridge.relaybot.private_chat.state_changes")
+            copy("bridge.relaybot.private_chat.message")
+            copy("bridge.relaybot.group_chat_invite")
+            copy("bridge.relaybot.ignore_unbridged_group_chat")
             copy("bridge.relaybot.authless_portals")
             copy("bridge.relaybot.whitelist_group_admins")
             copy("bridge.relaybot.whitelist")
+            copy("bridge.relaybot.ignore_own_incoming_events")
 
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
 
-        self._data = base._data
-        self.save()
+        copy("telegram.connection.timeout")
+        copy("telegram.connection.retries")
+        copy("telegram.connection.retry_delay")
+        copy("telegram.connection.flood_sleep_threshold")
+        copy("telegram.connection.request_retries")
 
-    def _get_permissions(self, key):
+        copy("telegram.device_info.device_model")
+        copy("telegram.device_info.system_version")
+        copy("telegram.device_info.app_version")
+        copy("telegram.device_info.lang_code")
+        copy("telegram.device_info.system_lang_code")
+
+        copy("telegram.server.enabled")
+        copy("telegram.server.dc")
+        copy("telegram.server.ip")
+        copy("telegram.server.port")
+
+        copy("telegram.proxy.type")
+        copy("telegram.proxy.address")
+        copy("telegram.proxy.port")
+        copy("telegram.proxy.rdns")
+        copy("telegram.proxy.username")
+        copy("telegram.proxy.password")
+
+    def _get_permissions(self, key: str) -> Permissions:
         level = self["bridge.permissions"].get(key, "")
         admin = level == "admin"
-        whitelisted = level == "full" or admin
-        relaybot = level == "relaybot" or whitelisted
-        return relaybot, whitelisted, admin
+        matrix_puppeting = level == "full" or admin
+        puppeting = level == "puppeting" or matrix_puppeting
+        user = level == "user" or puppeting
+        relaybot = level == "relaybot" or user
+        return Permissions(relaybot, user, puppeting, matrix_puppeting, admin, level)
 
-    def get_permissions(self, mxid):
-        permissions = self["bridge.permissions"] or {}
+    def get_permissions(self, mxid: UserID) -> Permissions:
+        permissions = self["bridge.permissions"]
         if mxid in permissions:
             return self._get_permissions(mxid)
 
-        homeserver = mxid[mxid.index(":") + 1:]
+        _, homeserver = Client.parse_user_id(mxid)
         if homeserver in permissions:
             return self._get_permissions(homeserver)
 
         return self._get_permissions("*")
-
-    def generate_registration(self):
-        homeserver = self["homeserver.domain"]
-
-        username_format = self.get("bridge.username_template", "telegram_{userid}") \
-            .format(userid=".+")
-        alias_format = self.get("bridge.alias_template", "telegram_{groupname}") \
-            .format(groupname=".+")
-
-        self.set("appservice.as_token", self._new_token())
-        self.set("appservice.hs_token", self._new_token())
-
-        url = (f"{self['appservice.protocol']}://"
-               f"{self['appservice.hostname']}:{self['appservice.port']}")
-        self._registration = {
-            "id": self.get("appservice.id", "telegram"),
-            "as_token": self["appservice.as_token"],
-            "hs_token": self["appservice.hs_token"],
-            "namespaces": {
-                "users": [{
-                    "exclusive": True,
-                    "regex": f"@{username_format}:{homeserver}"
-                }],
-                "aliases": [{
-                    "exclusive": True,
-                    "regex": f"#{alias_format}:{homeserver}"
-                }]
-            },
-            "url": url,
-            "sender_localpart": self["appservice.bot_username"],
-            "rate_limited": False
-        }

mautrix_telegram/context.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,21 +13,45 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Tuple, TYPE_CHECKING
+import asyncio
+
+from alchemysession import AlchemySessionContainer
+
+from mautrix.appservice import AppService
+
+if TYPE_CHECKING:
+    from .web import PublicBridgeWebsite, ProvisioningAPI
+    from .config import Config
+    from .bot import Bot
+    from .matrix import MatrixHandler
+    from .__main__ import TelegramBridge
 
 
 class Context:
-    def __init__(self, az, db, config, loop, bot, mx, telethon_session_container):
+    az: AppService
+    config: 'Config'
+    loop: asyncio.AbstractEventLoop
+    bridge: 'TelegramBridge'
+    bot: Optional['Bot']
+    mx: Optional['MatrixHandler']
+    session_container: AlchemySessionContainer
+    public_website: Optional['PublicBridgeWebsite']
+    provisioning_api: Optional['ProvisioningAPI']
+
+    def __init__(self, az: AppService, config: 'Config', loop: asyncio.AbstractEventLoop,
+                 session_container: AlchemySessionContainer, bridge: 'TelegramBridge',
+                 bot: Optional['Bot']) -> None:
         self.az = az
-        self.db = db
         self.config = config
         self.loop = loop
+        self.bridge = bridge
         self.bot = bot
-        self.mx = mx
-        self.telethon_session_container = telethon_session_container
+        self.mx = None
+        self.session_container = session_container
+        self.public_website = None
+        self.provisioning_api = None
 
-    def __iter__(self):
-        yield self.az
-        yield self.db
-        yield self.config
-        yield self.loop
-        yield self.bot
+    @property
+    def core(self) -> Tuple[AppService, 'Config', asyncio.AbstractEventLoop, Optional['Bot']]:
+        return self.az, self.config, self.loop, self.bot

mautrix_telegram/db.py

@@ -1,132 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
-                        BigInteger, String, Boolean)
-from sqlalchemy.orm import relationship
-
-from .base import Base
-
-
-class Portal(Base):
-    query = None
-    __tablename__ = "portal"
-
-    # Telegram chat information
-    tgid = Column(Integer, primary_key=True)
-    tg_receiver = Column(Integer, primary_key=True)
-    peer_type = Column(String)
-    megagroup = Column(Boolean)
-
-    # Matrix portal information
-    mxid = Column(String, unique=True, nullable=True)
-
-    # Telegram chat metadata
-    username = Column(String, nullable=True)
-    title = Column(String, nullable=True)
-    about = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-
-
-class Message(Base):
-    query = None
-    __tablename__ = "message"
-
-    mxid = Column(String)
-    mx_room = Column(String)
-    tgid = Column(Integer, primary_key=True)
-    tg_space = Column(Integer, primary_key=True)
-
-    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
-
-
-class UserPortal(Base):
-    query = None
-    __tablename__ = "user_portal"
-
-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
-    portal = Column(Integer, primary_key=True)
-    portal_receiver = Column(Integer, primary_key=True)
-
-    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
-                                           ("portal.tgid", "portal.tg_receiver")),)
-
-
-class User(Base):
-    query = None
-    __tablename__ = "user"
-
-    mxid = Column(String, primary_key=True)
-    tgid = Column(Integer, nullable=True)
-    tg_username = Column(String, nullable=True)
-    saved_contacts = Column(Integer, default=0)
-    contacts = relationship("Contact", uselist=True,
-                            cascade="save-update, merge, delete, delete-orphan")
-    portals = relationship("Portal", secondary="user_portal")
-
-
-class Contact(Base):
-    query = None
-    __tablename__ = "contact"
-
-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
-    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
-
-
-class Puppet(Base):
-    query = None
-    __tablename__ = "puppet"
-
-    id = Column(Integer, primary_key=True)
-    displayname = Column(String, nullable=True)
-    displayname_source = Column(Integer, nullable=True)
-    username = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-    is_bot = Column(Boolean, nullable=True)
-
-
-# Fucking Telegram not telling bots what chats they are in 3:<
-class BotChat(Base):
-    query = None
-    __tablename__ = "bot_chat"
-    id = Column(Integer, primary_key=True)
-    type = Column(String, nullable=False)
-
-
-class TelegramFile(Base):
-    query = None
-    __tablename__ = "telegram_file"
-
-    id = Column(String, primary_key=True)
-    mxc = Column(String)
-    mime_type = Column(String)
-    was_converted = Column(Boolean)
-    timestamp = Column(BigInteger)
-    size = Column(Integer, nullable=True)
-    width = Column(Integer, nullable=True)
-    height = Column(Integer, nullable=True)
-    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
-    thumbnail = relationship("TelegramFile", uselist=False)
-
-
-def init(db_session):
-    Portal.query = db_session.query_property()
-    Message.query = db_session.query_property()
-    UserPortal.query = db_session.query_property()
-    User.query = db_session.query_property()
-    Puppet.query = db_session.query_property()
-    BotChat.query = db_session.query_property()
-    TelegramFile.query = db_session.query_property()

mautrix_telegram/db/__init__.py

@@ -0,0 +1,31 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy.engine.base import Engine
+
+from mautrix.client.state_store.sqlalchemy import UserProfile, RoomState
+
+from .bot_chat import BotChat
+from .message import Message
+from .portal import Portal
+from .puppet import Puppet
+from .telegram_file import TelegramFile
+from .user import User, UserPortal, Contact
+
+
+def init(db_engine: Engine) -> None:
+    for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
+                  RoomState, BotChat):
+        table.bind(db_engine)

mautrix_telegram/db/bot_chat.py

@@ -0,0 +1,38 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Iterable
+
+from sqlalchemy import Column, Integer, String
+
+from mautrix.util.db import Base
+
+from ..types import TelegramID
+
+
+# Fucking Telegram not telling bots what chats they are in 3:<
+class BotChat(Base):
+    __tablename__ = "bot_chat"
+    id: TelegramID = Column(Integer, primary_key=True)
+    type: str = Column(String, nullable=False)
+
+    @classmethod
+    def delete_by_id(cls, chat_id: TelegramID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.id == chat_id))
+
+    @classmethod
+    def all(cls) -> Iterable['BotChat']:
+        return cls._select_all()

mautrix_telegram/db/message.py

@@ -0,0 +1,94 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Iterator
+
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+
+from mautrix.types import RoomID, EventID
+from mautrix.util.db import Base
+
+from ..types import TelegramID
+
+
+class Message(Base):
+    __tablename__ = "message"
+
+    mxid: EventID = Column(String)
+    mx_room: RoomID = Column(String)
+    tgid: TelegramID = Column(Integer, primary_key=True)
+    tg_space: TelegramID = Column(Integer, primary_key=True)
+    edit_index: int = Column(Integer, primary_key=True)
+
+    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"),)
+
+    @classmethod
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Iterator['Message']:
+        return cls._select_all(cls.c.tgid == tgid, cls.c.tg_space == tg_space)
+
+    @classmethod
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
+        if edit_index < 0:
+            return cls._one_or_none(cls.db.execute(
+                cls.t.select()
+                    .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                    .order_by(desc(cls.c.edit_index))
+                    .limit(1).offset(-edit_index - 1)))
+        else:
+            return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                           cls.c.edit_index == edit_index)
+
+    @classmethod
+    def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
+        rows = cls.db.execute(select([func.count(cls.c.tg_space)])
+                              .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
+        try:
+            count, = next(rows)
+            return count
+        except StopIteration:
+            return 0
+
+    @classmethod
+    def find_last(cls, mx_room: RoomID, tg_space: TelegramID) -> Optional['Message']:
+        return cls._one_or_none(cls.db.execute(
+            cls._make_simple_select(cls.c.mx_room == mx_room, cls.c.tg_space == tg_space)
+                .order_by(desc(cls.c.tgid)).limit(1)))
+
+    @classmethod
+    def delete_all(cls, mx_room: RoomID) -> None:
+        cls.db.execute(cls.t.delete().where(cls.c.mx_room == mx_room))
+
+    @classmethod
+    def get_by_mxid(cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
+                    ) -> Optional['Message']:
+        return cls._select_one_or_none(cls.c.mxid == mxid, cls.c.mx_room == mx_room,
+                                       cls.c.tg_space == tg_space)
+
+    @classmethod
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
+                         .values(**values))
+
+    @classmethod
+    def update_by_mxid(cls, s_mxid: EventID, s_mx_room: RoomID, **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
+                         .values(**values))

mautrix_telegram/db/portal.py

@@ -0,0 +1,66 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Iterable
+
+from sqlalchemy import Column, Integer, String, Boolean, Text, func, sql
+
+from mautrix.types import RoomID, ContentURI
+from mautrix.util.db import Base
+
+from ..types import TelegramID
+
+
+class Portal(Base):
+    __tablename__ = "portal"
+
+    # Telegram chat information
+    tgid: TelegramID = Column(Integer, primary_key=True)
+    tg_receiver: TelegramID = Column(Integer, primary_key=True)
+    peer_type: str = Column(String, nullable=False)
+    megagroup: bool = Column(Boolean)
+
+    # Matrix portal information
+    mxid: Optional[RoomID] = Column(String, unique=True, nullable=True)
+    avatar_url: Optional[ContentURI] = Column(String, nullable=True)
+    encrypted: bool = Column(Boolean, nullable=False, server_default=sql.expression.false())
+
+    config: str = Column(Text, nullable=True)
+
+    # Telegram chat metadata
+    username: str = Column(String, nullable=True)
+    title: str = Column(String, nullable=True)
+    about: str = Column(String, nullable=True)
+    photo_id: str = Column(String, nullable=True)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver)
+
+    @classmethod
+    def find_private_chats(cls, tg_receiver: TelegramID) -> Iterable['Portal']:
+        yield from cls._select_all(cls.c.tg_receiver == tg_receiver, cls.c.peer_type == "user")
+
+    @classmethod
+    def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Portal']:
+        return cls._select_one_or_none(func.lower(cls.c.username) == username)
+
+    @classmethod
+    def all(cls) -> Iterable['Portal']:
+        yield from cls._select_all()

mautrix_telegram/db/puppet.py

@@ -0,0 +1,61 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Iterable
+
+from sqlalchemy import Column, Integer, String, Text, Boolean
+from sqlalchemy.sql import expression, func
+
+from mautrix.types import UserID, SyncToken
+from mautrix.util.db import Base
+
+from ..types import TelegramID
+
+
+class Puppet(Base):
+    __tablename__ = "puppet"
+
+    id: TelegramID = Column(Integer, primary_key=True)
+    custom_mxid: UserID = Column(String, nullable=True)
+    access_token: str = Column(String, nullable=True)
+    next_batch: SyncToken = Column(String, nullable=True)
+    base_url: str = Column(Text, nullable=True)
+    displayname: str = Column(String, nullable=True)
+    displayname_source: TelegramID = Column(Integer, nullable=True)
+    username: str = Column(String, nullable=True)
+    photo_id: str = Column(String, nullable=True)
+    is_bot: bool = Column(Boolean, nullable=True)
+    matrix_registered: bool = Column(Boolean, nullable=False, server_default=expression.false())
+    disable_updates: bool = Column(Boolean, nullable=False, server_default=expression.false())
+
+    @classmethod
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        yield from cls._select_all(cls.c.custom_mxid != None)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.id == tgid)
+
+    @classmethod
+    def get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.custom_mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(func.lower(cls.c.username) == username)
+
+    @classmethod
+    def get_by_displayname(cls, displayname: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.displayname == displayname)

mautrix_telegram/db/telegram_file.py

@@ -0,0 +1,79 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, cast, Dict, Any
+
+from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
+                        TypeDecorator)
+from sqlalchemy.engine.result import RowProxy
+
+from mautrix.types import ContentURI, EncryptedFile
+from mautrix.util.db import Base
+
+
+class DBEncryptedFile(TypeDecorator):
+    impl = Text
+
+    @property
+    def python_type(self):
+        return EncryptedFile
+
+    def process_bind_param(self, value: EncryptedFile, dialect) -> Optional[str]:
+        if value is not None:
+            return value.json()
+        return None
+
+    def process_result_value(self, value: str, dialect) -> Optional[EncryptedFile]:
+        if value is not None:
+            return EncryptedFile.parse_json(value)
+        return None
+
+    def process_literal_param(self, value, dialect):
+        return value
+
+
+class TelegramFile(Base):
+    __tablename__ = "telegram_file"
+
+    id: str = Column(String, primary_key=True)
+    mxc: ContentURI = Column(String)
+    mime_type: str = Column(String)
+    was_converted: bool = Column(Boolean)
+    timestamp: int = Column(BigInteger)
+    size: Optional[int] = Column(Integer, nullable=True)
+    width: Optional[int] = Column(Integer, nullable=True)
+    height: Optional[int] = Column(Integer, nullable=True)
+    decryption_info: Optional[Dict[str, Any]] = Column(DBEncryptedFile, nullable=True)
+    thumbnail_id: str = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+    thumbnail: Optional['TelegramFile'] = None
+
+    @classmethod
+    def scan(cls, row: RowProxy) -> 'TelegramFile':
+        telegram_file = cast(TelegramFile, super().scan(row))
+        if isinstance(telegram_file.thumbnail, str):
+            telegram_file.thumbnail = cls.get(telegram_file.thumbnail)
+        return telegram_file
+
+    @classmethod
+    def get(cls, loc_id: str) -> Optional['TelegramFile']:
+        return cls._select_one_or_none(cls.c.id == loc_id)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, mxc=self.mxc, mime_type=self.mime_type,
+                was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
+                width=self.width, height=self.height, decryption_info=self.decryption_info,
+                thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/user.py

@@ -0,0 +1,108 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Iterable, Tuple
+
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String, func
+
+from mautrix.types import UserID
+from mautrix.util.db import Base
+
+from ..types import TelegramID
+
+
+class User(Base):
+    __tablename__ = "user"
+
+    mxid: UserID = Column(String, primary_key=True)
+    tgid: Optional[TelegramID] = Column(Integer, nullable=True, unique=True)
+    tg_username: str = Column(String, nullable=True)
+    tg_phone: str = Column(String, nullable=True)
+    saved_contacts: int = Column(Integer, default=0, nullable=False)
+
+    @classmethod
+    def all_with_tgid(cls) -> Iterable['User']:
+        return cls._select_all(cls.c.tgid != None)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tgid == tgid)
+
+    @classmethod
+    def get_by_mxid(cls, mxid: UserID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['User']:
+        return cls._select_one_or_none(func.lower(cls.c.tg_username) == username)
+
+    @property
+    def contacts(self) -> Iterable[TelegramID]:
+        rows = self.db.execute(Contact.t.select().where(Contact.c.user == self.tgid))
+        for row in rows:
+            user, contact = row
+            yield contact
+
+    @contacts.setter
+    def contacts(self, puppets: Iterable[TelegramID]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
+            insert_puppets = [{"user": self.tgid, "contact": tgid} for tgid in puppets]
+            if insert_puppets:
+                conn.execute(Contact.t.insert(), insert_puppets)
+
+    @property
+    def portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        rows = self.db.execute(UserPortal.t.select().where(UserPortal.c.user == self.tgid))
+        for row in rows:
+            user, portal, portal_receiver = row
+            yield (portal, portal_receiver)
+
+    @portals.setter
+    def portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
+            insert_portals = [{
+                "user": self.tgid,
+                "portal": tgid,
+                "portal_receiver": tg_receiver
+            } for tgid, tg_receiver in portals]
+            if insert_portals:
+                conn.execute(UserPortal.t.insert(), insert_portals)
+
+    def delete(self) -> None:
+        super().delete()
+        self.portals = []
+        self.contacts = []
+
+
+class UserPortal(Base):
+    __tablename__ = "user_portal"
+
+    user: TelegramID = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE",
+                                                  ondelete="CASCADE"), primary_key=True)
+    portal: TelegramID = Column(Integer, primary_key=True)
+    portal_receiver: TelegramID = Column(Integer, primary_key=True)
+
+    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
+                                           ("portal.tgid", "portal.tg_receiver"),
+                                           onupdate="CASCADE", ondelete="CASCADE"),)
+
+
+class Contact(Base):
+    __tablename__ = "contact"
+
+    user: TelegramID = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
+    contact: TelegramID = Column(Integer, ForeignKey("puppet.id"), primary_key=True)

mautrix_telegram/example-config.yaml

@@ -0,0 +1,522 @@
+# Homeserver details
+homeserver:
+    # The address that this appservice can use to connect to the homeserver.
+    address: https://example.com
+    # The domain of the homeserver (for MXIDs, etc).
+    domain: example.com
+    # Whether or not to verify the SSL certificate of the homeserver.
+    # Only applies if address starts with https://
+    verify_ssl: true
+    asmux: false
+
+# Application service host/registration related details
+# Changing these values requires regeneration of the registration.
+appservice:
+    # The address that the homeserver can use to connect to this appservice.
+    address: http://localhost:29317
+    # When using https:// the TLS certificate and key files for the address.
+    tls_cert: false
+    tls_key: false
+
+    # The hostname and port where this appservice should listen.
+    hostname: 0.0.0.0
+    port: 29317
+    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+    max_body_size: 1
+
+    # The full URI to the database. SQLite and Postgres are fully supported.
+    # Other DBMSes supported by SQLAlchemy may or may not work.
+    # Format examples:
+    #   SQLite:   sqlite:///filename.db
+    #   Postgres: postgres://username:password@hostname/dbname
+    database: sqlite:///mautrix-telegram.db
+    # Optional extra arguments for SQLAlchemy's create_engine
+    database_opts: {}
+
+    # Public part of web server for out-of-Matrix interaction with the bridge.
+    # Used for things like login if the user wants to make sure the 2FA password isn't stored in
+    # the HS database.
+    public:
+        # Whether or not the public-facing endpoints should be enabled.
+        enabled: false
+        # The prefix to use in the public-facing endpoints.
+        prefix: /public
+        # The base URL where the public-facing endpoints are available. The prefix is not added
+        # implicitly.
+        external: https://example.com/public
+
+    # Provisioning API part of the web server for automated portal creation and fetching information.
+    # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager).
+    provisioning:
+        # Whether or not the provisioning API should be enabled.
+        enabled: true
+        # The prefix to use in the provisioning API endpoints.
+        prefix: /_matrix/provision/v1
+        # The shared secret to authorize users of the API.
+        # Set to "generate" to generate and save a new token.
+        shared_secret: generate
+
+    # The unique ID of this appservice.
+    id: telegram
+    # Username of the appservice bot.
+    bot_username: telegrambot
+    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+    # to leave display name/avatar as-is.
+    bot_displayname: Telegram bridge bot
+    bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
+
+    # Community ID for bridged users (changes registration file) and rooms.
+    # Must be created manually.
+    #
+    # Example: "+telegram:example.com". Set to false to disable.
+    community_id: false
+
+    # Whether or not to receive ephemeral events via appservice transactions.
+    # Requires MSC2409 support (i.e. Synapse 1.22+).
+    # You should disable bridge -> sync_with_custom_puppets when this is enabled.
+    ephemeral_events: false
+
+    # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
+    as_token: "This value is generated when generating the registration"
+    hs_token: "This value is generated when generating the registration"
+
+# Prometheus telemetry config. Requires prometheus-client to be installed.
+metrics:
+    enabled: false
+    listen_port: 8000
+
+# Manhole config.
+manhole:
+    # Whether or not opening the manhole is allowed.
+    enabled: false
+    # The path for the unix socket.
+    path: /var/tmp/mautrix-telegram.manhole
+    # The list of UIDs who can be added to the whitelist.
+    # If empty, any UIDs can be specified in the open-manhole command.
+    whitelist:
+    - 0
+
+# Bridge config
+bridge:
+    # Localpart template of MXIDs for Telegram users.
+    # {userid} is replaced with the user ID of the Telegram user.
+    username_template: "telegram_{userid}"
+    # Localpart template of room aliases for Telegram portal rooms.
+    # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} )
+    alias_template: "telegram_{groupname}"
+    # Displayname template for Telegram users.
+    # {displayname} is replaced with the display name of the Telegram user.
+    displayname_template: "{displayname} (Telegram)"
+
+    # Set the preferred order of user identifiers which to use in the Matrix puppet display name.
+    # In the (hopefully unlikely) scenario that none of the given keys are found, the numeric user
+    # ID is used.
+    #
+    # If the bridge is working properly, a phone number or an username should always be known, but
+    # the other one can very well be empty.
+    #
+    # Valid keys:
+    #   "full name"          (First and/or last name)
+    #   "full name reversed" (Last and/or first name)
+    #   "first name"
+    #   "last name"
+    #   "username"
+    #   "phone number"
+    displayname_preference:
+    - full name
+    - username
+    - phone number
+    # Maximum length of displayname
+    displayname_max_length: 100
+    # Remove avatars from Telegram ghost users when removed on Telegram. This is disabled by default
+    # as there's no way to determine whether an avatar is removed or just hidden from some users. If
+    # you're on a single-user instance, this should be safe to enable.
+    allow_avatar_remove: false
+
+    # Maximum number of members to sync per portal when starting up. Other members will be
+    # synced when they send messages. The maximum is 10000, after which the Telegram server
+    # will not send any more members.
+    # -1 means no limit (which means it's limited to 10000 by the server)
+    max_initial_member_sync: 100
+    # Whether or not to sync the member list in channels.
+    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
+    # list regardless of this setting.
+    sync_channel_members: true
+    # Whether or not to skip deleted members when syncing members.
+    skip_deleted_members: true
+    # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
+    # their Telegram account at startup.
+    startup_sync: true
+    # Number of most recently active dialogs to check when syncing chats.
+    # Set to 0 to remove limit.
+    sync_update_limit: 0
+    # Number of most recently active dialogs to create portals for when syncing chats.
+    # Set to 0 to remove limit.
+    sync_create_limit: 30
+    # Whether or not to sync and create portals for direct chats at startup.
+    sync_direct_chats: false
+    # The maximum number of simultaneous Telegram deletions to handle.
+    # A large number of simultaneous redactions could put strain on your homeserver.
+    max_telegram_delete: 10
+    # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
+    # at startup and when creating a bridge.
+    sync_matrix_state: true
+    # Allow logging in within Matrix. If false, users can only log in using login-qr or the
+    # out-of-Matrix login website (see appservice.public config section)
+    allow_matrix_login: true
+    # Whether or not to bridge plaintext highlights.
+    # Only enable this if your displayname_template has some static part that the bridge can use to
+    # reliably identify what is a plaintext highlight.
+    plaintext_highlights: false
+    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
+    public_portals: true
+    # Whether or not to use /sync to get presence, read receipts and typing notifications
+    # when double puppeting is enabled
+    sync_with_custom_puppets: true
+    # Whether or not to update the m.direct account data event when double puppeting is enabled.
+    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
+    # and is therefore prone to race conditions.
+    sync_direct_chat_list: false
+    # Servers to always allow double puppeting from
+    double_puppet_server_map:
+        example.com: https://example.com
+    # Allow using double puppeting from any server with a valid client .well-known file.
+    double_puppet_allow_discovery: false
+    # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth
+    #
+    # If set, custom puppets will be enabled automatically for local users
+    # instead of users having to find an access token and run `login-matrix`
+    # manually.
+    # If using this for other servers than the bridge's server,
+    # you must also set the URL in the double_puppet_server_map.
+    login_shared_secret_map:
+        example.com: foobar
+    # Set to false to disable link previews in messages sent to Telegram.
+    telegram_link_preview: true
+    # Use inline images instead of a separate message for the caption.
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    inline_images: false
+    # Maximum size of image in megabytes before sending to Telegram as a document.
+    image_as_file_size: 10
+    # Maximum size of Telegram documents in megabytes to bridge.
+    max_document_size: 100
+    # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
+    # streaming from/to Matrix and using many connections for Telegram.
+    # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
+    parallel_file_transfer: false
+    # Whether or not created rooms should have federation enabled.
+    # If false, created portal rooms will never be federated.
+    federate_rooms: true
+    # Settings for converting animated stickers.
+    animated_sticker:
+        # Format to which animated stickers should be converted.
+        # disable - No conversion, send as-is (gzipped lottie)
+        # png - converts to non-animated png (fastest),
+        # gif - converts to animated gif, but loses transparency
+        # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support
+        target: gif
+        # Arguments for converter. All converters take width and height.
+        # GIF converter takes background as a hex color.
+        args:
+            width: 256
+            height: 256
+            background: "020202"  # only for gif
+            fps: 30               # only for webm
+    # End-to-bridge encryption support options. These require matrix-nio to be installed with pip
+    # and login_shared_secret to be configured in order to get a device for the bridge bot.
+    #
+    # Additionally, https://github.com/matrix-org/synapse/pull/5758 is required if using a normal
+    # application service.
+    encryption:
+        # Allow encryption, work in group chat rooms with e2ee enabled
+        allow: false
+        # Default to encryption, force-enable encryption in all portals the bridge creates
+        # This will cause the bridge bot to be in private chats for the encryption to work properly.
+        default: false
+        # Database for the encryption data. Currently only supports Postgres and an in-memory
+        # store that's persisted as a pickle.
+        # If set to `default`, will use the appservice postgres database
+        # or a pickle file if the appservice database is sqlite.
+        #
+        # Format examples:
+        #   Pickle:   pickle:///filename.pickle
+        #   Postgres: postgres://username:password@hostname/dbname
+        database: default
+        # Options for automatic key sharing.
+        key_sharing:
+            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
+            # You must use a client that supports requesting keys from other users to use this feature.
+            allow: false
+            # Require the requesting device to have a valid cross-signing signature?
+            # This doesn't require that the bridge has verified the device, only that the user has verified it.
+            # Not yet implemented.
+            require_cross_signing: false
+            # Require devices to be verified by the bridge?
+            # Verification by the bridge is not yet implemented.
+            require_verification: true
+    # Whether or not to explicitly set the avatar and room name for private
+    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
+    private_chat_portal_meta: false
+    # Whether or not the bridge should send a read receipt from the bridge bot when a message has
+    # been sent to Telegram.
+    delivery_receipts: false
+    # Whether or not delivery errors should be reported as messages in the Matrix room.
+    delivery_error_reports: false
+    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
+    # This field will automatically be changed back to false after it,
+    # except if the config file is not writable.
+    resend_bridge_info: false
+    # Settings for backfilling messages from Telegram.
+    backfill:
+        # Whether or not the Telegram ghosts of logged in Matrix users should be
+        # invited to private chats when backfilling history from Telegram. This is
+        # usually needed to prevent rate limits and to allow timestamp massaging.
+        invite_own_puppet: true
+        # Maximum number of messages to backfill without using a takeout.
+        # The first time a takeout is used, the user has to manually approve it from a different
+        # device. If initial_limit or missed_limit are higher than this value, the bridge will ask
+        # the user to accept the takeout after logging in before syncing any chats.
+        takeout_limit: 100
+        # Maximum number of messages to backfill initially.
+        # Set to 0 to disable backfilling when creating portal, or -1 to disable the limit.
+        #
+        # N.B. Initial backfill will only start after member sync. Make sure your
+        #      max_initial_member_sync is set to a low enough value so it doesn't take forever.
+        initial_limit: 0
+        # Maximum number of messages to backfill if messages were missed while the bridge was
+        # disconnected. Note that this only works for logged in users and only if the chat isn't
+        # older than sync_update_limit
+        # Set to 0 to disable backfilling missed messages.
+        missed_limit: 50
+        # If using double puppeting, should notifications be disabled
+        # while the initial backfill is in progress?
+        disable_notifications: false
+        # Whether or not to enable backfilling in normal groups.
+        # Normal groups have numerous technical problems in Telegram, and backfilling normal groups
+        # will likely cause problems if there are multiple Matrix users in the group.
+        normal_groups: false
+
+    # Overrides for base power levels.
+    initial_power_level_overrides:
+        user: {}
+        group: {}
+
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    bridge_notices:
+        # Whether or not Matrix bot messages (type m.notice) should be bridged.
+        default: false
+        # List of user IDs for whom the previous flag is flipped.
+        # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
+        #      notices from users listed here will be bridged.
+        exceptions:
+        - "@importantbot:example.com"
+
+    # Some config options related to Telegram message deduplication.
+    # The default values are usually fine, but some debug messages/warnings might recommend you
+    # change these.
+    deduplication:
+        # Whether or not to check the database if the message about to be sent is a duplicate.
+        pre_db_check: false
+        # The number of latest events to keep when checking for duplicates.
+        # You might need to increase this on high-traffic bridge instances.
+        cache_queue_length: 20
+
+    # The formats to use when sending messages to Telegram via the relay bot.
+    # Text msgtypes (m.text, m.notice and m.emote) support HTML, media msgtypes don't.
+    #
+    # Available variables:
+    #   $sender_displayname - The display name of the sender (e.g. Example User)
+    #   $sender_username    - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
+    #   $sender_mxid        - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $message            - The message content
+    message_formats:
+        m.text: "<b>$sender_displayname</b>: $message"
+        m.notice: "<b>$sender_displayname</b>: $message"
+        m.emote: "* <b>$sender_displayname</b> $message"
+        m.file: "<b>$sender_displayname</b> sent a file: $message"
+        m.image: "<b>$sender_displayname</b> sent an image: $message"
+        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "<b>$sender_displayname</b> sent a video: $message"
+        m.location: "<b>$sender_displayname</b> sent a location: $message"
+    # Telegram doesn't have built-in emotes, this field specifies how m.emote's from authenticated
+    # users are sent to telegram. All fields in message_formats are supported. Additionally, the
+    # Telegram user info is available in the following variables:
+    #    $displayname - Telegram displayname
+    #    $username    - Telegram username (may not exist)
+    #    $mention     - Telegram @username or displayname mention (depending on which exists)
+    emote_format: "* $mention $formatted_body"
+
+    # The formats to use when sending state events to Telegram via the relay bot.
+    #
+    # Variables from `message_formats` that have the `sender_` prefix are available without the prefix.
+    # In name_change events, `$prev_displayname` is the previous displayname.
+    #
+    # Set format to an empty string to disable the messages for that event.
+    state_event_formats:
+        join: "<b>$displayname</b> joined the room."
+        leave: "<b>$displayname</b> left the room."
+        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+
+    # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
+    # `filter-mode` management commands.
+    #
+    # Filters do not affect direct chats.
+    # An empty blacklist will essentially disable the filter.
+    filter:
+        # Filter mode to use. Either "blacklist" or "whitelist".
+        # If the mode is "blacklist", the listed chats will never be bridged.
+        # If the mode is "whitelist", only the listed chats can be bridged.
+        mode: blacklist
+        # The list of group/channel IDs to filter.
+        list: []
+
+    # The prefix for commands. Only required in non-management rooms.
+    command_prefix: "!tg"
+
+    # Permissions for using the bridge.
+    # Permitted values:
+    #   relaybot - Only use the bridge via the relaybot, no access to commands.
+    #       user - Relaybot level + access to commands to create bridges.
+    #  puppeting - User level + logging in with a Telegram account.
+    #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
+    #      admin - Full access to use the bridge and some extra administration commands.
+    # Permitted keys:
+    #        * - All Matrix users
+    #   domain - All users on that homeserver
+    #     mxid - Specific user
+    permissions:
+        "*": "relaybot"
+        "public.example.com": "user"
+        "example.com": "full"
+        "@admin:example.com": "admin"
+
+    # Options related to the message relay Telegram bot.
+    relaybot:
+        private_chat:
+            # List of users to invite to the portal when someone starts a private chat with the bot.
+            # If empty, private chats with the bot won't create a portal.
+            invite: []
+            # Whether or not to bridge state change messages in relaybot private chats.
+            state_changes: true
+            # When private_chat_invite is empty, this message is sent to users /starting the
+            # relaybot. Telegram's "markdown" is supported.
+            message: This is a Matrix bridge relaybot and does not support direct chats
+        # List of users to invite to all group chat portals created by the bridge.
+        group_chat_invite: []
+        # Whether or not the relaybot should not bridge events in unbridged group chats.
+        # If false, portals will be created when the relaybot receives messages, just like normal
+        # users. This behavior is usually not desirable, as it interferes with manually bridging
+        # the chat to another room.
+        ignore_unbridged_group_chat: true
+        # Whether or not to allow creating portals from Telegram.
+        authless_portals: true
+        # Whether or not to allow Telegram group admins to use the bot commands.
+        whitelist_group_admins: true
+        # Whether or not to ignore incoming events sent by the relay bot.
+        ignore_own_incoming_events: true
+        # List of usernames/user IDs who are also allowed to use the bot commands.
+        whitelist:
+        - myusername
+        - 12345678
+
+# Telegram config
+telegram:
+    # Get your own API keys at https://my.telegram.org/apps
+    api_id: 12345
+    api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
+    # (Optional) Create your own bot at https://t.me/BotFather
+    bot_token: disabled
+
+    # Telethon connection options.
+    connection:
+        # The timeout in seconds to be used when connecting.
+        timeout: 120
+        # How many times the reconnection should retry, either on the initial connection or when
+        # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
+        # this is not recommended, since the program can get stuck in an infinite loop.
+        retries: 5
+        # The delay in seconds to sleep between automatic reconnections.
+        retry_delay: 1
+        # The threshold below which the library should automatically sleep on flood wait errors
+        # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
+        # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
+        # the error instead. Values larger than a day (86400) will be changed to a day.
+        flood_sleep_threshold: 60
+        # How many times a request should be retried. Request are retried when Telegram is having
+        # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
+        # there's a migrate error. May take a negative or null value for infinite retries, but this
+        # is not recommended, since some requests can always trigger a call fail (such as searching
+        # for messages).
+        request_retries: 5
+
+    # Device info sent to Telegram.
+    device_info:
+        # "auto" = OS name+version.
+        device_model: auto
+        # "auto" = Telethon version.
+        system_version: auto
+        # "auto" = mautrix-telegram version.
+        app_version: auto
+        lang_code: en
+        system_lang_code: en
+
+    # Custom server to connect to.
+    server:
+        # Set to true to use these server settings. If false, will automatically
+        # use production server assigned by Telegram. Set to false in production.
+        enabled: false
+        # The DC ID to connect to.
+        dc: 2
+        # The IP to connect to.
+        ip: 149.154.167.40
+        # The port to connect to. 443 may not work, 80 is better and both are equally secure.
+        port: 80
+
+    # Telethon proxy configuration.
+    # You must install PySocks from pip for proxies to work.
+    proxy:
+        # Allowed types: disabled, socks4, socks5, http, mtproxy
+        type: disabled
+        # Proxy IP address and port.
+        address: 127.0.0.1
+        port: 1080
+        # Whether or not to perform DNS resolving remotely. Only for socks/http proxies.
+        rdns: true
+        # Proxy authentication (optional). Put MTProxy secret in password field.
+        username: ""
+        password: ""
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        colored:
+            (): mautrix_telegram.util.ColorFormatter
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+        normal:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: normal
+            filename: ./mautrix-telegram.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: colored
+    loggers:
+        mau:
+            level: DEBUG
+        telethon:
+            level: INFO
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,7 @@
-from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
-                          init_mx)
-from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
-from ..context import Context
+from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram, init_mx
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
+from .. import context as c
 
 
-def init(context: Context):
+def init(context: c.Context) -> None:
     init_mx(context)
-    init_tg(context)

mautrix_telegram/formatter/from_matrix.py

@@ -1,339 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from html import unescape
-from html.parser import HTMLParser
-from collections import deque
-from typing import Optional, List, Tuple, Type, Callable, Dict, Any
-import math
-import re
-import logging
-
-from telethon.tl.types import (MessageEntityMention,
-                               InputMessageEntityMentionName, MessageEntityEmail,
-                               MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
-                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
-                               MessageEntityBotCommand, InputUser, TypeMessageEntity)
-
-from ..context import Context
-from .. import user as u, puppet as pu, portal as po
-from ..db import Message as DBMessage
-from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text, html_to_unicode)
-
-log = logging.getLogger("mau.fmt.mx")
-should_bridge_plaintext_highlights = False
-
-
-class MatrixParser(HTMLParser):
-    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")
-    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")
-    block_tags = ("br", "p", "pre", "blockquote",
-                  "ol", "ul", "li",
-                  "h1", "h2", "h3", "h4", "h5", "h6",
-                  "div", "hr", "table")
-
-    def __init__(self):
-        super().__init__()
-        self.text = ""
-        self.entities = []
-        self._building_entities = {}
-        self._list_counter = 0
-        self._open_tags = deque()
-        self._open_tags_meta = deque()
-        self._line_is_new = True
-        self._list_entry_is_new = False
-
-    def _parse_url(self, url: str, args: Dict[str, Any]
-                   ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]:
-        mention = self.mention_regex.match(url)
-        if mention:
-            mxid = mention.group(1)
-            user = (pu.Puppet.get_by_mxid(mxid)
-                    or u.User.get_by_mxid(mxid, create=False))
-            if not user:
-                return None, None
-            if user.username:
-                return MessageEntityMention, f"@{user.username}"
-            elif user.tgid:
-                args["user_id"] = InputUser(user.tgid, 0)
-                return InputMessageEntityMentionName, user.displayname or None
-            else:
-                return None, None
-
-        room = self.room_regex.match(url)
-        if room:
-            username = po.Portal.get_username_from_mx_alias(room.group(1))
-            portal = po.Portal.find_by_username(username)
-            if portal and portal.username:
-                return MessageEntityMention, f"@{portal.username}"
-
-        if url.startswith("mailto:"):
-            return MessageEntityEmail, url[len("mailto:"):]
-        elif self.get_starttag_text() == url:
-            return MessageEntityUrl, url
-        else:
-            args["url"] = url
-            return MessageEntityTextUrl, None
-
-    def handle_starttag(self, tag: str, attrs: List[Tuple[str, str]]):
-        self._open_tags.appendleft(tag)
-        self._open_tags_meta.appendleft(0)
-
-        attrs = dict(attrs)
-        entity_type = None
-        args = {}
-        if tag in ("strong", "b"):
-            entity_type = MessageEntityBold
-        elif tag in ("em", "i"):
-            entity_type = MessageEntityItalic
-        elif tag == "code":
-            try:
-                pre = self._building_entities["pre"]
-                try:
-                    # Pre tag and language found, add language to MessageEntityPre
-                    pre.language = attrs["class"][len("language-"):]
-                except KeyError:
-                    # Pre tag found, but language not found, keep pre as-is
-                    pass
-            except KeyError:
-                # No pre tag found, this is inline code
-                entity_type = MessageEntityCode
-        elif tag == "pre":
-            entity_type = MessageEntityPre
-            args["language"] = ""
-        elif tag == "command":
-            entity_type = MessageEntityBotCommand
-        elif tag == "li":
-            self._list_entry_is_new = True
-        elif tag == "a":
-            try:
-                url = attrs["href"]
-            except KeyError:
-                return
-            entity_type, url = self._parse_url(url, args)
-            self._open_tags_meta.popleft()
-            self._open_tags_meta.appendleft(url)
-
-        if tag in self.block_tags and ("blockquote" not in self._open_tags or tag == "br"):
-            self._newline()
-
-        if entity_type and tag not in self._building_entities:
-            offset = len(self.text)
-            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
-
-    @property
-    def _list_indent(self) -> int:
-        indent = 0
-        first_skipped = False
-        for index, tag in enumerate(self._open_tags):
-            if not first_skipped and tag in ("ol", "ul"):
-                # The first list level isn't indented, so skip it.
-                first_skipped = True
-                continue
-            if tag == "ol":
-                n = self._open_tags_meta[index]
-                extra_length_for_long_index = (int(math.log(n, 10)) - 1) * 3
-                indent += 4 + extra_length_for_long_index
-            elif tag == "ul":
-                indent += 3
-        return indent
-
-    def _newline(self, allow_multi: bool = False):
-        if self._line_is_new and not allow_multi:
-            return
-        self.text += "\n"
-        self._line_is_new = True
-        for entity in self._building_entities.values():
-            entity.length += 1
-
-    def _handle_special_previous_tags(self, text: str) -> str:
-        if "pre" not in self._open_tags and "code" not in self._open_tags:
-            text = text.replace("\n", "")
-        else:
-            text = text.strip()
-
-        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
-        if previous_tag == "a":
-            url = self._open_tags_meta[0]
-            if url:
-                text = url
-        elif previous_tag == "command":
-            text = f"/{text}"
-        return text
-
-    def _html_to_unicode(self, text: str) -> str:
-        strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags
-        if strikethrough and underline:
-            text = html_to_unicode(text, "\u0336\u0332")
-        elif strikethrough:
-            text = html_to_unicode(text, "\u0336")
-        elif underline:
-            text = html_to_unicode(text, "\u0332")
-        return text
-
-    def _handle_tags_for_data(self, text: str) -> Tuple[str, int]:
-        extra_offset = 0
-        list_entry_handled_once = False
-        # In order to maintain order of things like blockquotes in lists or lists in blockquotes,
-        # we can't just have ifs/elses and we need to actually loop through the open tags in order.
-        for index, tag in enumerate(self._open_tags):
-            if tag == "blockquote" and self._line_is_new:
-                text = f"> {text}"
-                extra_offset += 2
-            elif tag == "li" and not list_entry_handled_once:
-                list_type_index = index + 1
-                list_type = self._open_tags[list_type_index]
-                indent = self._list_indent * " " if self._line_is_new else ""
-                if list_type == "ol":
-                    n = self._open_tags_meta[list_type_index]
-                    if self._list_entry_is_new:
-                        n += 1
-                        self._open_tags_meta[list_type_index] = n
-                        prefix = f"{n}. "
-                    else:
-                        prefix = int(math.log(n, 10)) * 3 * " " + 4 * " "
-                else:
-                    prefix = "* " if self._list_entry_is_new else 3 * " "
-                if not self._list_entry_is_new and not self._line_is_new:
-                    prefix = ""
-                extra_offset += len(indent) + len(prefix)
-                text = indent + prefix + text
-                self._list_entry_is_new = False
-                list_entry_handled_once = True
-        return text, extra_offset
-
-    def _extend_entities_in_construction(self, text: str, extra_offset: int):
-        for tag, entity in self._building_entities.items():
-            entity.length += len(text) - extra_offset
-            entity.offset += extra_offset
-
-    def handle_data(self, text: str):
-        text = unescape(text)
-        text = self._handle_special_previous_tags(text)
-        text = self._html_to_unicode(text)
-        text, extra_offset = self._handle_tags_for_data(text)
-        self._extend_entities_in_construction(text, extra_offset)
-        self._line_is_new = False
-        self.text += text
-
-    def handle_endtag(self, tag: str):
-        try:
-            self._open_tags.popleft()
-            self._open_tags_meta.popleft()
-        except IndexError:
-            pass
-
-        entity = self._building_entities.pop(tag, None)
-        if entity:
-            self.entities.append(entity)
-
-        if tag in self.block_tags and tag != "br" and "blockquote" not in self._open_tags:
-            self._newline(allow_multi=tag == "br")
-
-
-command_regex = re.compile(r"^!([A-Za-z0-9@]+)")
-not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")
-plain_mention_regex = None
-
-
-def plain_mention_to_html(match):
-    puppet = pu.Puppet.find_by_displayname(match.group(2))
-    if puppet:
-        return (f"{match.group(1)}"
-                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                f"{puppet.displayname}"
-                "</a>")
-    return "".join(match.groups())
-
-
-def matrix_to_telegram(html: str) -> Tuple[str, List[TypeMessageEntity]]:
-    try:
-        parser = MatrixParser()
-        html = command_regex.sub(r"<command>\1</command>", html)
-        html = not_command_regex.sub(r"\1", html)
-        if should_bridge_plaintext_highlights:
-            html = plain_mention_regex.sub(plain_mention_to_html, html)
-        parser.feed(add_surrogates(html))
-        return remove_surrogates(parser.text.strip()), parser.entities
-    except Exception:
-        log.exception("Failed to convert Matrix format:\nhtml=%s", html)
-
-
-def matrix_reply_to_telegram(content: dict, tg_space: int, room_id: Optional[str] = None
-                             ) -> Optional[int]:
-    try:
-        reply = content["m.relates_to"]["m.in_reply_to"]
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.query.filter(DBMessage.mxid == event_id,
-                                         DBMessage.tg_space == tg_space,
-                                         DBMessage.mx_room == room_id).one_or_none()
-        if message:
-            return message.tgid
-    except KeyError:
-        pass
-    return None
-
-
-def matrix_text_to_telegram(text: str) -> Tuple[str, List[TypeMessageEntity]]:
-    text = command_regex.sub(r"/\1", text)
-    text = not_command_regex.sub(r"\1", text)
-    if should_bridge_plaintext_highlights:
-        entities, pmr_replacer = plain_mention_to_text()
-        text = plain_mention_regex.sub(pmr_replacer, text)
-    else:
-        entities = []
-    return text, entities
-
-
-def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
-    entities = []
-
-    def replacer(match):
-        puppet = pu.Puppet.find_by_displayname(match.group(2))
-        if puppet:
-            offset = match.start()
-            length = match.end() - offset
-            if puppet.username:
-                entity = MessageEntityMention(offset, length)
-                text = f"@{puppet.username}"
-            else:
-                entity = InputMessageEntityMentionName(offset, length,
-                                                       user_id=InputUser(puppet.tgid, 0))
-                text = puppet.displayname
-            entities.append(entity)
-            return text
-        return "".join(match.groups())
-
-    return entities, replacer
-
-
-def init_mx(context: Context):
-    global plain_mention_regex, should_bridge_plaintext_highlights
-    config = context.config
-    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
-    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
-    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
-    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -0,0 +1,171 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple, Callable, Pattern, Match, TYPE_CHECKING
+import re
+import logging
+
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
+                               TypeMessageEntity, InputMessageEntityMentionName)
+from telethon.helpers import add_surrogate, del_surrogate
+from telethon import TelegramClient
+
+from mautrix.types import RoomID, MessageEventContent
+from mautrix.util.logging import TraceLogger
+
+from ... import puppet as pu
+from ...types import TelegramID
+from ...db import Message as DBMessage
+from .parser import ParsedMessage, parse_html
+
+if TYPE_CHECKING:
+    from ...context import Context
+
+log: TraceLogger = logging.getLogger("mau.fmt.mx")
+should_bridge_plaintext_highlights: bool = False
+
+command_regex: Pattern = re.compile(r"^!([A-Za-z0-9@]+)")
+not_command_regex: Pattern = re.compile(r"^\\(![A-Za-z0-9@]+)")
+plain_mention_regex: Optional[Pattern] = None
+
+MAX_LENGTH = 4096
+CUTOFF_TEXT = " [message cut]"
+CUT_MAX_LENGTH = MAX_LENGTH - len(CUTOFF_TEXT)
+
+
+def _cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+    if len(message) > MAX_LENGTH:
+        message = message[0:CUT_MAX_LENGTH] + CUTOFF_TEXT
+        new_entities = []
+        for entity in entities:
+            if entity.offset > CUT_MAX_LENGTH:
+                continue
+            if entity.offset + entity.length > CUT_MAX_LENGTH:
+                entity.length = CUT_MAX_LENGTH - entity.offset
+            new_entities.append(entity)
+        new_entities.append(MessageEntityItalic(CUT_MAX_LENGTH, len(CUTOFF_TEXT)))
+        entities = new_entities
+    return message, entities
+
+
+class FormatError(Exception):
+    pass
+
+
+def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
+                             room_id: Optional[RoomID] = None) -> Optional[TelegramID]:
+    event_id = content.get_reply_to()
+    if not event_id:
+        return
+    content.trim_reply_fallback()
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
+    return None
+
+
+async def matrix_to_telegram(client: TelegramClient, *, text: Optional[str] = None,
+                             html: Optional[str] = None) -> ParsedMessage:
+    if html is not None:
+        text, entities = _matrix_html_to_telegram(html)
+    elif text is not None:
+        text, entities = _matrix_text_to_telegram(text)
+    else:
+        raise ValueError("text or html must be provided to convert formatting")
+    await _fix_name_mentions(client, entities)
+    return text, entities
+
+
+def _matrix_html_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(_plain_mention_to_html, html)
+
+        text, entities = parse_html(add_surrogate(html))
+        text = del_surrogate(text.strip())
+        text, entities = _cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def _matrix_text_to_telegram(text: str) -> ParsedMessage:
+    text = command_regex.sub(r"/\1", text)
+    text = text.replace("\t", " " * 4)
+    text = not_command_regex.sub(r"\1", text)
+    if should_bridge_plaintext_highlights:
+        entities, pmr_replacer = _plain_mention_to_text()
+        text = plain_mention_regex.sub(pmr_replacer, text)
+    else:
+        entities = []
+    return text, entities
+
+
+async def _fix_name_mentions(client: TelegramClient, entities: List[TypeMessageEntity]) -> None:
+    for index in reversed(range(len(entities))):
+        entity = entities[index]
+        if isinstance(entity, (MessageEntityMentionName, InputMessageEntityMentionName)):
+            try:
+                user = await client.get_input_entity(entity.user_id)
+            except (ValueError, TypeError) as e:
+                log.trace(f"Dropping mention of {entity.user_id}: {e}")
+                del entities[index]
+            else:
+                entities[index] = InputMessageEntityMentionName(entity.offset, entity.length, user)
+
+
+def _plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
+    entities = []
+
+    def replacer(match: Match) -> str:
+        puppet = pu.Puppet.find_by_displayname(match.group(2))
+        if puppet:
+            offset = match.start()
+            length = match.end() - offset
+            if puppet.username:
+                entity = MessageEntityMention(offset, length)
+                text = f"@{puppet.username}"
+            else:
+                entity = MessageEntityMentionName(offset, length, user_id=puppet.tgid)
+                text = puppet.displayname
+            entities.append(entity)
+            return text
+        return "".join(match.groups())
+
+    return entities, replacer
+
+
+def _plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
+def init_mx(context: "Context") -> None:
+    global plain_mention_regex, should_bridge_plaintext_highlights
+    config = context.config
+    dn_template = config["bridge.displayname_template"]
+    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
+    plain_mention_regex = re.compile(f"^({dn_template})")
+    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"]

mautrix_telegram/formatter/from_matrix/parser.py

@@ -0,0 +1,89 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Tuple, Optional
+
+from telethon.tl.types import TypeMessageEntity
+
+from mautrix.types import UserID, RoomID
+from mautrix.util.formatter import MatrixParser as BaseMatrixParser, RecursionContext
+from mautrix.util.formatter.html_reader_htmlparser import read_html, HTMLNode
+
+from ... import user as u, puppet as pu, portal as po
+from .telegram_message import TelegramMessage, TelegramEntityType
+
+
+ParsedMessage = Tuple[str, List[TypeMessageEntity]]
+
+
+def parse_html(input_html: str) -> ParsedMessage:
+    msg = MatrixParser.parse(input_html)
+    return msg.text, msg.telegram_entities
+
+
+class MatrixParser(BaseMatrixParser[TelegramMessage]):
+    e = TelegramEntityType
+    fs = TelegramMessage
+    read_html = read_html
+
+    @classmethod
+    def custom_node_to_fstring(cls, node: HTMLNode, ctx: RecursionContext
+                               ) -> Optional[TelegramMessage]:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        if node.tag == "command":
+            msg.format(TelegramEntityType.COMMAND)
+        return None
+
+    @classmethod
+    def user_pill_to_fstring(cls, msg: TelegramMessage, user_id: UserID) -> TelegramMessage:
+        user = (pu.Puppet.deprecated_sync_get_by_mxid(user_id)
+                or u.User.get_by_mxid(user_id, create=False))
+        if not user:
+            return msg
+        if user.username:
+            return TelegramMessage(f"@{user.username}").format(TelegramEntityType.MENTION)
+        elif user.tgid:
+            displayname = user.plain_displayname or msg.text
+            return TelegramMessage(displayname).format(TelegramEntityType.MENTION_NAME,
+                                                       user_id=user.tgid)
+        return msg
+
+    @classmethod
+    def url_to_fstring(cls, msg: TelegramMessage, url: str) -> TelegramMessage:
+        if url == msg.text:
+            return msg.format(cls.e.URL)
+        else:
+            return msg.format(cls.e.INLINE_URL, url=url)
+
+    @classmethod
+    def room_pill_to_fstring(cls, msg: TelegramMessage, room_id: RoomID) -> TelegramMessage:
+        username = po.Portal.get_username_from_mx_alias(room_id)
+        portal = po.Portal.find_by_username(username)
+        if portal and portal.username:
+            return TelegramMessage(f"@{portal.username}").format(TelegramEntityType.MENTION)
+
+    @classmethod
+    def header_to_fstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        children = cls.node_to_fstrings(node, ctx)
+        length = int(node.tag[1])
+        prefix = "#" * length + " "
+        return TelegramMessage.join(children, "").prepend(prefix).format(TelegramEntityType.BOLD)
+
+    @classmethod
+    def blockquote_to_fstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -0,0 +1,99 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Union, Any, List, Type, Dict
+from enum import Enum
+
+from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
+                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
+                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
+                               MessageEntityBold as Bold, MessageEntityItalic as Italic,
+                               MessageEntityCode as Code, MessageEntityPre as Pre,
+                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
+                               MessageEntityBlockquote as Blockquote, TypeMessageEntity,
+                               InputMessageEntityMentionName as InputMentionName)
+
+from mautrix.util.formatter import EntityString, SemiAbstractEntity
+
+
+class TelegramEntityType(Enum):
+    """EntityType is a Matrix formatting entity type."""
+    BOLD = Bold
+    ITALIC = Italic
+    STRIKETHROUGH = Strike
+    UNDERLINE = Underline
+    URL = URL
+    INLINE_URL = TextURL
+    EMAIL = Email
+    PREFORMATTED = Pre
+    INLINE_CODE = Code
+    BLOCKQUOTE = Blockquote
+    MENTION = Mention
+    MENTION_NAME = MentionName
+    COMMAND = Command
+
+    USER_MENTION = 1
+    ROOM_MENTION = 2
+    HEADER = 3
+
+
+class TelegramEntity(SemiAbstractEntity):
+    internal: TypeMessageEntity
+
+    def __init__(self, type: Union[TelegramEntityType, Type[TypeMessageEntity]],
+                 offset: int, length: int, extra_info: Dict[str, Any]) -> None:
+        if isinstance(type, TelegramEntityType):
+            if isinstance(type.value, int):
+                raise ValueError(f"Can't create Entity with non-Telegram EntityType {type}")
+            type = type.value
+        self.internal = type(offset=offset, length=length, **extra_info)
+
+    def copy(self) -> Optional['TelegramEntity']:
+        extra_info = {}
+        if isinstance(self.internal, Pre):
+            extra_info["language"] = self.internal.language
+        elif isinstance(self.internal, TextURL):
+            extra_info["url"] = self.internal.url
+        elif isinstance(self.internal, (MentionName, InputMentionName)):
+            extra_info["user_id"] = self.internal.user_id
+        return TelegramEntity(type(self.internal), offset=self.internal.offset,
+                              length=self.internal.length, extra_info=extra_info)
+
+    def __repr__(self) -> str:
+        return str(self.internal)
+
+    @property
+    def offset(self) -> int:
+        return self.internal.offset
+
+    @offset.setter
+    def offset(self, value: int) -> None:
+        self.internal.offset = value
+
+    @property
+    def length(self) -> int:
+        return self.internal.length
+
+    @length.setter
+    def length(self, value: int) -> None:
+        self.internal.length = value
+
+
+class TelegramMessage(EntityString[TelegramEntity, TelegramEntityType]):
+    entity_class = TelegramEntity
+
+    @property
+    def telegram_entities(self) -> List[TypeMessageEntity]:
+        return [entity.internal for entity in self.entities]

mautrix_telegram/formatter/from_telegram.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,192 +13,172 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Optional, TYPE_CHECKING
 from html import escape
-from typing import Optional, List, Tuple
-
-try:
-    from lxml.html.diff import htmldiff
-except ImportError:
-    htmldiff = None  # type: function
 import logging
 import re
 
-from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName,
-                               MessageEntityEmail, MessageEntityUrl, MessageEntityTextUrl,
-                               MessageEntityBold, MessageEntityItalic, MessageEntityCode,
-                               MessageEntityPre, MessageEntityBotCommand, Message, PeerChannel,
-                               MessageEntityHashtag, TypeMessageEntity, MessageFwdHeader, PeerUser)
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityUrl,
+                               MessageEntityEmail, MessageEntityTextUrl, MessageEntityBold,
+                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
+                               MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
+                               MessageEntityPhone, TypeMessageEntity, PeerChannel, PeerChat,
+                               MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
+                               MessageEntityUnderline, PeerUser)
+from telethon.tl.custom import Message
+from telethon.errors import RPCError
+from telethon.helpers import add_surrogate, del_surrogate
 
-from mautrix_appservice import MatrixRequestError
-from mautrix_appservice.intent_api import IntentAPI
+from mautrix.errors import MatrixRequestError
+from mautrix.appservice import IntentAPI
+from mautrix.types import (TextMessageEventContent, RelatesTo, RelationType, Format, MessageType,
+                           MessageEvent)
 
 from .. import user as u, puppet as pu, portal as po
-from ..context import Context
+from ..types import TelegramID
 from ..db import Message as DBMessage
-from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text, unicode_to_html)
 
-log = logging.getLogger("mau.fmt.tg")
-should_highlight_edits = False
+if TYPE_CHECKING:
+    from ..abstract_user import AbstractUser
+
+log: logging.Logger = logging.getLogger("mau.fmt.tg")
 
 
-def telegram_reply_to_matrix(evt: Message, source: u.User) -> dict:
-    if evt.reply_to_msg_id:
-        space = (evt.to_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Optional[RelatesTo]:
+    if evt.reply_to:
+        space = (evt.peer_id.channel_id
+                 if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+        msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
         if msg:
-            return {
-                "m.in_reply_to": {
-                    "event_id": msg.mxid,
-                    "room_id": msg.mx_room,
-                }
-            }
-    return {}
+            return RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
+    return None
 
 
-async def _add_forward_header(source, text: str, html: Optional[str],
-                              fwd_from: MessageFwdHeader) -> Tuple[str, str]:
-    if not html:
-        html = escape(text)
+async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventContent,
+                              fwd_from: MessageFwdHeader) -> None:
+    if not content.formatted_body or content.format != Format.HTML:
+        content.format = Format.HTML
+        content.formatted_body = escape(content.body)
     fwd_from_html, fwd_from_text = None, None
-    if fwd_from.from_id:
-        user = u.User.get_by_tgid(fwd_from.from_id)
+    if isinstance(fwd_from.from_id, PeerUser):
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id.user_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
-            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
+            fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
+                             f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
-            puppet = pu.Puppet.get(fwd_from.from_id, create=False)
+            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id.user_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
-                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
+                fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                                 f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
-            user = await source.client.get_entity(PeerUser(fwd_from.from_id))
-            if user:
-                fwd_from_text = pu.Puppet.get_displayname(user, format=False)
-                fwd_from_html = f"<b>{fwd_from_text}</b>"
-
-    if not fwd_from_text:
-        if fwd_from.from_id:
-            fwd_from_text = "Unknown user"
+            try:
+                user = await source.client.get_entity(fwd_from.from_id)
+                if user:
+                    fwd_from_text = pu.Puppet.get_displayname(user, False)
+                    fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+            except (ValueError, RPCError):
+                fwd_from_text = fwd_from_html = "unknown user"
+    elif isinstance(fwd_from.from_id, (PeerChannel, PeerChat)):
+        from_id = (fwd_from.from_id.chat_id if isinstance(fwd_from.from_id, PeerChat)
+                   else fwd_from.from_id.channel_id)
+        portal = po.Portal.get_by_tgid(TelegramID(from_id))
+        if portal:
+            fwd_from_text = portal.title
+            if portal.alias:
+                fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
+                                 f"{escape(fwd_from_text)}</a>")
+            else:
+                fwd_from_html = f"channel <b>{escape(fwd_from_text)}</b>"
         else:
-            fwd_from_text = "Unknown source"
-        fwd_from_html = f"<b>{fwd_from_text}</b>"
+            try:
+                channel = await source.client.get_entity(fwd_from.from_id)
+                if channel:
+                    fwd_from_text = f"channel {channel.title}"
+                    fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
+            except (ValueError, RPCError):
+                fwd_from_text = fwd_from_html = "unknown channel"
+    elif fwd_from.from_name:
+        fwd_from_text = fwd_from.from_name
+        fwd_from_html = f"<b>{escape(fwd_from.from_name)}</b>"
+    else:
+        fwd_from_text = "unknown source"
+        fwd_from_html = f"unknown source"
 
-    text = "\n".join([f"> {line}" for line in text.split("\n")])
-    text = f"Forwarded from {fwd_from_text}:\n{text}"
-    html = (f"Forwarded message from {fwd_from_html}<br/>"
-            f"<tg-forward><blockquote>{html}</blockquote></tg-forward>")
-    return text, html
+    content.body = "\n".join([f"> {line}" for line in content.body.split("\n")])
+    content.body = f"Forwarded from {fwd_from_text}:\n{content.body}"
+    content.formatted_body = (
+        f"Forwarded message from {fwd_from_html}<br/>"
+        f"<tg-forward><blockquote>{content.formatted_body}</blockquote></tg-forward>")
 
 
-def highlight_edits(new_html: str, old_html: str) -> str:
-    # Don't include `Edit:` text in diff.
-    if old_html.startswith("<u>Edit:</u> "):
-        old_html = old_html[len("<u>Edit:</u> "):]
-
-    # Generate diff with lxml
-    new_html = htmldiff(old_html, new_html)
-
-    # Replace <ins> with <u> since Riot doesn't allow <ins>
-    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
-    # Remove <del>s since we just want to hide deletions.
-    new_html = re.sub("<del>.+?</del>", "", new_html)
-    return new_html
-
-
-async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, relates_to: dict,
-                            main_intent: IntentAPI, is_edit: bool) -> Tuple[str, str]:
-    space = (evt.to_id.channel_id
-             if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+async def _add_reply_header(source: 'AbstractUser', content: TextMessageEventContent, evt: Message,
+                            main_intent: IntentAPI):
+    space = (evt.peer_id.channel_id
+             if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+    msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
     if not msg:
-        return text, html
+        return
 
-    relates_to["m.in_reply_to"] = {
-        "event_id": msg.mxid,
-        "room_id": msg.mx_room,
-    }
+    content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
 
     try:
-        event = await main_intent.get_event(msg.mx_room, msg.mxid)
-
-        content = event["content"]
-        r_sender = event["sender"]
-
-        r_text_body = trim_reply_fallback_text(content["body"])
-        r_html_body = trim_reply_fallback_html(content["formatted_body"]
-                                               if "formatted_body" in content
-                                               else escape(content["body"]))
-
-        puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
-        r_displayname = puppet.displayname if puppet else r_sender
-        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
-
-        if is_edit and should_highlight_edits:
-            html = highlight_edits(html or escape(text), r_html_body)
-    except (ValueError, KeyError, MatrixRequestError) as e:
-        r_sender_link = "unknown user"
-        r_displayname = "unknown user"
-        r_text_body = "Failed to fetch message"
-        r_html_body = "<em>Failed to fetch message</em>"
-
-    if is_edit:
-        html = f"<u>Edit:</u> {html or escape(text)}"
-        text = f"Edit: {text}"
-
-    r_keyword = "In reply to" if not is_edit else "Edit to"
-    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
-    html = (f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
-            + (html or escape(text)))
-
-    lines = r_text_body.strip().split("\n")
-    text_with_quote = f"> <{r_displayname}> {lines.pop(0)}"
-    for line in lines:
-        if line:
-            text_with_quote += f"\n> {line}"
-    text_with_quote += "\n\n"
-    text_with_quote += text
-    return text_with_quote, html
+        event: MessageEvent = await main_intent.get_event(msg.mx_room, msg.mxid)
+        if isinstance(event.content, TextMessageEventContent):
+            event.content.trim_reply_fallback()
+        puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
+        content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
+    except MatrixRequestError:
+        log.exception("Failed to get event to add reply fallback")
 
 
-async def telegram_to_matrix(evt: Message, source: u.User, main_intent: Optional[IntentAPI] = None,
-                             is_edit: bool = False, prefix_text: Optional[str] = None,
-                             prefix_html: Optional[str] = None) -> Tuple[str, str, dict]:
-    text = add_surrogates(evt.message)
-    html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None
-    relates_to = {}
+async def telegram_to_matrix(evt: Message, source: "AbstractUser",
+                             main_intent: Optional[IntentAPI] = None,
+                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
+                             override_text: str = None,
+                             override_entities: List[TypeMessageEntity] = None,
+                             no_reply_fallback: bool = False) -> TextMessageEventContent:
+    content = TextMessageEventContent(
+        msgtype=MessageType.TEXT,
+        body=add_surrogate(override_text or evt.message),
+    )
+    entities = override_entities or evt.entities
+    if entities:
+        content.format = Format.HTML
+        content.formatted_body = _telegram_entities_to_matrix_catch(content.body, entities)
 
     if prefix_html:
-        html = prefix_html + (html or escape(text))
+        if not content.formatted_body:
+            content.format = Format.HTML
+            content.formatted_body = escape(content.body)
+        content.formatted_body = prefix_html + content.formatted_body
     if prefix_text:
-        text = prefix_text + text
+        content.body = prefix_text + content.body
 
     if evt.fwd_from:
-        text, html = await _add_forward_header(source, text, html, evt.fwd_from)
+        await _add_forward_header(source, content, evt.fwd_from)
 
-    if evt.reply_to_msg_id:
-        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
-                                             is_edit)
+    if evt.reply_to and not no_reply_fallback:
+        await _add_reply_header(source, content, evt, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
-        if not html:
-            html = escape(text)
-        text += f"\n- {evt.post_author}"
-        html += f"<br/><i>- <u>{evt.post_author}</u></i>"
+        if not content.formatted_body:
+            content.formatted_body = escape(content.body)
+        content.body += f"\n- {evt.post_author}"
+        content.formatted_body += f"<br/><i>- <u>{evt.post_author}</u></i>"
 
-    html = unicode_to_html(text, html, "\u0336", "del")
-    html = unicode_to_html(text, html, "\u0332", "u")
+    content.body = del_surrogate(content.body)
 
-    if html:
-        html = html.replace("\n", "<br/>")
+    if content.formatted_body:
+        content.formatted_body = del_surrogate(content.formatted_body.replace("\n", "<br/>"))
 
-    return remove_surrogates(text), remove_surrogates(html), relates_to
+    return content
 
 
 def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEntity]) -> str:
@@ -210,48 +189,65 @@ def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEnti
                       "message=%s\n"
                       "entities=%s",
                       text, entities)
+    return "[failed conversion in _telegram_entities_to_matrix]"
 
 
-def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity],
+                                 offset: int = 0, length: int = None) -> str:
     if not entities:
-        return text
+        return escape(text)
+    if length is None:
+        length = len(text)
     html = []
     last_offset = 0
-    for entity in entities:
-        if entity.offset > last_offset:
-            html.append(escape(text[last_offset:entity.offset]))
-        elif entity.offset < last_offset:
+    for i, entity in enumerate(entities):
+        if entity.offset > offset + length:
+            break
+        relative_offset = entity.offset - offset
+        if relative_offset > last_offset:
+            html.append(escape(text[last_offset:relative_offset]))
+        elif relative_offset < last_offset:
             continue
 
         skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length])
+        entity_text = _telegram_entities_to_matrix(
+            text=text[relative_offset:relative_offset + entity.length],
+            entities=entities[i + 1:], offset=entity.offset, length=entity.length)
         entity_type = type(entity)
 
         if entity_type == MessageEntityBold:
             html.append(f"<strong>{entity_text}</strong>")
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
+        elif entity_type == MessageEntityUnderline:
+            html.append(f"<u>{entity_text}</u>")
+        elif entity_type == MessageEntityStrike:
+            html.append(f"<del>{entity_text}</del>")
+        elif entity_type == MessageEntityBlockquote:
+            html.append(f"<blockquote>{entity_text}</blockquote>")
         elif entity_type == MessageEntityCode:
-            html.append(f"<code>{entity_text}</code>")
+            html.append(f"<pre><code>{entity_text}</code></pre>"
+                        if "\n" in entity_text
+                        else f"<code>{entity_text}</code>")
         elif entity_type == MessageEntityPre:
             skip_entity = _parse_pre(html, entity_text, entity.language)
         elif entity_type == MessageEntityMention:
             skip_entity = _parse_mention(html, entity_text)
         elif entity_type == MessageEntityMentionName:
-            skip_entity = _parse_name_mention(html, entity_text, entity.user_id)
+            skip_entity = _parse_name_mention(html, entity_text, TelegramID(entity.user_id))
         elif entity_type == MessageEntityEmail:
             html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
-        elif entity_type in {MessageEntityTextUrl, MessageEntityUrl}:
+        elif entity_type in (MessageEntityTextUrl, MessageEntityUrl):
             skip_entity = _parse_url(html, entity_text,
                                      entity.url if entity_type == MessageEntityTextUrl else None)
         elif entity_type == MessageEntityBotCommand:
             html.append(f"<font color='blue'>!{entity_text[1:]}</font>")
-        elif entity_type == MessageEntityHashtag:
+        elif entity_type in (MessageEntityHashtag, MessageEntityCashtag, MessageEntityPhone):
             html.append(f"<font color='blue'>{entity_text}</font>")
         else:
             skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:])
+        last_offset = relative_offset + (0 if skip_entity else entity.length)
+    html.append(escape(text[last_offset:]))
 
     return "".join(html)
 
@@ -283,7 +279,7 @@ def _parse_mention(html: List[str], entity_text: str) -> bool:
     return False
 
 
-def _parse_name_mention(html: List[str], entity_text: str, user_id: int) -> bool:
+def _parse_name_mention(html: List[str], entity_text: str, user_id: TelegramID) -> bool:
     user = u.User.get_by_tgid(user_id)
     if user:
         mxid = user.mxid
@@ -297,8 +293,8 @@ def _parse_name_mention(html: List[str], entity_text: str, user_id: int) -> bool
     return False
 
 
-message_link_regex = re.compile(
-    r"https?://t(?:elegram)?\.(?:me|dog)/([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})")
+message_link_regex = re.compile(r"https?://t(?:elegram)?\.(?:me|dog)/"
+                                r"([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})")
 
 
 def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
@@ -308,19 +304,14 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
 
     message_link_match = message_link_regex.match(url)
     if message_link_match:
-        group, msgid = message_link_match.groups()
-        msgid = int(msgid)
+        group, msgid_str = message_link_match.groups()
+        msgid = int(msgid_str)
 
         portal = po.Portal.find_by_username(group)
         if portal:
-            message = DBMessage.query.get((msgid, portal.tgid))
+            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 
     html.append(f"<a href='{url}'>{entity_text}</a>")
     return False
-
-
-def init_tg(context: Context):
-    global should_highlight_edits
-    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/formatter/util.py

@@ -1,86 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from html import escape
-from typing import Optional
-import struct
-import re
-
-
-# add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
-# Licensed under the MIT license.
-# https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py
-def add_surrogates(text: Optional[str]) -> Optional[str]:
-    if text is None:
-        return None
-    return "".join("".join(chr(y) for y in struct.unpack("<HH", x.encode("utf-16-le")))
-                   if (0x10000 <= ord(x) <= 0x10FFFF) else x for x in text)
-
-
-def remove_surrogates(text: Optional[str]) -> Optional[str]:
-    if text is None:
-        return None
-    return text.encode("utf-16", "surrogatepass").decode("utf-16")
-
-
-def trim_reply_fallback_text(text: str) -> str:
-    if not text.startswith("> ") or "\n" not in text:
-        return text
-    lines = text.split("\n")
-    while len(lines) > 0 and lines[0].startswith("> "):
-        lines.pop(0)
-    return "\n".join(lines)
-
-
-html_reply_fallback_regex = re.compile("^<mx-reply>"
-                                       r"[\s\S]+?"
-                                       "</mx-reply>")
-
-
-def trim_reply_fallback_html(html: str) -> str:
-    return html_reply_fallback_regex.sub("", html)
-
-
-def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
-    if ctrl not in text:
-        return html
-    if not html:
-        html = escape(text)
-    tag_start = f"<{tag}>"
-    tag_end = f"</{tag}>"
-    characters = html.split(ctrl)
-    html = ""
-    in_tag = False
-    for char in characters:
-        if not in_tag:
-            if len(char) > 1:
-                html += char[0:-1]
-                char = char[-1]
-            html += tag_start
-            in_tag = True
-            html += char
-        else:
-            if len(char) > 1:
-                html += tag_end
-                in_tag = False
-            html += char
-    if in_tag:
-        html += tag_end
-    return html
-
-
-def html_to_unicode(text: str, ctrl: str) -> str:
-    return ctrl.join(text) + ctrl

mautrix_telegram/get_version.py

@@ -0,0 +1,49 @@
+import subprocess
+import shutil
+import os
+
+from . import __version__
+
+cmd_env = {
+    "PATH": os.environ["PATH"],
+    "HOME": os.environ["HOME"],
+    "LANG": "C",
+    "LC_ALL": "C",
+}
+
+
+def run(cmd):
+    return subprocess.check_output(cmd, stderr=subprocess.DEVNULL, env=cmd_env)
+
+if os.path.exists(".git") and shutil.which("git"):
+    try:
+        git_revision = run(["git", "rev-parse", "HEAD"]).strip().decode("ascii")
+        git_revision_url = f"https://github.com/tulir/mautrix-telegram/commit/{git_revision}"
+        git_revision = git_revision[:8]
+    except (subprocess.SubprocessError, OSError):
+        git_revision = "unknown"
+        git_revision_url = None
+
+    try:
+        git_tag = run(["git", "describe", "--exact-match", "--tags"]).strip().decode("ascii")
+    except (subprocess.SubprocessError, OSError):
+        git_tag = None
+else:
+    git_revision = "unknown"
+    git_revision_url = None
+    git_tag = None
+
+git_tag_url = (f"https://github.com/tulir/mautrix-telegram/releases/tag/{git_tag}"
+               if git_tag else None)
+
+if git_tag and __version__ == git_tag[1:].replace("-", ""):
+    version = __version__
+    linkified_version = f"[{version}]({git_tag_url})"
+else:
+    if not __version__.endswith("+dev"):
+        __version__ += "+dev"
+    version = f"{__version__}.{git_revision}"
+    if git_revision_url:
+        linkified_version = f"{__version__}.[{git_revision}]({git_revision_url})"
+    else:
+        linkified_version = version

mautrix_telegram/matrix.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,277 +13,390 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import logging
+from typing import Dict, Set, Tuple, Union, Iterable, TYPE_CHECKING
 
-from mautrix_appservice import MatrixRequestError
+from mautrix.bridge import BaseMatrixHandler
+from mautrix.types import (Event, EventType, RoomID, UserID, EventID, ReceiptEvent, ReceiptType,
+                           ReceiptEventContent, PresenceEvent, PresenceState, TypingEvent,
+                           MessageEvent, StateEvent, RedactionEvent, RoomNameStateEventContent,
+                           RoomAvatarStateEventContent, RoomTopicStateEventContent,
+                           MemberStateEventContent, EncryptedEvent, TextMessageEventContent,
+                           MessageType)
+from mautrix.errors import MatrixError
 
-from .user import User
-from .portal import Portal
-from .puppet import Puppet
-from .commands import CommandHandler
+from . import user as u, portal as po, puppet as pu, commands as com
+
+if TYPE_CHECKING:
+    from .context import Context
+    from .bot import Bot
+
+RoomMetaStateEventContent = Union[RoomNameStateEventContent, RoomAvatarStateEventContent,
+                                  RoomTopicStateEventContent]
 
 
-class MatrixHandler:
-    log = logging.getLogger("mau.mx")
+class MatrixHandler(BaseMatrixHandler):
+    bot: 'Bot'
+    commands: 'com.CommandProcessor'
+    previously_typing: Dict[RoomID, Set[UserID]]
 
-    def __init__(self, context):
-        self.az, self.db, self.config, _, self.tgbot = context
-        self.commands = CommandHandler(context)
+    def __init__(self, context: 'Context') -> None:
+        prefix, suffix = context.config["bridge.username_template"].format(userid=":").split(":")
+        homeserver = context.config["homeserver.domain"]
+        self.user_id_prefix = f"@{prefix}"
+        self.user_id_suffix = f"{suffix}:{homeserver}"
 
-        self.az.matrix_event_handler(self.handle_event)
+        super().__init__(command_processor=com.CommandProcessor(context), bridge=context.bridge)
 
-    async def init_as_bot(self):
-        await self.az.intent.set_display_name(
-            self.config.get("appservice.bot_displayname", "Telegram bridge bot"))
+        self.bot = context.bot
+        self.previously_typing = {}
 
-    async def handle_puppet_invite(self, room, puppet, inviter):
-        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}")
-        if not inviter.logged_in:
-            await puppet.intent.error_and_leave(
-                room, text="Please log in before inviting Telegram puppets.")
+    async def handle_puppet_invite(self, room_id: RoomID, puppet: pu.Puppet, inviter: u.User,
+                                   event_id: EventID) -> None:
+        intent = puppet.default_mxid_intent
+        self.log.debug(f"{inviter.mxid} invited puppet for {puppet.tgid} to {room_id}")
+        if not await inviter.is_logged_in():
+            await intent.error_and_leave(
+                room_id, text="Please log in before inviting Telegram puppets.")
             return
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if portal:
             if portal.peer_type == "user":
-                await puppet.intent.error_and_leave(
-                    room, text="You can not invite additional users to private chats.")
+                await intent.error_and_leave(
+                    room_id, text="You can not invite additional users to private chats.")
                 return
             await portal.invite_telegram(inviter, puppet)
-            await puppet.intent.join_room(room)
+            await intent.join_room(room_id)
             return
         try:
-            members = await self.az.intent.get_room_members(room)
-        except MatrixRequestError:
-            members = []
+            members = await intent.get_room_members(room_id)
+        except MatrixError:
+            self.log.exception(f"Failed to get members after joining {room_id} as {intent.mxid}")
+            return
         if self.az.bot_mxid not in members:
-            if len(members) > 1:
-                await puppet.intent.error_and_leave(room, text=None, html=(
+            if len(members) > 2:
+                await intent.error_and_leave(room_id, text=None, html=(
                     f"Please invite "
                     f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
                     f"first if you want to create a Telegram chat."))
                 return
 
-            await puppet.intent.join_room(room)
-            portal = Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
+            await intent.join_room(room_id)
+            portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
             if portal.mxid:
                 try:
-                    await puppet.intent.invite(portal.mxid, inviter.mxid)
-                    await puppet.intent.send_notice(room, text=None, html=(
-                        "You already have a private chat with me: "
-                        f"<a href='https://matrix.to/#/{portal.mxid}'>"
-                        "Link to room"
-                        "</a>"))
-                    await puppet.intent.leave_room(room)
+                    await intent.invite_user(portal.mxid, inviter.mxid)
+                    await intent.send_notice(
+                        room_id, text=f"You already have a private chat with me: {portal.mxid}",
+                        html=("You already have a private chat with me: "
+                              f"<a href='https://matrix.to/#/{portal.mxid}'>Link to room</a>"))
+                    await intent.leave_room(room_id)
                     return
-                except MatrixRequestError:
+                except MatrixError:
                     pass
-            portal.mxid = room
-            portal.save()
-            inviter.register_portal(portal)
-            await puppet.intent.send_notice(room, "Portal to private chat created.")
+            portal.mxid = room_id
+            e2be_ok = None
+            if self.config["bridge.encryption.default"] and self.e2ee:
+                e2be_ok = await portal.enable_dm_encryption()
+            await portal.save()
+            await inviter.register_portal(portal)
+            if e2be_ok is True:
+                evt_type, content = await self.e2ee.encrypt(
+                    room_id, EventType.ROOM_MESSAGE,
+                    TextMessageEventContent(msgtype=MessageType.NOTICE,
+                                            body="Portal to private chat created and end-to-bridge"
+                                                 " encryption enabled."))
+                await intent.send_message_event(room_id, evt_type, content)
+            else:
+                message = "Portal to private chat created."
+                if e2be_ok is False:
+                    message += "\n\nWarning: Failed to enable end-to-bridge encryption"
+                await intent.send_notice(room_id, message)
         else:
-            await puppet.intent.join_room(room)
-            await puppet.intent.send_notice(room, "This puppet will remain inactive until a "
-                                                  "Telegram chat is created for this room.")
+            await intent.join_room(room_id)
+            await intent.send_notice(room_id, "This puppet will remain inactive until a "
+                                              "Telegram chat is created for this room.")
 
-    async def handle_invite(self, room, user, inviter):
-        self.log.debug(f"{inviter} invited {user} to {room}")
-        inviter = await User.get_by_mxid(inviter).ensure_started()
-        if user == self.az.bot_mxid:
-            await self.az.intent.join_room(room)
-            if not inviter.whitelisted:
-                await self.az.intent.send_notice(
-                    room, text=None,
-                    html="You are not whitelisted to use this bridge.<br/><br/>"
-                         "If you are the owner of this bridge, see the "
-                         "<code>bridge.permissions</code> section in your config file.")
-                await self.az.intent.leave_room(room)
-            return
-        elif not inviter.whitelisted:
+    async def send_welcome_message(self, room_id: RoomID, inviter: 'u.User') -> None:
+        try:
+            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
+        except MatrixError:
+            # The AS bot is not in the room.
             return
+        cmd_prefix = self.commands.command_prefix
+        text = html = "Hello, I'm a Telegram bridge bot. "
+        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
+            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
+            html += (f"Use <code>{cmd_prefix} help</code> for help"
+                     f" or <code>{cmd_prefix} login</code> to log in.")
+        else:
+            text += f"Use `{cmd_prefix} help` for help."
+            html += f"Use <code>{cmd_prefix} help</code> for help."
+        await self.az.intent.send_notice(room_id, text=text, html=html)
 
-        puppet = Puppet.get_by_mxid(user)
-        if puppet:
-            await self.handle_puppet_invite(room, puppet, inviter)
-            return
-
-        user = User.get_by_mxid(user, create=False)
+    async def handle_invite(self, room_id: RoomID, user_id: UserID, inviter: 'u.User',
+                            event_id: EventID) -> None:
+        user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
-        portal = Portal.get_by_mxid(room)
-        if user and user.has_full_access and portal:
+        portal = po.Portal.get_by_mxid(room_id)
+        if user and await user.has_full_access(allow_bot=True) and portal:
             await portal.invite_telegram(inviter, user)
-            return
 
-        # The rest can probably be ignored
+    async def handle_join(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None:
+        user = await u.User.get_by_mxid(user_id).ensure_started()
 
-    async def handle_join(self, room, user, event_id):
-        user = await User.get_by_mxid(user).ensure_started()
-
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
         if not user.relaybot_whitelisted:
-            await portal.main_intent.kick(room, user.mxid,
-                                          "You are not whitelisted on this Telegram bridge.")
+            await portal.main_intent.kick_user(room_id, user.mxid,
+                                               "You are not whitelisted on this Telegram bridge.")
             return
-        elif not user.logged_in and not portal.has_bot:
-            await portal.main_intent.kick(room, user.mxid,
-                                          "This chat does not have a bot relaying "
-                                          "messages for unauthenticated users.")
+        elif not await user.is_logged_in() and not portal.has_bot:
+            await portal.main_intent.kick_user(room_id, user.mxid,
+                                               "This chat does not have a bot relaying "
+                                               "messages for unauthenticated users.")
             return
 
-        self.log.debug(f"{user} joined {room}")
-        if user.logged_in or portal.has_bot:
+        self.log.debug(f"{user.mxid} joined {room_id}")
+        if await user.is_logged_in() or portal.has_bot:
             await portal.join_matrix(user, event_id)
 
-    async def handle_part(self, room, user, sender, event_id):
-        self.log.debug(f"{user} left {room}")
+    async def get_leave_handle_info(self) -> Tuple[po.Portal, u.User]:
+        pass
 
-        sender = User.get_by_mxid(sender, create=False)
+    async def handle_leave(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None:
+        self.log.debug(f"{user_id} left {room_id}")
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        user = u.User.get_by_mxid(user_id, create=False)
+        if not user:
+            return
+        await user.ensure_started()
+        await portal.leave_matrix(user, event_id)
+
+    async def handle_kick_ban(self, ban: bool, room_id: RoomID, user_id: UserID, sender: UserID,
+                              reason: str, event_id: EventID) -> None:
+        action = "banned" if ban else "kicked"
+        self.log.debug(f"{user_id} was {action} from {room_id} by {sender} for {reason}")
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        if user_id == self.az.bot_mxid:
+            # Direct chat portal unbridging is handled in portal.kick_matrix
+            if portal.peer_type != "user":
+                await portal.unbridge()
+            return
+
+        sender = u.User.get_by_mxid(sender, create=False)
         if not sender:
             return
         await sender.ensure_started()
 
-        portal = Portal.get_by_mxid(room)
-        if not portal:
+        puppet = await pu.Puppet.get_by_mxid(user_id)
+        if puppet:
+            if ban:
+                await portal.ban_matrix(puppet, sender)
+            else:
+                await portal.kick_matrix(puppet, sender)
             return
 
-        puppet = Puppet.get_by_mxid(user)
-        if sender and puppet:
-            await portal.leave_matrix(puppet, sender)
-
-        user = User.get_by_mxid(user, create=False)
+        user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
-        if user.logged_in or portal.has_bot:
-            await portal.leave_matrix(user, sender, event_id)
+        if ban:
+            await portal.ban_matrix(user, sender)
+        else:
+            await portal.kick_matrix(user, sender)
 
-    def is_command(self, message):
-        text = message.get("body", "")
-        prefix = self.config["bridge.command_prefix"]
-        is_command = text.startswith(prefix)
-        if is_command:
-            text = text[len(prefix) + 1:]
-        return is_command, text
+    async def handle_kick(self, room_id: RoomID, user_id: UserID, kicked_by: UserID, reason: str,
+                          event_id: EventID) -> None:
+        await self.handle_kick_ban(False, room_id, user_id, kicked_by, reason, event_id)
 
-    async def handle_message(self, room, sender, message, event_id):
-        self.log.debug(f"{sender} sent {message} to ${room}")
+    async def handle_unban(self, room_id: RoomID, user_id: UserID, unbanned_by: UserID,
+                           reason: str, event_id: EventID) -> None:
+        # TODO handle unbans properly instead of handling it as a kick
+        await self.handle_kick_ban(False, room_id, user_id, unbanned_by, reason, event_id)
 
-        is_command, text = self.is_command(message)
-        sender = await User.get_by_mxid(sender).ensure_started()
+    async def handle_ban(self, room_id: RoomID, user_id: UserID, banned_by: UserID, reason: str,
+                         event_id: EventID) -> None:
+        await self.handle_kick_ban(True, room_id, user_id, banned_by, reason, event_id)
+
+    @staticmethod
+    async def allow_message(user: 'u.User') -> bool:
+        return user.relaybot_whitelisted
+
+    @staticmethod
+    async def allow_command(user: 'u.User') -> bool:
+        return user.whitelisted
+
+    @staticmethod
+    async def allow_bridging_message(user: 'u.User', portal: 'po.Portal') -> bool:
+        return await user.is_logged_in() or portal.has_bot
+
+    @staticmethod
+    async def handle_redaction(evt: RedactionEvent) -> None:
+        sender = await u.User.get_by_mxid(evt.sender).ensure_started()
         if not sender.relaybot_whitelisted:
             return
 
-        portal = Portal.get_by_mxid(room)
-        if not is_command and portal and (sender.logged_in or portal.has_bot):
-            await portal.handle_matrix_message(sender, message, event_id)
-            return
-
-        if not sender.whitelisted or message["msgtype"] != "m.text":
-            return
-
-        try:
-            is_management = len(await self.az.intent.get_room_members(room)) == 2
-        except MatrixRequestError:
-            # The AS bot is not in the room.
-            return
-
-        if is_command or is_management:
-            try:
-                command, arguments = text.split(" ", 1)
-                args = arguments.split(" ")
-            except ValueError:
-                # Not enough values to unpack, i.e. no arguments
-                command = text
-                args = []
-            await self.commands.handle(room, sender, command, args, is_management,
-                                       is_portal=portal is not None)
-
-    async def handle_redaction(self, room, sender, event_id):
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if not sender.relaybot_whitelisted:
-            return
-
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(evt.room_id)
         if not portal:
             return
 
-        await portal.handle_matrix_deletion(sender, event_id)
+        await portal.handle_matrix_deletion(sender, evt.redacts, evt.event_id)
 
-    async def handle_power_levels(self, room, sender, new, old):
-        portal = Portal.get_by_mxid(room)
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if sender.has_full_access and portal:
-            await portal.handle_matrix_power_levels(sender, new["users"], old["users"])
+    @staticmethod
+    async def handle_power_levels(evt: StateEvent) -> None:
+        portal = po.Portal.get_by_mxid(evt.room_id)
+        sender = await u.User.get_by_mxid(evt.sender).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
+            await portal.handle_matrix_power_levels(sender, evt.content.users,
+                                                    evt.unsigned.prev_content.users,
+                                                    evt.event_id)
 
-    async def handle_room_meta(self, type, room, sender, content):
-        portal = Portal.get_by_mxid(room)
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if sender.has_full_access and portal:
-            handler, content_key = {
-                "m.room.name": (portal.handle_matrix_title, "name"),
-                "m.room.topic": (portal.handle_matrix_about, "topic"),
-                "m.room.avatar": (portal.handle_matrix_avatar, "url"),
-            }[type]
-            if content_key not in content:
+    @staticmethod
+    async def handle_room_meta(evt_type: EventType, room_id: RoomID, sender_mxid: UserID,
+                               content: RoomMetaStateEventContent, event_id: EventID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
+            handler, content_type, content_key = {
+                EventType.ROOM_NAME: (portal.handle_matrix_title, RoomNameStateEventContent, "name"),
+                EventType.ROOM_TOPIC: (portal.handle_matrix_about, RoomTopicStateEventContent, "topic"),
+                EventType.ROOM_AVATAR: (portal.handle_matrix_avatar, RoomAvatarStateEventContent, "url"),
+            }[evt_type]
+            if not isinstance(content, content_type):
                 return
-            await handler(sender, content[content_key])
+            await handler(sender, content[content_key], event_id)
 
-    async def handle_room_pin(self, room, sender, new_events, old_events):
-        portal = Portal.get_by_mxid(room)
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if sender.has_full_access and portal:
+    @staticmethod
+    async def handle_room_pin(room_id: RoomID, sender_mxid: UserID,
+                              new_events: Set[str], old_events: Set[str],
+                              event_id: EventID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             events = new_events - old_events
             if len(events) > 0:
                 # New event pinned, set that as pinned in Telegram.
-                await portal.handle_matrix_pin(sender, events.pop())
+                await portal.handle_matrix_pin(sender, EventID(events.pop()), event_id)
             elif len(new_events) == 0:
                 # All pinned events removed, remove pinned event in Telegram.
-                await portal.handle_matrix_pin(sender, None)
+                await portal.handle_matrix_pin(sender, None, event_id)
 
-    def filter_matrix_event(self, event):
-        return (event["sender"] == self.az.bot_mxid
-                or Puppet.get_id_from_mxid(event["sender"]) is not None)
+    @staticmethod
+    async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID,
+                                  event_id: EventID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if portal:
+            await portal.handle_matrix_upgrade(sender, new_room_id, event_id)
 
-    async def handle_event(self, evt):
-        if self.filter_matrix_event(evt):
+    async def handle_member_info_change(self, room_id: RoomID, user_id: UserID,
+                                        profile: MemberStateEventContent,
+                                        prev_profile: MemberStateEventContent,
+                                        event_id: EventID) -> None:
+        if profile.displayname == prev_profile.displayname:
             return
-        self.log.debug("Received event: %s", evt)
-        type = evt["type"]
-        content = evt.get("content", {})
-        if type == "m.room.member":
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})
-            membership = content.get("membership", "")
-            prev_membership = prev_content.get("membership", "leave")
-            if membership == prev_membership:
-                # TODO handle displayname/avatar changes
-                pass
-            elif membership == "invite":
-                await self.handle_invite(evt["room_id"], evt["state_key"], evt["sender"])
-            elif prev_membership == "join" and membership == "leave":
-                await self.handle_part(evt["room_id"], evt["state_key"], evt["sender"],
-                                       evt["event_id"])
-            elif membership == "join":
-                await self.handle_join(evt["room_id"], evt["state_key"], evt["event_id"])
-        elif type in ("m.room.message", "m.sticker"):
-            if type != "m.room.message":
-                content["msgtype"] = type
-            await self.handle_message(evt["room_id"], evt["sender"], content, evt["event_id"])
-        elif type == "m.room.redaction":
-            await self.handle_redaction(evt["room_id"], evt["sender"], evt["redacts"])
-        elif type == "m.room.power_levels":
-            await self.handle_power_levels(evt["room_id"], evt["sender"], evt["content"],
-                                           evt["prev_content"])
-        elif type in ("m.room.name", "m.room.avatar", "m.room.topic"):
-            await self.handle_room_meta(type, evt["room_id"], evt["sender"], evt["content"])
-        elif type == "m.room.pinned_events":
-            new_events = set(evt["content"]["pinned"])
+
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.has_bot:
+            return
+
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if await user.needs_relaybot(portal):
+            await portal.name_change_matrix(user, profile.displayname, prev_profile.displayname,
+                                            event_id)
+
+    @staticmethod
+    def parse_read_receipts(content: ReceiptEventContent) -> Iterable[Tuple[UserID, EventID]]:
+        return ((user_id, event_id)
+                for event_id, receipts in content.items()
+                for user_id in receipts.get(ReceiptType.READ, {}))
+
+    @staticmethod
+    async def handle_read_receipts(room_id: RoomID, receipts: Iterable[Tuple[UserID, EventID]]
+                                   ) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id, event_id in receipts:
+            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
+            if user and await user.is_logged_in():
+                await portal.mark_read(user, event_id)
+
+    @staticmethod
+    async def handle_presence(user_id: UserID, presence: PresenceState) -> None:
+        user = u.User.get_by_mxid(user_id, check_db=False, create=False)
+        if user and await user.is_logged_in():
+            await user.set_presence(presence == PresenceState.ONLINE)
+
+    async def handle_typing(self, room_id: RoomID, now_typing: Set[UserID]) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        previously_typing = self.previously_typing.get(room_id, set())
+
+        for user_id in set(previously_typing | now_typing):
+            is_typing = user_id in now_typing
+            was_typing = user_id in previously_typing
+            if is_typing and was_typing:
+                continue
+
+            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
+            if user and await user.is_logged_in():
+                await portal.set_typing(user, is_typing)
+
+        self.previously_typing[room_id] = now_typing
+
+    def filter_matrix_event(self, evt: Event) -> bool:
+        if isinstance(evt, (TypingEvent, ReceiptEvent, PresenceEvent)):
+            return False
+        elif not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent, EncryptedEvent)):
+            return True
+        if evt.content.get(self.az.real_user_content_key, False):
+            puppet = pu.Puppet.deprecated_sync_get_by_custom_mxid(evt.sender)
+            if puppet:
+                self.log.debug("Ignoring puppet-sent event %s", evt.event_id)
+                return True
+        return evt.sender and (evt.sender == self.az.bot_mxid
+                               or pu.Puppet.get_id_from_mxid(evt.sender) is not None)
+
+    async def handle_ephemeral_event(self, evt: Union[ReceiptEvent, PresenceEvent, TypingEvent]
+                                     ) -> None:
+        if evt.type == EventType.RECEIPT:
+            await self.handle_read_receipts(evt.room_id, self.parse_read_receipts(evt.content))
+        elif evt.type == EventType.PRESENCE:
+            await self.handle_presence(evt.sender, evt.content.presence)
+        elif evt.type == EventType.TYPING:
+            await self.handle_typing(evt.room_id, set(evt.content.user_ids))
+
+    async def handle_event(self, evt: Event) -> None:
+        if evt.type == EventType.ROOM_REDACTION:
+            await self.handle_redaction(evt)
+
+    async def handle_state_event(self, evt: StateEvent) -> None:
+        if evt.type == EventType.ROOM_POWER_LEVELS:
+            await self.handle_power_levels(evt)
+        elif evt.type in (EventType.ROOM_NAME, EventType.ROOM_AVATAR, EventType.ROOM_TOPIC):
+            await self.handle_room_meta(evt.type, evt.room_id, evt.sender, evt.content,
+                                        evt.event_id)
+        elif evt.type == EventType.ROOM_PINNED_EVENTS:
+            new_events = set(evt.content.pinned)
             try:
-                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
-            except KeyError:
+                old_events = set(evt.unsigned.prev_content.pinned)
+            except (KeyError, ValueError, TypeError, AttributeError):
                 old_events = set()
-            await self.handle_room_pin(evt["room_id"], evt["sender"], new_events, old_events)
+            await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events,
+                                       evt.event_id)
+        elif evt.type == EventType.ROOM_TOMBSTONE:
+            await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room,
+                                           evt.event_id)