Bump version to 0.10.2

SQLite is now supported for the crypto store instead of pickle (via aiosqlite)

.editorconfig

@@ -13,3 +13,6 @@ max_line_length = 99
 
 [*.{yaml,yml,py}]
 indent_style = space
+
+[.gitlab-ci.yml]
+indent_size = 2

.gitignore

@@ -14,4 +14,5 @@ __pycache__
 /registration.yaml
 *.log*
 *.db
+*.pickle
 *.bak

.gitlab-ci.yml

@@ -2,37 +2,65 @@ image: docker:stable
 
 stages:
 - build
-- push
+- manifest
 
 default:
   before_script:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
-build:
+build amd64:
   stage: build
+  tags:
+  - amd64
   script:
   - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  after_script:
+  - |
+    if [[ "$CI_COMMIT_BRANCH" == "master" && "$CI_JOB_STATUS" == "success" ]]; then
+      apk add --update curl jq
+      rm -rf /var/cache/apk/*
 
-push latest:
-  stage: push
-  only:
-  - master
-  variables:
-    GIT_STRATEGY: none
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
-  - docker push $CI_REGISTRY_IMAGE:latest
+      jq -n '
+         {
+           password:  env.BEEPER_DEV_ADMIN_NIGHTLY_PASS,
+           bridge:    env.BEEPER_BRIDGE_TYPE,
+           image:     "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
+           channel:   "STABLE"
+         }
+        ' | curl "$BEEPER_DEV_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
 
-push tag:
-  stage: push
-  variables:
-    GIT_STRATEGY: none
-  except:
-  - master
+      jq -n '
+         {
+           password:    env.BEEPER_PROD_ADMIN_NIGHTLY_PASS,
+           bridge:      env.BEEPER_BRIDGE_TYPE,
+           image:       "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
+           channel:     "INTERNAL",
+           deployNext:  true
+         }
+         ' | curl "$BEEPER_PROD_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
+    fi
+
+build arm64:
+  stage: build
+  tags:
+  - arm64
   script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+
+manifest:
+  stage: manifest
+  before_script:
+  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
+  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64

Dockerfile

@@ -1,77 +1,70 @@
-FROM docker.io/alpine:3.10 AS lottieconverter
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.14
 
-WORKDIR /build
+ARG TARGETARCH=amd64
 
-RUN apk add --no-cache git build-base cmake \
-  && git clone https://github.com/Samsung/rlottie.git \
-  && cd rlottie \
-  && mkdir build \
-  && cd build \
-  && cmake .. \
-  && make -j2 \
-  && make install \
-  && cd ../..
-
-RUN apk add --no-cache libpng libpng-dev zlib zlib-dev \
-  && git clone https://github.com/Eramde/LottieConverter.git \
-  && cd LottieConverter \
-  && git checkout 543c1d23ac9322f4f03c7fb6612ea7d026d44ac0 \
-  && make
-
-FROM docker.io/alpine:3.11
-
-ENV UID=1337 \
-    GID=1337 \
-    FFMPEG_BINARY=/usr/bin/ffmpeg
-
-COPY --from=lottieconverter /usr/lib/librlottie* /usr/lib/
-COPY --from=lottieconverter /build/LottieConverter/dist/Debug/GNU-Linux/lottieconverter /usr/local/bin/lottieconverter
-
-COPY . /opt/mautrix-telegram
-WORKDIR /opt/mautrix-telegram
-RUN apk add --no-cache --virtual .build-deps \
-      python3-dev \
-      libffi-dev \
-      build-base \
-      git \
-  && apk add --no-cache \
+RUN apk add --no-cache \
+      python3 py3-pip py3-setuptools py3-wheel \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-magic \
       py3-sqlalchemy \
+      py3-telethon-session-sqlalchemy \
+      py3-alembic \
       py3-psycopg2 \
       py3-ruamel.yaml \
+      py3-commonmark \
+      py3-prometheus-client \
       # Indirect dependencies
       py3-idna \
-      #commonmark
-        py3-future \
-      #alembic
-        py3-mako \
-        py3-dateutil \
-        py3-markupsafe \
       #moviepy
         py3-decorator \
         py3-tqdm \
         py3-requests \
         #imageio
           py3-numpy \
-      #telethon
-        py3-rsa \
+      #py3-telethon \ (outdated)
+        # Optional for socks proxies
+        py3-pysocks \
+        py3-pyaes \
         # cryptg
           py3-cffi \
+	  py3-qrcode \
+      py3-brotli \
       # Other dependencies
       ffmpeg \
       ca-certificates \
       su-exec \
       netcat-openbsd \
-      # lottieconverter
-      zlib libpng \
- && pip3 install .[speedups,hq_thumbnails,metrics] \
- # pip installs the sources to /usr/lib/python3.8/site-packages, so we don't need them here
- && rm -rf /opt/mautrix-telegram/mautrix_telegram \
+      # encryption
+      py3-olm \
+      py3-pycryptodome \
+      py3-unpaddedbase64 \
+      py3-future \
+      bash \
+      curl \
+      jq \
+      yq
+
+COPY requirements.txt /opt/mautrix-telegram/requirements.txt
+COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
+WORKDIR /opt/mautrix-telegram
+RUN apk add --virtual .build-deps \
+      python3-dev \
+      libffi-dev \
+      build-base \
+ && sed -Ei 's/psycopg2-binary.+//' optional-requirements.txt \
+ # TODO: unpin Pillow here after it's updated in Alpine
+ && pip3 install -r requirements.txt -r optional-requirements.txt 'pillow==8.2' \
  && apk del .build-deps
 
+COPY . /opt/mautrix-telegram
+RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics,e2be] && apk del git \
+  # This doesn't make the image smaller, but it's needed so that the `version` command works properly
+  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
+
 VOLUME /data
+ENV UID=1337 GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
 
 CMD ["/opt/mautrix-telegram/docker-run.sh"]

MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include LICENSE
+include requirements.txt
+include optional-requirements.txt

README.md

@@ -1,15 +1,28 @@
 # mautrix-telegram
-![Languages](https://img.shields.io/github/languages/top/tulir/mautrix-telegram.svg)
-[![License](https://img.shields.io/github/license/tulir/mautrix-telegram.svg)](LICENSE)
-[![Release](https://img.shields.io/github/release/tulir/mautrix-telegram/all.svg)](https://github.com/tulir/mautrix-telegram/releases)
-[![GitLab CI](https://mau.dev/tulir/mautrix-telegram/badges/master/pipeline.svg)](https://mau.dev/tulir/mautrix-telegram/container_registry)
-[![Maintainability](https://img.shields.io/codeclimate/maintainability/tulir/mautrix-telegram.svg)](https://codeclimate.com/github/tulir/mautrix-telegram)
+![Languages](https://img.shields.io/github/languages/top/mautrix/telegram.svg)
+[![License](https://img.shields.io/github/license/mautrix/telegram.svg)](LICENSE)
+[![Release](https://img.shields.io/github/release/mautrix/telegram/all.svg)](https://github.com/mautrix/telegram/releases)
+[![GitLab CI](https://mau.dev/mautrix/telegram/badges/master/pipeline.svg)](https://mau.dev/mautrix/telegram/container_registry)
 
 A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
-### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)
+## Sponsors
+* [Joel Lehtonen / Zouppen](https://github.com/zouppen)
 
-### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+### Documentation
+All setup and usage instructions are located on
+[docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
+Some quick links:
+
+* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
+  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
+* Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
+  [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
+  [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)
+
+### Features & Roadmap
+[ROADMAP.md](https://github.com/mautrix/telegram/blob/master/ROADMAP.md)
+contains a general overview of what is supported by the bridge.
 
 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)
@@ -17,4 +30,4 @@ Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net
 Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
 
 ## Preview
-![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)
+![Preview](preview.png)

ROADMAP.md

@@ -28,7 +28,10 @@
 	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
-  * [ ] Message history
+  * [x] Message history
+    * [x] Manually (`!tg backfill`)
+    * [x] Automatically when creating portal
+    * [x] Automatically for missed messages
   * [x] Avatars
   * [x] Presence
   * [x] Typing notifications
@@ -50,11 +53,11 @@
     * [x] At startup
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [x] Option to use bot to relay messages for unauthenticated Matrix users
-  * [x] Option to use own Matrix account for messages sent from other Telegram clients
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
-  * [ ] ‡ Secret chats (not yet supported by Telethon)
-  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
+  * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
+  * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/env.py

@@ -21,7 +21,6 @@ mxtg_config = Config(mxtg_config_path, None, None)
 mxtg_config.load()
 config.set_main_option("sqlalchemy.url", mxtg_config["appservice.database"].replace("%", "%%"))
 
-
 AlchemySessionContainer.create_table_classes(None, "telethon_", Base)
 
 # Interpret the config file for Python logging.
@@ -55,7 +54,8 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True)
+        url=url, target_metadata=target_metadata, literal_binds=True,
+        render_as_batch=True)
 
     with context.begin_transaction():
         context.run_migrations()
@@ -76,7 +76,8 @@ def run_migrations_online():
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
-            target_metadata=target_metadata
+            target_metadata=target_metadata,
+            render_as_batch=True
         )
 
         with context.begin_transaction():

alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py

@@ -0,0 +1,27 @@
+"""Add encrypted field for portals
+
+Revision ID: 24f31fc8a72b
+Revises: a7c04a56041b
+Create Date: 2020-03-28 20:14:29.046699
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "24f31fc8a72b"
+down_revision = "a7c04a56041b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column("encrypted", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column("encrypted")

alembic/versions/3e3745baa458_store_matrix_avatar_url_in_database.py

@@ -0,0 +1,32 @@
+"""Store Matrix avatar URL in database
+
+Revision ID: 3e3745baa458
+Revises: dff56c93da8d
+Create Date: 2020-06-15 14:32:10.454033
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3e3745baa458'
+down_revision = 'dff56c93da8d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('portal', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('avatar_url', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('portal', schema=None) as batch_op:
+        batch_op.drop_column('avatar_url')
+
+    # ### end Alembic commands ###

alembic/versions/7de69cf5809e_add_matrix_redaction_state_to_message_.py

@@ -0,0 +1,25 @@
+"""Add Matrix redaction state to message table
+
+Revision ID: 7de69cf5809e
+Revises: 888275d58e57
+Create Date: 2020-12-19 12:39:57.368568
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '7de69cf5809e'
+down_revision = '888275d58e57'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('message', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('redacted', sa.Boolean(), server_default=sa.false(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table('message', schema=None) as batch_op:
+        batch_op.drop_column('redacted')

alembic/versions/888275d58e57_add_double_puppet_base_url_to_puppet_.py

@@ -0,0 +1,30 @@
+"""Add double puppet base URL to puppet table
+
+Revision ID: 888275d58e57
+Revises: a328bf4f0932
+Create Date: 2020-10-14 18:52:00.730666
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '888275d58e57'
+down_revision = 'a328bf4f0932'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('base_url', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('base_url')
+    # ### end Alembic commands ###

alembic/versions/990f4395afc6_store_displayname_contact_status_in_.py

@@ -0,0 +1,32 @@
+"""Store displayname contact status in puppet table
+
+Revision ID: 990f4395afc6
+Revises: 7de69cf5809e
+Create Date: 2021-01-01 11:56:54.610681
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '990f4395afc6'
+down_revision = '7de69cf5809e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('displayname_contact', sa.Boolean(), server_default=sa.true(), nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('displayname_contact')
+
+    # ### end Alembic commands ###

alembic/versions/a328bf4f0932_store_encryption_state_event_in_db.py

@@ -0,0 +1,38 @@
+"""Store encryption state event in db
+
+Revision ID: a328bf4f0932
+Revises: ccbaff858240
+Create Date: 2020-07-11 21:31:27.059813
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from mautrix.client.state_store.sqlalchemy import SerializableType
+from mautrix.types import RoomEncryptionStateEventContent
+
+# revision identifiers, used by Alembic.
+revision = 'a328bf4f0932'
+down_revision = 'ccbaff858240'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('encryption',
+                                      SerializableType(RoomEncryptionStateEventContent),
+                                      nullable=True))
+        batch_op.add_column(sa.Column('has_full_member_list', sa.Boolean(), nullable=True))
+        batch_op.add_column(sa.Column('is_encrypted', sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
+        batch_op.drop_column('is_encrypted')
+        batch_op.drop_column('has_full_member_list')
+        batch_op.drop_column('encryption')
+    # ### end Alembic commands ###

alembic/versions/bfc0a39bfe02_store_displayname_quality_in_puppet_.py

@@ -0,0 +1,32 @@
+"""Store displayname quality in puppet table
+
+Revision ID: bfc0a39bfe02
+Revises: ec1d3dcc77e9
+Create Date: 2021-03-23 20:03:08.825333
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'bfc0a39bfe02'
+down_revision = 'ec1d3dcc77e9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('displayname_quality', sa.Integer(), server_default='0', nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('displayname_quality')
+
+    # ### end Alembic commands ###

alembic/versions/ccbaff858240_switch_to_mautrix_python_crypto.py

@@ -0,0 +1,71 @@
+"""Switch to mautrix-python crypto
+
+Revision ID: ccbaff858240
+Revises: 3e3745baa458
+Create Date: 2020-07-08 19:06:12.588047
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'ccbaff858240'
+down_revision = '3e3745baa458'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('nio_account')
+    op.drop_table('nio_device_key')
+    op.drop_table('nio_outgoing_key_request')
+    op.drop_table('nio_olm_session')
+    op.drop_table('nio_megolm_inbound_session')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('nio_megolm_inbound_session',
+    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('fp_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.Column('forwarded_chains', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('session_id', name='nio_megolm_inbound_session_pkey')
+    )
+    op.create_table('nio_olm_session',
+    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.Column('last_used', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('session_id', name='nio_olm_session_pkey')
+    )
+    op.create_table('nio_outgoing_key_request',
+    sa.Column('request_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('algorithm', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('request_id', name='nio_outgoing_key_request_pkey')
+    )
+    op.create_table('nio_device_key',
+    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('display_name', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('deleted', sa.BOOLEAN(), autoincrement=False, nullable=False),
+    sa.Column('keys', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_device_key_pkey')
+    )
+    op.create_table('nio_account',
+    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('shared', sa.BOOLEAN(), autoincrement=False, nullable=False),
+    sa.Column('sync_token', sa.TEXT(), autoincrement=False, nullable=False),
+    sa.Column('account', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_account_pkey')
+    )
+    # ### end Alembic commands ###

alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py

@@ -0,0 +1,26 @@
+"""Add decryption info field for reuploaded telegram files
+
+Revision ID: d3c922a6acd2
+Revises: 24f31fc8a72b
+Create Date: 2020-03-30 20:07:17.340346
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd3c922a6acd2'
+down_revision = '24f31fc8a72b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.add_column(sa.Column("decryption_info", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column("decryption_info")

alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py

@@ -0,0 +1,71 @@
+"""Add matrix-nio state store to main db
+
+Revision ID: dff56c93da8d
+Revises: d3c922a6acd2
+Create Date: 2020-03-31 22:04:04.014048
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'dff56c93da8d'
+down_revision = 'd3c922a6acd2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('nio_account',
+    sa.Column('user_id', sa.String(length=255), nullable=False),
+    sa.Column('device_id', sa.String(length=255), nullable=False),
+    sa.Column('shared', sa.Boolean(), nullable=False),
+    sa.Column('sync_token', sa.Text(), nullable=False),
+    sa.Column('account', sa.LargeBinary(), nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id')
+    )
+    op.create_table('nio_device_key',
+    sa.Column('user_id', sa.String(length=255), nullable=False),
+    sa.Column('device_id', sa.String(length=255), nullable=False),
+    sa.Column('display_name', sa.String(length=255), nullable=False),
+    sa.Column('deleted', sa.Boolean(), nullable=False),
+    sa.Column('keys', sa.PickleType(), nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id')
+    )
+    op.create_table('nio_megolm_inbound_session',
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('sender_key', sa.String(length=255), nullable=False),
+    sa.Column('fp_key', sa.String(length=255), nullable=False),
+    sa.Column('room_id', sa.String(length=255), nullable=False),
+    sa.Column('session', sa.LargeBinary(), nullable=False),
+    sa.Column('forwarded_chains', sa.PickleType(), nullable=False),
+    sa.PrimaryKeyConstraint('session_id')
+    )
+    op.create_table('nio_olm_session',
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('sender_key', sa.String(length=255), nullable=False),
+    sa.Column('session', sa.LargeBinary(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('last_used', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('session_id')
+    )
+    op.create_table('nio_outgoing_key_request',
+    sa.Column('request_id', sa.String(length=255), nullable=False),
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('room_id', sa.String(length=255), nullable=False),
+    sa.Column('algorithm', sa.String(length=255), nullable=False),
+    sa.PrimaryKeyConstraint('request_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('nio_outgoing_key_request')
+    op.drop_table('nio_olm_session')
+    op.drop_table('nio_megolm_inbound_session')
+    op.drop_table('nio_device_key')
+    op.drop_table('nio_account')
+    # ### end Alembic commands ###

alembic/versions/ec1d3dcc77e9_switch_telegram_ids_to_bigints.py

@@ -0,0 +1,44 @@
+"""Switch Telegram IDs to bigints
+
+Revision ID: ec1d3dcc77e9
+Revises: 990f4395afc6
+Create Date: 2021-03-09 21:36:58.443727
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'ec1d3dcc77e9'
+down_revision = '990f4395afc6'
+branch_labels = None
+depends_on = None
+
+
+columns_to_upgrade = (
+    ("bot_chat", "id"),
+    ("message", "tgid"),
+    ("message", "tg_space"),
+    ("portal", "tgid"),
+    ("portal", "tg_receiver"),
+    ("puppet", "id"),
+    ("puppet", "displayname_source"),
+    ("user", "tgid"),
+    ("user_portal", "user"),
+    ("user_portal", "portal"),
+    ("user_portal", "portal_receiver"),
+    ("contact", "user"),
+    ("contact", "contact"),
+)
+
+
+def upgrade():
+    if op.get_context().dialect.name == "postgresql":
+        for table, column in columns_to_upgrade:
+            op.alter_column(table, column, existing_type=sa.Integer, type_=sa.BigInteger)
+
+
+def downgrade():
+    if op.get_context().dialect.name == "postgresql":
+        for table, column in columns_to_upgrade:
+            op.alter_column(table, column, existing_type=sa.BigInteger, type_=sa.Integer)

docker-run.sh

@@ -13,8 +13,6 @@ sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /d
 if [ -f /data/mx-state.json ]; then
 	ln -s /data/mx-state.json
 fi
-# Check that database is in the right state
-alembic -x config=/data/config.yaml upgrade head
 
 if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
@@ -28,12 +26,12 @@ fi
 
 if [ ! -f /data/registration.yaml ]; then
 	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
-	echo "Didn't find a registration file."
-	echo "Generated one for you."
-	echo "Copy that over to synapses app service directory."
 	fixperms
 	exit
 fi
 
+# Check that database is in the right state
+alembic -x config=/data/config.yaml upgrade head
+
 fixperms
 exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.7.1rc1"
+__version__ = "0.10.2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -13,17 +13,17 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
-from itertools import chain
+from typing import Dict, Any
 
+from telethon import __version__ as __telethon_version__
 from alchemysession import AlchemySessionContainer
 
+from mautrix.types import UserID, RoomID
 from mautrix.bridge import Bridge
 from mautrix.util.db import Base
 
 from .web.provisioning import ProvisioningAPI
 from .web.public import PublicBridgeWebsite
-from .commands.manhole import ManholeState
 from .abstract_user import init as init_abstract_user
 from .bot import Bot, init as init_bot
 from .config import Config
@@ -31,9 +31,8 @@ from .context import Context
 from .db import init as init_db
 from .formatter import init as init_formatter
 from .matrix import MatrixHandler
-from .portal import init as init_portal
+from .portal import Portal, init as init_portal
 from .puppet import Puppet, init as init_puppet
-from .sqlstatestore import SQLStateStore
 from .user import User, init as init_user
 from .version import version, linkified_version
 
@@ -44,21 +43,20 @@ except ImportError:
 
 
 class TelegramBridge(Bridge):
+    module = "mautrix_telegram"
     name = "mautrix-telegram"
     command = "python -m mautrix-telegram"
     description = "A Matrix-Telegram puppeting bridge."
-    repo_url = "https://github.com/tulir/mautrix-telegram"
+    repo_url = "https://github.com/mautrix/telegram"
     real_user_content_key = "net.maunium.telegram.puppet"
     version = version
     markdown_version = linkified_version
     config_class = Config
     matrix_class = MatrixHandler
-    state_store_class = SQLStateStore
 
     config: Config
     session_container: AlchemySessionContainer
     bot: Bot
-    manhole: Optional[ManholeState]
 
     def prepare_db(self) -> None:
         super().prepare_db()
@@ -79,35 +77,67 @@ class TelegramBridge(Bridge):
                                    provisioning_api.app)
             context.provisioning_api = provisioning_api
 
-        if self.config["metrics.enabled"]:
-            if prometheus:
-                prometheus.start_http_server(self.config["metrics.listen_port"])
-            else:
-                self.log.warning("Metrics are enabled in the config, "
-                                 "but prometheus_client is not installed.")
-
     def prepare_bridge(self) -> None:
         self.bot = init_bot(self.config)
         context = Context(self.az, self.config, self.loop, self.session_container, self, self.bot)
         self._prepare_website(context)
         self.matrix = context.mx = MatrixHandler(context)
-        self.manhole = None
 
         init_abstract_user(context)
         init_formatter(context)
         init_portal(context)
-        puppet_startup = init_puppet(context)
-        user_startup = init_user(context)
-        bot_startup = [self.bot.start()] if self.bot else []
-        self.startup_actions = chain(puppet_startup, user_startup, bot_startup)
+        self.add_startup_actions(init_puppet(context))
+        self.add_startup_actions(init_user(context))
+        if self.bot:
+            self.add_startup_actions(self.bot.start())
+        if self.config["bridge.resend_bridge_info"]:
+            self.add_startup_actions(self.resend_bridge_info())
+
+    async def resend_bridge_info(self) -> None:
+        self.config["bridge.resend_bridge_info"] = False
+        self.config.save()
+        self.log.info("Re-sending bridge info state event to all portals")
+        for portal in Portal.all():
+            await portal.update_bridge_info()
+        self.log.info("Finished re-sending bridge info state events")
 
     def prepare_stop(self) -> None:
         for puppet in Puppet.by_custom_mxid.values():
             puppet.stop()
         self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
-        if self.manhole:
-            self.manhole.close()
-            self.manhole = None
+
+    async def get_user(self, user_id: UserID, create: bool = True) -> User:
+        user = User.get_by_mxid(user_id, create=create)
+        if user:
+            await user.ensure_started()
+        return user
+
+    async def get_portal(self, room_id: RoomID) -> Portal:
+        return Portal.get_by_mxid(room_id)
+
+    async def get_puppet(self, user_id: UserID, create: bool = False) -> Puppet:
+        return await Puppet.get_by_mxid(user_id, create=create)
+
+    async def get_double_puppet(self, user_id: UserID) -> Puppet:
+        return await Puppet.get_by_custom_mxid(user_id)
+
+    def is_bridge_ghost(self, user_id: UserID) -> bool:
+        return bool(Puppet.get_id_from_mxid(user_id))
+
+    async def count_logged_in_users(self) -> int:
+        return len([user for user in User.by_tgid.values() if user.tgid])
+
+    async def manhole_global_namespace(self, user_id: UserID) -> Dict[str, Any]:
+        return {
+            **await super().manhole_global_namespace(user_id),
+            "User": User,
+            "Portal": Portal,
+            "Puppet": Puppet,
+        }
+
+    @property
+    def manhole_banner_program_version(self) -> str:
+        return f"{super().manhole_banner_program_version} and Telethon {__telethon_version__}"
 
 
 TelegramBridge().run()

mautrix_telegram/abstract_user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,9 +15,9 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Tuple, Optional, Union, Dict, Type, Any, TYPE_CHECKING
 from abc import ABC, abstractmethod
+import platform
 import asyncio
 import logging
-import platform
 import time
 
 from telethon.sessions import Session
@@ -25,16 +25,20 @@ from telethon.network import (ConnectionTcpMTProxyRandomizedIntermediate, Connec
                               Connection)
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
-    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdateChatPinnedMessage,
-    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants,
+    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdatePinnedMessages,
+    UpdatePinnedChannelMessages, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
     UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
     UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
     UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
-    UpdateUserTyping, User, UserStatusOffline, UserStatusOnline)
+    UpdateUserTyping, User, UserStatusOffline, UserStatusOnline, UpdateReadHistoryInbox,
+    UpdateReadChannelInbox, MessageEmpty, UpdateFolderPeers, UpdatePinnedDialogs,
+    UpdateNotifySettings, UpdateChannelUserTyping)
 
 from mautrix.types import UserID, PresenceState
 from mautrix.errors import MatrixError
 from mautrix.appservice import AppService
+from mautrix.util.logging import TraceLogger
+from mautrix.util.opt_prometheus import Histogram, Counter
 from alchemysession import AlchemySessionContainer
 
 from . import portal as po, puppet as pu, __version__
@@ -46,6 +50,7 @@ if TYPE_CHECKING:
     from .context import Context
     from .config import Config
     from .bot import Bot
+    from .__main__ import TelegramBridge
 
 config: Optional['Config'] = None
 # Value updated from config in init()
@@ -54,22 +59,20 @@ MAX_DELETIONS: int = 10
 UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
+UpdateTyping = Union[UpdateUserTyping, UpdateChatUserTyping, UpdateChannelUserTyping]
 
-try:
-    from prometheus_client import Histogram
-
-    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
-                            ["update_type"])
-except ImportError:
-    Histogram = None
-    UPDATE_TIME = None
+UPDATE_TIME = Histogram("bridge_telegram_update", "Time spent processing Telegram updates",
+                        ("update_type",))
+UPDATE_ERRORS = Counter("bridge_telegram_update_error",
+                        "Number of fatal errors while handling Telegram updates", ("update_type",))
 
 
 class AbstractUser(ABC):
     session_container: AlchemySessionContainer = None
     loop: asyncio.AbstractEventLoop = None
-    log: logging.Logger
+    log: TraceLogger
     az: AppService
+    bridge: 'TelegramBridge'
     relaybot: Optional['Bot']
     ignore_incoming_bot_events: bool = True
 
@@ -97,7 +100,6 @@ class AbstractUser(ABC):
         self.client = None
         self.is_relaybot = False
         self.is_bot = False
-        self.relaybot = None
 
     @property
     def connected(self) -> bool:
@@ -129,9 +131,9 @@ class AbstractUser(ABC):
     def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
 
-        self.session = self.session_container.new_session(self.name)
+        session = self.session_container.new_session(self.name)
         if config["telegram.server.enabled"]:
-            self.session.set_dc(config["telegram.server.dc"],
+            session.set_dc(config["telegram.server.dc"],
                                 config["telegram.server.ip"],
                                 config["telegram.server.port"])
 
@@ -145,10 +147,10 @@ class AbstractUser(ABC):
         appversion = config["telegram.device_info.app_version"]
         connection, proxy = self._proxy_settings
 
-        assert isinstance(self.session, Session)
+        assert isinstance(session, Session)
 
         self.client = MautrixTelegramClient(
-            session=self.session,
+            session=session,
 
             api_id=config["telegram.api_id"],
             api_hash=config["telegram.api_hash"],
@@ -166,6 +168,7 @@ class AbstractUser(ABC):
             request_retries=config["telegram.connection.request_retries"],
             connection=connection,
             proxy=proxy,
+            raise_last_call_error=True,
 
             loop=self.loop,
             base_logger=base_logger
@@ -181,22 +184,23 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     @abstractmethod
-    def register_portal(self, portal: po.Portal) -> None:
+    async def register_portal(self, portal: po.Portal) -> None:
         raise NotImplementedError()
 
     @abstractmethod
-    def unregister_portal(self, portal: po.Portal) -> None:
+    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
         raise NotImplementedError()
 
     async def _update_catch(self, update: TypeUpdate) -> None:
         start_time = time.time()
+        update_type = type(update).__name__
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
-            self.log.exception(f"Failed to handle Telegram update {update}")
-        if UPDATE_TIME:
-            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
+            self.log.exception("Failed to handle Telegram update")
+            UPDATE_ERRORS.labels(update_type=update_type).inc()
+        UPDATE_TIME.labels(update_type=update_type).observe(time.time() - start_time)
 
     @property
     @abstractmethod
@@ -235,8 +239,7 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
-        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
-                              loop=self.loop)
+        asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -244,7 +247,7 @@ class AbstractUser(ABC):
             await self.delete_message(update)
         elif isinstance(update, UpdateDeleteChannelMessages):
             await self.delete_channel_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
+        elif isinstance(update, (UpdateChatUserTyping, UpdateChannelUserTyping, UpdateUserTyping)):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
@@ -252,29 +255,47 @@ class AbstractUser(ABC):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
+        elif isinstance(update, (UpdatePinnedMessages, UpdatePinnedChannelMessages)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
         elif isinstance(update, UpdateReadHistoryOutbox):
             await self.update_read_receipt(update)
+        elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
+            await self.update_own_read_receipt(update)
+        elif isinstance(update, UpdateFolderPeers):
+            await self.update_folder_peers(update)
+        elif isinstance(update, UpdatePinnedDialogs):
+            await self.update_pinned_dialogs(update)
+        elif isinstance(update, UpdateNotifySettings):
+            await self.update_notify_settings(update)
         else:
-            self.log.debug("Unhandled update: %s", update)
+            self.log.trace("Unhandled update: %s", update)
 
-    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
-                                                         UpdateChatPinnedMessage]) -> None:
-        if isinstance(update, UpdateChatPinnedMessage):
-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
+        pass
+
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        pass
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        pass
+
+    async def update_pinned_messages(self, update: Union[UpdatePinnedMessages,
+                                                         UpdatePinnedChannelMessages]) -> None:
+        if isinstance(update, UpdatePinnedMessages):
+            portal = po.Portal.get_by_entity(update.peer, receiver_id=self.tgid)
         else:
             portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id, self.tgid)
+            await portal.receive_telegram_pin_ids(update.messages, self.tgid,
+                                                  remove=not update.pinned)
 
     @staticmethod
     async def update_participants(update: UpdateChatParticipants) -> None:
         portal = po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
         if portal and portal.mxid:
-            await portal.update_telegram_participants(update.participants.participants)
+            await portal.update_power_levels(update.participants.participants)
 
     async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
         if not isinstance(update.peer, PeerUser):
@@ -293,6 +314,32 @@ class AbstractUser(ABC):
         puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
+    async def update_own_read_receipt(self, update: Union[UpdateReadHistoryInbox,
+                                                          UpdateReadChannelInbox]) -> None:
+        puppet = pu.Puppet.get(self.tgid)
+        if not puppet.is_real_user:
+            return
+
+        if isinstance(update, UpdateReadChannelInbox):
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update.peer, PeerChat):
+            portal = po.Portal.get_by_tgid(TelegramID(update.peer.chat_id))
+        elif isinstance(update.peer, PeerUser):
+            portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
+        else:
+            self.log.debug("Unexpected own read receipt peer: %s", update.peer)
+            return
+
+        if not portal or not portal.mxid:
+            return
+
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), tg_space, edit_index=-1)
+        if not message:
+            return
+
+        await puppet.intent.mark_read(portal.mxid, message.mxid)
+
     async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
         # TODO duplication not checked
         portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
@@ -301,16 +348,27 @@ class AbstractUser(ABC):
 
         await portal.set_telegram_admin(TelegramID(update.user_id))
 
-    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+    async def update_typing(self, update: UpdateTyping) -> None:
+        sender = None
         if isinstance(update, UpdateUserTyping):
             portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
-        else:
+            sender = pu.Puppet.get(TelegramID(update.user_id))
+        elif isinstance(update, UpdateChannelUserTyping):
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update, UpdateChatUserTyping):
             portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
-
-        if not portal or not portal.mxid:
+        else:
+            return
+
+        if isinstance(update, (UpdateChannelUserTyping, UpdateChatUserTyping)):
+            # Can typing notifications come from non-user peers?
+            if not update.from_id.user_id:
+                return
+            sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
+
+        if not sender or not portal or not portal.mxid:
             return
 
-        sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
     async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
@@ -329,12 +387,12 @@ class AbstractUser(ABC):
         if isinstance(update, UpdateUserName):
             puppet.username = update.username
             if await puppet.update_displayname(self, update):
-                puppet.save()
+                await puppet.save()
         elif isinstance(update, UpdateUserPhoto):
             if await puppet.update_avatar(self, update.photo):
-                puppet.save()
+                await puppet.save()
         else:
-            self.log.warning("Unexpected other user info update: %s", update)
+            self.log.warning(f"Unexpected other user info update: {type(update)}")
 
     async def update_status(self, update: UpdateUserStatus) -> None:
         puppet = pu.Puppet.get(TelegramID(update.user_id))
@@ -343,7 +401,7 @@ class AbstractUser(ABC):
         elif isinstance(update.status, UserStatusOffline):
             await puppet.default_mxid_intent.set_presence(PresenceState.OFFLINE)
         else:
-            self.log.warning("Unexpected user status update: %s", update)
+            self.log.warning(f"Unexpected user status update: type({update})")
         return
 
     def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
@@ -360,15 +418,18 @@ class AbstractUser(ABC):
         elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
                                  UpdateEditMessage, UpdateEditChannelMessage)):
             update = update.message
-            if isinstance(update.to_id, PeerUser) and not update.out:
-                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
+            if isinstance(update, MessageEmpty):
+                return update, None, None
+            portal = po.Portal.get_by_entity(update.peer_id, receiver_id=self.tgid)
+            if update.out:
+                sender = pu.Puppet.get(self.tgid)
+            elif isinstance(update.from_id, PeerUser):
+                sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
             else:
-                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
-            sender = pu.Puppet.get(update.from_id) if update.from_id else None
+                sender = None
         else:
-            self.log.warning(
-                f"Unexpected message type in User#get_message_details: {type(update)}")
+            self.log.warning("Unexpected message type in User#get_message_details: "
+                             f"{type(update)}")
             return update, None, None
         return update, sender, portal
 
@@ -388,6 +449,8 @@ class AbstractUser(ABC):
 
         for message_id in update.messages:
             for message in DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
+                if message.redacted:
+                    continue
                 message.delete()
                 number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
                 if number_left == 0:
@@ -401,6 +464,8 @@ class AbstractUser(ABC):
 
         for message_id in update.messages:
             for message in DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
+                if message.redacted:
+                    continue
                 message.delete()
                 await self._try_redact(message)
 
@@ -422,18 +487,22 @@ class AbstractUser(ABC):
                                f" in unbridged chat {portal.tgid_log}")
                 return
 
-        if self.ignore_incoming_bot_events and self.relaybot and sender.id == self.relaybot.tgid:
-            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
+        if ((self.ignore_incoming_bot_events and self.relaybot
+             and sender and sender.id == self.relaybot.tgid)):
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
             return
 
+        await portal.backfill_lock.wait(update.id)
+
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.debug(f"Ignoring action %s to %s by %d", update.action,
-                               portal.tgid_log,
-                               sender.id)
+                self.log.trace(f"Received %s in %s by %d, unregistering portal...",
+                               update.action, portal.tgid_log, sender.id)
+                await self.unregister_portal(update.action.chat_id, update.action.chat_id)
+                await self.register_portal(portal)
                 return
-            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
-                           sender.id)
+            self.log.trace("Handling action %s to %s by %d", update.action, portal.tgid_log,
+                           (sender.id if sender else 0))
             return await portal.handle_telegram_action(self, sender, update)
 
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
@@ -446,6 +515,7 @@ class AbstractUser(ABC):
 def init(context: 'Context') -> None:
     global config, MAX_DELETIONS
     AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
+    AbstractUser.bridge = context.bridge
     AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
     AbstractUser.session_container = context.session_container
     MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/bot.py

@@ -117,11 +117,11 @@ class Bot(AbstractUser):
             except (ChannelPrivateError, ChannelInvalidError):
                 self.remove_chat(TelegramID(channel_id.channel_id))
 
-    def register_portal(self, portal: po.Portal) -> None:
+    async def register_portal(self, portal: po.Portal) -> None:
         self.add_chat(portal.tgid, portal.peer_type)
 
-    def unregister_portal(self, portal: po.Portal) -> None:
-        self.remove_chat(portal.tgid)
+    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
+        self.remove_chat(tgid)
 
     def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
         if chat_id not in self.chats:
@@ -147,7 +147,7 @@ class Bot(AbstractUser):
         if self.whitelist_group_admins:
             if isinstance(chat, PeerChannel):
                 p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
+                return isinstance(p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin))
             elif isinstance(chat, PeerChat):
                 chat = await self.client(GetFullChatRequest(chat.chat_id))
                 participants = chat.full_chat.participants.participants
@@ -195,7 +195,7 @@ class Bot(AbstractUser):
             return await reply("That user seems to be logged in. "
                                f"Just invite [{displayname}](tg://user?id={user.tgid})")
         else:
-            await portal.main_intent.invite_user(portal.mxid, user.mxid)
+            await portal.invite_to_matrix(user.mxid)
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
     @staticmethod
@@ -226,7 +226,7 @@ class Bot(AbstractUser):
 
         return False
 
-    async def handle_command(self, message: Message) -> Optional[bool]:
+    async def handle_command(self, message: Message) -> None:
         def reply(reply_text: str) -> Awaitable[Message]:
             return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)
 
@@ -234,9 +234,8 @@ class Bot(AbstractUser):
 
         if self.match_command(text, "start"):
             pcm = config["bridge.relaybot.private_chat.message"]
-            if not pcm:
-                return True
-            await reply(pcm)
+            if pcm:
+                await reply(pcm)
             return
         elif self.match_command(text, "id"):
             await self.handle_command_id(message, reply)
@@ -246,18 +245,19 @@ class Bot(AbstractUser):
 
         portal = po.Portal.get_by_entity(message.to_id)
 
-        if self.match_command(text, "portal"):
+        is_portal_cmd = self.match_command(text, "portal")
+        is_invite_cmd = self.match_command(text, "invite")
+        if is_portal_cmd or is_invite_cmd:
             if not await self.check_can_use_commands(message, reply):
                 return
-            await self.handle_command_portal(portal, reply)
-        elif self.match_command(text, "invite"):
-            if not await self.check_can_use_commands(message, reply):
-                return
-            try:
-                mxid = text[text.index(" ") + 1:]
-            except ValueError:
-                mxid = ""
-            await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
+            if is_portal_cmd:
+                await self.handle_command_portal(portal, reply)
+            elif is_invite_cmd:
+                try:
+                    mxid = text[text.index(" ") + 1:]
+                except ValueError:
+                    mxid = ""
+                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
 
     def handle_service_message(self, message: MessageService) -> None:
         to_peer = message.to_id
@@ -288,9 +288,10 @@ class Bot(AbstractUser):
 
         is_command = (isinstance(update.message, Message)
                       and update.message.entities and len(update.message.entities) > 0
-                      and isinstance(update.message.entities[0], MessageEntityBotCommand))
+                      and isinstance(update.message.entities[0], MessageEntityBotCommand)
+                      and update.message.entities[0].offset == 0)
         if is_command:
-            return not await self.handle_command(update.message)
+            await self.handle_command(update.message)
         return False
 
     def is_in_chat(self, peer_id) -> bool:

mautrix_telegram/commands/__init__.py

@@ -1,7 +1,7 @@
 from .handler import (command_handler, CommandHandler, CommandProcessor, CommandEvent,
                       SECTION_AUTH, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT,
                       SECTION_MISC, SECTION_ADMIN)
-from . import portal, telegram, clean_rooms, matrix_auth, manhole
+from . import portal, telegram, matrix_auth
 
 __all__ = ["command_handler", "CommandHandler", "CommandProcessor", "CommandEvent",
            "SECTION_AUTH", "SECTION_MISC", "SECTION_ADMIN", "SECTION_CREATING_PORTALS",

mautrix_telegram/commands/clean_rooms.py

@@ -1,177 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, NamedTuple, Tuple, Union
-
-from mautrix.appservice import IntentAPI
-from mautrix.errors import MatrixRequestError
-from mautrix.types import RoomID, UserID, EventID
-
-from . import command_handler, CommandEvent, SECTION_ADMIN
-from .. import puppet as pu, portal as po
-
-ManagementRoom = NamedTuple('ManagementRoom', room_id=RoomID, user_id=UserID)
-
-
-async def _find_rooms(intent: IntentAPI) -> Tuple[List[ManagementRoom], List[RoomID],
-                                                  List['po.Portal'], List['po.Portal']]:
-    management_rooms: List[ManagementRoom] = []
-    unidentified_rooms: List[RoomID] = []
-    portals: List[po.Portal] = []
-    empty_portals: List[po.Portal] = []
-
-    rooms = await intent.get_joined_rooms()
-    for room_id in rooms:
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
-            try:
-                members = await intent.get_room_members(room_id)
-            except MatrixRequestError:
-                members = []
-            if len(members) == 2:
-                other_member = members[0] if members[0] != intent.mxid else members[1]
-                if pu.Puppet.get_id_from_mxid(other_member):
-                    unidentified_rooms.append(room_id)
-                else:
-                    management_rooms.append(ManagementRoom(room_id, other_member))
-            else:
-                unidentified_rooms.append(room_id)
-        else:
-            members = await portal.get_authenticated_matrix_users()
-            if len(members) == 0:
-                empty_portals.append(portal)
-            else:
-                portals.append(portal)
-
-    return management_rooms, unidentified_rooms, portals, empty_portals
-
-
-@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
-                 help_section=SECTION_ADMIN,
-                 help_text="Clean up unused portal/management rooms.")
-async def clean_rooms(evt: CommandEvent) -> EventID:
-    management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
-
-    reply = ["#### Management rooms (M)"]
-    reply += ([f"{n+1}. [M{n+1}](https://matrix.to/#/{room}) (with {other_member}"
-               for n, (room, other_member) in enumerate(management_rooms)]
-              or ["No management rooms found."])
-    reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(portals)]
-              or ["No active portal rooms found."])
-    reply.append("#### Unidentified rooms (U)")
-    reply += ([f"{n+1}. [U{n+1}](https://matrix.to/#/{room})"
-               for n, room in enumerate(unidentified_rooms)]
-              or ["No unidentified rooms found."])
-    reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(empty_portals)]
-              or ["No inactive portal rooms found."])
-
-    reply += ["#### Usage",
-              ("To clean the recommended set of rooms (unidentified & inactive portals), "
-               "type `$cmdprefix+sp clean-recommended`"),
-              "",
-              ("To clean other groups of rooms, type `$cmdprefix+sp clean-groups <letters>` "
-               "where `letters` are the first letters of the group names (M, A, U, I)"),
-              "",
-              ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
-               "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name. (e.g. `I2-6`)"),
-              "",
-              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
-               "between each use of the commands above.")]
-
-    evt.sender.command_status = {
-        "next": lambda clean_evt: set_rooms_to_clean(clean_evt, management_rooms,
-                                                     unidentified_rooms, portals, empty_portals),
-        "action": "Room cleaning",
-    }
-
-    return await evt.reply("\n".join(reply))
-
-
-async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
-                             unidentified_rooms: List[RoomID], portals: List["po.Portal"],
-                             empty_portals: List["po.Portal"]) -> None:
-    command = evt.args[0]
-    rooms_to_clean: List[Union[po.Portal, RoomID]] = []
-    if command == "clean-recommended":
-        rooms_to_clean += empty_portals
-        rooms_to_clean += unidentified_rooms
-    elif command == "clean-groups":
-        if len(evt.args) < 2:
-            return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1].upper()
-        if "M" in groups_to_clean:
-            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
-        if "A" in groups_to_clean:
-            rooms_to_clean += portals
-        if "U" in groups_to_clean:
-            rooms_to_clean += unidentified_rooms
-        if "I" in groups_to_clean:
-            rooms_to_clean += empty_portals
-    elif command == "clean-range":
-        try:
-            clean_range = evt.args[1]
-            group, clean_range = clean_range[0], clean_range[1:]
-            start, end = clean_range.split("-")
-            start, end = int(start), int(end)
-            if group == "M":
-                group = [room_id for (room_id, user_id) in management_rooms]
-            elif group == "A":
-                group = portals
-            elif group == "U":
-                group = unidentified_rooms
-            elif group == "I":
-                group = empty_portals
-            else:
-                raise ValueError("Unknown group")
-            rooms_to_clean = group[start - 1:end]
-        except (KeyError, ValueError):
-            return await evt.reply(
-                "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
-    else:
-        return await evt.reply(f"Unknown room cleaning action `{command}`. "
-                               "Use `$cmdprefix+sp cancel` to cancel room "
-                               "cleaning.")
-
-    evt.sender.command_status = {
-        "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
-        "action": "Room cleaning",
-    }
-    await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type "
-                    "`$cmdprefix+sp confirm-clean`.")
-
-
-async def execute_room_cleanup(evt, rooms_to_clean: List[Union[po.Portal, RoomID]]) -> None:
-    if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
-        await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
-                        "This might take a while.")
-        cleaned = 0
-        for room in rooms_to_clean:
-            if isinstance(room, po.Portal):
-                await room.cleanup_and_delete()
-                cleaned += 1
-            else:
-                await po.Portal.cleanup_room(evt.az.intent, room, "Room deleted")
-                cleaned += 1
-        evt.sender.command_status = None
-        await evt.reply(f"{cleaned} rooms cleaned up successfully.")
-    else:
-        await evt.reply("Room cleaning cancelled.")

mautrix_telegram/commands/handler.py

@@ -25,11 +25,17 @@ from mautrix.bridge.commands import (HelpSection, CommandEvent as BaseCommandEve
                                      CommandHandlerFunc, command_handler as base_command_handler)
 
 from ..util import format_duration
-from .. import user as u, context as c
+from .. import user as u, context as c, portal as po
+
+
+class HelpCacheKey(NamedTuple):
+    is_management: bool
+    is_portal: bool
+    puppet_whitelisted: bool
+    matrix_puppet_whitelisted: bool
+    is_admin: bool
+    is_logged_in: bool
 
-HelpCacheKey = NamedTuple('HelpCacheKey',
-                          is_management=bool, is_portal=bool, puppet_whitelisted=bool,
-                          matrix_puppet_whitelisted=bool, is_admin=bool, is_logged_in=bool)
 
 SECTION_AUTH = HelpSection("Authentication", 10, "")
 SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
@@ -40,12 +46,13 @@ SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 class CommandEvent(BaseCommandEvent):
     sender: u.User
+    portal: po.Portal
 
     def __init__(self, processor: 'CommandProcessor', room_id: RoomID, event_id: EventID,
                  sender: u.User, command: str, args: List[str], content: MessageEventContent,
-                 is_management: bool, is_portal: bool) -> None:
+                 portal: Optional['po.Portal'], is_management: bool, has_bridge_bot: bool) -> None:
         super().__init__(processor, room_id, event_id, sender, command, args, content,
-                         is_management, is_portal)
+                         portal, is_management, has_bridge_bot)
         self.bridge = processor.bridge
         self.tgbot = processor.tgbot
         self.config = processor.config
@@ -56,19 +63,16 @@ class CommandEvent(BaseCommandEvent):
         return self.sender.is_admin
 
     async def get_help_key(self) -> HelpCacheKey:
-        return HelpCacheKey(self.is_management, self.is_portal, self.sender.puppet_whitelisted,
-                            self.sender.matrix_puppet_whitelisted, self.sender.is_admin,
-                            await self.sender.is_logged_in())
+        return HelpCacheKey(self.is_management, self.portal is not None,
+                            self.sender.puppet_whitelisted, self.sender.matrix_puppet_whitelisted,
+                            self.sender.is_admin, await self.sender.is_logged_in())
 
 
 class CommandHandler(BaseCommandHandler):
     name: str
 
-    management_only: bool
-    needs_auth: bool
     needs_puppeting: bool
     needs_matrix_puppeting: bool
-    needs_admin: bool
 
     def __init__(self, handler: Callable[[CommandEvent], Awaitable[EventID]],
                  management_only: bool, name: str, help_text: str, help_args: str,
@@ -79,25 +83,16 @@ class CommandHandler(BaseCommandHandler):
                          needs_matrix_puppeting=needs_matrix_puppeting, needs_admin=needs_admin)
 
     async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
-        if self.management_only and not evt.is_management:
-            return (f"`{evt.command}` is a restricted command: "
-                    "you may only run it in management rooms.")
-        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
+        if self.needs_puppeting and not evt.sender.puppet_whitelisted:
             return "This command requires puppeting privileges."
         elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
             return "This command requires Matrix puppeting privileges."
-        elif self.needs_admin and not evt.sender.is_admin:
-            return "This command requires administrator privileges."
-        elif self.needs_auth and not await evt.sender.is_logged_in():
-            return "This command requires you to be logged in."
-        return None
+        return await super().get_permission_error(evt)
 
     def has_permission(self, key: HelpCacheKey) -> bool:
-        return ((not self.management_only or key.is_management) and
+        return (super().has_permission(key) and
                 (not self.needs_puppeting or key.puppet_whitelisted) and
-                (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted) and
-                (not self.needs_admin or key.is_admin) and
-                (not self.needs_auth or key.is_logged_in))
+                (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted))
 
 
 def command_handler(_func: Optional[CommandHandlerFunc] = None, *, needs_auth: bool = True,
@@ -115,13 +110,9 @@ def command_handler(_func: Optional[CommandHandlerFunc] = None, *, needs_auth: b
 
 class CommandProcessor(BaseCommandProcessor):
     def __init__(self, context: c.Context) -> None:
-        super().__init__(az=context.az, config=context.config, event_class=CommandEvent,
-                         loop=context.loop, bridge=context.bridge)
+        super().__init__(event_class=CommandEvent, bridge=context.bridge)
         self.tgbot = context.bot
-        self.bridge = context.bridge
-        self.az, self.config, self.loop, self.tgbot = context.core
         self.public_website = context.public_website
-        self.command_prefix = self.config["bridge.command_prefix"]
 
     @staticmethod
     async def _run_handler(handler: Callable[[CommandEvent], Awaitable[Any]], evt: CommandEvent

mautrix_telegram/commands/manhole.py

@@ -1,128 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Set, Callable
-import asyncio
-import sys
-import os
-
-from attr import dataclass
-
-from telethon import __version__ as __telethon_version__
-
-from mautrix import __version__ as __mautrix_version__
-from mautrix.types import UserID
-from mautrix.errors import MatrixConnectionError
-from mautrix.util.manhole import start_manhole
-
-from .. import __version__
-from . import command_handler, CommandEvent, SECTION_ADMIN
-
-
-@dataclass
-class ManholeState:
-    server: asyncio.AbstractServer
-    opened_by: UserID
-    close: Callable[[], None]
-    whitelist: Set[int]
-
-
-@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
-                 help_text="Open a manhole into the bridge.", help_args="<_uid..._>")
-async def open_manhole(evt: CommandEvent) -> None:
-    if not evt.config["manhole.enabled"]:
-        await evt.reply("The manhole has been disabled in the config.")
-        return
-    elif len(evt.args) == 0:
-        await evt.reply("**Usage:** `$cmdprefix+sp open-manhole <uid...>`")
-        return
-
-    whitelist = set()
-    whitelist_whitelist = evt.config["manhole.whitelist"]
-    for arg in evt.args:
-        try:
-            uid = int(arg)
-        except ValueError:
-            await evt.reply(f"{arg} is not an integer.")
-            return
-        if whitelist_whitelist and uid not in whitelist_whitelist:
-            await evt.reply(f"{uid} is not in the list of allowed UIDs.")
-            return
-        whitelist.add(uid)
-
-    if evt.bridge.manhole:
-        added = [uid for uid in whitelist
-                 if uid not in evt.bridge.manhole.whitelist]
-        evt.bridge.manhole.whitelist |= set(added)
-        if len(added) == 0:
-            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
-                            " and all the given UIDs are already whitelisted.")
-        else:
-            added_str = (f"{', '.join(str(uid) for uid in added[:-1])} and {added[-1]}"
-                         if len(added) > 1 else added[0])
-            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
-                            f". Added {added_str} to the whitelist.")
-            evt.log.info(f"{evt.sender.mxid} added {added_str} to the manhole whitelist.")
-        return
-
-    from ..portal import Portal
-    from ..puppet import Puppet
-    from ..user import User
-    namespace = {
-        "bridge": evt.bridge,
-        "User": User,
-        "Portal": Portal,
-        "Puppet": Puppet,
-    }
-    banner = (f"Python {sys.version} on {sys.platform}\n"
-              f"mautrix-telegram {__version__} with mautrix-python {__mautrix_version__} "
-              f"and Telethon {__telethon_version__}\n\nManhole opened by {evt.sender.mxid}\n")
-    path = evt.config["manhole.path"]
-
-    wl_list = list(whitelist)
-    whitelist_str = (f"{', '.join(str(uid) for uid in wl_list[:-1])} and {wl_list[-1]}"
-                     if len(wl_list) > 1 else wl_list[0])
-    evt.log.info(f"{evt.sender.mxid} opened a manhole with {whitelist_str} whitelisted.")
-    server, close = await start_manhole(path=path, banner=banner, namespace=namespace,
-                                        loop=evt.loop, whitelist=whitelist)
-    evt.bridge.manhole = ManholeState(server=server, opened_by=evt.sender.mxid, close=close,
-                                      whitelist=whitelist)
-    plrl = "s" if len(whitelist) != 1 else ""
-    await evt.reply(f"Opened manhole at unix://{path} with UID{plrl} {whitelist_str} whitelisted")
-    await server.wait_closed()
-    evt.bridge.manhole = None
-    try:
-        os.unlink(path)
-    except FileNotFoundError:
-        pass
-    evt.log.info(f"{evt.sender.mxid}'s manhole was closed.")
-    try:
-        await evt.reply("Your manhole was closed.")
-    except (AttributeError, MatrixConnectionError) as e:
-        evt.log.warning(f"Failed to send manhole close notification: {e}")
-
-
-@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
-                 help_text="Close an open manhole.")
-async def close_manhole(evt: CommandEvent) -> None:
-    if not evt.bridge.manhole:
-        await evt.reply("There is no open manhole.")
-        return
-
-    opened_by = evt.bridge.manhole.opened_by
-    evt.bridge.manhole.close()
-    evt.bridge.manhole = None
-    if opened_by != evt.sender.mxid:
-        await evt.reply(f"Closed manhole opened by {opened_by}")

mautrix_telegram/commands/portal/admin.py

@@ -15,34 +15,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import asyncio
 
-from mautrix.errors import MatrixRequestError
 from mautrix.types import EventID
 
 from ... import portal as po, puppet as pu, user as u
 from .. import command_handler, CommandEvent, SECTION_ADMIN
 
 
-@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
-                 help_section=SECTION_ADMIN,
-                 help_args="<_level_> [_mxid_]",
-                 help_text="Set a temporary power level without affecting Telegram.")
-async def set_power_level(evt: CommandEvent) -> EventID:
-    try:
-        level = int(evt.args[0])
-    except (KeyError, IndexError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp set-pl <level> [mxid]`")
-    except ValueError:
-        return await evt.reply("The level must be an integer.")
-    levels = await evt.az.intent.get_power_levels(evt.room_id)
-    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
-    levels.users[mxid] = level
-    try:
-        return await evt.az.intent.set_power_levels(evt.room_id, levels)
-    except MatrixRequestError:
-        evt.log.exception("Failed to set power level.")
-        return await evt.reply("Failed to set power level.")
-
-
 @command_handler(needs_admin=True, needs_auth=False,
                  help_section=SECTION_ADMIN,
                  help_args="<`portal`|`puppet`|`user`>",
@@ -86,7 +64,7 @@ async def reload_user(evt: CommandEvent) -> EventID:
     user = u.User.get_by_mxid(mxid, create=False)
     if not user:
         return await evt.reply("User not found")
-    puppet = pu.Puppet.get_by_custom_mxid(mxid)
+    puppet = await pu.Puppet.get_by_custom_mxid(mxid)
     if puppet:
         puppet.sync_task.cancel()
     await user.stop()

mautrix_telegram/commands/portal/bridge.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Tuple, Coroutine
+from typing import Optional, Tuple, Awaitable
 import asyncio
 
 from telethon.tl.types import ChatForbidden, ChannelForbidden
@@ -23,7 +23,7 @@ from mautrix.types import EventID, RoomID
 from ...types import TelegramID
 from ... import portal as po
 from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
-from .util import user_has_power_level, get_initial_state
+from .util import user_has_power_level, get_initial_state, warn_missing_power
 
 
 @command_handler(needs_auth=False, needs_puppeting=False,
@@ -105,18 +105,17 @@ async def bridge(evt: CommandEvent) -> EventID:
 
 
 async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
-                                            ) -> Tuple[
-    bool, Optional[Coroutine[None, None, None]]]:
+                                            ) -> Tuple[bool, Optional[Awaitable[None]]]:
     if not portal.mxid:
         await evt.reply("The portal seems to have lost its Matrix room between you"
                         "calling `$cmdprefix+sp bridge` and this command.\n\n"
                         "Continuing without touching previous Matrix room...")
         return True, None
     elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_portal("Portal deleted (moving to another room)")
+        return True, portal.cleanup_portal("Portal deleted (moving to another room)", delete=False)
     elif evt.args[0] == "unbridge-and-continue":
         return True, portal.cleanup_portal("Room unbridged (portal moving to another room)",
-                                           puppets_only=True)
+                                           puppets_only=True, delete=False)
     else:
         await evt.reply(
             "The chat you were trying to bridge already has a Matrix portal room.\n\n"
@@ -137,6 +136,9 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
         return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
                                "This shouldn't happen unless you're messing with the command "
                                "handler code.")
+
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+
     if "mxid" in status:
         ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
         if not ok:
@@ -154,7 +156,13 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
                                "`$cmdprefix+sp cancel` to cancel.")
 
     evt.sender.command_status = None
-    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+    async with portal._room_create_lock:
+        await _locked_confirm_bridge(evt, portal=portal, room_id=bridge_to_mxid,
+                                     is_logged_in=is_logged_in)
+
+
+async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id: RoomID,
+                                 is_logged_in: bool) -> Optional[EventID]:
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)
@@ -172,14 +180,17 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
         else:
             return await evt.reply("The bot doesn't seem to be in that chat.")
 
-    direct = False
-
-    portal.mxid = bridge_to_mxid
-    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    portal.mxid = room_id
+    portal.by_mxid[portal.mxid] = portal
+    (portal.title, portal.about, levels,
+     portal.encrypted) = await get_initial_state(evt.az.intent, evt.room_id)
     portal.photo_id = ""
-    portal.save()
+    await portal.save()
+    await portal.update_bridge_info()
 
-    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
+    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
                           loop=evt.loop)
 
+    await warn_missing_power(levels, evt)
+
     return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -13,9 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable
+from typing import Awaitable, Any
 from io import StringIO
 
+from ruamel.yaml import YAMLError
+
 from mautrix.util.config import yaml
 from mautrix.types import EventID
 
@@ -48,7 +50,11 @@ async def config(evt: CommandEvent) -> None:
         return
 
     key = evt.args[1] if len(evt.args) > 1 else None
-    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    try:
+        value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    except YAMLError as e:
+        await evt.reply(f"Invalid value provided. Values must be valid YAML.\n{e}")
+        return
     if cmd == "set":
         await config_set(evt, portal, key, value)
     elif cmd == "unset":
@@ -57,7 +63,7 @@ async def config(evt: CommandEvent) -> None:
         await config_add_del(evt, portal, key, value, cmd)
     else:
         return
-    portal.save()
+    await portal.save()
 
 
 def config_help(evt: CommandEvent) -> Awaitable[EventID]:
@@ -74,14 +80,11 @@ def config_help(evt: CommandEvent) -> Awaitable[EventID]:
 
 
 def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[EventID]:
-    stream = StringIO()
-    yaml.dump(portal.local_config, stream)
-    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
+    return evt.reply(f"Room-specific config:\n{_str_value(portal.local_config).rstrip()}")
 
 
 def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
-    stream = StringIO()
-    yaml.dump({
+    value = _str_value({
         "bridge_notices": {
             "default": evt.config["bridge.bridge_notices.default"],
             "exceptions": evt.config["bridge.bridge_notices.exceptions"],
@@ -92,15 +95,25 @@ def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
         "emote_format": evt.config["bridge.emote_format"],
         "state_event_formats": evt.config["bridge.state_event_formats"],
         "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
-    }, stream)
-    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
+    })
+    return evt.reply(f"Bridge instance wide config:\n{value.rstrip()}")
 
 
-def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[EventID]:
+def _str_value(value: Any) -> str:
+    stream = StringIO()
+    yaml.dump(value, stream)
+    value_str = stream.getvalue()
+    if "\n" in value_str:
+        return f"\n```yaml\n{value_str}\n```\n"
+    else:
+        return f"`{value_str}`"
+
+
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: Any) -> Awaitable[EventID]:
     if not key or value is None:
         return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
     elif util.recursive_set(portal.local_config, key, value):
-        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
+        return evt.reply(f"Successfully set the value of `{key}` to {_str_value(value)}".rstrip())
     else:
         return evt.reply(f"Failed to set value of `{key}`. "
                          "Does the path contain non-map types?")
@@ -128,11 +141,11 @@ def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, c
         return evt.reply("`{key}` does not seem to be an array.")
     elif cmd == "add":
         if value in arr:
-            return evt.reply(f"The array at `{key}` already contains `{value}`.")
+            return evt.reply(f"The array at `{key}` already contains {_str_value(value)}".rstrip())
         arr.append(value)
-        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
+        return evt.reply(f"Successfully added {_str_value(value)} to the array at `{key}`")
     else:
         if value not in arr:
-            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
+            return evt.reply(f"The array at `{key}` does not contain {_str_value(value)}")
         arr.remove(value)
-        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")
+        return evt.reply(f"Successfully removed {_str_value(value)} from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -18,17 +18,17 @@ from mautrix.types import EventID
 from ... import portal as po
 from ...types import TelegramID
 from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
-from .util import user_has_power_level, get_initial_state
+from .util import user_has_power_level, get_initial_state, warn_missing_power
 
 
 @command_handler(help_section=SECTION_CREATING_PORTALS,
                  help_args="[_type_]",
                  help_text="Create a Telegram chat of the given type for the current Matrix room. "
                            "The type is either `group`, `supergroup` or `channel` (defaults to "
-                           "`group`).")
+                           "`supergroup`).")
 async def create(evt: CommandEvent) -> EventID:
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
+    type = evt.args[0] if len(evt.args) > 0 else "supergroup"
+    if type not in ("chat", "group", "supergroup", "channel"):
         return await evt.reply(
             "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
 
@@ -38,7 +38,7 @@ async def create(evt: CommandEvent) -> EventID:
     if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
         return await evt.reply("You do not have the permissions to bridge this room.")
 
-    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    title, about, levels, encrypted = await get_initial_state(evt.az.intent, evt.room_id)
     if not title:
         return await evt.reply("Please set a title before creating a Telegram chat.")
 
@@ -50,11 +50,20 @@ async def create(evt: CommandEvent) -> EventID:
         "group": "chat",
     }[type]
 
-    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
-                       mxid=evt.room_id, title=title, about=about)
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type, mxid=evt.room_id,
+                       title=title, about=about, encrypted=encrypted)
+    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender)
+    if len(errors) > 0:
+        error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
+        await evt.reply(f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
+                        "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
+                        "those users.")
+
+    await warn_missing_power(levels, evt)
+
     try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+        await portal.create_telegram_chat(evt.sender, invites=invites, supergroup=supergroup)
     except ValueError as e:
-        portal.delete()
+        await portal.delete()
         return await evt.reply(e.args[0])
     return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/misc.py

@@ -13,6 +13,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple
+from datetime import timedelta, datetime
+import re
+
 from telethon.tl.functions.channels import GetFullChannelRequest
 from telethon.tl.functions.messages import GetFullChatRequest
 from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
@@ -35,7 +39,7 @@ async def sync_state(evt: CommandEvent) -> EventID:
     elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
         return await evt.reply(f"You do not have the permissions to synchronize this room.")
 
-    await portal.sync_matrix_members()
+    await portal.main_intent.get_joined_members(portal.mxid)
     await evt.reply("Synchronization complete")
 
 
@@ -80,9 +84,81 @@ async def get_id(evt: CommandEvent) -> EventID:
     await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
 
 
+invite_link_usage = ("**Usage:** `$cmdprefix+sp invite-link [--uses=<amount>] [--expire=<delta>]`"
+                     "\n\n"
+                     "* `--uses`: the number of times the invite link can be used."
+                     "            Defaults to unlimited.\n"
+                     "* `--expire`: the duration after which the link will expire."
+                     "              A number suffixed with d(ay), h(our), m(inute) or s(econd)")
+
+
+def _parse_flag(args: List[str]) -> Tuple[str, str]:
+    arg = args.pop(0).lower()
+    if arg.startswith("--"):
+        value_start = arg.index("=")
+        if value_start:
+            flag = arg[2:value_start]
+            value = arg[value_start+1:]
+        else:
+            flag = arg[2:]
+            value = args.pop(0).lower()
+    elif arg.startswith("-"):
+        flag = arg[1]
+        if len(arg) > 3 and arg[2] == "=":
+            value = arg[3:]
+        else:
+            value = args.pop(0).lower()
+    else:
+        raise ValueError("invalid flag")
+    return flag, value
+
+
+delta_regex = re.compile("([0-9]+)(w(?:eek)?|d(?:ay)?|h(?:our)?|m(?:in(?:ute)?)?|s(?:ec(?:ond)?)?)")
+
+
+def _parse_delta(value: str) -> Optional[timedelta]:
+    match = delta_regex.fullmatch(value)
+    if not match:
+        return None
+    number = int(match.group(1))
+    unit = match.group(2)[0]
+    if unit == "w":
+        return timedelta(weeks=number)
+    elif unit == "d":
+        return timedelta(days=number)
+    elif unit == "h":
+        return timedelta(hours=number)
+    elif unit == "m":
+        return timedelta(minutes=number)
+    elif unit == "s":
+        return timedelta(seconds=number)
+    else:
+        return None
+
+
 @command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Get a Telegram invite link to the current chat.")
+                 help_text="Get a Telegram invite link to the current chat.",
+                 help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>]")
 async def invite_link(evt: CommandEvent) -> EventID:
+    # TODO once we switch to Python 3.9 minimum, use argparse with exit_on_error=False
+    uses = None
+    expire = None
+    while evt.args:
+        try:
+            flag, value = _parse_flag(evt.args)
+        except (ValueError, IndexError):
+            return await evt.reply(invite_link_usage)
+        if flag in ("uses", "u"):
+            try:
+                uses = int(value)
+            except ValueError:
+                await evt.reply("The number of uses must be an integer")
+        elif flag in ("expire", "e"):
+            expire_delta = _parse_delta(value)
+            if not expire_delta:
+                await evt.reply("Invalid format for expiry time delta")
+            expire = datetime.now() + expire_delta
+
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
@@ -91,7 +167,7 @@ async def invite_link(evt: CommandEvent) -> EventID:
         return await evt.reply("You can't invite users to private chats.")
 
     try:
-        link = await portal.get_invite_link(evt.sender)
+        link = await portal.get_invite_link(evt.sender, uses=uses, expire=expire)
         return await evt.reply(f"Invite link to {portal.title}: {link}")
     except ValueError as e:
         return await evt.reply(e.args[0])

mautrix_telegram/commands/portal/unbridge.py

@@ -22,9 +22,7 @@ from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
 from .util import user_has_power_level
 
 
-async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
-                                           action: Optional[str] = None
-                                           ) -> Optional[po.Portal]:
+async def _get_portal_and_check_permission(evt: CommandEvent) -> Optional[po.Portal]:
     room_id = RoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
 
     portal = po.Portal.get_by_mxid(room_id)
@@ -33,9 +31,14 @@ async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
         await evt.reply(f"{that_this} is not a portal room.")
         return None
 
-    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
-        action = action or f"{permission.replace('_', ' ')}s"
-        await evt.reply(f"You do not have the permissions to {action} that portal.")
+    if portal.peer_type == "user":
+        if portal.tg_receiver != evt.sender.tgid:
+            await evt.reply("You do not have the permissions to unbridge that portal.")
+            return None
+        return portal
+
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+        await evt.reply("You do not have the permissions to unbridge that portal.")
         return None
     return portal
 
@@ -64,7 +67,7 @@ def _get_portal_murder_function(action: str, room_id: str, function: Callable, c
                            "Only works for group chats; to delete a private chat portal, simply "
                            "leave the room.")
 async def delete_portal(evt: CommandEvent) -> Optional[EventID]:
-    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    portal = await _get_portal_and_check_permission(evt)
     if not portal:
         return None
 
@@ -85,7 +88,7 @@ async def delete_portal(evt: CommandEvent) -> Optional[EventID]:
                  help_section=SECTION_PORTAL_MANAGEMENT,
                  help_text="Remove puppets from the current portal room and forget the portal.")
 async def unbridge(evt: CommandEvent) -> Optional[EventID]:
-    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    portal = await _get_portal_and_check_permission(evt)
     if not portal:
         return None
 

mautrix_telegram/commands/portal/util.py

@@ -18,18 +18,21 @@ from typing import Tuple, Optional
 from mautrix.errors import MatrixRequestError
 from mautrix.appservice import IntentAPI
 from mautrix.types import RoomID, EventType, PowerLevelStateEventContent
+from .. import CommandEvent
 
 from ... import user as u
 
 OptStr = Optional[str]
 
 
-async def get_initial_state(intent: IntentAPI, room_id: RoomID
-                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent]]:
+async def get_initial_state(
+    intent: IntentAPI, room_id: RoomID
+) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent], bool]:
     state = await intent.get_state(room_id)
     title: OptStr = None
     about: OptStr = None
     levels: Optional[PowerLevelStateEventContent] = None
+    encrypted: bool = False
     for event in state:
         try:
             if event.type == EventType.ROOM_NAME:
@@ -40,10 +43,20 @@ async def get_initial_state(intent: IntentAPI, room_id: RoomID
                 levels = event.content
             elif event.type == EventType.ROOM_CANONICAL_ALIAS:
                 title = title or event.content.canonical_alias
+            elif event.type == EventType.ROOM_ENCRYPTION:
+                encrypted = True
         except KeyError:
             # Some state event probably has empty content
             pass
-    return title, about, levels
+    return title, about, levels, encrypted
+
+
+async def warn_missing_power(levels: PowerLevelStateEventContent, evt: CommandEvent) -> None:
+    if levels.get_user_level(evt.az.bot_mxid) < levels.redact:
+        await evt.reply("Warning: The bot does not have privileges to redact messages on Matrix. "
+                        "Message deletions from Telegram will not be bridged unless you give "
+                        "redaction permissions to "
+                        f"[{evt.az.bot_mxid}](https://matrix.to/#/{evt.az.bot_mxid})")
 
 
 async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.User,
@@ -55,6 +68,5 @@ async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.Use
         await intent.get_power_levels(room_id)
     except MatrixRequestError:
         return False
-    event_type = EventType.find(f"net.maunium.telegram.{event}")
-    event_type.t_class = EventType.Class.STATE
-    return intent.state_store.has_power_level(room_id, sender.mxid, event_type)
+    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
+    return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)

mautrix_telegram/commands/telegram/account.py

@@ -16,7 +16,7 @@
 from typing import Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError, AboutTooLongError)
 from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
                                            ResetAuthorizationRequest, UpdateProfileRequest)
@@ -53,6 +53,23 @@ async def username(evt: CommandEvent) -> EventID:
     else:
         await evt.reply(f"Username changed to {evt.sender.username}")
 
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new about_>",
+                 help_text="Change your Telegram about section.")
+async def about(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp about <new about>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own about section.")
+    new_about = " ".join(evt.args)
+    if new_about == "-":
+        new_about = ""
+    try:
+        await evt.sender.client(UpdateProfileRequest(about=new_about))
+    except AboutTooLongError:
+        return await evt.reply("The provided about section is too long")
+    return await evt.reply("About section updated")
 
 @command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
                  help_text="Change your Telegram displayname.")

mautrix_telegram/commands/telegram/auth.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Any, Dict, Optional
 import asyncio
+import io
 
 from telethon.errors import (  # isort: skip
     AccessTokenExpiredError, AccessTokenInvalidError, FirstNameInvalidError, FloodWaitError,
@@ -22,22 +23,36 @@ from telethon.errors import (  # isort: skip
     PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
     PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError,
     PhoneNumberInvalidError)
+from telethon.tl.types import User
 
-from mautrix.types import EventID
+from mautrix.types import (EventID, UserID, MediaMessageEventContent, ImageInfo, MessageType,
+                           TextMessageEventContent)
 
 from ... import user as u
+from ...types import TelegramID
 from ...commands import command_handler, CommandEvent, SECTION_AUTH
-from ...util import format_duration
+from ...util import format_duration as fmt_duration
+
+try:
+    import qrcode
+    import PIL as _
+    from telethon.tl.custom import QRLogin
+except ImportError:
+    qrcode = None
+    QRLogin = None
 
 
 @command_handler(needs_auth=False,
                  help_section=SECTION_AUTH,
                  help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> EventID:
-    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
-    if me:
-        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
-        return await evt.reply(f"You're logged in as {human_tg_id}")
+    if await evt.sender.is_logged_in():
+        me = await evt.sender.get_me()
+        if me:
+            human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
+            return await evt.reply(f"You're logged in as {human_tg_id}")
+        else:
+            return await evt.reply("You were logged in, but there appears to have been an error.")
     else:
         return await evt.reply("You're not logged in.")
 
@@ -58,7 +73,7 @@ async def ping_bot(evt: CommandEvent) -> EventID:
                  help_section=SECTION_AUTH,
                  help_args="<_phone_> <_full name_>",
                  help_text="Register to Telegram")
-async def register(evt: CommandEvent) -> Optional[EventID]:
+async def register(evt: CommandEvent) -> EventID:
     if await evt.sender.is_logged_in():
         return await evt.reply("You are already logged in.")
     elif len(evt.args) < 1:
@@ -75,7 +90,8 @@ async def register(evt: CommandEvent) -> Optional[EventID]:
         "action": "Register",
         "full_name": full_name,
     })
-    return None
+    return await evt.reply("By signing up for Telegram, you agree to "
+                           "the terms of service: https://telegram.org/tos")
 
 
 async def enter_code_register(evt: CommandEvent) -> EventID:
@@ -104,18 +120,76 @@ async def enter_code_register(evt: CommandEvent) -> EventID:
                                "Check console for more details.")
 
 
+@command_handler(needs_auth=False, management_only=True, help_section=SECTION_AUTH,
+                 help_text="Log in by scanning a QR code.")
+async def login_qr(evt: CommandEvent) -> EventID:
+    login_as = evt.sender
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        login_as = u.User.get_by_mxid(UserID(evt.args[0]))
+    if not qrcode or not QRLogin:
+        return await evt.reply("This bridge instance does not support logging in with a QR code.")
+    if await login_as.is_logged_in():
+        return await evt.reply(f"You are already logged in as {login_as.human_tg_id}.")
+
+    await login_as.ensure_started(even_if_no_session=True)
+    qr_login = QRLogin(login_as.client, ignored_ids=[])
+    qr_event_id: Optional[EventID] = None
+
+    async def upload_qr() -> None:
+        nonlocal qr_event_id
+        buffer = io.BytesIO()
+        image = qrcode.make(qr_login.url)
+        size = image.pixel_size
+        image.save(buffer, "PNG")
+        qr = buffer.getvalue()
+        mxc = await evt.az.intent.upload_media(qr, "image/png", "login-qr.png", len(qr))
+        content = MediaMessageEventContent(body=qr_login.url, url=mxc, msgtype=MessageType.IMAGE,
+                                           info=ImageInfo(mimetype="image/png", size=len(qr),
+                                                          width=size, height=size))
+        if qr_event_id:
+            content.set_edit(qr_event_id)
+            await evt.az.intent.send_message(evt.room_id, content)
+        else:
+            content.set_reply(evt.event_id)
+            qr_event_id = await evt.az.intent.send_message(evt.room_id, content)
+
+    retries = 4
+    while retries > 0:
+        await qr_login.recreate()
+        await upload_qr()
+        try:
+            user = await qr_login.wait()
+            break
+        except asyncio.TimeoutError:
+            retries -= 1
+        except SessionPasswordNeededError:
+            evt.sender.command_status = {
+                "next": enter_password,
+                "login_as": login_as if login_as != evt.sender else None,
+                "action": "Login (password entry)",
+            }
+            return await evt.reply("Your account has two-factor authentication. "
+                                   "Please send your password here.")
+    else:
+        timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
+        timeout.set_edit(qr_event_id)
+        return await evt.az.intent.send_message(evt.room_id, timeout)
+
+    return await _finish_sign_in(evt, user, login_as=login_as)
+
+
 @command_handler(needs_auth=False, management_only=True,
                  help_section=SECTION_AUTH,
                  help_text="Get instructions on how to log in.")
 async def login(evt: CommandEvent) -> EventID:
     override_sender = False
     if len(evt.args) > 0 and evt.sender.is_admin:
-        evt.sender = await u.User.get_by_mxid(evt.args[0]).ensure_started()
+        evt.sender = await u.User.get_by_mxid(UserID(evt.args[0])).ensure_started()
         override_sender = True
     if await evt.sender.is_logged_in():
         return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 
-    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    allow_matrix_login = evt.config["bridge.allow_matrix_login"]
     if allow_matrix_login and not override_sender:
         evt.sender.command_status = {
             "next": enter_phone_or_token,
@@ -152,21 +226,18 @@ async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[
         ok = True
         return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
     except PhoneNumberAppSignupForbiddenError:
-        return await evt.reply(
-            "Your phone number does not allow 3rd party apps to sign in.")
+        return await evt.reply("Your phone number does not allow 3rd party apps to sign in.")
     except PhoneNumberFloodError:
-        return await evt.reply(
-            "Your phone number has been temporarily blocked for flooding. "
-            "The ban is usually applied for around a day.")
+        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
+                               "The ban is usually applied for around a day.")
     except FloodWaitError as e:
-        return await evt.reply(
-            "Your phone number has been temporarily blocked for flooding. "
-            f"Please wait for {format_duration(e.seconds)} before trying again.")
+        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
+                               f"Please wait for {fmt_duration(e.seconds)} before trying again.")
     except PhoneNumberBannedError:
-        return await  evt.reply("Your phone number has been banned from Telegram.")
+        return await evt.reply("Your phone number has been banned from Telegram.")
     except PhoneNumberUnoccupiedError:
-        return await  evt.reply("That phone number has not been registered. "
-                                "Please register with `$cmdprefix+sp register <phone>`.")
+        return await evt.reply("That phone number has not been registered. "
+                               "Please register with `$cmdprefix+sp register <phone>`.")
     except PhoneNumberInvalidError:
         return await evt.reply("That phone number is not valid.")
     except Exception:
@@ -225,7 +296,8 @@ async def enter_password(evt: CommandEvent) -> Optional[EventID]:
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await _sign_in(evt, password=" ".join(evt.args))
+        await _sign_in(evt, login_as=evt.sender.command_status.get("login_as", None),
+                       password=" ".join(evt.args))
     except AccessTokenInvalidError:
         return await evt.reply("That bot token is not valid.")
     except AccessTokenExpiredError:
@@ -237,20 +309,12 @@ async def enter_password(evt: CommandEvent) -> Optional[EventID]:
     return None
 
 
-async def _sign_in(evt: CommandEvent, **sign_in_info) -> EventID:
+async def _sign_in(evt: CommandEvent, login_as: 'u.User' = None, **sign_in_info) -> EventID:
+    login_as = login_as or evt.sender
     try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        user = await evt.sender.client.sign_in(**sign_in_info)
-        existing_user = u.User.get_by_tgid(user.id)
-        if existing_user and existing_user != evt.sender:
-            await existing_user.log_out()
-            await evt.reply(f"[{existing_user.displayname}]"
-                            f"(https://matrix.to/#/{existing_user.mxid})"
-                            " was logged out from the account.")
-        asyncio.ensure_future(evt.sender.post_login(user, first_login=True), loop=evt.loop)
-        evt.sender.command_status = None
-        name = f"@{user.username}" if user.username else f"+{user.phone}"
-        return await evt.reply(f"Successfully logged in as {name}")
+        await login_as.ensure_started(even_if_no_session=True)
+        user = await login_as.client.sign_in(**sign_in_info)
+        await _finish_sign_in(evt, user)
     except PhoneCodeExpiredError:
         return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
     except PhoneCodeInvalidError:
@@ -266,10 +330,31 @@ async def _sign_in(evt: CommandEvent, **sign_in_info) -> EventID:
                                "Please send your password here.")
 
 
-@command_handler(needs_auth=True,
+async def _finish_sign_in(evt: CommandEvent, user: User, login_as: 'u.User' = None) -> EventID:
+    login_as = login_as or evt.sender
+    existing_user = u.User.get_by_tgid(TelegramID(user.id))
+    if existing_user and existing_user != login_as:
+        await existing_user.log_out()
+        await evt.reply(f"[{existing_user.displayname}]"
+                        f"(https://matrix.to/#/{existing_user.mxid})"
+                        " was logged out from the account.")
+    asyncio.ensure_future(login_as.post_login(user, first_login=True), loop=evt.loop)
+    evt.sender.command_status = None
+    name = f"@{user.username}" if user.username else f"+{user.phone}"
+    if login_as != evt.sender:
+        msg = (f"Successfully logged in [{login_as.mxid}](https://matrix.to/#/{login_as.mxid})"
+               f" as {name}")
+    else:
+        msg = f"Successfully logged in as {name}"
+    return await evt.reply(msg)
+
+
+@command_handler(needs_auth=False,
                  help_section=SECTION_AUTH,
                  help_text="Log out from Telegram.")
 async def logout(evt: CommandEvent) -> EventID:
+    if not evt.sender.tgid:
+        return await evt.reply("You're not logged in")
     if await evt.sender.log_out():
         return await evt.reply("Logged out successfully.")
     return await evt.reply("Failed to log out.")

mautrix_telegram/commands/telegram/misc.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,16 +14,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import List, Optional, Tuple, cast
-import logging
 import codecs
 import base64
 import re
 
+from aiohttp import ClientSession, InvalidURL
+
 from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
-                             UserAlreadyParticipantError, ChatIdInvalidError)
+                             UserAlreadyParticipantError, ChatIdInvalidError,
+                             TakeoutInitDelayError, EmoticonInvalidError)
 from telethon.tl.patched import Message
 from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
-                               TypeInputPeer)
+                               TypeInputPeer, InputMediaDice)
 from telethon.tl.types.messages import BotCallbackAnswer
 from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
                                             GetBotCallbackAnswerRequest, SendVoteRequest)
@@ -35,7 +37,8 @@ from ... import puppet as pu, portal as po
 from ...abstract_user import AbstractUser
 from ...db import Message as DBMessage
 from ...types import TelegramID
-from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
+from ...commands import (command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS,
+                         SECTION_PORTAL_MANAGEMENT)
 
 
 @command_handler(needs_auth=False,
@@ -102,7 +105,8 @@ async def pm(evt: CommandEvent) -> EventID:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
 
     try:
-        user = await evt.sender.client.get_entity(evt.args[0])
+        id = "".join(evt.args).translate({ord(c): None for c in "+()- "})
+        user = await evt.sender.client.get_entity(id)
     except ValueError:
         return await evt.reply("Invalid user identifier or user not found.")
 
@@ -112,25 +116,25 @@ async def pm(evt: CommandEvent) -> EventID:
         return await evt.reply("That doesn't seem to be a user.")
     portal = po.Portal.get_by_entity(user, evt.sender.tgid)
     await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply("Created private chat room with "
-                           f"{pu.Puppet.get_displayname(user, False)}")
+    displayname, _ = pu.Puppet.get_displayname(user, False)
+    return await evt.reply(f"Created private chat room with {displayname}")
 
 
-async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
-    if arg.startswith("joinchat/"):
-        invite_hash = arg[len("joinchat/"):]
+async def _join(evt: CommandEvent, identifier: str, link_type: str
+                ) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
+    if link_type == "joinchat":
         try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
+            await evt.sender.client(CheckChatInviteRequest(identifier))
         except InviteHashInvalidError:
             return None, await evt.reply("Invalid invite link.")
         except InviteHashExpiredError:
             return None, await evt.reply("Invite link expired.")
         try:
-            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
+            return (await evt.sender.client(ImportChatInviteRequest(identifier))), None
         except UserAlreadyParticipantError:
             return None, await evt.reply("You are already in that chat.")
     else:
-        channel = await evt.sender.client.get_entity(arg)
+        channel = await evt.sender.client.get_entity(identifier)
         if not channel:
             return None, await evt.reply("Channel/supergroup not found.")
         return await evt.sender.client(JoinChannelRequest(channel)), None
@@ -143,12 +147,26 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
 
-    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
-    arg = regex.match(evt.args[0])
+    url = evt.args[0]
+    if evt.config["bridge.invite_link_resolve"]:
+        try:
+            async with ClientSession() as sess, sess.get(url) as resp:
+                url = str(resp.url)
+        except InvalidURL:
+            return await evt.reply("That doesn't look like a Telegram invite link.")
+
+    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)"
+                       r"(?:/(?P<type>joinchat|s))?/(?P<id>[^/]+)/?", flags=re.IGNORECASE)
+    arg = regex.match(url)
     if not arg:
         return await evt.reply("That doesn't look like a Telegram invite link.")
 
-    updates, _ = await _join(evt, arg.group(1))
+    data = arg.groupdict()
+    identifier = data["id"]
+    link_type = data["type"]
+    if link_type:
+        link_type = link_type.lower()
+    updates, _ = await _join(evt, identifier, link_type)
     if not updates:
         return None
 
@@ -162,7 +180,8 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
             try:
                 await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
             except ChatIdInvalidError as e:
-                logging.getLogger("mau.commands").info(updates.stringify())
+                evt.log.trace("ChatIdInvalidError while creating portal from !tg join command: %s",
+                              updates.stringify())
                 raise e
             return await evt.reply(f"Created room for {portal.title}")
     return None
@@ -180,8 +199,10 @@ async def sync(evt: CommandEvent) -> EventID:
         sync_only = None
 
     if not sync_only or sync_only == "chats":
-        await evt.sender.sync_dialogs(synchronous_create=True)
+        await evt.reply("Synchronizing chats...")
+        await evt.sender.sync_dialogs()
     if not sync_only or sync_only == "contacts":
+        await evt.reply("Synchronizing contacts...")
         await evt.sender.sync_contacts()
     if not sync_only or sync_only == "me":
         await evt.sender.update_info()
@@ -303,3 +324,52 @@ async def vote(evt: CommandEvent) -> EventID:
         return await evt.reply("You passed too many options.")
     # TODO use response
     return await evt.mark_read()
+
+
+@command_handler(help_section=SECTION_MISC, help_args="<_emoji_>",
+                 help_text="Roll a dice (\U0001F3B2), kick a football (\u26BD\uFE0F) or throw a "
+                           "dart (\U0001F3AF) or basketball (\U0001F3C0) on the Telegram servers.")
+async def random(evt: CommandEvent) -> EventID:
+    if not evt.is_portal:
+        return await evt.reply("You can only randomize values in portal rooms")
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    arg = evt.args[0] if len(evt.args) > 0 else "dice"
+    emoticon = {
+        "dart": "\U0001F3AF",
+        "dice": "\U0001F3B2",
+        "ball": "\U0001F3C0",
+        "basketball": "\U0001F3C0",
+        "football": "\u26BD",
+        "soccer": "\u26BD",
+    }.get(arg, arg)
+    try:
+        await evt.sender.client.send_media(await portal.get_input_entity(evt.sender),
+                                           InputMediaDice(emoticon))
+    except EmoticonInvalidError:
+        return await evt.reply("Invalid emoji for randomization")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT, help_args="[_limit_]",
+                 help_text="Backfill messages from Telegram history.")
+async def backfill(evt: CommandEvent) -> None:
+    if not evt.is_portal:
+        await evt.reply("You can only use backfill in portal rooms")
+        return
+    try:
+        limit = int(evt.args[0])
+    except (ValueError, IndexError):
+        limit = -1
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
+        await evt.reply("Backfilling normal groups is disabled in the bridge config")
+        return
+    try:
+        await portal.backfill(evt.sender, limit=limit)
+    except TakeoutInitDelayError:
+        msg = ("Please accept the data export request from a mobile device, "
+               "then re-run the backfill command.")
+        if portal.peer_type == "user":
+            from mautrix.appservice import IntentAPI
+            await portal.main_intent.send_notice(evt.room_id, msg)
+        else:
+            await evt.reply(msg)

mautrix_telegram/config.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,14 +13,14 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Any, Dict, List, NamedTuple
+from typing import Any, List, NamedTuple
 from ruamel.yaml.comments import CommentedMap
 import os
 
 from mautrix.types import UserID
 from mautrix.client import Client
-from mautrix.bridge.config import (BaseBridgeConfig, ConfigUpdateHelper, ForbiddenKey,
-                                   ForbiddenDefault)
+from mautrix.bridge.config import BaseBridgeConfig
+from mautrix.util.config import ForbiddenKey, ForbiddenDefault, ConfigUpdateHelper
 
 Permissions = NamedTuple("Permissions", relaybot=bool, user=bool, puppeting=bool,
                          matrix_puppeting=bool, admin=bool, level=str)
@@ -45,23 +45,20 @@ class Config(BaseBridgeConfig):
         ]
 
     def do_update(self, helper: ConfigUpdateHelper) -> None:
+        super().do_update(helper)
         copy, copy_dict, base = helper
 
-        copy("homeserver.address")
-        copy("homeserver.domain")
-        copy("homeserver.verify_ssl")
+        copy("homeserver.asmux")
 
         if "appservice.protocol" in self and "appservice.address" not in self:
             protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
                                         self["appservice.port"])
             base["appservice.address"] = f"{protocol}://{hostname}:{port}"
-        else:
-            copy("appservice.address")
-        copy("appservice.hostname")
-        copy("appservice.port")
-        copy("appservice.max_body_size")
-
-        copy("appservice.database")
+        if "appservice.debug" in self and "logging" not in self:
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
+            base["logging.root.level"] = level
+            base["logging.loggers.mau.level"] = level
+            base["logging.loggers.telethon.level"] = level
 
         copy("appservice.public.enabled")
         copy("appservice.public.prefix")
@@ -73,35 +70,29 @@ class Config(BaseBridgeConfig):
         if base["appservice.provisioning.shared_secret"] == "generate":
             base["appservice.provisioning.shared_secret"] = self._new_token()
 
-        copy("appservice.id")
-        copy("appservice.bot_username")
-        copy("appservice.bot_displayname")
-        copy("appservice.bot_avatar")
-
         copy("appservice.community_id")
 
-        copy("appservice.as_token")
-        copy("appservice.hs_token")
-
         copy("metrics.enabled")
         copy("metrics.listen_port")
 
-        copy("manhole.enabled")
-        copy("manhole.path")
-        copy("manhole.whitelist")
-
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
 
         copy("bridge.displayname_preference")
         copy("bridge.displayname_max_length")
+        copy("bridge.allow_avatar_remove")
 
         copy("bridge.max_initial_member_sync")
         copy("bridge.sync_channel_members")
         copy("bridge.skip_deleted_members")
         copy("bridge.startup_sync")
-        copy("bridge.sync_dialog_limit")
+        if "bridge.sync_dialog_limit" in self:
+            base["bridge.sync_create_limit"] = self["bridge.sync_dialog_limit"]
+            base["bridge.sync_update_limit"] = self["bridge.sync_dialog_limit"]
+        else:
+            copy("bridge.sync_update_limit")
+            copy("bridge.sync_create_limit")
         copy("bridge.sync_direct_chats")
         copy("bridge.max_telegram_delete")
         copy("bridge.sync_matrix_state")
@@ -109,8 +100,17 @@ class Config(BaseBridgeConfig):
         copy("bridge.plaintext_highlights")
         copy("bridge.public_portals")
         copy("bridge.sync_with_custom_puppets")
-        copy("bridge.login_shared_secret")
+        copy("bridge.sync_direct_chat_list")
+        copy("bridge.double_puppet_server_map")
+        copy("bridge.double_puppet_allow_discovery")
+        if "bridge.login_shared_secret" in self:
+            base["bridge.login_shared_secret_map"] = {
+                base["homeserver.domain"]: self["bridge.login_shared_secret"]
+            }
+        else:
+            copy("bridge.login_shared_secret_map")
         copy("bridge.telegram_link_preview")
+        copy("bridge.invite_link_resolve")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
         copy("bridge.max_document_size")
@@ -118,6 +118,26 @@ class Config(BaseBridgeConfig):
         copy("bridge.federate_rooms")
         copy("bridge.animated_sticker.target")
         copy("bridge.animated_sticker.args")
+        copy("bridge.encryption.allow")
+        copy("bridge.encryption.default")
+        copy("bridge.encryption.database")
+        copy("bridge.encryption.key_sharing.allow")
+        copy("bridge.encryption.key_sharing.require_cross_signing")
+        copy("bridge.encryption.key_sharing.require_verification")
+        copy("bridge.private_chat_portal_meta")
+        copy("bridge.delivery_receipts")
+        copy("bridge.delivery_error_reports")
+        copy("bridge.resend_bridge_info")
+        copy("bridge.mute_bridging")
+        copy("bridge.pinned_tag")
+        copy("bridge.archive_tag")
+        copy("bridge.tag_only_on_create")
+        copy("bridge.backfill.invite_own_puppet")
+        copy("bridge.backfill.takeout_limit")
+        copy("bridge.backfill.initial_limit")
+        copy("bridge.backfill.missed_limit")
+        copy("bridge.backfill.disable_notifications")
+        copy("bridge.backfill.normal_groups")
 
         copy("bridge.initial_power_level_overrides.group")
         copy("bridge.initial_power_level_overrides.user")
@@ -202,14 +222,6 @@ class Config(BaseBridgeConfig):
         copy("telegram.proxy.username")
         copy("telegram.proxy.password")
 
-        if "appservice.debug" in self and "logging" not in self:
-            level = "DEBUG" if self["appservice.debug"] else "INFO"
-            base["logging.root.level"] = level
-            base["logging.loggers.mau.level"] = level
-            base["logging.loggers.telethon.level"] = level
-        else:
-            copy("logging")
-
     def _get_permissions(self, key: str) -> Permissions:
         level = self["bridge.permissions"].get(key, "")
         admin = level == "admin"
@@ -229,24 +241,3 @@ class Config(BaseBridgeConfig):
             return self._get_permissions(homeserver)
 
         return self._get_permissions("*")
-
-    @property
-    def namespaces(self) -> Dict[str, List[Dict[str, Any]]]:
-        homeserver = self["homeserver.domain"]
-
-        username_format = self["bridge.username_template"].format(userid=".+")
-        alias_format = self["bridge.alias_template"].format(groupname=".+")
-        group_id = ({"group_id": self["appservice.community_id"]}
-                    if self["appservice.community_id"] else {})
-
-        return {
-            "users": [{
-                "exclusive": True,
-                "regex": f"@{username_format}:{homeserver}",
-                **group_id,
-            }],
-            "aliases": [{
-                "exclusive": True,
-                "regex": f"#{alias_format}:{homeserver}",
-            }]
-        }

mautrix_telegram/db/__init__.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from sqlalchemy.engine.base import Engine
 
-from mautrix.bridge.db import UserProfile, RoomState
+from mautrix.client.state_store.sqlalchemy import UserProfile, RoomState
 
 from .bot_chat import BotChat
 from .message import Message
@@ -28,7 +28,4 @@ from .user import User, UserPortal, Contact
 def init(db_engine: Engine) -> None:
     for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
                   RoomState, BotChat):
-        table.db = db_engine
-        table.t = table.__table__
-        table.c = table.t.c
-        table.column_names = table.c.keys()
+        table.bind(db_engine)

mautrix_telegram/db/bot_chat.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Iterable
 
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, BigInteger, String
 
 from mautrix.util.db import Base
 
@@ -25,7 +25,7 @@ from ..types import TelegramID
 # Fucking Telegram not telling bots what chats they are in 3:<
 class BotChat(Base):
     __tablename__ = "bot_chat"
-    id: TelegramID = Column(Integer, primary_key=True)
+    id: TelegramID = Column(BigInteger, primary_key=True)
     type: str = Column(String, nullable=False)
 
     @classmethod

mautrix_telegram/db/message.py

@@ -13,9 +13,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Iterator
+from typing import Optional, Iterator, List
 
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+from sqlalchemy import (Column, UniqueConstraint, BigInteger, Integer, String, Boolean, and_, func,
+                        desc, select, false)
 
 from mautrix.types import RoomID, EventID
 from mautrix.util.db import Base
@@ -28,9 +29,10 @@ class Message(Base):
 
     mxid: EventID = Column(String)
     mx_room: RoomID = Column(String)
-    tgid: TelegramID = Column(Integer, primary_key=True)
-    tg_space: TelegramID = Column(Integer, primary_key=True)
+    tgid: TelegramID = Column(BigInteger, primary_key=True)
+    tg_space: TelegramID = Column(BigInteger, primary_key=True)
     edit_index: int = Column(Integer, primary_key=True)
+    redacted: bool = Column(Boolean, server_default=false())
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"),)
 
@@ -51,6 +53,12 @@ class Message(Base):
             return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
                                            cls.c.edit_index == edit_index)
 
+    @classmethod
+    def get_first_by_tgids(cls, tgids: List[TelegramID], tg_space: TelegramID
+                           ) -> Iterator['Message']:
+        return cls._select_all(cls.c.tgid.in_(tgids), cls.c.tg_space == tg_space,
+                               cls.c.edit_index == 0)
+
     @classmethod
     def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
         rows = cls.db.execute(select([func.count(cls.c.tg_space)])
@@ -61,12 +69,28 @@ class Message(Base):
         except StopIteration:
             return 0
 
+    @classmethod
+    def find_last(cls, mx_room: RoomID, tg_space: TelegramID) -> Optional['Message']:
+        return cls._one_or_none(cls.db.execute(
+            cls._make_simple_select(cls.c.mx_room == mx_room, cls.c.tg_space == tg_space)
+                .order_by(desc(cls.c.tgid)).limit(1)))
+
+    @classmethod
+    def delete_all(cls, mx_room: RoomID) -> None:
+        cls.db.execute(cls.t.delete().where(cls.c.mx_room == mx_room))
+
     @classmethod
     def get_by_mxid(cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
                     ) -> Optional['Message']:
         return cls._select_one_or_none(cls.c.mxid == mxid, cls.c.mx_room == mx_room,
                                        cls.c.tg_space == tg_space)
 
+    @classmethod
+    def get_by_mxids(cls, mxids: List[EventID], mx_room: RoomID, tg_space: TelegramID
+                     ) -> Iterator['Message']:
+        return cls._select_all(cls.c.mxid.in_(mxids), cls.c.mx_room == mx_room,
+                               cls.c.tg_space == tg_space)
+
     @classmethod
     def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
                        **values) -> None:

mautrix_telegram/db/portal.py

@@ -13,11 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Boolean, Text, func
+from sqlalchemy import Column, BigInteger, String, Boolean, Text, func, sql
 
-from mautrix.types import RoomID
+from mautrix.types import RoomID, ContentURI
 from mautrix.util.db import Base
 
 from ..types import TelegramID
@@ -27,13 +27,15 @@ class Portal(Base):
     __tablename__ = "portal"
 
     # Telegram chat information
-    tgid: TelegramID = Column(Integer, primary_key=True)
-    tg_receiver: TelegramID = Column(Integer, primary_key=True)
+    tgid: TelegramID = Column(BigInteger, primary_key=True)
+    tg_receiver: TelegramID = Column(BigInteger, primary_key=True)
     peer_type: str = Column(String, nullable=False)
     megagroup: bool = Column(Boolean)
 
     # Matrix portal information
-    mxid: RoomID = Column(String, unique=True, nullable=True)
+    mxid: Optional[RoomID] = Column(String, unique=True, nullable=True)
+    avatar_url: Optional[ContentURI] = Column(String, nullable=True)
+    encrypted: bool = Column(Boolean, nullable=False, server_default=sql.expression.false())
 
     config: str = Column(Text, nullable=True)
 
@@ -47,6 +49,10 @@ class Portal(Base):
     def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
         return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver)
 
+    @classmethod
+    def find_private_chats(cls, tg_receiver: TelegramID) -> Iterable['Portal']:
+        yield from cls._select_all(cls.c.tg_receiver == tg_receiver, cls.c.peer_type == "user")
+
     @classmethod
     def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
         return cls._select_one_or_none(cls.c.mxid == mxid)
@@ -54,3 +60,7 @@ class Portal(Base):
     @classmethod
     def get_by_username(cls, username: str) -> Optional['Portal']:
         return cls._select_one_or_none(func.lower(cls.c.username) == username)
+
+    @classmethod
+    def all(cls) -> Iterable['Portal']:
+        yield from cls._select_all()

mautrix_telegram/db/puppet.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy import Column, Integer, BigInteger, String, Text, Boolean
 from sqlalchemy.sql import expression, func
 
 from mautrix.types import UserID, SyncToken
@@ -27,12 +27,15 @@ from ..types import TelegramID
 class Puppet(Base):
     __tablename__ = "puppet"
 
-    id: TelegramID = Column(Integer, primary_key=True)
+    id: TelegramID = Column(BigInteger, primary_key=True)
     custom_mxid: UserID = Column(String, nullable=True)
     access_token: str = Column(String, nullable=True)
     next_batch: SyncToken = Column(String, nullable=True)
+    base_url: str = Column(Text, nullable=True)
     displayname: str = Column(String, nullable=True)
-    displayname_source: TelegramID = Column(Integer, nullable=True)
+    displayname_source: TelegramID = Column(BigInteger, nullable=True)
+    displayname_contact: bool = Column(Boolean, nullable=False, server_default=expression.true())
+    displayname_quality: int = Column(Integer, nullable=False, server_default="0")
     username: str = Column(String, nullable=True)
     photo_id: str = Column(String, nullable=True)
     is_bot: bool = Column(Boolean, nullable=True)

mautrix_telegram/db/telegram_file.py

@@ -13,14 +13,38 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from typing import Optional, cast, Dict, Any, TYPE_CHECKING
 
-from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
-from sqlalchemy.engine.result import RowProxy
+from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
+                        TypeDecorator)
 
-from mautrix.types import ContentURI
+from mautrix.types import ContentURI, EncryptedFile
 from mautrix.util.db import Base
 
+if TYPE_CHECKING:
+    from sqlalchemy.engine.result import RowProxy
+
+
+class DBEncryptedFile(TypeDecorator):
+    impl = Text
+
+    @property
+    def python_type(self):
+        return EncryptedFile
+
+    def process_bind_param(self, value: EncryptedFile, dialect) -> Optional[str]:
+        if value is not None:
+            return value.json()
+        return None
+
+    def process_result_value(self, value: str, dialect) -> Optional[EncryptedFile]:
+        if value is not None:
+            return EncryptedFile.parse_json(value)
+        return None
+
+    def process_literal_param(self, value, dialect):
+        return value
+
 
 class TelegramFile(Base):
     __tablename__ = "telegram_file"
@@ -33,12 +57,13 @@ class TelegramFile(Base):
     size: Optional[int] = Column(Integer, nullable=True)
     width: Optional[int] = Column(Integer, nullable=True)
     height: Optional[int] = Column(Integer, nullable=True)
+    decryption_info: Optional[Dict[str, Any]] = Column(DBEncryptedFile, nullable=True)
     thumbnail_id: str = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
     thumbnail: Optional['TelegramFile'] = None
 
     @classmethod
-    def scan(cls, row: RowProxy) -> 'TelegramFile':
-        telegram_file: TelegramFile = super().scan(row)
+    def scan(cls, row: 'RowProxy') -> 'TelegramFile':
+        telegram_file = cast(TelegramFile, super().scan(row))
         if isinstance(telegram_file.thumbnail, str):
             telegram_file.thumbnail = cls.get(telegram_file.thumbnail)
         return telegram_file
@@ -52,5 +77,5 @@ class TelegramFile(Base):
             conn.execute(self.t.insert().values(
                 id=self.id, mxc=self.mxc, mime_type=self.mime_type,
                 was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
-                width=self.width, height=self.height,
+                width=self.width, height=self.height, decryption_info=self.decryption_info,
                 thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/user.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable, Tuple
 
-from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String, func
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, BigInteger, Integer, String, func
 
 from mautrix.types import UserID
 from mautrix.util.db import Base
@@ -27,7 +27,7 @@ class User(Base):
     __tablename__ = "user"
 
     mxid: UserID = Column(String, primary_key=True)
-    tgid: Optional[TelegramID] = Column(Integer, nullable=True, unique=True)
+    tgid: Optional[TelegramID] = Column(BigInteger, nullable=True, unique=True)
     tg_username: str = Column(String, nullable=True)
     tg_phone: str = Column(String, nullable=True)
     saved_contacts: int = Column(Integer, default=0, nullable=False)
@@ -91,10 +91,10 @@ class User(Base):
 class UserPortal(Base):
     __tablename__ = "user_portal"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE",
+    user: TelegramID = Column(BigInteger, ForeignKey("user.tgid", onupdate="CASCADE",
                                                   ondelete="CASCADE"), primary_key=True)
-    portal: TelegramID = Column(Integer, primary_key=True)
-    portal_receiver: TelegramID = Column(Integer, primary_key=True)
+    portal: TelegramID = Column(BigInteger, primary_key=True)
+    portal_receiver: TelegramID = Column(BigInteger, primary_key=True)
 
     __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
                                            ("portal.tgid", "portal.tg_receiver"),
@@ -104,5 +104,5 @@ class UserPortal(Base):
 class Contact(Base):
     __tablename__ = "contact"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
-    contact: TelegramID = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
+    user: TelegramID = Column(BigInteger, ForeignKey("user.tgid"), primary_key=True)
+    contact: TelegramID = Column(BigInteger, ForeignKey("puppet.id"), primary_key=True)

mautrix_telegram/example-config.yaml

@@ -7,12 +7,22 @@ homeserver:
     # Whether or not to verify the SSL certificate of the homeserver.
     # Only applies if address starts with https://
     verify_ssl: true
+    asmux: false
+    # Number of retries for all HTTP requests if the homeserver isn't reachable.
+    http_retry_count: 4
+    # The URL to push real-time bridge status to.
+    # If set, the bridge will make POST requests to this URL whenever a user's Telegram connection state changes.
+    # The bridge will use the appservice as_token to authorize requests.
+    status_endpoint: null
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
 appservice:
     # The address that the homeserver can use to connect to this appservice.
     address: http://localhost:29317
+    # When using https:// the TLS certificate and key files for the address.
+    tls_cert: false
+    tls_key: false
 
     # The hostname and port where this appservice should listen.
     hostname: 0.0.0.0
@@ -27,6 +37,8 @@ appservice:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
     database: sqlite:///mautrix-telegram.db
+    # Optional extra arguments for SQLAlchemy's create_engine
+    database_opts: {}
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in
@@ -41,7 +53,7 @@ appservice:
         external: https://example.com/public
 
     # Provisioning API part of the web server for automated portal creation and fetching information.
-    # Used by things like Dimension (https://dimension.t2bot.io/).
+    # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager).
     provisioning:
         # Whether or not the provisioning API should be enabled.
         enabled: true
@@ -62,8 +74,15 @@ appservice:
 
     # Community ID for bridged users (changes registration file) and rooms.
     # Must be created manually.
+    #
+    # Example: "+telegram:example.com". Set to false to disable.
     community_id: false
 
+    # Whether or not to receive ephemeral events via appservice transactions.
+    # Requires MSC2409 support (i.e. Synapse 1.22+).
+    # You should disable bridge -> sync_with_custom_puppets when this is enabled.
+    ephemeral_events: false
+
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
@@ -116,12 +135,16 @@ bridge:
     - phone number
     # Maximum length of displayname
     displayname_max_length: 100
+    # Remove avatars from Telegram ghost users when removed on Telegram. This is disabled by default
+    # as there's no way to determine whether an avatar is removed or just hidden from some users. If
+    # you're on a single-user instance, this should be safe to enable.
+    allow_avatar_remove: false
 
     # Maximum number of members to sync per portal when starting up. Other members will be
     # synced when they send messages. The maximum is 10000, after which the Telegram server
     # will not send any more members.
-    # Defaults to no local limit (-> limited to 10000 by server)
-    max_initial_member_sync: -1
+    # -1 means no limit (which means it's limited to 10000 by the server)
+    max_initial_member_sync: 100
     # Whether or not to sync the member list in channels.
     # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
     # list regardless of this setting.
@@ -133,7 +156,10 @@ bridge:
     startup_sync: true
     # Number of most recently active dialogs to check when syncing chats.
     # Set to 0 to remove limit.
-    sync_dialog_limit: 30
+    sync_update_limit: 0
+    # Number of most recently active dialogs to create portals for when syncing chats.
+    # Set to 0 to remove limit.
+    sync_create_limit: 30
     # Whether or not to sync and create portals for direct chats at startup.
     sync_direct_chats: false
     # The maximum number of simultaneous Telegram deletions to handle.
@@ -142,8 +168,8 @@ bridge:
     # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
     # at startup and when creating a bridge.
     sync_matrix_state: true
-    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
-    # login website (see appservice.public config section)
+    # Allow logging in within Matrix. If false, users can only log in using login-qr or the
+    # out-of-Matrix login website (see appservice.public config section)
     allow_matrix_login: true
     # Whether or not to bridge plaintext highlights.
     # Only enable this if your displayname_template has some static part that the bridge can use to
@@ -151,19 +177,34 @@ bridge:
     plaintext_highlights: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
-    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
-    # your own Matrix account as the Matrix puppet for your Telegram account.
+    # Whether or not to use /sync to get presence, read receipts and typing notifications
+    # when double puppeting is enabled
     sync_with_custom_puppets: true
-    # Shared secret for https://github.com/devture/matrix-synapse-shared-secret-auth
+    # Whether or not to update the m.direct account data event when double puppeting is enabled.
+    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
+    # and is therefore prone to race conditions.
+    sync_direct_chat_list: false
+    # Servers to always allow double puppeting from
+    double_puppet_server_map:
+        example.com: https://example.com
+    # Allow using double puppeting from any server with a valid client .well-known file.
+    double_puppet_allow_discovery: false
+    # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth
     #
     # If set, custom puppets will be enabled automatically for local users
     # instead of users having to find an access token and run `login-matrix`
     # manually.
-    login_shared_secret: null
+    # If using this for other servers than the bridge's server,
+    # you must also set the URL in the double_puppet_server_map.
+    login_shared_secret_map:
+        example.com: foobar
     # Set to false to disable link previews in messages sent to Telegram.
     telegram_link_preview: true
+    # Whether or not the !tg join command should do a HTTP request
+    # to resolve redirects in invite links.
+    invite_link_resolve: false
     # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    # N.B. Inline images are not supported on all clients (e.g. Element iOS/Android).
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
@@ -172,6 +213,7 @@ bridge:
     # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
     # streaming from/to Matrix and using many connections for Telegram.
     # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
+    # This option uses internal Telethon implementation details and may break with minor updates.
     parallel_file_transfer: false
     # Whether or not created rooms should have federation enabled.
     # If false, created portal rooms will never be federated.
@@ -191,6 +233,81 @@ bridge:
             height: 256
             background: "020202"  # only for gif
             fps: 30               # only for webm
+    # End-to-bridge encryption support options. These require matrix-nio to be installed with pip
+    # and login_shared_secret to be configured in order to get a device for the bridge bot.
+    #
+    # Additionally, https://github.com/matrix-org/synapse/pull/5758 is required if using a normal
+    # application service.
+    encryption:
+        # Allow encryption, work in group chat rooms with e2ee enabled
+        allow: false
+        # Default to encryption, force-enable encryption in all portals the bridge creates
+        # This will cause the bridge bot to be in private chats for the encryption to work properly.
+        default: false
+        # Database for the encryption data. If set to `default`, will use the appservice database.
+        database: default
+        # Options for automatic key sharing.
+        key_sharing:
+            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
+            # You must use a client that supports requesting keys from other users to use this feature.
+            allow: false
+            # Require the requesting device to have a valid cross-signing signature?
+            # This doesn't require that the bridge has verified the device, only that the user has verified it.
+            # Not yet implemented.
+            require_cross_signing: false
+            # Require devices to be verified by the bridge?
+            # Verification by the bridge is not yet implemented.
+            require_verification: true
+    # Whether or not to explicitly set the avatar and room name for private
+    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
+    private_chat_portal_meta: false
+    # Whether or not the bridge should send a read receipt from the bridge bot when a message has
+    # been sent to Telegram.
+    delivery_receipts: false
+    # Whether or not delivery errors should be reported as messages in the Matrix room.
+    delivery_error_reports: false
+    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
+    # This field will automatically be changed back to false after it,
+    # except if the config file is not writable.
+    resend_bridge_info: false
+    # When using double puppeting, should muted chats be muted in Matrix?
+    mute_bridging: false
+    # When using double puppeting, should pinned chats be moved to a specific tag in Matrix?
+    # The favorites tag is `m.favourite`.
+    pinned_tag: null
+    # Same as above for archived chats, the low priority tag is `m.lowpriority`.
+    archive_tag: null
+    # Whether or not mute status and tags should only be bridged when the portal room is created.
+    tag_only_on_create: true
+    # Settings for backfilling messages from Telegram.
+    backfill:
+        # Whether or not the Telegram ghosts of logged in Matrix users should be
+        # invited to private chats when backfilling history from Telegram. This is
+        # usually needed to prevent rate limits and to allow timestamp massaging.
+        invite_own_puppet: true
+        # Maximum number of messages to backfill without using a takeout.
+        # The first time a takeout is used, the user has to manually approve it from a different
+        # device. If initial_limit or missed_limit are higher than this value, the bridge will ask
+        # the user to accept the takeout after logging in before syncing any chats.
+        takeout_limit: 100
+        # Maximum number of messages to backfill initially.
+        # Set to 0 to disable backfilling when creating portal, or -1 to disable the limit.
+        #
+        # N.B. Initial backfill will only start after member sync. Make sure your
+        #      max_initial_member_sync is set to a low enough value so it doesn't take forever.
+        initial_limit: 0
+        # Maximum number of messages to backfill if messages were missed while the bridge was
+        # disconnected. Note that this only works for logged in users and only if the chat isn't
+        # older than sync_update_limit
+        # Set to 0 to disable backfilling missed messages.
+        missed_limit: 50
+        # If using double puppeting, should notifications be disabled
+        # while the initial backfill is in progress?
+        disable_notifications: false
+        # Whether or not to enable backfilling in normal groups.
+        # Normal groups have numerous technical problems in Telegram, and backfilling normal groups
+        # will likely cause problems if there are multiple Matrix users in the group.
+        normal_groups: false
 
     # Overrides for base power levels.
     initial_power_level_overrides:
@@ -270,6 +387,21 @@ bridge:
     # The prefix for commands. Only required in non-management rooms.
     command_prefix: "!tg"
 
+    # Messages sent upon joining a management room.
+    # Markdown is supported. The defaults are listed below.
+    management_room_text:
+        # Sent when joining a room.
+        welcome: "Hello, I'm a Telegram bridge bot."
+        # Sent when joining a management room and the user is already logged in.
+        welcome_connected: "Use `help` for help."
+        # Sent when joining a management room and the user is not logged in.
+        welcome_unconnected: "Use `help` for help or `login` to log in."
+        # Optional extra text sent when joining a management room.
+        additional_help: ""
+
+    # Send each message separately (for readability in some clients)
+    management_room_multiple_messages: false
+
     # Permissions for using the bridge.
     # Permitted values:
     #   relaybot - Only use the bridge via the relaybot, no access to commands.
@@ -409,7 +541,7 @@ logging:
         mau:
             level: DEBUG
         telethon:
-            level: DEBUG
+            level: INFO
         aiohttp:
             level: INFO
     root:

mautrix_telegram/formatter/__init__.py

@@ -1,5 +1,4 @@
-from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
-                          init_mx)
+from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram, init_mx
 from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -18,10 +18,12 @@ import re
 import logging
 
 from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
-                               TypeMessageEntity)
+                               TypeMessageEntity, InputMessageEntityMentionName)
 from telethon.helpers import add_surrogate, del_surrogate
+from telethon import TelegramClient
 
 from mautrix.types import RoomID, MessageEventContent
+from mautrix.util.logging import TraceLogger
 
 from ... import puppet as pu
 from ...types import TelegramID
@@ -31,30 +33,19 @@ from .parser import ParsedMessage, parse_html
 if TYPE_CHECKING:
     from ...context import Context
 
-log: logging.Logger = logging.getLogger("mau.fmt.mx")
+log: TraceLogger = logging.getLogger("mau.fmt.mx")
 should_bridge_plaintext_highlights: bool = False
 
 command_regex: Pattern = re.compile(r"^!([A-Za-z0-9@]+)")
 not_command_regex: Pattern = re.compile(r"^\\(![A-Za-z0-9@]+)")
 plain_mention_regex: Optional[Pattern] = None
 
-
-def plain_mention_to_html(match: Match) -> str:
-    puppet = pu.Puppet.find_by_displayname(match.group(2))
-    if puppet:
-        return (f"{match.group(1)}"
-                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                f"{puppet.displayname}"
-                "</a>")
-    return "".join(match.groups())
-
-
 MAX_LENGTH = 4096
 CUTOFF_TEXT = " [message cut]"
 CUT_MAX_LENGTH = MAX_LENGTH - len(CUTOFF_TEXT)
 
 
-def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+def _cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
     if len(message) > MAX_LENGTH:
         message = message[0:CUT_MAX_LENGTH] + CUTOFF_TEXT
         new_entities = []
@@ -73,23 +64,6 @@ class FormatError(Exception):
     pass
 
 
-def matrix_to_telegram(html: str) -> ParsedMessage:
-    try:
-        html = command_regex.sub(r"<command>\1</command>", html)
-        html = html.replace("\t", " " * 4)
-        html = not_command_regex.sub(r"\1", html)
-        if should_bridge_plaintext_highlights:
-            html = plain_mention_regex.sub(plain_mention_to_html, html)
-
-        text, entities = parse_html(add_surrogate(html))
-        text = del_surrogate(text.strip())
-        text, entities = cut_long_message(text, entities)
-
-        return text, entities
-    except Exception as e:
-        raise FormatError(f"Failed to convert Matrix format: {html}") from e
-
-
 def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
                              room_id: Optional[RoomID] = None) -> Optional[TelegramID]:
     event_id = content.get_reply_to()
@@ -103,19 +77,61 @@ def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
     return None
 
 
-def matrix_text_to_telegram(text: str) -> ParsedMessage:
+async def matrix_to_telegram(client: TelegramClient, *, text: Optional[str] = None,
+                             html: Optional[str] = None) -> ParsedMessage:
+    if html is not None:
+        text, entities = _matrix_html_to_telegram(html)
+    elif text is not None:
+        text, entities = _matrix_text_to_telegram(text)
+    else:
+        raise ValueError("text or html must be provided to convert formatting")
+    await _fix_name_mentions(client, entities)
+    return text, entities
+
+
+def _matrix_html_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(_plain_mention_to_html, html)
+
+        text, entities = parse_html(add_surrogate(html))
+        text = del_surrogate(text.strip())
+        text, entities = _cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def _matrix_text_to_telegram(text: str) -> ParsedMessage:
     text = command_regex.sub(r"/\1", text)
     text = text.replace("\t", " " * 4)
     text = not_command_regex.sub(r"\1", text)
     if should_bridge_plaintext_highlights:
-        entities, pmr_replacer = plain_mention_to_text()
+        entities, pmr_replacer = _plain_mention_to_text()
         text = plain_mention_regex.sub(pmr_replacer, text)
     else:
         entities = []
     return text, entities
 
 
-def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
+async def _fix_name_mentions(client: TelegramClient, entities: List[TypeMessageEntity]) -> None:
+    for index in reversed(range(len(entities))):
+        entity = entities[index]
+        if isinstance(entity, (MessageEntityMentionName, InputMessageEntityMentionName)):
+            try:
+                user = await client.get_input_entity(entity.user_id)
+            except (ValueError, TypeError) as e:
+                log.trace(f"Dropping mention of {entity.user_id}: {e}")
+                del entities[index]
+            else:
+                entities[index] = InputMessageEntityMentionName(entity.offset, entity.length, user)
+
+
+def _plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
     entities = []
 
     def replacer(match: Match) -> str:
@@ -136,6 +152,16 @@ def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match],
     return entities, replacer
 
 
+def _plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
 def init_mx(context: "Context") -> None:
     global plain_mention_regex, should_bridge_plaintext_highlights
     config = context.config

mautrix_telegram/formatter/from_matrix/parser.py

@@ -48,7 +48,7 @@ class MatrixParser(BaseMatrixParser[TelegramMessage]):
 
     @classmethod
     def user_pill_to_fstring(cls, msg: TelegramMessage, user_id: UserID) -> TelegramMessage:
-        user = (pu.Puppet.get_by_mxid(user_id)
+        user = (pu.Puppet.deprecated_sync_get_by_mxid(user_id)
                 or u.User.get_by_mxid(user_id, create=False))
         if not user:
             return msg

mautrix_telegram/formatter/from_telegram.py

@@ -22,7 +22,7 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, M
                                MessageEntityEmail, MessageEntityTextUrl, MessageEntityBold,
                                MessageEntityItalic, MessageEntityCode, MessageEntityPre,
                                MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
-                               MessageEntityPhone, TypeMessageEntity, PeerChannel,
+                               MessageEntityPhone, TypeMessageEntity, PeerChannel, PeerChat,
                                MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
                                MessageEntityUnderline, PeerUser)
 from telethon.tl.custom import Message
@@ -32,7 +32,7 @@ from telethon.helpers import add_surrogate, del_surrogate
 from mautrix.errors import MatrixRequestError
 from mautrix.appservice import IntentAPI
 from mautrix.types import (TextMessageEventContent, RelatesTo, RelationType, Format, MessageType,
-                           MessageEvent)
+                           MessageEvent, EventType)
 
 from .. import user as u, puppet as pu, portal as po
 from ..types import TelegramID
@@ -45,13 +45,13 @@ log: logging.Logger = logging.getLogger("mau.fmt.tg")
 
 
 def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Optional[RelatesTo]:
-    if evt.reply_to_msg_id:
-        space = (evt.to_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+    if evt.reply_to:
+        space = (evt.peer_id.channel_id
+                 if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to_msg_id), space)
+        msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
         if msg:
-            return RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
+            return RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
     return None
 
 
@@ -61,15 +61,15 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
         content.format = Format.HTML
         content.formatted_body = escape(content.body)
     fwd_from_html, fwd_from_text = None, None
-    if fwd_from.from_id:
-        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
+    if isinstance(fwd_from.from_id, PeerUser):
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id.user_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
             fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
                              f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
-            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
+            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id.user_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
                 fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
@@ -77,15 +77,17 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
 
         if not fwd_from_text:
             try:
-                user = await source.client.get_entity(PeerUser(fwd_from.from_id))
+                user = await source.client.get_entity(fwd_from.from_id)
                 if user:
-                    fwd_from_text = pu.Puppet.get_displayname(user, False)
+                    fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
                     fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
             except (ValueError, RPCError):
                 fwd_from_text = fwd_from_html = "unknown user"
-    elif fwd_from.channel_id:
-        portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
-        if portal:
+    elif isinstance(fwd_from.from_id, (PeerChannel, PeerChat)):
+        from_id = (fwd_from.from_id.chat_id if isinstance(fwd_from.from_id, PeerChat)
+                   else fwd_from.from_id.channel_id)
+        portal = po.Portal.get_by_tgid(TelegramID(from_id))
+        if portal and portal.title:
             fwd_from_text = portal.title
             if portal.alias:
                 fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
@@ -94,7 +96,7 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
                 fwd_from_html = f"channel <b>{escape(fwd_from_text)}</b>"
         else:
             try:
-                channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
+                channel = await source.client.get_entity(fwd_from.from_id)
                 if channel:
                     fwd_from_text = f"channel {channel.title}"
                     fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
@@ -116,23 +118,25 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
 
 async def _add_reply_header(source: 'AbstractUser', content: TextMessageEventContent, evt: Message,
                             main_intent: IntentAPI):
-    space = (evt.to_id.channel_id
-             if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+    space = (evt.peer_id.channel_id
+             if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to_msg_id), space)
+    msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
     if not msg:
         return
 
-    content.relates_to = RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
+    content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
 
     try:
-        event: MessageEvent = await main_intent.get_event(msg.mx_room, msg.mxid)
+        event = await main_intent.get_event(msg.mx_room, msg.mxid)
+        if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
+            event = await source.bridge.matrix.e2ee.decrypt(event)
         if isinstance(event.content, TextMessageEventContent):
             event.content.trim_reply_fallback()
-        puppet = pu.Puppet.get_by_mxid(event.sender, create=False)
+        puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
         content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
-    except MatrixRequestError:
+    except Exception:
         log.exception("Failed to get event to add reply fallback")
 
 
@@ -162,7 +166,7 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
     if evt.fwd_from:
         await _add_forward_header(source, content, evt.fwd_from)
 
-    if evt.reply_to_msg_id and not no_reply_fallback:
+    if evt.reply_to and not no_reply_fallback:
         await _add_reply_header(source, content, evt, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:

mautrix_telegram/get_version.py

@@ -18,7 +18,7 @@ def run(cmd):
 if os.path.exists(".git") and shutil.which("git"):
     try:
         git_revision = run(["git", "rev-parse", "HEAD"]).strip().decode("ascii")
-        git_revision_url = f"https://github.com/tulir/mautrix-telegram/commit/{git_revision}"
+        git_revision_url = f"https://github.com/mautrix/telegram/commit/{git_revision}"
         git_revision = git_revision[:8]
     except (subprocess.SubprocessError, OSError):
         git_revision = "unknown"
@@ -33,7 +33,7 @@ else:
     git_revision_url = None
     git_tag = None
 
-git_tag_url = (f"https://github.com/tulir/mautrix-telegram/releases/tag/{git_tag}"
+git_tag_url = (f"https://github.com/mautrix/telegram/releases/tag/{git_tag}"
                if git_tag else None)
 
 if git_tag and __version__ == git_tag[1:].replace("-", ""):

mautrix_telegram/matrix.py

@@ -20,7 +20,8 @@ from mautrix.types import (Event, EventType, RoomID, UserID, EventID, ReceiptEve
                            ReceiptEventContent, PresenceEvent, PresenceState, TypingEvent,
                            MessageEvent, StateEvent, RedactionEvent, RoomNameStateEventContent,
                            RoomAvatarStateEventContent, RoomTopicStateEventContent,
-                           MemberStateEventContent)
+                           MemberStateEventContent, EncryptedEvent, TextMessageEventContent,
+                           MessageType)
 from mautrix.errors import MatrixError
 
 from . import user as u, portal as po, puppet as pu, commands as com
@@ -29,14 +30,6 @@ if TYPE_CHECKING:
     from .context import Context
     from .bot import Bot
 
-try:
-    from prometheus_client import Histogram
-
-    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events", ["event_type"])
-except ImportError:
-    Histogram = None
-    EVENT_TIME = None
-
 RoomMetaStateEventContent = Union[RoomNameStateEventContent, RoomAvatarStateEventContent,
                                   RoomTopicStateEventContent]
 
@@ -47,24 +40,20 @@ class MatrixHandler(BaseMatrixHandler):
     previously_typing: Dict[RoomID, Set[UserID]]
 
     def __init__(self, context: 'Context') -> None:
-        super(MatrixHandler, self).__init__(context.az, context.config, loop=context.loop,
-                                            command_processor=com.CommandProcessor(context))
+        prefix, suffix = context.config["bridge.username_template"].format(userid=":").split(":")
+        homeserver = context.config["homeserver.domain"]
+        self.user_id_prefix = f"@{prefix}"
+        self.user_id_suffix = f"{suffix}:{homeserver}"
+
+        super().__init__(command_processor=com.CommandProcessor(context), bridge=context.bridge)
+
         self.bot = context.bot
         self.previously_typing = {}
 
-    async def get_user(self, user_id: UserID) -> 'u.User':
-        return await u.User.get_by_mxid(user_id).ensure_started()
-
-    async def get_portal(self, room_id: RoomID) -> 'po.Portal':
-        return po.Portal.get_by_mxid(room_id)
-
-    async def get_puppet(self, user_id: UserID) -> 'pu.Puppet':
-        return pu.Puppet.get_by_mxid(user_id)
-
     async def handle_puppet_invite(self, room_id: RoomID, puppet: pu.Puppet, inviter: u.User,
                                    event_id: EventID) -> None:
         intent = puppet.default_mxid_intent
-        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}")
+        self.log.debug(f"{inviter.mxid} invited puppet for {puppet.tgid} to {room_id}")
         if not await inviter.is_logged_in():
             await intent.error_and_leave(
                 room_id, text="Please log in before inviting Telegram puppets.")
@@ -79,11 +68,12 @@ class MatrixHandler(BaseMatrixHandler):
             await intent.join_room(room_id)
             return
         try:
-            members = await self.az.intent.get_room_members(room_id)
+            members = await intent.get_room_members(room_id)
         except MatrixError:
-            members = []
+            self.log.exception(f"Failed to get members after joining {room_id} as {intent.mxid}")
+            return
         if self.az.bot_mxid not in members:
-            if len(members) > 1:
+            if len(members) > 2:
                 await intent.error_and_leave(room_id, text=None, html=(
                     f"Please invite "
                     f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
@@ -94,7 +84,7 @@ class MatrixHandler(BaseMatrixHandler):
             portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
             if portal.mxid:
                 try:
-                    await intent.invite_user(portal.mxid, inviter.mxid)
+                    await portal.invite_to_matrix(inviter.mxid)
                     await intent.send_notice(
                         room_id, text=f"You already have a private chat with me: {portal.mxid}",
                         html=("You already have a private chat with me: "
@@ -104,31 +94,27 @@ class MatrixHandler(BaseMatrixHandler):
                 except MatrixError:
                     pass
             portal.mxid = room_id
-            portal.save()
-            inviter.register_portal(portal)
-            await intent.send_notice(room_id, "Portal to private chat created.")
+            e2be_ok = await portal.check_dm_encryption()
+            await portal.save()
+            await inviter.register_portal(portal)
+            if e2be_ok is True:
+                evt_type, content = await self.e2ee.encrypt(
+                    room_id, EventType.ROOM_MESSAGE,
+                    TextMessageEventContent(msgtype=MessageType.NOTICE,
+                                            body="Portal to private chat created and end-to-bridge"
+                                                 " encryption enabled."))
+                await intent.send_message_event(room_id, evt_type, content)
+            else:
+                message = "Portal to private chat created."
+                if e2be_ok is False:
+                    message += "\n\nWarning: Failed to enable end-to-bridge encryption"
+                await intent.send_notice(room_id, message)
+            await portal.update_bridge_info()
         else:
             await intent.join_room(room_id)
             await intent.send_notice(room_id, "This puppet will remain inactive until a "
                                               "Telegram chat is created for this room.")
 
-    async def send_welcome_message(self, room_id: RoomID, inviter: 'u.User') -> None:
-        try:
-            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
-        except MatrixError:
-            # The AS bot is not in the room.
-            return
-        cmd_prefix = self.commands.command_prefix
-        text = html = "Hello, I'm a Telegram bridge bot. "
-        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
-            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
-            html += (f"Use <code>{cmd_prefix} help</code> for help"
-                     f" or <code>{cmd_prefix} login</code> to log in.")
-        else:
-            text += f"Use `{cmd_prefix} help` for help."
-            html += f"Use <code>{cmd_prefix} help</code> for help."
-        await self.az.intent.send_notice(room_id, text=text, html=html)
-
     async def handle_invite(self, room_id: RoomID, user_id: UserID, inviter: 'u.User',
                             event_id: EventID) -> None:
         user = u.User.get_by_mxid(user_id, create=False)
@@ -156,7 +142,7 @@ class MatrixHandler(BaseMatrixHandler):
                                                "messages for unauthenticated users.")
             return
 
-        self.log.debug(f"{user} joined {room_id}")
+        self.log.debug(f"{user.mxid} joined {room_id}")
         if await user.is_logged_in() or portal.has_bot:
             await portal.join_matrix(user, event_id)
 
@@ -194,7 +180,7 @@ class MatrixHandler(BaseMatrixHandler):
             return
         await sender.ensure_started()
 
-        puppet = pu.Puppet.get_by_mxid(user_id)
+        puppet = await pu.Puppet.get_by_mxid(user_id)
         if puppet:
             if ban:
                 await portal.ban_matrix(puppet, sender)
@@ -246,7 +232,7 @@ class MatrixHandler(BaseMatrixHandler):
         if not portal:
             return
 
-        await portal.handle_matrix_deletion(sender, evt.redacts)
+        await portal.handle_matrix_deletion(sender, evt.redacts, evt.event_id)
 
     @staticmethod
     async def handle_power_levels(evt: StateEvent) -> None:
@@ -254,11 +240,12 @@ class MatrixHandler(BaseMatrixHandler):
         sender = await u.User.get_by_mxid(evt.sender).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
             await portal.handle_matrix_power_levels(sender, evt.content.users,
-                                                    evt.unsigned.prev_content.users)
+                                                    evt.unsigned.prev_content.users,
+                                                    evt.event_id)
 
     @staticmethod
     async def handle_room_meta(evt_type: EventType, room_id: RoomID, sender_mxid: UserID,
-                               content: RoomMetaStateEventContent) -> None:
+                               content: RoomMetaStateEventContent, event_id: EventID) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
@@ -269,27 +256,28 @@ class MatrixHandler(BaseMatrixHandler):
             }[evt_type]
             if not isinstance(content, content_type):
                 return
-            await handler(sender, content[content_key])
+            await handler(sender, content[content_key], event_id)
 
     @staticmethod
     async def handle_room_pin(room_id: RoomID, sender_mxid: UserID,
-                              new_events: Set[str], old_events: Set[str]) -> None:
+                              new_events: Set[str], old_events: Set[str],
+                              event_id: EventID) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
-            events = new_events - old_events
-            if len(events) > 0:
-                # New event pinned, set that as pinned in Telegram.
-                await portal.handle_matrix_pin(sender, EventID(events.pop()))
-            elif len(new_events) == 0:
-                # All pinned events removed, remove pinned event in Telegram.
-                await portal.handle_matrix_pin(sender, None)
+            if not new_events:
+                await portal.handle_matrix_unpin_all(sender, event_id)
+            else:
+                changes = {event_id: event_id in new_events
+                           for event_id in new_events ^ old_events}
+                await portal.handle_matrix_pin(sender, changes, event_id)
 
     @staticmethod
-    async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID) -> None:
+    async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID,
+                                  event_id: EventID) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         if portal:
-            await portal.handle_matrix_upgrade(sender, new_room_id)
+            await portal.handle_matrix_upgrade(sender, new_room_id, event_id)
 
     async def handle_member_info_change(self, room_id: RoomID, user_id: UserID,
                                         profile: MemberStateEventContent,
@@ -321,17 +309,15 @@ class MatrixHandler(BaseMatrixHandler):
             return
 
         for user_id, event_id in receipts:
-            user = await u.User.get_by_mxid(user_id).ensure_started()
-            if not await user.is_logged_in():
-                continue
-            await portal.mark_read(user, event_id)
+            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
+            if user and await user.is_logged_in():
+                await portal.mark_read(user, event_id)
 
     @staticmethod
     async def handle_presence(user_id: UserID, presence: PresenceState) -> None:
-        user = await u.User.get_by_mxid(user_id).ensure_started()
-        if not await user.is_logged_in():
-            return
-        await user.set_presence(presence == PresenceState.ONLINE)
+        user = u.User.get_by_mxid(user_id, check_db=False, create=False)
+        if user and await user.is_logged_in():
+            await user.set_presence(presence == PresenceState.ONLINE)
 
     async def handle_typing(self, room_id: RoomID, now_typing: Set[UserID]) -> None:
         portal = po.Portal.get_by_mxid(room_id)
@@ -346,17 +332,22 @@ class MatrixHandler(BaseMatrixHandler):
             if is_typing and was_typing:
                 continue
 
-            user = await u.User.get_by_mxid(user_id).ensure_started()
-            if not await user.is_logged_in():
-                continue
-
-            await portal.set_typing(user, is_typing)
+            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
+            if user and await user.is_logged_in():
+                await portal.set_typing(user, is_typing)
 
         self.previously_typing[room_id] = now_typing
 
     def filter_matrix_event(self, evt: Event) -> bool:
-        if not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent)):
+        if isinstance(evt, (TypingEvent, ReceiptEvent, PresenceEvent)):
+            return False
+        elif not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent, EncryptedEvent)):
             return True
+        if evt.content.get(self.az.real_user_content_key, False):
+            puppet = pu.Puppet.deprecated_sync_get_by_custom_mxid(evt.sender)
+            if puppet:
+                self.log.debug("Ignoring puppet-sent event %s", evt.event_id)
+                return True
         return evt.sender and (evt.sender == self.az.bot_mxid
                                or pu.Puppet.get_id_from_mxid(evt.sender) is not None)
 
@@ -377,17 +368,16 @@ class MatrixHandler(BaseMatrixHandler):
         if evt.type == EventType.ROOM_POWER_LEVELS:
             await self.handle_power_levels(evt)
         elif evt.type in (EventType.ROOM_NAME, EventType.ROOM_AVATAR, EventType.ROOM_TOPIC):
-            await self.handle_room_meta(evt.type, evt.room_id, evt.sender, evt.content)
+            await self.handle_room_meta(evt.type, evt.room_id, evt.sender, evt.content,
+                                        evt.event_id)
         elif evt.type == EventType.ROOM_PINNED_EVENTS:
             new_events = set(evt.content.pinned)
             try:
                 old_events = set(evt.unsigned.prev_content.pinned)
             except (KeyError, ValueError, TypeError, AttributeError):
                 old_events = set()
-            await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events)
+            await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events,
+                                       evt.event_id)
         elif evt.type == EventType.ROOM_TOMBSTONE:
-            await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room)
-
-    async def log_event_handle_duration(self, evt: Event, duration: float) -> None:
-        if EVENT_TIME:
-            EVENT_TIME.labels(event_type=str(evt.type)).observe(duration)
+            await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room,
+                                           evt.event_id)

mautrix_telegram/portal/__init__.pyi

@@ -1,8 +1,8 @@
 from typing import Union
 from .base import BasePortal
-from .portal_matrix import PortalMatrix
-from .portal_metadata import PortalMetadata
-from .portal_telegram import PortalTelegram
+from .matrix import PortalMatrix
+from .metadata import PortalMetadata
+from .telegram import PortalTelegram
 from ..context import Context
 
 Portal = Union[BasePortal, PortalMatrix, PortalMetadata, PortalTelegram]

mautrix_telegram/portal/base.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,29 +13,34 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
+from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, Set, Iterable, TYPE_CHECKING
 from abc import ABC, abstractmethod
+from datetime import datetime
 import asyncio
 import logging
 import json
 
 from telethon.tl.functions.messages import ExportChatInviteRequest
-from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, ChatInviteEmpty, InputChannel,
+from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, InputChannel,
                                InputPeerChannel, InputPeerChat, InputPeerUser, InputUser,
                                PeerChannel, PeerChat, PeerUser, TypeChat, TypeInputPeer, TypePeer,
                                TypeUser, TypeUserFull, User, UserFull, TypeInputChannel, Photo,
                                Document, TypePhotoSize, PhotoSize, InputPhotoFileLocation,
                                TypeChatParticipant, TypeChannelParticipant, PhotoEmpty, ChatPhoto,
-                               ChatPhotoEmpty)
+                               ChatPhotoEmpty, PhotoSizeProgressive, PhotoSizeEmpty)
 
 from mautrix.errors import MatrixRequestError, IntentError
 from mautrix.appservice import AppService, IntentAPI
-from mautrix.types import RoomID, RoomAlias, UserID, EventType, PowerLevelStateEventContent
+from mautrix.types import (RoomID, RoomAlias, UserID, EventID, EventType,
+                           PowerLevelStateEventContent, ContentURI)
 from mautrix.util.simple_template import SimpleTemplate
+from mautrix.util.simple_lock import SimpleLock
+from mautrix.util.logging import TraceLogger
+from mautrix.bridge import BasePortal as MautrixBasePortal
 
 from ..types import TelegramID
 from ..context import Context
-from ..db import Portal as DBPortal
+from ..db import Portal as DBPortal, Message as DBMessage
 from .. import puppet as p, user as u, util
 from .deduplication import PortalDedup
 from .send_lock import PortalSendLock
@@ -44,6 +49,7 @@ if TYPE_CHECKING:
     from ..bot import Bot
     from ..abstract_user import AbstractUser
     from ..config import Config
+    from ..matrix import MatrixHandler
     from . import Portal
 
 TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
@@ -53,11 +59,12 @@ InviteList = Union[UserID, List[UserID]]
 config: Optional['Config'] = None
 
 
-class BasePortal(ABC):
-    base_log: logging.Logger = logging.getLogger("mau.portal")
+class BasePortal(MautrixBasePortal, ABC):
+    base_log: TraceLogger = logging.getLogger("mau.portal")
     az: AppService = None
     bot: 'Bot' = None
     loop: asyncio.AbstractEventLoop = None
+    matrix: 'MatrixHandler' = None
 
     # Config cache
     filter_mode: str = None
@@ -67,6 +74,7 @@ class BasePortal(ABC):
     sync_channel_members: bool = True
     sync_matrix_state: bool = True
     public_portals: bool = False
+    private_chat_portal_meta: bool = False
 
     alias_template: SimpleTemplate[str]
     hs_domain: str
@@ -85,22 +93,30 @@ class BasePortal(ABC):
     about: Optional[str]
     photo_id: Optional[str]
     local_config: Dict[str, Any]
+    avatar_url: Optional[ContentURI]
+    encrypted: bool
     deleted: bool
-    log: logging.Logger
+    backfill_lock: SimpleLock
+    backfill_method_lock: asyncio.Lock
+    backfill_leave: Optional[Set[IntentAPI]]
+    log: TraceLogger
 
     alias: Optional[RoomAlias]
 
     dedup: PortalDedup
     send_lock: PortalSendLock
+    _pin_lock: asyncio.Lock
 
     _db_instance: DBPortal
     _main_intent: Optional[IntentAPI]
+    _room_create_lock: asyncio.Lock
 
     def __init__(self, tgid: TelegramID, peer_type: str, tg_receiver: Optional[TelegramID] = None,
                  mxid: Optional[RoomID] = None, username: Optional[str] = None,
                  megagroup: Optional[bool] = False, title: Optional[str] = None,
                  about: Optional[str] = None, photo_id: Optional[str] = None,
-                 local_config: Optional[str] = None, db_instance: DBPortal = None) -> None:
+                 local_config: Optional[str] = None, avatar_url: Optional[ContentURI] = None,
+                 encrypted: Optional[bool] = False, db_instance: DBPortal = None) -> None:
         self.mxid = mxid
         self.tgid = tgid
         self.tg_receiver = tg_receiver or tgid
@@ -111,20 +127,27 @@ class BasePortal(ABC):
         self.about = about
         self.photo_id = photo_id
         self.local_config = json.loads(local_config or "{}")
+        self.avatar_url = avatar_url
+        self.encrypted = encrypted
         self._db_instance = db_instance
         self._main_intent = None
         self.deleted = False
         self.log = self.base_log.getChild(self.tgid_log if self.tgid else self.mxid)
+        self.backfill_lock = SimpleLock("Waiting for backfilling to finish before handling %s",
+                                        log=self.log)
+        self.backfill_method_lock = asyncio.Lock()
+        self.backfill_leave = None
 
         self.dedup = PortalDedup(self)
         self.send_lock = PortalSendLock()
+        self._pin_lock = asyncio.Lock()
 
         if tgid:
             self.by_tgid[self.tgid_full] = self
         if mxid:
             self.by_mxid[mxid] = self
 
-    # region Propegrties
+    # region Properties
 
     @property
     def tgid_full(self) -> Tuple[TelegramID, TelegramID]:
@@ -136,6 +159,10 @@ class BasePortal(ABC):
             return str(self.tgid)
         return f"{self.tg_receiver}<->{self.tgid}"
 
+    @property
+    def name(self) -> str:
+        return self.title
+
     @property
     def alias(self) -> Optional[RoomAlias]:
         if not self.username:
@@ -157,6 +184,10 @@ class BasePortal(ABC):
         elif self.peer_type == "channel":
             return PeerChannel(channel_id=self.tgid)
 
+    @property
+    def is_direct(self) -> bool:
+        return self.peer_type == "user"
+
     @property
     def has_bot(self) -> bool:
         return (bool(self.bot)
@@ -191,18 +222,26 @@ class BasePortal(ABC):
         return config[f"bridge.{key}"]
 
     @staticmethod
-    def _get_largest_photo_size(photo: Union[Photo, Document]
+    def _photo_size_key(photo: TypePhotoSize) -> int:
+        if isinstance(photo, PhotoSize):
+            return photo.size
+        elif isinstance(photo, PhotoSizeProgressive):
+            return max(photo.sizes)
+        elif isinstance(photo, PhotoSizeEmpty):
+            return 0
+        else:
+            return len(photo.bytes)
+
+    @classmethod
+    def _get_largest_photo_size(cls, photo: Union[Photo, Document]
                                 ) -> Tuple[Optional[InputPhotoFileLocation],
                                            Optional[TypePhotoSize]]:
-        if not photo:
-            return None, None
-        if isinstance(photo, Document) and not photo.thumbs:
+        if not photo or isinstance(photo, PhotoEmpty) or (isinstance(photo, Document)
+                                                          and not photo.thumbs):
             return None, None
 
         largest = max(photo.thumbs if isinstance(photo, Document) else photo.sizes,
-                      key=(lambda photo2: (len(photo2.bytes)
-                                           if not isinstance(photo2, PhotoSize)
-                                           else photo2.size)))
+                      key=cls._photo_size_key)
         return InputPhotoFileLocation(
             id=photo.id,
             access_hash=photo.access_hash,
@@ -220,9 +259,8 @@ class BasePortal(ABC):
             await self.main_intent.get_power_levels(self.mxid)
         except MatrixRequestError:
             return False
-        evt_type = EventType.find(f"net.maunium.telegram.{event}")
-        evt_type.t_class = EventType.Class.STATE
-        return self.main_intent.state_store.has_power_level(self.mxid, user.mxid, evt_type)
+        evt_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
+        return await self.main_intent.state_store.has_power_level(self.mxid, user.mxid, evt_type)
 
     def get_input_entity(self, user: 'AbstractUser'
                          ) -> Awaitable[Union[TypeInputPeer, TypeInputChannel]]:
@@ -233,8 +271,7 @@ class BasePortal(ABC):
             return await user.client.get_entity(self.peer)
         except ValueError:
             if user.is_bot:
-                self.log.warning(f"Could not find entity with bot {user.tgid}. "
-                                 "Failing...")
+                self.log.warning(f"Could not find entity with bot {user.tgid}. Failing...")
                 raise
             self.log.warning(f"Could not find entity with user {user.tgid}. "
                              "falling back to get_dialogs.")
@@ -243,73 +280,45 @@ class BasePortal(ABC):
                     return dialog.entity
             raise
 
-    async def get_invite_link(self, user: 'u.User') -> str:
+    async def get_invite_link(self, user: 'u.User', uses: Optional[int] = None,
+                              expire: Optional[datetime] = None) -> str:
         if self.peer_type == "user":
             raise ValueError("You can't invite users to private chats.")
         if self.username:
             return f"https://t.me/{self.username}"
-        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user)))
-        if isinstance(link, ChatInviteEmpty):
-            raise ValueError("Failed to get invite link.")
+        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user),
+                                                         expire_date=expire, usage_limit=uses))
         return link.link
 
     # endregion
     # region Matrix room cleanup
 
-    async def get_authenticated_matrix_users(self) -> List['u.User']:
+    async def get_authenticated_matrix_users(self) -> List[UserID]:
         try:
             members = await self.main_intent.get_room_members(self.mxid)
         except MatrixRequestError:
             return []
-        authenticated: List[u.User] = []
+        authenticated: List[UserID] = []
         has_bot = self.has_bot
-        for member_str in members:
-            member = UserID(member_str)
-            if p.Puppet.get_id_from_mxid(member) or member == self.main_intent.mxid:
+        for member in members:
+            if p.Puppet.get_id_from_mxid(member) or member == self.az.bot_mxid:
                 continue
             user = await u.User.get_by_mxid(member).ensure_started()
             authenticated_through_bot = has_bot and user.relaybot_whitelisted
             if authenticated_through_bot or await user.has_full_access(allow_bot=True):
-                authenticated.append(user)
+                authenticated.append(user.mxid)
         return authenticated
 
-    @staticmethod
-    async def cleanup_room(intent: IntentAPI, room_id: RoomID, message: str,
-                           puppets_only: bool = False) -> None:
-        try:
-            members = await intent.get_room_members(room_id)
-        except MatrixRequestError:
-            members = []
-        for user in members:
-            puppet = p.Puppet.get_by_mxid(UserID(user), create=False)
-            if user != intent.mxid and (not puppets_only or puppet):
-                try:
-                    if puppet:
-                        await puppet.default_mxid_intent.leave_room(room_id)
-                    else:
-                        await intent.kick_user(room_id, user, message)
-                except (MatrixRequestError, IntentError):
-                    pass
-        try:
-            await intent.leave_room(room_id)
-        except (MatrixRequestError, IntentError):
-            self.log.warning("Failed to leave room when cleaning up room", exc_info=True)
-
-    async def cleanup_portal(self, message: str, puppets_only: bool = False) -> None:
+    async def cleanup_portal(self, message: str, puppets_only: bool = False, delete: bool = True
+                             ) -> None:
         if self.username:
             try:
                 await self.main_intent.remove_room_alias(self.alias_localpart)
             except (MatrixRequestError, IntentError):
                 self.log.warning("Failed to remove alias when cleaning up room", exc_info=True)
         await self.cleanup_room(self.main_intent, self.mxid, message, puppets_only)
-
-    async def unbridge(self) -> None:
-        await self.cleanup_portal("Room unbridged", puppets_only=True)
-        self.delete()
-
-    async def cleanup_and_delete(self) -> None:
-        await self.cleanup_portal("Portal deleted")
-        self.delete()
+        if delete:
+            await self.delete()
 
     # endregion
     # region Database conversion
@@ -324,14 +333,19 @@ class BasePortal(ABC):
         return DBPortal(tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
                         mxid=self.mxid, username=self.username, megagroup=self.megagroup,
                         title=self.title, about=self.about, photo_id=self.photo_id,
-                        config=json.dumps(self.local_config))
+                        config=json.dumps(self.local_config), avatar_url=self.avatar_url,
+                        encrypted=self.encrypted)
 
-    def save(self) -> None:
+    async def save(self) -> None:
         self.db_instance.edit(mxid=self.mxid, username=self.username, title=self.title,
                               about=self.about, photo_id=self.photo_id, megagroup=self.megagroup,
-                              config=json.dumps(self.local_config))
+                              config=json.dumps(self.local_config), avatar_url=self.avatar_url,
+                              encrypted=self.encrypted)
 
-    def delete(self) -> None:
+    async def delete(self) -> None:
+        self.delete_sync()
+
+    def delete_sync(self) -> None:
         try:
             del self.by_tgid[self.tgid_full]
         except KeyError:
@@ -342,19 +356,29 @@ class BasePortal(ABC):
             pass
         if self._db_instance:
             self._db_instance.delete()
+        DBMessage.delete_all(self.mxid)
         self.deleted = True
 
     @classmethod
     def from_db(cls, db_portal: DBPortal) -> 'Portal':
         return cls(tgid=db_portal.tgid, tg_receiver=db_portal.tg_receiver,
-                   peer_type=db_portal.peer_type, mxid=db_portal.mxid,
-                   username=db_portal.username, megagroup=db_portal.megagroup,
-                   title=db_portal.title, about=db_portal.about, photo_id=db_portal.photo_id,
-                   local_config=db_portal.config, db_instance=db_portal)
+                   peer_type=db_portal.peer_type, mxid=db_portal.mxid, username=db_portal.username,
+                   megagroup=db_portal.megagroup, title=db_portal.title, about=db_portal.about,
+                   photo_id=db_portal.photo_id, local_config=db_portal.config,
+                   avatar_url=db_portal.avatar_url, encrypted=db_portal.encrypted,
+                   db_instance=db_portal)
 
     # endregion
     # region Class instance lookup
 
+    @classmethod
+    def all(cls) -> Iterable['Portal']:
+        for db_portal in DBPortal.all():
+            try:
+                yield cls.by_tgid[(db_portal.tgid, db_portal.tg_receiver)]
+            except KeyError:
+                yield cls.from_db(db_portal)
+
     @classmethod
     def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
         try:
@@ -392,6 +416,8 @@ class BasePortal(ABC):
     @classmethod
     def get_by_tgid(cls, tgid: TelegramID, tg_receiver: Optional[TelegramID] = None,
                     peer_type: str = None) -> Optional['Portal']:
+        if peer_type == "user" and tg_receiver is None:
+            raise ValueError("tg_receiver is required when peer_type is \"user\"")
         tg_receiver = tg_receiver or tgid
         tgid_full = (tgid, tg_receiver)
         try:
@@ -486,9 +512,24 @@ class BasePortal(ABC):
     def _migrate_and_save_telegram(self, new_id: TelegramID) -> None:
         pass
 
+    @abstractmethod
+    async def update_bridge_info(self) -> None:
+        pass
+
     @abstractmethod
     def handle_matrix_power_levels(self, sender: 'u.User', new_levels: Dict[UserID, int],
-                                   old_levels: Dict[UserID, int]) -> Awaitable[None]:
+                                   old_levels: Dict[UserID, int], event_id: Optional[EventID]
+                                   ) -> Awaitable[None]:
+        pass
+
+    @abstractmethod
+    def backfill(self, source: 'AbstractUser', is_initial: bool = False,
+                 limit: Optional[int] = None, last_id: Optional[int] = None) -> Awaitable[None]:
+        pass
+
+    @abstractmethod
+    async def _send_delivery_receipt(self, event_id: EventID, room_id: Optional[RoomID] = None
+                                     ) -> None:
         pass
 
     # endregion
@@ -497,10 +538,13 @@ class BasePortal(ABC):
 def init(context: Context) -> None:
     global config
     BasePortal.az, config, BasePortal.loop, BasePortal.bot = context.core
+    BasePortal.matrix = context.mx
+    MautrixBasePortal.bridge = context.bridge
     BasePortal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
     BasePortal.sync_channel_members = config["bridge.sync_channel_members"]
     BasePortal.sync_matrix_state = config["bridge.sync_matrix_state"]
     BasePortal.public_portals = config["bridge.public_portals"]
+    BasePortal.private_chat_portal_meta = config["bridge.private_chat_portal_meta"]
     BasePortal.filter_mode = config["bridge.filter.mode"]
     BasePortal.filter_list = config["bridge.filter.list"]
     BasePortal.hs_domain = config["homeserver.domain"]

mautrix_telegram/portal/deduplication.py

@@ -50,7 +50,7 @@ class PortalDedup:
 
     @property
     def _always_force_hash(self) -> bool:
-        return self._portal.peer_type != 'channel'
+        return self._portal.peer_type == 'chat'
 
     @staticmethod
     def _hash_event(event: TypeMessage) -> str:
@@ -61,15 +61,15 @@ class PortalDedup:
         if isinstance(event, MessageService):
             hash_content = [event.date.timestamp(), event.from_id, event.action]
         else:
-            hash_content = [event.date.timestamp(), event.message]
+            hash_content = [event.date.timestamp(), event.message.strip()]
             if event.fwd_from:
-                hash_content += [event.fwd_from.from_id, event.fwd_from.channel_id]
+                hash_content += [event.fwd_from.from_id]
             elif isinstance(event, Message) and event.media:
                 try:
                     hash_content += {
                         MessageMediaContact: lambda media: [media.user_id],
                         MessageMediaDocument: lambda media: [media.document.id],
-                        MessageMediaPhoto: lambda media: [media.photo.id],
+                        MessageMediaPhoto: lambda media: [media.photo.id if media.photo else 0],
                         MessageMediaGeo: lambda media: [media.geo.long, media.geo.lat],
                     }[type(event.media)](event.media)
                 except KeyError:

mautrix_telegram/portal/matrix.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
+from typing import Awaitable, Dict, Optional, Union, Any, TYPE_CHECKING
 from html import escape as escape_html
 from string import Template
 from abc import ABC
@@ -22,21 +22,20 @@ import magic
 
 from telethon.tl.functions.messages import (EditChatPhotoRequest, EditChatTitleRequest,
                                             UpdatePinnedMessageRequest, SetTypingRequest,
-                                            EditChatAboutRequest)
+                                            EditChatAboutRequest, UnpinAllMessagesRequest)
 from telethon.tl.functions.channels import EditPhotoRequest, EditTitleRequest, JoinChannelRequest
-from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError,
-                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError)
+from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError, MessageIdInvalidError,
+                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, RPCError)
 from telethon.tl.patched import Message, MessageService
-from telethon.tl.types import (
-    DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
-    InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
-    SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer, TypeMessageEntity,
-    UpdateNewMessage, InputMediaUploadedDocument, InputMediaUploadedPhoto)
+from telethon.tl.types import (DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
+                               InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
+                               SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer,
+                               UpdateNewMessage, InputMediaUploadedDocument,
+                               InputMediaUploadedPhoto)
 
 from mautrix.types import (EventID, RoomID, UserID, ContentURI, MessageType, MessageEventContent,
                            TextMessageEventContent, MediaMessageEventContent, Format,
-                           LocationMessageEventContent)
-from mautrix.bridge import BasePortal as MautrixBasePortal
+                           LocationMessageEventContent, ImageInfo, VideoInfo)
 
 from ..types import TelegramID
 from ..db import Message as DBMessage
@@ -50,12 +49,17 @@ if TYPE_CHECKING:
     from ..tgclient import MautrixTelegramClient
     from ..config import Config
 
+try:
+    from mautrix.crypto.attachments import decrypt_attachment
+except ImportError:
+    decrypt_attachment = None
+
 TypeMessage = Union[Message, MessageService]
 
 config: Optional['Config'] = None
 
 
-class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
+class PortalMatrix(BasePortal, ABC):
     async def _get_state_change_message(self, event: str, user: 'u.User', **kwargs: Any
                                         ) -> Optional[str]:
         tpl = self.get_config(f"state_event_formats.{event}")
@@ -82,9 +86,9 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             message = await self._get_state_change_message(event, user, **kwargs)
             if not message:
                 return
-            response = await self.bot.client.send_message(
-                self.peer, message,
-                parse_mode=self._matrix_event_to_entities)
+            message, entities = await formatter.matrix_to_telegram(self.bot.client, html=message)
+            response = await self.bot.client.send_message(self.peer, message,
+                                                          formatting_entities=entities)
             space = self.tgid if self.peer_type == "channel" else self.bot.tgid
             self.dedup.check(response, (event_id, space))
 
@@ -108,7 +112,19 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
         space = self.tgid if self.peer_type == "channel" else user.tgid
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
-            return
+            message = DBMessage.find_last(self.mxid, space)
+            if not message:
+                self.log.debug(f"Dropping Matrix read receipt from {user.mxid}: "
+                               f"target message {event_id} not known and last message"
+                               " in chat not found")
+                return
+            else:
+                self.log.debug(f"Matrix read receipt target {event_id} not known, marking "
+                               f"messages up to most recent ({message.mxid}/{message.tgid}) "
+                               f"as read by {user.mxid}/{user.tgid}")
+        else:
+            self.log.debug("Handling Matrix read receipt: marking messages up to "
+                           f"{message.mxid}/{message.tgid} as read by {user.mxid}/{user.tgid}")
         await user.client.send_read_acknowledge(self.peer, max_id=message.tgid,
                                                 clear_mentions=True)
 
@@ -117,7 +133,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
         if user.tgid == source.tgid:
             return None
         if self.peer_type == "user" and user.tgid == self.tgid:
-            self.delete()
+            await self.delete()
             return None
         if isinstance(user, u.User) and await user.needs_relaybot(self):
             if not self.bot:
@@ -147,7 +163,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
 
         if self.peer_type == "user":
             await self.main_intent.leave_room(self.mxid)
-            self.delete()
+            await self.delete()
             try:
                 del self.by_tgid[self.tgid_full]
                 del self.by_mxid[self.mxid]
@@ -209,52 +225,53 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
         elif content.msgtype == MessageType.EMOTE:
             await self._apply_emote_format(sender, content)
 
-    @staticmethod
-    def _matrix_event_to_entities(event: Union[str, MessageEventContent]
-                                  ) -> Tuple[str, Optional[List[TypeMessageEntity]]]:
-        try:
-            if isinstance(event, str):
-                message, entities = formatter.matrix_to_telegram(event)
-            elif isinstance(event, TextMessageEventContent) and event.format == Format.HTML:
-                message, entities = formatter.matrix_to_telegram(event.formatted_body)
-            else:
-                message, entities = formatter.matrix_text_to_telegram(event.body)
-        except KeyError:
-            message, entities = None, None
-        return message, entities
-
     async def _handle_matrix_text(self, sender_id: TelegramID, event_id: EventID,
                                   space: TelegramID, client: 'MautrixTelegramClient',
                                   content: TextMessageEventContent, reply_to: TelegramID) -> None:
+        message, entities = await formatter.matrix_to_telegram(client, text=content.body,
+                                                               html=content.formatted(Format.HTML))
         async with self.send_lock(sender_id):
             lp = self.get_config("telegram_link_preview")
             if content.get_edit():
                 orig_msg = DBMessage.get_by_mxid(content.get_edit(), self.mxid, space)
                 if orig_msg:
-                    response = await client.edit_message(self.peer, orig_msg.tgid, content,
-                                                         parse_mode=self._matrix_event_to_entities,
+                    response = await client.edit_message(self.peer, orig_msg.tgid, message,
+                                                         formatting_entities=entities,
                                                          link_preview=lp)
                     self._add_telegram_message_to_db(event_id, space, -1, response)
                     return
-            response = await client.send_message(self.peer, content, reply_to=reply_to,
-                                                 parse_mode=self._matrix_event_to_entities,
+            response = await client.send_message(self.peer, message, reply_to=reply_to,
+                                                 formatting_entities=entities,
                                                  link_preview=lp)
             self._add_telegram_message_to_db(event_id, space, 0, response)
+        await self._send_delivery_receipt(event_id)
 
     async def _handle_matrix_file(self, sender_id: TelegramID, event_id: EventID,
                                   space: TelegramID, client: 'MautrixTelegramClient',
                                   content: MediaMessageEventContent, reply_to: TelegramID,
                                   caption: TextMessageEventContent = None) -> None:
         mime = content.info.mimetype
-        w, h = content.info.width, content.info.height
+        if isinstance(content.info, (ImageInfo, VideoInfo)):
+            w, h = content.info.width, content.info.height
+        else:
+            w = h = None
         file_name = content["net.maunium.telegram.internal.filename"]
         max_image_size = config["bridge.image_as_file_size"] * 1000 ** 2
 
-        if config["bridge.parallel_file_transfer"]:
+        if config["bridge.parallel_file_transfer"] and content.url:
             file_handle, file_size = await parallel_transfer_to_telegram(client, self.main_intent,
                                                                          content.url, sender_id)
         else:
-            file = await self.main_intent.download_media(content.url)
+            if content.file:
+                if not decrypt_attachment:
+                    self.log.warning(f"Can't bridge encrypted media event {event_id}:"
+                                     " matrix-nio not installed")
+                    return
+                file = await self.main_intent.download_media(content.file.url)
+                file = decrypt_attachment(file, content.file.key.key,
+                                          content.file.hashes.get("sha256"), content.file.iv)
+            else:
+                file = await self.main_intent.download_media(content.url)
 
             if content.msgtype == MessageType.STICKER:
                 if mime != "image/gif":
@@ -279,20 +296,23 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             media = InputMediaUploadedDocument(file=file_handle, attributes=attributes,
                                                mime_type=mime or "application/octet-stream")
 
-        caption, entities = self._matrix_event_to_entities(caption) if caption else (None, None)
+        capt, entities = (await formatter.matrix_to_telegram(client, text=caption.body,
+                                                             html=caption.formatted(Format.HTML))
+                          if caption else (None, None))
 
         async with self.send_lock(sender_id):
-            if await self._matrix_document_edit(client, content, space, caption, media, event_id):
+            if await self._matrix_document_edit(client, content, space, capt, media, event_id):
                 return
             try:
                 response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption, entities=entities)
+                                                   caption=capt, entities=entities)
             except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
                 media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
                                                    attributes=attributes)
                 response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption, entities=entities)
+                                                   caption=capt, entities=entities)
             self._add_telegram_message_to_db(event_id, space, 0, response)
+        await self._send_delivery_receipt(event_id)
 
     async def _matrix_document_edit(self, client: 'MautrixTelegramClient',
                                     content: MessageEventContent, space: TelegramID,
@@ -303,6 +323,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
                 response = await client.edit_message(self.peer, orig_msg.tgid,
                                                      caption, file=media)
                 self._add_telegram_message_to_db(event_id, space, -1, response)
+                await self._send_delivery_receipt(event_id)
                 return True
         return False
 
@@ -311,13 +332,13 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
                                       content: LocationMessageEventContent, reply_to: TelegramID
                                       ) -> None:
         try:
-            lat, long = content.geo_uri[len("geo:"):].split(",")
+            lat, long = content.geo_uri[len("geo:"):].split(";")[0].split(",")
             lat, long = float(lat), float(long)
         except (KeyError, ValueError):
             self.log.exception("Failed to parse location")
             return None
-        caption, entities = self._matrix_event_to_entities(content)
-        media = MessageMediaGeo(geo=GeoPoint(lat, long, access_hash=0))
+        caption, entities = await formatter.matrix_to_telegram(client, text=content.body)
+        media = MessageMediaGeo(geo=GeoPoint(lat=lat, long=long, access_hash=0))
 
         async with self.send_lock(sender_id):
             if await self._matrix_document_edit(client, content, space, caption, media, event_id):
@@ -325,10 +346,11 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             response = await client.send_media(self.peer, media, reply_to=reply_to,
                                                caption=caption, entities=entities)
             self._add_telegram_message_to_db(event_id, space, 0, response)
+        await self._send_delivery_receipt(event_id)
 
     def _add_telegram_message_to_db(self, event_id: EventID, space: TelegramID,
                                     edit_index: int, response: TypeMessage) -> None:
-        self.log.debug("Handled Matrix message: %s", response)
+        self.log.trace("Handled Matrix message: %s", response)
         self.dedup.check(response, (event_id, space), force_hash=edit_index != 0)
         if edit_index < 0:
             prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1)
@@ -340,17 +362,27 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             mxid=event_id,
             edit_index=edit_index).insert()
 
+    async def _send_bridge_error(self, msg: str) -> None:
+        if config["bridge.delivery_error_reports"]:
+            await self._send_message(self.main_intent,
+                                     TextMessageEventContent(msgtype=MessageType.NOTICE, body=msg))
+
     async def handle_matrix_message(self, sender: 'u.User', content: MessageEventContent,
                                     event_id: EventID) -> None:
+        try:
+            await self._handle_matrix_message(sender, content, event_id)
+        except RPCError as e:
+            if config["bridge.delivery_error_reports"]:
+                await self._send_bridge_error(
+                    f"\u26a0 Your message may not have been bridged: {e}")
+            raise
+
+    async def _handle_matrix_message(self, sender: 'u.User', content: MessageEventContent,
+                                     event_id: EventID) -> None:
         if not content.body or not content.msgtype:
             self.log.debug(f"Ignoring message {event_id} in {self.mxid} without body or msgtype")
             return
 
-        puppet = p.Puppet.get_by_custom_mxid(sender.mxid)
-        if puppet and content.get("net.maunium.telegram.puppet", False):
-            self.log.debug("Ignoring puppet-sent message by confirmed puppet user %s", sender.mxid)
-            return
-
         logged_in = not await sender.needs_relaybot(self)
         client = sender.client if logged_in else self.bot.client
         sender_id = sender.tgid if logged_in else self.bot.tgid
@@ -389,35 +421,45 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             await self._handle_matrix_file(sender_id, event_id, space, client, content, reply_to,
                                            caption_content)
         else:
-            self.log.debug(f"Unhandled Matrix event: {content}")
+            self.log.debug("Didn't handle Matrix event {event_id} due to unknown msgtype {content.msgtype}")
+            self.log.trace("Unhandled Matrix event content: %s", content)
 
-    async def handle_matrix_pin(self, sender: 'u.User',
-                                pinned_message: Optional[EventID]) -> None:
-        if self.peer_type != "chat" and self.peer_type != "channel":
-            return
-        try:
-            if not pinned_message:
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=0))
-            else:
-                tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
-                message = DBMessage.get_by_mxid(pinned_message, self.mxid, tg_space)
-                if message is None:
-                    self.log.warning(f"Could not find pinned {pinned_message} in {self.mxid}")
-                    return
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=message.tgid))
-        except ChatNotModifiedError:
-            pass
+    async def handle_matrix_unpin_all(self, sender: 'u.User', pin_event_id: EventID) -> None:
+        await sender.client(UnpinAllMessagesRequest(peer=self.peer))
+        await self._send_delivery_receipt(pin_event_id)
 
-    async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID) -> None:
+    async def handle_matrix_pin(self, sender: 'u.User', changes: Dict[EventID, bool],
+                                pin_event_id: EventID) -> None:
+        tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
+        ids = {msg.mxid: msg.tgid
+               for msg in DBMessage.get_by_mxids(list(changes.keys()),
+                                                 mx_room=self.mxid, tg_space=tg_space)}
+        for event_id, pinned in changes.items():
+            try:
+                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=ids[event_id],
+                                                               unpin=not pinned))
+            except (ChatNotModifiedError, MessageIdInvalidError, KeyError):
+                pass
+        await self._send_delivery_receipt(pin_event_id)
+
+    async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID,
+                                     redaction_event_id: EventID) -> None:
         real_deleter = deleter if not await deleter.needs_relaybot(self) else self.bot
         space = self.tgid if self.peer_type == "channel" else real_deleter.tgid
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
-            return
-        if message.edit_index == 0:
-            await real_deleter.client.delete_messages(self.peer, [message.tgid])
+            self.log.trace(f"Ignoring Matrix redaction of unknown event {event_id}")
+        elif message.redacted:
+            self.log.debug("Ignoring Matrix redaction of already redacted event "
+                           f"{message.mxid} in {message.mx_room}")
+        elif message.edit_index != 0:
+            message.edit(redacted=True)
+            self.log.debug("Ignoring Matrix redaction of edit event "
+                           f"{message.mxid} in {message.mx_room}")
         else:
-            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
+            message.edit(redacted=True)
+            await real_deleter.client.delete_messages(self.peer, [message.tgid])
+            await self._send_delivery_receipt(redaction_event_id)
 
     async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
                                            level: int) -> None:
@@ -430,7 +472,8 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
                                        pin_messages=moderator, add_admins=admin)
 
     async def handle_matrix_power_levels(self, sender: 'u.User', new_users: Dict[UserID, int],
-                                         old_users: Dict[UserID, int]) -> None:
+                                         old_users: Dict[UserID, int], event_id: Optional[EventID]
+                                         ) -> None:
         # TODO handle all power level changes and bridge exact admin rights to supergroups/channels
         for user, level in new_users.items():
             if not user or user == self.main_intent.mxid or user == sender.mxid:
@@ -446,15 +489,16 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             if user not in old_users or level != old_users[user]:
                 await self._update_telegram_power_level(sender, user_id, level)
 
-    async def handle_matrix_about(self, sender: 'u.User', about: str) -> None:
+    async def handle_matrix_about(self, sender: 'u.User', about: str, event_id: EventID) -> None:
         if self.peer_type not in ("chat", "channel"):
             return
         peer = await self.get_input_entity(sender)
         await sender.client(EditChatAboutRequest(peer=peer, about=about))
         self.about = about
-        self.save()
+        await self.save()
+        await self._send_delivery_receipt(event_id)
 
-    async def handle_matrix_title(self, sender: 'u.User', title: str) -> None:
+    async def handle_matrix_title(self, sender: 'u.User', title: str, event_id: EventID) -> None:
         if self.peer_type not in ("chat", "channel"):
             return
 
@@ -465,13 +509,19 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             response = await sender.client(EditTitleRequest(channel=channel, title=title))
         self.dedup.register_outgoing_actions(response)
         self.title = title
-        self.save()
+        await self.save()
+        await self._send_delivery_receipt(event_id)
+        await self.update_bridge_info()
 
-    async def handle_matrix_avatar(self, sender: 'u.User', url: ContentURI) -> None:
+    async def handle_matrix_avatar(self, sender: 'u.User', url: ContentURI, event_id: EventID
+                                   ) -> None:
         if self.peer_type not in ("chat", "channel"):
             # Invalid peer type
             return
+        elif self.avatar_url == url:
+            return
 
+        self.avatar_url = url
         file = await self.main_intent.download_media(url)
         mime = magic.from_buffer(file, mime=True)
         ext = sane_mimetypes.guess_extension(mime)
@@ -491,10 +541,13 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             if is_photo_update:
                 loc, size = self._get_largest_photo_size(update.message.action.photo)
                 self.photo_id = f"{size.location.volume_id}-{size.location.local_id}"
-                self.save()
+                await self.save()
                 break
+        await self._send_delivery_receipt(event_id)
+        await self.update_bridge_info()
 
-    async def handle_matrix_upgrade(self, sender: UserID, new_room: RoomID) -> None:
+    async def handle_matrix_upgrade(self, sender: UserID, new_room: RoomID, event_id: EventID
+                                    ) -> None:
         _, server = self.main_intent.parse_user_id(sender)
         old_room = self.mxid
         self.migrate_and_save_matrix(new_room)
@@ -521,6 +574,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             return
         await self.update_matrix_room(user, entity, direct=self.peer_type == "user")
         self.log.info(f"{sender} upgraded room from {old_room} to {self.mxid}")
+        await self._send_delivery_receipt(event_id, room_id=old_room)
 
     def migrate_and_save_matrix(self, new_id: RoomID) -> None:
         try:
@@ -531,6 +585,16 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
         self.db_instance.edit(mxid=self.mxid)
         self.by_mxid[self.mxid] = self
 
+    async def enable_dm_encryption(self) -> bool:
+        ok = await super().enable_dm_encryption()
+        if ok:
+            try:
+                puppet = p.Puppet.get(self.tgid)
+                await self.main_intent.set_room_name(self.mxid, puppet.displayname)
+            except Exception:
+                self.log.warning(f"Failed to set room name", exc_info=True)
+        return ok
+
 
 def init(context: Context) -> None:
     global config

mautrix_telegram/portal/metadata.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Optional, Tuple, Union, Callable, TYPE_CHECKING
+from typing import List, Optional, Iterable, Union, Dict, Any, Tuple, TYPE_CHECKING
 from abc import ABC
 import asyncio
 
@@ -26,12 +26,14 @@ from telethon.tl.types import (
     Channel, ChatBannedRights, ChannelParticipantsRecent, ChannelParticipantsSearch, ChatPhoto,
     PhotoEmpty, InputChannel, InputUser, ChatPhotoEmpty, PeerUser, Photo, TypeChat, TypeInputPeer,
     TypeUser, User, InputPeerPhotoFileLocation, ChatParticipantAdmin, ChannelParticipantAdmin,
-    ChatParticipantCreator, ChannelParticipantCreator)
+    ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty,
+    InputPeerUser, ChannelParticipantBanned)
 
 from mautrix.errors import MForbidden
-from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership, Member,
-                           PowerLevelStateEventContent, RoomAlias)
-from mautrix.appservice import IntentAPI
+from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership,
+                           PowerLevelStateEventContent, RoomTopicStateEventContent,
+                           RoomNameStateEventContent, RoomAvatarStateEventContent,
+                           StateEventContent, EventID)
 
 from ..types import TelegramID
 from ..context import Context
@@ -44,6 +46,9 @@ if TYPE_CHECKING:
 
 config: Optional['Config'] = None
 
+StateBridge = EventType.find("m.bridge", EventType.Class.STATE)
+StateHalfShotBridge = EventType.find("uk.half-shot.bridge", EventType.Class.STATE)
+
 
 class PortalMetadata(BasePortal, ABC):
     _room_create_lock: asyncio.Lock
@@ -54,21 +59,30 @@ class PortalMetadata(BasePortal, ABC):
 
     # region Matrix -> Telegram
 
-    async def _get_telegram_users_in_matrix_room(self) -> List[Union[InputUser, PeerUser]]:
-        user_tgids = set()
+    async def get_telegram_users_in_matrix_room(self, source: 'u.User'
+                                                ) -> Tuple[List[InputPeerUser], List[UserID]]:
+        user_tgids = {}
         user_mxids = await self.main_intent.get_room_members(self.mxid, (Membership.JOIN,
                                                                          Membership.INVITE))
-        for user_str in user_mxids:
-            user = UserID(user_str)
-            if user == self.az.bot_mxid:
+        for mxid in user_mxids:
+            if mxid == self.az.bot_mxid:
                 continue
-            mx_user = u.User.get_by_mxid(user, create=False)
+            mx_user = u.User.get_by_mxid(mxid, create=False)
             if mx_user and mx_user.tgid:
-                user_tgids.add(mx_user.tgid)
-            puppet_id = p.Puppet.get_id_from_mxid(user)
+                user_tgids[mx_user.tgid] = mxid
+            puppet_id = p.Puppet.get_id_from_mxid(mxid)
             if puppet_id:
-                user_tgids.add(puppet_id)
-        return [PeerUser(user_id) for user_id in user_tgids]
+                user_tgids[puppet_id] = mxid
+        input_users = []
+        errors = []
+        for tgid, mxid in user_tgids.items():
+            try:
+                input_users.append(await source.client.get_input_entity(tgid))
+            except ValueError as e:
+                source.log.debug(f"Failed to find the input entity for {tgid} ({mxid}) for "
+                                 f"creating a group: {e}")
+                errors.append(mxid)
+        return input_users, errors
 
     async def upgrade_telegram_chat(self, source: 'u.User') -> None:
         if self.peer_type != "chat":
@@ -93,7 +107,7 @@ class PortalMetadata(BasePortal, ABC):
             pass
         try:
             existing = self.by_tgid[(new_id, new_id)]
-            existing.delete()
+            existing.delete_sync()
         except KeyError:
             pass
         self.db_instance.edit(tgid=new_id, tg_receiver=new_id, peer_type=self.peer_type)
@@ -110,15 +124,15 @@ class PortalMetadata(BasePortal, ABC):
         await source.client(
             UpdateUsernameRequest(await self.get_input_entity(source), username))
         if await self._update_username(username):
-            self.save()
+            await self.save()
 
-    async def create_telegram_chat(self, source: 'u.User', supergroup: bool = False) -> None:
+    async def create_telegram_chat(self, source: 'u.User', invites: List[InputUser],
+                                   supergroup: bool = False) -> None:
         if not self.mxid:
             raise ValueError("Can't create Telegram chat for portal without Matrix room.")
         elif self.tgid:
             raise ValueError("Can't create Telegram chat for portal with existing Telegram chat.")
 
-        invites = await self._get_telegram_users_in_matrix_room()
         if len(invites) < 2:
             if self.bot is not None:
                 info, mxid = await self.bot.get_me()
@@ -155,7 +169,8 @@ class PortalMetadata(BasePortal, ABC):
         if levels.get_user_level(self.main_intent.mxid) == 100:
             levels = self._get_base_power_levels(levels, entity)
             await self.main_intent.set_power_levels(self.mxid, levels)
-        await self.handle_matrix_power_levels(source, levels.users, {})
+        await self.handle_matrix_power_levels(source, levels.users, {}, None)
+        await self.update_bridge_info()
 
     async def invite_telegram(self, source: 'u.User',
                               puppet: Union[p.Puppet, 'AbstractUser']) -> None:
@@ -168,62 +183,81 @@ class PortalMetadata(BasePortal, ABC):
         elif not self.bot or self.tg_receiver != self.bot.tgid:
             raise ValueError("Invalid peer type for Telegram user invite")
 
-    async def sync_matrix_members(self) -> None:
-        resp = await self.main_intent.get_room_joined_memberships(self.mxid)
-        members = resp["joined"]
-        for mxid, info in members.items():
-            member = Member(membership=Membership.JOIN)
-            if "display_name" in info:
-                member.displayname = info["display_name"]
-            if "avatar_url" in info:
-                member.avatar_url = info["avatar_url"]
-            self.az.state_store.set_member(self.mxid, mxid, member)
-
     # endregion
     # region Telegram -> Matrix
 
+    def _get_invite_content(self, double_puppet: Optional['p.Puppet']) -> Dict[str, Any]:
+        invite_content = {}
+        if double_puppet:
+            invite_content["fi.mau.will_auto_accept"] = True
+        if self.is_direct:
+            invite_content["is_direct"] = True
+        return invite_content
+
     async def invite_to_matrix(self, users: InviteList) -> None:
         if isinstance(users, list):
             for user in users:
-                await self.main_intent.invite_user(self.mxid, user, check_cache=True)
+                await self.invite_to_matrix(user)
         else:
-            await self.main_intent.invite_user(self.mxid, users, check_cache=True)
+            puppet = await p.Puppet.get_by_custom_mxid(users)
+            await self.main_intent.invite_user(self.mxid, users, check_cache=True,
+                                               extra_content=self._get_invite_content(puppet))
+            if puppet:
+                try:
+                    await puppet.intent.ensure_joined(self.mxid)
+                except Exception:
+                    self.log.exception("Failed to ensure %s is joined to portal", users)
 
     async def update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
                                  direct: bool = None, puppet: p.Puppet = None,
                                  levels: PowerLevelStateEventContent = None,
-                                 users: List[User] = None,
-                                 participants: List[TypeParticipant] = None) -> None:
+                                 users: List[User] = None) -> None:
         if direct is None:
             direct = self.peer_type == "user"
         try:
-            await self._update_matrix_room(user, entity, direct, puppet, levels, users,
-                                           participants)
+            await self._update_matrix_room(user, entity, direct, puppet, levels, users)
         except Exception:
             self.log.exception("Fatal error updating Matrix room")
 
     async def _update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
                                   direct: bool, puppet: p.Puppet = None,
                                   levels: PowerLevelStateEventContent = None,
-                                  users: List[User] = None,
-                                  participants: List[TypeParticipant] = None) -> None:
+                                  users: List[User] = None) -> None:
         if not direct:
             await self.update_info(user, entity)
-            if not users or not participants:
-                users, participants = await self._get_users(user, entity)
+            if not users:
+                users = await self._get_users(user, entity)
             await self._sync_telegram_users(user, users)
-            await self.update_telegram_participants(participants, levels)
+            await self.update_power_levels(users, levels)
         else:
             if not puppet:
                 puppet = p.Puppet.get(self.tgid)
             await puppet.update_info(user, entity)
             await puppet.intent_for(self).join_room(self.mxid)
-        if self.sync_matrix_state:
-            await self.sync_matrix_members()
+            if self.encrypted or self.private_chat_portal_meta:
+                # The bridge bot needs to join for e2ee, but that messes up the default name
+                # generation. If/when canonical DMs happen, this might not be necessary anymore.
+                changed = await self._update_title(puppet.displayname)
+                changed = await self._update_avatar(user, entity.photo) or changed
+                if changed:
+                    await self.save()
+                    await self.update_bridge_info()
 
-    async def create_matrix_room(self, user: 'AbstractUser', entity: TypeChat = None,
-                                 invites: InviteList = None, update_if_exists: bool = True,
-                                 synchronous: bool = False) -> Optional[str]:
+            puppet = await p.Puppet.get_by_custom_mxid(user.mxid)
+            if puppet:
+                try:
+                    did_join = await puppet.intent.ensure_joined(self.mxid)
+                    if isinstance(user, u.User) and did_join and self.peer_type == "user":
+                        await user.update_direct_chats({self.main_intent.mxid: [self.mxid]})
+                except Exception:
+                    self.log.exception("Failed to ensure %s is joined to portal", user.mxid)
+
+        if self.sync_matrix_state:
+            await self.main_intent.get_joined_members(self.mxid)
+
+    async def create_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User] = None,
+                                 invites: InviteList = None, update_if_exists: bool = True
+                                 ) -> Optional[RoomID]:
         if self.mxid:
             if update_if_exists:
                 if not entity:
@@ -233,10 +267,7 @@ class PortalMetadata(BasePortal, ABC):
                         self.log.exception(f"Failed to get entity through {user.tgid} for update")
                         return self.mxid
                 update = self.update_matrix_room(user, entity, self.peer_type == "user")
-                if synchronous:
-                    await update
-                else:
-                    asyncio.ensure_future(update, loop=self.loop)
+                self.loop.create_task(update)
                 await self.invite_to_matrix(invites or [])
             return self.mxid
         async with self._room_create_lock:
@@ -245,19 +276,62 @@ class PortalMetadata(BasePortal, ABC):
             except Exception:
                 self.log.exception("Fatal error creating Matrix room")
 
-    async def _create_matrix_room(self, user: 'AbstractUser', entity: TypeChat, invites: InviteList
-                                  ) -> Optional[RoomID]:
-        direct = self.peer_type == "user"
+    @property
+    def bridge_info_state_key(self) -> str:
+        return f"net.maunium.telegram://telegram/{self.tgid}"
 
+    @property
+    def bridge_info(self) -> Dict[str, Any]:
+        info = {
+            "bridgebot": self.az.bot_mxid,
+            "creator": self.main_intent.mxid,
+            "protocol": {
+                "id": "telegram",
+                "displayname": "Telegram",
+                "avatar_url": config["appservice.bot_avatar"],
+                "external_url": "https://telegram.org",
+            },
+            "channel": {
+                "id": str(self.tgid),
+                "displayname": self.title,
+                "avatar_url": self.avatar_url,
+            }
+        }
+        if self.username:
+            info["channel"]["external_url"] = f"https://t.me/{self.username}"
+        elif self.peer_type == "user":
+            puppet = p.Puppet.get(self.tgid)
+            if puppet and puppet.username:
+                info["channel"]["external_url"] = f"https://t.me/{puppet.username}"
+        return info
+
+    async def update_bridge_info(self) -> None:
+        if not self.mxid:
+            self.log.debug("Not updating bridge info: no Matrix room created")
+            return
+        try:
+            self.log.debug("Updating bridge info...")
+            await self.main_intent.send_state_event(self.mxid, StateBridge,
+                                                    self.bridge_info, self.bridge_info_state_key)
+            # TODO remove this once https://github.com/matrix-org/matrix-doc/pull/2346 is in spec
+            await self.main_intent.send_state_event(self.mxid, StateHalfShotBridge,
+                                                    self.bridge_info, self.bridge_info_state_key)
+        except Exception:
+            self.log.warning("Failed to update bridge info", exc_info=True)
+
+    async def _create_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
+                                  invites: InviteList) -> Optional[RoomID]:
         if self.mxid:
             return self.mxid
-
-        if not self.allow_bridging:
+        elif not self.allow_bridging:
             return None
 
+        direct = self.peer_type == "user"
+        invites = invites or []
+
         if not entity:
             entity = await self.get_entity(user)
-            self.log.debug(f"Fetched data: {entity}")
+            self.log.trace("Fetched data: %s", entity)
 
         self.log.debug("Creating room")
 
@@ -271,17 +345,20 @@ class PortalMetadata(BasePortal, ABC):
             self.about = "Your Telegram cloud storage chat"
 
         puppet = p.Puppet.get(self.tgid) if direct else None
+        if puppet:
+            await puppet.update_info(user, entity)
         self._main_intent = puppet.intent_for(self) if direct else self.az.intent
 
         if self.peer_type == "channel":
             self.megagroup = entity.megagroup
 
+        preset = RoomCreatePreset.PRIVATE
         if self.peer_type == "channel" and entity.username:
-            preset = RoomCreatePreset.PUBLIC
+            if self.public_portals:
+                preset = RoomCreatePreset.PUBLIC
             self.username = entity.username
             alias = self.alias_localpart
         else:
-            preset = RoomCreatePreset.PRIVATE
             # TODO invite link alias?
             alias = None
 
@@ -290,24 +367,45 @@ class PortalMetadata(BasePortal, ABC):
             await self.main_intent.remove_room_alias(alias)
 
         power_levels = self._get_base_power_levels(entity=entity)
-        users = participants = None
+        users = None
         if not direct:
-            users, participants = await self._get_users(user, entity)
+            users = await self._get_users(user, entity)
             if self.has_bot:
                 extra_invites = config["bridge.relaybot.group_chat_invite"]
                 invites += extra_invites
                 for invite in extra_invites:
                     power_levels.users.setdefault(invite, 100)
-            self._participants_to_power_levels(participants, power_levels)
+            await self._participants_to_power_levels(users, power_levels)
         elif self.bot and self.tg_receiver == self.bot.tgid:
             invites = config["bridge.relaybot.private_chat.invite"]
             for invite in invites:
                 power_levels.users.setdefault(invite, 100)
             self.title = puppet.displayname
+
         initial_state = [{
             "type": EventType.ROOM_POWER_LEVELS.serialize(),
             "content": power_levels.serialize(),
+        }, {
+            "type": str(StateBridge),
+            "state_key": self.bridge_info_state_key,
+            "content": self.bridge_info,
+        }, {
+            # TODO remove this once https://github.com/matrix-org/matrix-doc/pull/2346 is in spec
+            "type": str(StateHalfShotBridge),
+            "state_key": self.bridge_info_state_key,
+            "content": self.bridge_info,
         }]
+        create_invites = []
+        if config["bridge.encryption.default"] and self.matrix.e2ee:
+            self.encrypted = True
+            initial_state.append({
+                "type": "m.room.encryption",
+                "content": {"algorithm": "m.megolm.v1.aes-sha2"},
+            })
+            if direct:
+                create_invites.append(self.az.bot_mxid)
+        if direct and (self.encrypted or self.private_chat_portal_meta):
+            self.title = puppet.displayname
         if config["appservice.community_id"]:
             initial_state.append({
                 "type": "m.room.related_groups",
@@ -317,22 +415,42 @@ class PortalMetadata(BasePortal, ABC):
         if not config["bridge.federate_rooms"]:
             creation_content["m.federate"] = False
 
-        room_id = await self.main_intent.create_room(alias_localpart=alias, preset=preset,
-                                                     is_direct=direct, invitees=invites or [],
-                                                     name=self.title, topic=self.about,
-                                                     initial_state=initial_state,
-                                                     creation_content=creation_content)
-        if not room_id:
-            raise Exception(f"Failed to create room")
+        with self.backfill_lock:
+            room_id = await self.main_intent.create_room(alias_localpart=alias, preset=preset,
+                                                         is_direct=direct, invitees=create_invites,
+                                                         name=self.title, topic=self.about,
+                                                         initial_state=initial_state,
+                                                         creation_content=creation_content)
+            if not room_id:
+                raise Exception(f"Failed to create room")
 
-        self.mxid = RoomID(room_id)
-        self.by_mxid[self.mxid] = self
-        self.save()
-        self.az.state_store.set_power_levels(self.mxid, power_levels)
-        user.register_portal(self)
-        asyncio.ensure_future(self.update_matrix_room(user, entity, direct, puppet,
-                                                      levels=power_levels, users=users,
-                                                      participants=participants), loop=self.loop)
+            if self.encrypted and self.matrix.e2ee and direct:
+                try:
+                    await self.az.intent.ensure_joined(room_id)
+                except Exception:
+                    self.log.warning(f"Failed to add bridge bot to new private chat {room_id}")
+
+            self.mxid = room_id
+            self.by_mxid[self.mxid] = self
+            await self.save()
+            await self.az.state_store.set_power_levels(self.mxid, power_levels)
+            await user.register_portal(self)
+
+            await self.invite_to_matrix(invites)
+
+            update_room = self.loop.create_task(self.update_matrix_room(
+                user, entity, direct, puppet,
+                levels=power_levels, users=users))
+
+            if config["bridge.backfill.initial_limit"] > 0:
+                self.log.debug("Initial backfill is enabled. Waiting for room members to sync "
+                               "and then starting backfill")
+                await update_room
+
+                try:
+                    await self.backfill(user, is_initial=True)
+                except Exception:
+                    self.log.exception("Failed to backfill new portal")
 
         return self.mxid
 
@@ -362,7 +480,7 @@ class PortalMetadata(BasePortal, ABC):
             levels.kick = overrides.get("kick", 50)
             levels.redact = overrides.get("redact", 50)
             levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
-            levels.events[EventType.ROOM_ENCRYPTED] = 99
+            levels.events[EventType.ROOM_ENCRYPTION] = 50 if self.matrix.e2ee else 99
             levels.events[EventType.ROOM_TOMBSTONE] = 99
             levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
             levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0
@@ -385,14 +503,15 @@ class PortalMetadata(BasePortal, ABC):
             levels.users[self.main_intent.mxid] = 100
         return levels
 
-    @staticmethod
-    def _get_level_from_participant(participant: TypeParticipant) -> int:
+    @classmethod
+    def _get_level_from_participant(cls, participant: TypeParticipant,
+                                    levels: PowerLevelStateEventContent) -> int:
         # TODO use the power level requirements to get better precision in channels
         if isinstance(participant, (ChatParticipantAdmin, ChannelParticipantAdmin)):
-            return 50
+            return levels.state_default or 50
         elif isinstance(participant, (ChatParticipantCreator, ChannelParticipantCreator)):
-            return 95
-        return 0
+            return levels.get_user_level(cls.az.bot_mxid) - 5
+        return levels.users_default or 0
 
     @staticmethod
     def _participant_to_power_levels(levels: PowerLevelStateEventContent,
@@ -405,24 +524,28 @@ class PortalMetadata(BasePortal, ABC):
             return True
         return False
 
-    def _participants_to_power_levels(self, participants: List[TypeParticipant],
-                                      levels: PowerLevelStateEventContent) -> bool:
+    async def _participants_to_power_levels(self, users: List[Union[TypeUser, TypeParticipant]],
+                                            levels: PowerLevelStateEventContent) -> bool:
         bot_level = levels.get_user_level(self.main_intent.mxid)
         if bot_level < levels.get_event_level(EventType.ROOM_POWER_LEVELS):
             return False
         changed = False
         admin_power_level = min(75 if self.peer_type == "channel" else 50, bot_level)
-        if levels.events[EventType.ROOM_POWER_LEVELS] != admin_power_level:
+        if levels.get_event_level(EventType.ROOM_POWER_LEVELS) != admin_power_level:
             changed = True
             levels.events[EventType.ROOM_POWER_LEVELS] = admin_power_level
 
-        for participant in participants:
+        for user in users:
+            # The User objects we get from TelegramClient.get_participants have a custom
+            # participant property
+            participant = getattr(user, "participant", user)
+
             puppet = p.Puppet.get(TelegramID(participant.user_id))
             user = u.User.get_by_tgid(TelegramID(participant.user_id))
-            new_level = self._get_level_from_participant(participant)
+            new_level = self._get_level_from_participant(participant, levels)
 
             if user:
-                user.register_portal(self)
+                await user.register_portal(self)
                 changed = self._participant_to_power_levels(levels, user, new_level,
                                                             bot_level) or changed
 
@@ -431,34 +554,36 @@ class PortalMetadata(BasePortal, ABC):
                                                             bot_level) or changed
         return changed
 
-    async def update_telegram_participants(self, participants: List[TypeParticipant],
-                                           levels: PowerLevelStateEventContent = None) -> None:
+    async def update_power_levels(self, users: List[Union[TypeUser, TypeParticipant]],
+                                  levels: PowerLevelStateEventContent = None) -> None:
         if not levels:
             levels = await self.main_intent.get_power_levels(self.mxid)
-        if self._participants_to_power_levels(participants, levels):
+        if await self._participants_to_power_levels(users, levels):
             await self.main_intent.set_power_levels(self.mxid, levels)
 
-    def _add_bot_chat(self, bot: User) -> None:
+    async def _add_bot_chat(self, bot: User) -> None:
         if self.bot and bot.id == self.bot.tgid:
             self.bot.add_chat(self.tgid, self.peer_type)
             return
 
         user = u.User.get_by_tgid(TelegramID(bot.id))
         if user and user.is_bot:
-            user.register_portal(self)
+            await user.register_portal(self)
 
     async def _sync_telegram_users(self, source: 'AbstractUser', users: List[User]) -> None:
         allowed_tgids = set()
         skip_deleted = config["bridge.skip_deleted_members"]
         for entity in users:
-            if skip_deleted and entity.deleted:
-                continue
             puppet = p.Puppet.get(TelegramID(entity.id))
             if entity.bot:
-                self._add_bot_chat(entity)
+                await self._add_bot_chat(entity)
             allowed_tgids.add(entity.id)
-            await puppet.intent_for(self).ensure_joined(self.mxid)
+
             await puppet.update_info(source, entity)
+            if skip_deleted and entity.deleted:
+                continue
+
+            await puppet.intent_for(self).ensure_joined(self.mxid)
 
             user = u.User.get_by_tgid(TelegramID(entity.id))
             if user:
@@ -468,35 +593,42 @@ class PortalMetadata(BasePortal, ABC):
         #  * There are close to 10 000 users, because Telegram might not be sending all members.
         #  * The member sync count is limited, because then we might ignore some members.
         #  * It's a channel, because non-admins don't have access to the member list.
-        trust_member_list = (len(allowed_tgids) < 9900
-                             and self.max_initial_member_sync == -1
+        trust_member_list = ((len(allowed_tgids) < 9900
+                              if self.max_initial_member_sync < 0
+                              else len(allowed_tgids) < self.max_initial_member_sync - 10)
                              and (self.megagroup or self.peer_type != "channel"))
-        if trust_member_list:
-            joined_mxids = await self.main_intent.get_room_members(self.mxid)
-            for user_mxid in joined_mxids:
-                if user_mxid == self.az.bot_mxid:
-                    continue
-                puppet_id = p.Puppet.get_id_from_mxid(user_mxid)
-                if puppet_id and puppet_id not in allowed_tgids:
-                    if self.bot and puppet_id == self.bot.tgid:
-                        self.bot.remove_chat(self.tgid)
-                    try:
-                        await self.main_intent.kick_user(self.mxid, user_mxid,
-                                                         "User had left this Telegram chat.")
-                    except MForbidden:
-                        pass
-                    continue
-                mx_user = u.User.get_by_mxid(user_mxid, create=False)
-                if mx_user and mx_user.is_bot and mx_user.tgid not in allowed_tgids:
-                    mx_user.unregister_portal(self)
+        if not trust_member_list:
+            return
 
-                if mx_user and not self.has_bot and mx_user.tgid not in allowed_tgids:
+        for user_mxid in await self.main_intent.get_room_members(self.mxid):
+            if user_mxid == self.az.bot_mxid:
+                continue
+
+            puppet_id = p.Puppet.get_id_from_mxid(user_mxid)
+            if puppet_id:
+                if puppet_id in allowed_tgids:
+                    continue
+                if self.bot and puppet_id == self.bot.tgid:
+                    self.bot.remove_chat(self.tgid)
+                try:
+                    await self.main_intent.kick_user(self.mxid, user_mxid,
+                                                     "User had left this Telegram chat.")
+                except MForbidden:
+                    pass
+                continue
+
+            mx_user = u.User.get_by_mxid(user_mxid, create=False)
+            if mx_user:
+                if mx_user.tgid in allowed_tgids:
+                    continue
+                if mx_user.is_bot:
+                    await mx_user.unregister_portal(*self.tgid_full)
+                if not self.has_bot:
                     try:
                         await self.main_intent.kick_user(self.mxid, mx_user.mxid,
                                                          "You had left this Telegram chat.")
                     except MForbidden:
                         pass
-                    continue
 
     async def _add_telegram_user(self, user_id: TelegramID, source: Optional['AbstractUser'] = None
                                  ) -> None:
@@ -508,7 +640,7 @@ class PortalMetadata(BasePortal, ABC):
 
         user = u.User.get_by_tgid(user_id)
         if user:
-            user.register_portal(self)
+            await user.register_portal(self)
             await self.invite_to_matrix(user.mxid)
 
     async def _delete_telegram_user(self, user_id: TelegramID, sender: p.Puppet) -> None:
@@ -525,7 +657,7 @@ class PortalMetadata(BasePortal, ABC):
         else:
             await puppet.intent_for(self).leave_room(self.mxid)
         if user:
-            user.unregister_portal(self)
+            await user.unregister_portal(*self.tgid_full)
             if sender.tgid != puppet.tgid:
                 try:
                     await sender.intent_for(self).kick_user(self.mxid, puppet.mxid)
@@ -542,12 +674,12 @@ class PortalMetadata(BasePortal, ABC):
             self.log.warning("Called update_info() for direct chat portal")
             return
 
+        changed = False
         self.log.debug("Updating info")
         try:
             if not entity:
                 entity = await self.get_entity(user)
-                self.log.debug(f"Fetched data: {entity}")
-            changed = False
+                self.log.trace("Fetched data: %s", entity)
 
             if self.peer_type == "channel":
                 changed = self.megagroup != entity.megagroup or changed
@@ -565,7 +697,8 @@ class PortalMetadata(BasePortal, ABC):
             self.log.exception(f"Failed to update info from source {user.tgid}")
 
         if changed:
-            self.save()
+            await self.save()
+            await self.update_bridge_info()
 
     async def _update_username(self, username: str, save: bool = False) -> bool:
         if self.username == username:
@@ -582,18 +715,21 @@ class PortalMetadata(BasePortal, ABC):
             await self.main_intent.set_join_rule(self.mxid, "invite")
 
         if save:
-            self.save()
+            await self.save()
         return True
 
-    async def _try_use_intent(self, sender: Optional['p.Puppet'],
-                              action: Callable[[IntentAPI], None]) -> None:
+    async def _try_set_state(self, sender: Optional['p.Puppet'], evt_type: EventType,
+                             content: StateEventContent) -> None:
         if sender:
             try:
-                await action(sender.intent_for(self))
+                intent = sender.intent_for(self)
+                if sender.is_real_user:
+                    content[self.az.real_user_content_key] = True
+                await intent.send_state_event(self.mxid, evt_type, content)
             except MForbidden:
-                await action(self.main_intent)
+                await self.main_intent.send_state_event(self.mxid, evt_type, content)
         else:
-            await action(self.main_intent)
+            await self.main_intent.send_state_event(self.mxid, evt_type, content)
 
     async def _update_about(self, about: str, sender: Optional['p.Puppet'] = None,
                             save: bool = False) -> bool:
@@ -601,10 +737,10 @@ class PortalMetadata(BasePortal, ABC):
             return False
 
         self.about = about
-        await self._try_use_intent(sender,
-                                   lambda intent: intent.set_room_topic(self.mxid, self.about))
+        await self._try_set_state(sender, EventType.ROOM_TOPIC,
+                                  RoomTopicStateEventContent(topic=self.about))
         if save:
-            self.save()
+            await self.save()
         return True
 
     async def _update_title(self, title: str, sender: Optional['p.Puppet'] = None,
@@ -613,95 +749,125 @@ class PortalMetadata(BasePortal, ABC):
             return False
 
         self.title = title
-        await self._try_use_intent(sender,
-                                   lambda intent: intent.set_room_name(self.mxid, self.title))
+        await self._try_set_state(sender, EventType.ROOM_NAME,
+                                  RoomNameStateEventContent(name=self.title))
         if save:
-            self.save()
+            await self.save()
         return True
 
     async def _update_avatar(self, user: 'AbstractUser', photo: TypeChatPhoto,
                              sender: Optional['p.Puppet'] = None, save: bool = False) -> bool:
-        if isinstance(photo, ChatPhoto):
+        if isinstance(photo, (ChatPhoto, UserProfilePhoto)):
             loc = InputPeerPhotoFileLocation(
                 peer=await self.get_input_entity(user),
-                local_id=photo.photo_big.local_id,
-                volume_id=photo.photo_big.volume_id,
+                photo_id=photo.photo_id,
                 big=True
             )
-            photo_id = f"{loc.volume_id}-{loc.local_id}"
+            photo_id = str(photo.photo_id)
         elif isinstance(photo, Photo):
-            loc, largest = self._get_largest_photo_size(photo)
-            photo_id = f"{largest.location.volume_id}-{largest.location.local_id}"
-        elif isinstance(photo, (ChatPhotoEmpty, PhotoEmpty)):
+            loc, _ = self._get_largest_photo_size(photo)
+            photo_id = str(loc.id)
+        elif isinstance(photo, (UserProfilePhotoEmpty, ChatPhotoEmpty, PhotoEmpty, type(None))):
             photo_id = ""
             loc = None
         else:
             raise ValueError(f"Unknown photo type {type(photo)}")
+        if self.peer_type == "user" and not photo_id and not config["bridge.allow_avatar_remove"]:
+            return False
         if self.photo_id != photo_id:
             if not photo_id:
-                await self._try_use_intent(sender,
-                                           lambda intent: intent.set_room_avatar(self.mxid, None))
+                await self._try_set_state(sender, EventType.ROOM_AVATAR,
+                                          RoomAvatarStateEventContent(url=None))
                 self.photo_id = ""
+                self.avatar_url = None
                 if save:
-                    self.save()
+                    await self.save()
                 return True
             file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
             if file:
-                await self._try_use_intent(sender, lambda intent: intent.set_room_avatar(self.mxid,
-                                                                                         file.mxc))
+                await self._try_set_state(sender, EventType.ROOM_AVATAR,
+                                          RoomAvatarStateEventContent(url=file.mxc))
                 self.photo_id = photo_id
+                self.avatar_url = file.mxc
                 if save:
-                    self.save()
+                    await self.save()
                 return True
         return False
 
+    @staticmethod
+    def _filter_participants(users: List[TypeUser], participants: List[TypeParticipant]
+                             ) -> Iterable[TypeUser]:
+        participant_map = {part.user_id: part for part in participants
+                           if not isinstance(part, ChannelParticipantBanned)}
+        for user in users:
+            try:
+                user.participant = participant_map[user.id]
+            except KeyError:
+                pass
+            else:
+                yield user
+
+    async def _get_channel_users(self, user: 'AbstractUser', entity: InputChannel, limit: int
+                                 ) -> List[TypeUser]:
+        if 0 < limit <= 200:
+            response = await user.client(GetParticipantsRequest(
+                entity, ChannelParticipantsRecent(), offset=0, limit=limit, hash=0))
+            return list(self._filter_participants(response.users, response.participants))
+        elif limit > 200 or limit == -1:
+            users: List[TypeUser] = []
+            offset = 0
+            remaining_quota = limit if limit > 0 else 1000000
+            query = (ChannelParticipantsSearch("") if limit == -1
+                     else ChannelParticipantsRecent())
+            while True:
+                if remaining_quota <= 0:
+                    break
+                response = await user.client(GetParticipantsRequest(
+                    entity, query, offset=offset, limit=min(remaining_quota, 200), hash=0))
+                if not response.users:
+                    break
+                users += self._filter_participants(response.users, response.participants)
+                offset += len(response.participants)
+                remaining_quota -= len(response.participants)
+            return users
+
     async def _get_users(self, user: 'AbstractUser',
                          entity: Union[TypeInputPeer, InputUser, TypeChat, TypeUser, InputChannel]
-                         ) -> Tuple[List[TypeUser], List[TypeParticipant]]:
-        # TODO replace with client.get_participants
+                         ) -> List[TypeUser]:
+        limit = self.max_initial_member_sync
         if self.peer_type == "chat":
             chat = await user.client(GetFullChatRequest(chat_id=self.tgid))
-            return chat.users, chat.full_chat.participants.participants
+            return list(
+                self._filter_participants(chat.users, chat.full_chat.participants.participants)
+            )[:limit]
         elif self.peer_type == "channel":
             if not self.megagroup and not self.sync_channel_members:
-                return [], []
+                return []
 
-            limit = self.max_initial_member_sync
             if limit == 0:
-                return [], []
+                return []
 
             try:
-                if 0 < limit <= 200:
-                    response = await user.client(GetParticipantsRequest(
-                        entity, ChannelParticipantsRecent(), offset=0, limit=limit, hash=0))
-                    return response.users, response.participants
-                elif limit > 200 or limit == -1:
-                    users: List[TypeUser] = []
-                    participants: List[TypeParticipant] = []
-                    offset = 0
-                    remaining_quota = limit if limit > 0 else 1000000
-                    query = (ChannelParticipantsSearch("") if limit == -1
-                             else ChannelParticipantsRecent())
-                    while True:
-                        if remaining_quota <= 0:
-                            break
-                        response = await user.client(GetParticipantsRequest(
-                            entity, query, offset=offset, limit=min(remaining_quota, 100), hash=0))
-                        if not response.users:
-                            break
-                        participants += response.participants
-                        users += response.users
-                        offset += len(response.participants)
-                        remaining_quota -= len(response.participants)
-                    return users, participants
+                return await self._get_channel_users(user, entity, limit)
             except ChatAdminRequiredError:
-                return [], []
+                return []
         elif self.peer_type == "user":
-            return [entity], []
-        return [], []
+            return [entity]
+        else:
+            raise RuntimeError(f"Unexpected peer type {self.peer_type}")
 
     # endregion
 
+    async def _send_delivery_receipt(self, event_id: EventID, room_id: Optional[RoomID] = None
+                                     ) -> None:
+        # TODO maybe check if the bot is in the room rather than assuming based on self.encrypted
+        if event_id and config["bridge.delivery_receipts"] and (self.encrypted
+                                                                or self.peer_type != "user"):
+            try:
+                await self.az.intent.mark_read(room_id or self.mxid, event_id)
+            except Exception:
+                self.log.exception("Failed to send delivery receipt for %s", event_id)
+
 
 def init(context: Context) -> None:
     global config

mautrix_telegram/portal/telegram.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,13 +14,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Awaitable, Dict, List, Optional, Tuple, Union, NamedTuple, TYPE_CHECKING
-from html import escape as escape_html
 from abc import ABC
 import random
 import mimetypes
 import codecs
 import unicodedata
 import base64
+import asyncio
 
 from sqlalchemy.exc import IntegrityError
 
@@ -30,21 +30,24 @@ from telethon.tl.types import (
     MessageMediaPoll, MessageActionChannelCreate, MessageActionChatAddUser,
     MessageActionChatCreate, MessageActionChatDeletePhoto, MessageActionChatDeleteUser,
     MessageActionChatEditPhoto, MessageActionChatEditTitle, MessageActionChatJoinedByLink,
-    MessageActionChatMigrateTo, MessageActionPinMessage, MessageActionGameScore,
-    MessageMediaDocument, MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported,
-    MessageMediaGame, PeerUser, PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant,
-    TypeDocumentAttribute, TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping,
-    UpdateUserTyping, MessageEntityPre, ChatPhotoEmpty)
+    MessageActionChatMigrateTo, MessageActionGameScore, MessageMediaDocument, MessageMediaGeo,
+    MessageMediaPhoto, MessageMediaDice, MessageMediaGame, MessageMediaUnsupported, PeerUser,
+    PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant, TypeDocumentAttribute,
+    TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping, UpdateUserTyping,
+    MessageEntityPre, ChatPhotoEmpty, DocumentAttributeImageSize, DocumentAttributeAnimated,
+    UpdateChannelUserTyping, SendMessageTypingAction)
 
 from mautrix.appservice import IntentAPI
 from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType,
                            EventType, MediaMessageEventContent, TextMessageEventContent,
                            LocationMessageEventContent, Format)
+from mautrix.bridge import NotificationDisabler
 
 from ..types import TelegramID
 from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
 from ..util import sane_mimetypes
 from ..context import Context
+from ..tgclient import TelegramClient
 from .. import puppet as p, user as u, formatter, util
 from .base import BasePortal
 
@@ -54,16 +57,20 @@ if TYPE_CHECKING:
 
 InviteList = Union[UserID, List[UserID]]
 TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
+UpdateTyping = Union[UpdateUserTyping, UpdateChatUserTyping, UpdateChannelUserTyping]
 DocAttrs = NamedTuple("DocAttrs", name=Optional[str], mime_type=Optional[str], is_sticker=bool,
-                      sticker_alt=Optional[str], width=int, height=int)
+                      sticker_alt=Optional[str], width=int, height=int, is_gif=bool)
 
 config: Optional['Config'] = None
 
 
 class PortalTelegram(BasePortal, ABC):
-    async def handle_telegram_typing(self, user: p.Puppet,
-                                     _: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
-        await user.intent_for(self).set_typing(self.mxid, is_typing=True)
+    async def handle_telegram_typing(self, user: p.Puppet, update: UpdateTyping) -> None:
+        if user.is_real_user:
+            # Ignore typing notifications from double puppeted users to avoid echoing
+            return
+        is_typing = isinstance(update.action, SendMessageTypingAction)
+        await user.default_mxid_intent.set_typing(self.mxid, is_typing=is_typing)
 
     def _get_external_url(self, evt: Message) -> Optional[str]:
         if self.peer_type == "channel" and self.username is not None:
@@ -72,41 +79,67 @@ class PortalTelegram(BasePortal, ABC):
             return f"https://t.me/c/{self.tgid}/{evt.id}"
         return None
 
+    async def _expire_telegram_photo(self, intent: IntentAPI, event_id: EventID, ttl: int) -> None:
+        try:
+            content = TextMessageEventContent(msgtype=MessageType.NOTICE, body="Photo has expired")
+            content.set_edit(event_id)
+            await asyncio.sleep(ttl)
+            await self._send_message(intent, content)
+        except Exception:
+            self.log.warning("Failed to expire Telegram photo %s", event_id, exc_info=True)
+
     async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
-                                    relates_to: Dict = None) -> Optional[EventID]:
-        loc, largest_size = self._get_largest_photo_size(evt.media.photo)
-        file = await util.transfer_file_to_matrix(source.client, intent, loc)
+                                    relates_to: RelatesTo = None) -> Optional[EventID]:
+        media: MessageMediaPhoto = evt.media
+        if media.photo is None and media.ttl_seconds:
+            return await self._send_message(intent, TextMessageEventContent(
+                msgtype=MessageType.NOTICE, body="Photo has expired"))
+        loc, largest_size = self._get_largest_photo_size(media.photo)
+        if loc is None:
+            content = TextMessageEventContent(msgtype=MessageType.TEXT,
+                                              body="Failed to bridge image",
+                                              external_url=self._get_external_url(evt))
+            return await self._send_message(intent, content, timestamp=evt.date)
+        file = await util.transfer_file_to_matrix(source.client, intent, loc,
+                                                  encrypt=self.encrypted)
         if not file:
             return None
-        if self.get_config("inline_images") and (evt.message
-                                                 or evt.fwd_from or evt.reply_to_msg_id):
+        if self.get_config("inline_images") and (evt.message or evt.fwd_from or evt.reply_to):
             content = await formatter.telegram_to_matrix(
                 evt, source, self.main_intent,
                 prefix_html=f"<img src='{file.mxc}' alt='Inline Telegram photo'/><br/>",
                 prefix_text="Inline image: ")
             content.external_url = self._get_external_url(evt)
             await intent.set_typing(self.mxid, is_typing=False)
-            return await intent.send_message(self.mxid, content, timestamp=evt.date)
+            return await self._send_message(intent, content, timestamp=evt.date)
         info = ImageInfo(
             height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type,
-            size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize))
-                  else largest_size.size))
-        name = f"image{sane_mimetypes.guess_extension(file.mime_type)}"
+            size=self._photo_size_key(largest_size))
+        ext = sane_mimetypes.guess_extension(file.mime_type)
+        name = f"disappearing_image{ext}" if media.ttl_seconds else f"image{ext}"
         await intent.set_typing(self.mxid, is_typing=False)
-        content = MediaMessageEventContent(url=file.mxc, msgtype=MessageType.IMAGE, info=info,
+        content = MediaMessageEventContent(msgtype=MessageType.IMAGE, info=info,
                                            body=name, relates_to=relates_to,
                                            external_url=self._get_external_url(evt))
-        result = await intent.send_message(self.mxid, content, timestamp=evt.date)
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+        result = await self._send_message(intent, content, timestamp=evt.date)
+        if media.ttl_seconds:
+            self.loop.create_task(self._expire_telegram_photo(intent, result,
+                                                              media.ttl_seconds))
         if evt.message:
             caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
                                                                  no_reply_fallback=True)
             caption_content.external_url = content.external_url
-            result = await intent.send_message(self.mxid, caption_content, timestamp=evt.date)
+            result = await self._send_message(intent, caption_content, timestamp=evt.date)
         return result
 
     @staticmethod
     def _parse_telegram_document_attributes(attributes: List[TypeDocumentAttribute]) -> DocAttrs:
         name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
+        is_gif = False
         for attr in attributes:
             if isinstance(attr, DocumentAttributeFilename):
                 name = name or attr.file_name
@@ -114,15 +147,19 @@ class PortalTelegram(BasePortal, ABC):
             elif isinstance(attr, DocumentAttributeSticker):
                 is_sticker = True
                 sticker_alt = attr.alt
+            elif isinstance(attr, DocumentAttributeAnimated):
+                is_gif = True
             elif isinstance(attr, DocumentAttributeVideo):
                 width, height = attr.w, attr.h
-        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height)
+            elif isinstance(attr, DocumentAttributeImageSize):
+                width, height = attr.w, attr.h
+        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height, is_gif)
 
     @staticmethod
     def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: DocAttrs,
                                       thumb_size: TypePhotoSize) -> Tuple[ImageInfo, str]:
         document = evt.media.document
-        name = evt.message or attrs.name
+        name = attrs.name
         if attrs.is_sticker:
             alt = attrs.sticker_alt
             if len(alt) > 0:
@@ -134,6 +171,8 @@ class PortalTelegram(BasePortal, ABC):
         generic_types = ("text/plain", "application/octet-stream")
         if file.mime_type in generic_types and document.mime_type not in generic_types:
             mime_type = document.mime_type or file.mime_type
+        elif file.mime_type == 'application/ogg':
+            mime_type = 'audio/ogg'
         else:
             mime_type = file.mime_type or document.mime_type
         info = ImageInfo(size=file.size, mimetype=mime_type)
@@ -146,11 +185,21 @@ class PortalTelegram(BasePortal, ABC):
             info.width, info.height = attrs.width, attrs.height
 
         if file.thumbnail:
-            info.thumbnail_url = file.thumbnail.mxc
+            if file.thumbnail.decryption_info:
+                info.thumbnail_file = file.thumbnail.decryption_info
+            else:
+                info.thumbnail_url = file.thumbnail.mxc
             info.thumbnail_info = ThumbnailInfo(mimetype=file.thumbnail.mime_type,
                                                 height=file.thumbnail.height or thumb_size.h,
                                                 width=file.thumbnail.width or thumb_size.w,
                                                 size=file.thumbnail.size)
+        elif attrs.is_sticker:
+            # This is a hack for bad clients like Element iOS that require a thumbnail
+            info.thumbnail_info = ImageInfo.deserialize(info.serialize())
+            if file.decryption_info:
+                info.thumbnail_file = file.decryption_info
+            else:
+                info.thumbnail_url = file.mxc
 
         return info, name
 
@@ -164,6 +213,7 @@ class PortalTelegram(BasePortal, ABC):
         if document.size > config["bridge.max_document_size"] * 1000 ** 2:
             name = attrs.name or ""
             caption = f"\n{evt.message}" if evt.message else ""
+            # TODO encrypt
             return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
 
         thumb_loc, thumb_size = self._get_largest_photo_size(document)
@@ -175,7 +225,8 @@ class PortalTelegram(BasePortal, ABC):
         file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
                                                   is_sticker=attrs.is_sticker,
                                                   tgs_convert=config["bridge.animated_sticker"],
-                                                  filename=attrs.name, parallel_id=parallel_id)
+                                                  filename=attrs.name, parallel_id=parallel_id,
+                                                  encrypt=self.encrypted)
         if not file:
             return None
 
@@ -184,52 +235,87 @@ class PortalTelegram(BasePortal, ABC):
         await intent.set_typing(self.mxid, is_typing=False)
 
         event_type = EventType.ROOM_MESSAGE
-        # Riot only supports images as stickers, so send animated webm stickers as m.video
+        # Elements only support images as stickers, so send animated webm stickers as m.video
         if attrs.is_sticker and file.mime_type.startswith("image/"):
             event_type = EventType.STICKER
+            # Tell clients to render the stickers as 256x256 if they're bigger
+            if info.width > 256 or info.height > 256:
+                if info.width > info.height:
+                    info.height = int(info.height / (info.width / 256))
+                    info.width = 256
+                else:
+                    info.width = int(info.width / (info.height / 256))
+                    info.height = 256
+            if info.thumbnail_info:
+                info.thumbnail_info.width = info.width
+                info.thumbnail_info.height = info.height
+        if attrs.is_gif or (attrs.is_sticker and info.mimetype == "video/webm"):
+            if attrs.is_gif:
+                info["fi.mau.telegram.gif"] = True
+            else:
+                info["fi.mau.telegram.animated_sticker"] = True
+            info["fi.mau.loop"] = True
+            info["fi.mau.autoplay"] = True
+            info["fi.mau.hide_controls"] = True
+            info["fi.mau.no_audio"] = True
+        if not name:
+            ext = sane_mimetypes.guess_extension(file.mime_type)
+            name = "unnamed_file" + ext
+
         content = MediaMessageEventContent(
-            body=name or "unnamed file", info=info, url=file.mxc, relates_to=relates_to,
+            body=name, info=info, relates_to=relates_to,
             external_url=self._get_external_url(evt),
             msgtype={
                 "video/": MessageType.VIDEO,
                 "audio/": MessageType.AUDIO,
                 "image/": MessageType.IMAGE,
             }.get(info.mimetype[:6], MessageType.FILE))
-        return await intent.send_message_event(self.mxid, event_type, content, timestamp=evt.date)
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+        res = await self._send_message(intent, content, event_type=event_type, timestamp=evt.date)
+        if evt.message:
+            caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                                 no_reply_fallback=True)
+            caption_content.external_url = content.external_url
+            res = await self._send_message(intent, caption_content, timestamp=evt.date)
+        return res
 
-    def handle_telegram_location(self, _: 'AbstractUser', intent: IntentAPI, evt: Message,
-                                 relates_to: dict = None) -> Awaitable[EventID]:
+    def handle_telegram_location(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
+                                 relates_to: RelatesTo = None) -> Awaitable[EventID]:
         long = evt.media.geo.long
         lat = evt.media.geo.lat
         long_char = "E" if long > 0 else "W"
         lat_char = "N" if lat > 0 else "S"
+        geo = f"{round(lat, 6)},{round(long, 6)}"
 
-        body = f"{round(lat, 5)}° {lat_char}, {round(long, 5)}° {long_char}"
-        url = f"https://maps.google.com/?q={lat},{long}"
+        body = f"{round(abs(lat), 4)}° {lat_char}, {round(abs(long), 4)}° {long_char}"
+        url = f"https://maps.google.com/?q={geo}"
 
         content = LocationMessageEventContent(
-            msgtype=MessageType.LOCATION, geo_uri=f"geo:{lat},{long}",
+            msgtype=MessageType.LOCATION, geo_uri=f"geo:{geo}",
             body=f"Location: {body}\n{url}",
             relates_to=relates_to, external_url=self._get_external_url(evt))
         content["format"] = str(Format.HTML)
         content["formatted_body"] = f"Location: <a href='{url}'>{body}</a>"
 
-        return intent.send_message(self.mxid, content, timestamp=evt.date)
+        return self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_text(self, source: 'AbstractUser', intent: IntentAPI, is_bot: bool,
                                    evt: Message) -> EventID:
-        self.log.debug(f"Sending {evt.message} to {self.mxid} by {intent.mxid}")
+        self.log.trace(f"Sending {evt.message} to {self.mxid} by {intent.mxid}")
         content = await formatter.telegram_to_matrix(evt, source, self.main_intent)
         content.external_url = self._get_external_url(evt)
         if is_bot and self.get_config("bot_messages_as_notices"):
             content.msgtype = MessageType.NOTICE
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
-                                          evt: Message, relates_to: dict = None) -> EventID:
+                                          evt: Message, relates_to: RelatesTo = None) -> EventID:
         override_text = ("This message is not supported on your version of Mautrix-Telegram. "
-                         "Please check https://github.com/tulir/mautrix-telegram or ask your "
+                         "Please check https://github.com/mautrix/telegram or ask your "
                          "bridge administrator about possible updates.")
         content = await formatter.telegram_to_matrix(
             evt, source, self.main_intent, override_text=override_text)
@@ -237,7 +323,7 @@ class PortalTelegram(BasePortal, ABC):
         content.external_url = self._get_external_url(evt)
         content["net.maunium.telegram.unsupported"] = True
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_poll(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                    relates_to: RelatesTo) -> EventID:
@@ -263,11 +349,75 @@ class PortalTelegram(BasePortal, ABC):
             relates_to=relates_to, external_url=self._get_external_url(evt))
 
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
+
+    @staticmethod
+    def _format_dice(roll: MessageMediaDice) -> str:
+        if roll.emoticon == "\U0001F3B0":
+            emojis = {
+                0: "\U0001F36B",  # "🍫",
+                1: "\U0001F352",  # "🍒",
+                2: "\U0001F34B",  # "🍋",
+                3: "7\ufe0f\u20e3"  # "7️⃣",
+            }
+            res = roll.value - 1
+            slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
+            return f"{slot1} {slot2} {slot3} ({roll.value})"
+        elif roll.emoticon == "\u26BD":
+            results = {
+                1: "miss",
+                2: "hit the woodwork",
+                3: "goal",  # seems to go in through the center
+                4: "goal",
+                5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
+            }
+        elif roll.emoticon == "\U0001F3B3":
+            results = {
+                1: "miss",
+                2: "1 pin down",
+                3: "3 pins down, split",
+                4: "4 pins down, split",
+                5: "5 pins down",
+                6: "strike 🎉",
+            }
+        # elif roll.emoticon == "\U0001F3C0":
+        #     results = {
+        #         2: "rolled off",
+        #         3: "stuck",
+        #     }
+        # elif roll.emoticon == "\U0001F3AF":
+        #     results = {
+        #         1: "bounced off",
+        #         2: "outer rim",
+        #
+        #         6: "bullseye",
+        #     }
+        else:
+            return str(roll.value)
+        return f"{results[roll.value]} ({roll.value})"
+
+    async def handle_telegram_dice(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
+                                   relates_to: RelatesTo) -> EventID:
+        emoji_text = {
+            "\U0001F3AF": " Dart throw",
+            "\U0001F3B2": " Dice roll",
+            "\U0001F3C0": " Basketball throw",
+            "\U0001F3B0": " Slot machine",
+            "\U0001F3B3": " Bowling",
+            "\u26BD": " Football kick"
+        }
+        roll: MessageMediaDice = evt.media
+        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {self._format_dice(roll)}"
+        content = TextMessageEventContent(msgtype=MessageType.TEXT, format=Format.HTML, body=text,
+                                          formatted_body=f"<h4>{text}</h4>", relates_to=relates_to,
+                                          external_url=self._get_external_url(evt))
+        content["net.maunium.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     @staticmethod
     def _int_to_bytes(i: int) -> bytes:
-        hex_value = "{0:010x}".format(i)
+        hex_value = f"{i:010x}".encode("utf-8")
         return codecs.decode(hex_value, "hex_codec")
 
     def _encode_msgid(self, source: 'AbstractUser', evt: Message) -> str:
@@ -305,11 +455,12 @@ class PortalTelegram(BasePortal, ABC):
         content["net.maunium.telegram.game"] = play_id
 
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_edit(self, source: 'AbstractUser', sender: p.Puppet, evt: Message
                                    ) -> None:
         if not self.mxid:
+            self.log.trace("Ignoring edit to %d as chat has no Matrix room", evt.id)
             return
         elif hasattr(evt, "media") and isinstance(evt.media, MessageMediaGame):
             self.log.debug("Ignoring game message edit event")
@@ -349,21 +500,126 @@ class PortalTelegram(BasePortal, ABC):
 
         intent = sender.intent_for(self) if sender else self.main_intent
         await intent.set_typing(self.mxid, is_typing=False)
-        event_id = await intent.send_message(self.mxid, content)
+        event_id = await self._send_message(intent, content)
 
         prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
         DBMessage(mxid=event_id, mx_room=self.mxid, tg_space=tg_space, tgid=TelegramID(evt.id),
                   edit_index=prev_edit_msg.edit_index + 1).insert()
         DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=event_id)
 
+    @property
+    def _takeout_options(self) -> Dict[str, Union[bool, int]]:
+        return {
+            "files": True,
+            "megagroups": self.megagroup,
+            "chats": self.peer_type == "chat",
+            "users": self.peer_type == "user",
+            "channels": (self.peer_type == "channel" and not self.megagroup),
+            "max_file_size": min(config["bridge.max_document_size"], 2000) * 1024 * 1024
+        }
+
+    async def backfill(self, source: 'u.User', is_initial: bool = False,
+                       limit: Optional[int] = None, last_id: Optional[int] = None) -> None:
+        async with self.backfill_method_lock:
+            await self._locked_backfill(source, is_initial, limit, last_id)
+
+    async def _locked_backfill(self, source: 'u.User', is_initial: bool = False,
+                               limit: Optional[int] = None, last_id: Optional[int] = None) -> None:
+        limit = limit or (config["bridge.backfill.initial_limit"] if is_initial
+                          else config["bridge.backfill.missed_limit"])
+        if limit == 0:
+            return
+        if not config["bridge.backfill.normal_groups"] and self.peer_type == "chat":
+            return
+        last = DBMessage.find_last(self.mxid, (source.tgid if self.peer_type != "channel"
+                                               else self.tgid))
+        min_id = last.tgid if last else 0
+        if last_id is None:
+            messages = await source.client.get_messages(self.peer, limit=1)
+            if not messages:
+                # The chat seems empty
+                return
+            last_id = messages[0].id
+        if last_id <= min_id:
+            # Nothing to backfill
+            return
+        if limit < 0:
+            limit = last_id - min_id
+            self.log.debug(f"Backfilling approximately {last_id - min_id} messages "
+                           f"through {source.mxid}")
+        elif self.peer_type == "channel":
+            # This is a channel or supergroup, so we'll backfill messages based on the ID.
+            # There are some cases, such as deleted messages, where this may backfill less
+            # messages than the limit.
+            min_id = max(last_id - limit, min_id)
+            self.log.debug(f"Backfilling messages after ID {min_id} (last message: {last_id}) "
+                           f"through {source.mxid}")
+        else:
+            # Private chats and normal groups don't have their own message ID namespace,
+            # which means we'll have to fetch messages a different way.
+            # The _backfill_messages method will detect min_id=None and not use reverse=True
+            min_id = None
+            self.log.debug(f"Backfilling up to {limit} messages through {source.mxid}")
+        with self.backfill_lock:
+            await self._backfill(source, min_id, limit)
+
+    async def _backfill(self, source: 'u.User', min_id: Optional[int], limit: int) -> None:
+        self.backfill_leave = set()
+        if ((self.peer_type == "user" and self.tgid != source.tgid
+             and config["bridge.backfill.invite_own_puppet"])):
+            self.log.debug("Adding %s's default puppet to room for backfilling", source.mxid)
+            sender = p.Puppet.get(source.tgid)
+            await self.main_intent.invite_user(self.mxid, sender.default_mxid)
+            await sender.default_mxid_intent.join_room_by_id(self.mxid)
+            self.backfill_leave.add(sender.default_mxid_intent)
+
+        client = source.client
+        async with NotificationDisabler(self.mxid, source):
+            if limit > config["bridge.backfill.takeout_limit"]:
+                self.log.debug(f"Opening takeout client for {source.tgid}")
+                async with client.takeout(**self._takeout_options) as takeout:
+                    count = await self._backfill_messages(source, min_id, limit, takeout)
+            else:
+                count = await self._backfill_messages(source, min_id, limit, client)
+
+        for intent in self.backfill_leave:
+            self.log.trace("Leaving room with %s post-backfill", intent.mxid)
+            await intent.leave_room(self.mxid)
+        self.backfill_leave = None
+        self.log.info("Backfilled %d messages through %s", count, source.mxid)
+
+    async def _backfill_messages(self, source: 'AbstractUser', min_id: Optional[int], limit: int,
+                                 client: TelegramClient) -> int:
+        count = 0
+        entity = await self.get_input_entity(source)
+        if min_id is not None:
+            self.log.debug(f"Iterating all messages starting with {min_id} (approx: {limit})")
+            messages = client.iter_messages(entity, reverse=True, min_id=min_id)
+            async for message in messages:
+                sender = (p.Puppet.get(message.from_id.user_id)
+                          if isinstance(message.from_id, PeerUser) else None)
+                # TODO handle service messages?
+                await self.handle_telegram_message(source, sender, message)
+                count += 1
+        else:
+            self.log.debug(f"Fetching up to {limit} most recent messages")
+            messages = await client.get_messages(entity, limit=limit)
+            for message in reversed(messages):
+                sender = (p.Puppet.get(TelegramID(message.from_id.user_id))
+                          if isinstance(message.from_id, PeerUser) else None)
+                await self.handle_telegram_message(source, sender, message)
+                count += 1
+        return count
+
     async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
                                       evt: Message) -> None:
         if not self.mxid:
+            self.log.trace("Got telegram message %d, but no room exists, creating...", evt.id)
             await self.create_matrix_room(source, invites=[source.mxid], update_if_exists=False)
 
-        if (self.peer_type == "user" and sender.tgid == self.tg_receiver
-            and not sender.is_real_user and not self.az.state_store.is_joined(self.mxid,
-                                                                              sender.mxid)):
+        if (self.peer_type == "user" and sender and sender.tgid == self.tg_receiver
+            and not sender.is_real_user and not await self.az.state_store.is_joined(self.mxid,
+                                                                                    sender.mxid)):
             self.log.debug(f"Ignoring private chat message {evt.id}@{source.tgid} as receiver does"
                            " not have matrix puppeting and their default puppet isn't in the room")
             return
@@ -383,26 +639,39 @@ class PortalTelegram(BasePortal, ABC):
                               tg_space=tg_space, edit_index=0).insert()
                 return
 
-        if self.dedup.pre_db_check and self.peer_type == "channel":
+        if self.backfill_lock.locked or (self.dedup.pre_db_check and self.peer_type == "channel"):
             msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
             if msg:
-                self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
+                self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already "
                                f"handled into {msg.mxid}. This duplicate was catched in the db "
-                               "check. If you get this message often, consider increasing"
+                               "check. If you get this message often, consider increasing "
                                "bridge.deduplication.cache_queue_length in the config.")
                 return
 
+        self.log.trace("Handling Telegram message %s", evt)
+
         if sender and not sender.displayname:
             self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a "
                            "displayname, updating info...")
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
+            if not sender.displayname:
+                self.log.debug(f"Telegram user {sender.tgid} doesn't have a displayname even after"
+                               f" updating with data {entity!s}")
 
         allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
-                         MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
+                         MessageMediaGame, MessageMediaDice, MessageMediaPoll,
+                         MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
-        intent = sender.intent_for(self) if sender else self.main_intent
+        if sender:
+            intent = sender.intent_for(self)
+            if ((self.backfill_lock.locked and intent != sender.default_mxid_intent
+                 and config["bridge.backfill.invite_own_puppet"])):
+                intent = sender.default_mxid_intent
+                self.backfill_leave.add(intent)
+        else:
+            intent = self.main_intent
         if not media and evt.message:
             is_bot = sender.is_bot if sender else False
             event_id = await self.handle_telegram_text(source, intent, is_bot, evt)
@@ -412,12 +681,13 @@ class PortalTelegram(BasePortal, ABC):
                 MessageMediaDocument: self.handle_telegram_document,
                 MessageMediaGeo: self.handle_telegram_location,
                 MessageMediaPoll: self.handle_telegram_poll,
+                MessageMediaDice: self.handle_telegram_dice,
                 MessageMediaUnsupported: self.handle_telegram_unsupported,
                 MessageMediaGame: self.handle_telegram_game,
             }[type(media)](source, intent, evt,
                            relates_to=formatter.telegram_reply_to_matrix(evt, source))
         else:
-            self.log.debug("Unhandled Telegram message: %s", evt)
+            self.log.debug("Unhandled Telegram message %d", evt.id)
             return
 
         if not event_id:
@@ -434,7 +704,7 @@ class PortalTelegram(BasePortal, ABC):
             await intent.redact(self.mxid, event_id)
             return
 
-        self.log.debug("Handled Telegram message: %s", evt)
+        self.log.debug("Handled telegram message %d -> %s", evt.id, event_id)
         try:
             DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=event_id,
                       tg_space=tg_space, edit_index=0).insert()
@@ -445,6 +715,7 @@ class PortalTelegram(BasePortal, ABC):
                                "dedup cache queue. You can try enabling bridge.deduplication."
                                "pre_db_check in the config.")
             await intent.redact(self.mxid, event_id)
+        await self._send_delivery_receipt(event_id)
 
     async def _create_room_on_action(self, source: 'AbstractUser',
                                      action: TypeMessageAction) -> bool:
@@ -468,10 +739,13 @@ class PortalTelegram(BasePortal, ABC):
             return
         if isinstance(action, MessageActionChatEditTitle):
             await self._update_title(action.title, sender=sender, save=True)
+            await self.update_bridge_info()
         elif isinstance(action, MessageActionChatEditPhoto):
             await self._update_avatar(source, action.photo, sender=sender, save=True)
+            await self.update_bridge_info()
         elif isinstance(action, MessageActionChatDeletePhoto):
             await self._update_avatar(source, ChatPhotoEmpty(), sender=sender, save=True)
+            await self.update_bridge_info()
         elif isinstance(action, MessageActionChatAddUser):
             for user_id in action.users:
                 await self._add_telegram_user(TelegramID(user_id), source)
@@ -482,13 +756,15 @@ class PortalTelegram(BasePortal, ABC):
         elif isinstance(action, MessageActionChatMigrateTo):
             self.peer_type = "channel"
             self._migrate_and_save_telegram(TelegramID(action.channel_id))
+            # TODO encrypt
             await sender.intent_for(self).send_emote(self.mxid,
                                                      "upgraded this group to a supergroup.")
+            await self.update_bridge_info()
         elif isinstance(action, MessageActionGameScore):
             # TODO handle game score
             pass
         else:
-            self.log.debug("Unhandled Telegram action in %s: %s", self.title, action)
+            self.log.trace("Unhandled Telegram action in %s: %s", self.title, action)
 
     async def set_telegram_admin(self, user_id: TelegramID) -> None:
         puppet = p.Puppet.get(user_id)
@@ -501,13 +777,20 @@ class PortalTelegram(BasePortal, ABC):
             levels.users[puppet.mxid] = 50
         await self.main_intent.set_power_levels(self.mxid, levels)
 
-    async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
-        tg_space = receiver  if self.peer_type != "channel" else self.tgid
-        message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
-        if message:
-            await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
-        else:
-            await self.main_intent.set_pinned_messages(self.mxid, [])
+    async def receive_telegram_pin_ids(self, msg_ids: List[TelegramID], receiver: TelegramID,
+                                       remove: bool) -> None:
+        async with self._pin_lock:
+            tg_space = receiver if self.peer_type != "channel" else self.tgid
+            previously_pinned = await self.main_intent.get_pinned_messages(self.mxid)
+            currently_pinned_dict = {event_id: True for event_id in previously_pinned}
+            for message in DBMessage.get_first_by_tgids(msg_ids, tg_space):
+                if remove:
+                    currently_pinned_dict.pop(message.mxid, None)
+                else:
+                    currently_pinned_dict[message.mxid] = True
+            currently_pinned = list(currently_pinned_dict.keys())
+            if currently_pinned != previously_pinned:
+                await self.main_intent.set_pinned_messages(self.mxid, currently_pinned)
 
     async def set_telegram_admins_enabled(self, enabled: bool) -> None:
         level = 50 if enabled else 10
@@ -521,3 +804,5 @@ class PortalTelegram(BasePortal, ABC):
 def init(context: Context) -> None:
     global config
     config = context.config
+    NotificationDisabler.puppet_cls = p.Puppet
+    NotificationDisabler.config_enabled = config["bridge.backfill.disable_notifications"]

mautrix_telegram/puppet.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Any, Dict, Iterable, Optional, Union, TYPE_CHECKING
+from typing import Awaitable, Any, Dict, Iterable, Optional, Union, Tuple, TYPE_CHECKING
 from difflib import SequenceMatcher
 import unicodedata
 import asyncio
@@ -21,12 +21,14 @@ import logging
 
 from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
                                InputPeerPhotoFileLocation, UserProfilePhotoEmpty, TypeInputUser)
+from yarl import URL
 
 from mautrix.appservice import AppService, IntentAPI
-from mautrix.errors import MatrixRequestError
-from mautrix.bridge import CustomPuppetMixin
-from mautrix.types import UserID, SyncToken
+from mautrix.errors import MatrixRequestError, MatrixError
+from mautrix.bridge import BasePuppet
+from mautrix.types import UserID, SyncToken, RoomID, ContentURI
 from mautrix.util.simple_template import SimpleTemplate
+from mautrix.util.logging import TraceLogger
 
 from .types import TelegramID
 from .db import Puppet as DBPuppet
@@ -41,8 +43,8 @@ if TYPE_CHECKING:
 config: Optional['Config'] = None
 
 
-class Puppet(CustomPuppetMixin):
-    log: logging.Logger = logging.getLogger("mau.puppet")
+class Puppet(BasePuppet):
+    log: TraceLogger = logging.getLogger("mau.puppet")
     az: AppService
     mx: 'MatrixHandler'
     loop: asyncio.AbstractEventLoop
@@ -57,11 +59,14 @@ class Puppet(CustomPuppetMixin):
     access_token: Optional[str]
     custom_mxid: Optional[UserID]
     _next_batch: Optional[SyncToken]
+    base_url: Optional[URL]
     default_mxid: UserID
 
     username: Optional[str]
     displayname: Optional[str]
     displayname_source: Optional[TelegramID]
+    displayname_contact: bool
+    displayname_quality: int
     photo_id: Optional[str]
     is_bot: bool
     is_registered: bool
@@ -79,9 +84,12 @@ class Puppet(CustomPuppetMixin):
                  access_token: Optional[str] = None,
                  custom_mxid: Optional[UserID] = None,
                  next_batch: Optional[SyncToken] = None,
+                 base_url: Optional[str] = None,
                  username: Optional[str] = None,
                  displayname: Optional[str] = None,
                  displayname_source: Optional[TelegramID] = None,
+                 displayname_contact: bool = True,
+                 displayname_quality: int = 0,
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
@@ -91,11 +99,14 @@ class Puppet(CustomPuppetMixin):
         self.access_token = access_token
         self.custom_mxid = custom_mxid
         self._next_batch = next_batch
+        self.base_url = URL(base_url) if base_url else None
         self.default_mxid = self.get_mxid_from_id(self.id)
 
         self.username = username
         self.displayname = displayname
         self.displayname_source = displayname_source
+        self.displayname_contact = displayname_contact
+        self.displayname_quality = displayname_quality
         self.photo_id = photo_id
         self.is_bot = is_bot
         self.is_registered = is_registered
@@ -160,22 +171,25 @@ class Puppet(CustomPuppetMixin):
         return dict(access_token=self.access_token, next_batch=self._next_batch,
                     custom_mxid=self.custom_mxid, username=self.username, is_bot=self.is_bot,
                     displayname=self.displayname, displayname_source=self.displayname_source,
-                    photo_id=self.photo_id, matrix_registered=self.is_registered,
-                    disable_updates=self.disable_updates)
+                    displayname_contact=self.displayname_contact,
+                    displayname_quality=self.displayname_quality, photo_id=self.photo_id,
+                    matrix_registered=self.is_registered, disable_updates=self.disable_updates,
+                    base_url=str(self.base_url) if self.base_url else None)
 
     def new_db_instance(self) -> DBPuppet:
         return DBPuppet(id=self.id, **self._fields)
 
-    def save(self) -> None:
+    async def save(self) -> None:
         self.db_instance.edit(**self._fields)
 
     @classmethod
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
-                      db_puppet.next_batch, db_puppet.username, db_puppet.displayname,
-                      db_puppet.displayname_source, db_puppet.photo_id, db_puppet.is_bot,
-                      db_puppet.matrix_registered, db_puppet.disable_updates,
-                      db_instance=db_puppet)
+                      db_puppet.next_batch, db_puppet.base_url, db_puppet.username,
+                      db_puppet.displayname, db_puppet.displayname_source,
+                      db_puppet.displayname_contact, db_puppet.displayname_quality,
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
     # endregion
     # region Info updating
@@ -195,11 +209,13 @@ class Puppet(CustomPuppetMixin):
         whitespace = ("\t\n\r\v\f \u00a0\u034f\u180e\u2063\u202f\u205f\u2800\u3000\u3164\ufeff"
                       "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b"
                       "\u200c\u200d\u200e\u200f\ufe0f")
-        name = "".join(c for c in name.strip(whitespace) if unicodedata.category(c) != 'Cf')
+        allowed_other_format = ("\u200d", "\u200c")
+        name = "".join(c for c in name.strip(whitespace) if unicodedata.category(c) != 'Cf'
+                                                         or c in allowed_other_format)
         return name
 
     @classmethod
-    def get_displayname(cls, info: User, enable_format: bool = True) -> str:
+    def get_displayname(cls, info: User, enable_format: bool = True) -> Tuple[str, int]:
         fn = cls._filter_name(info.first_name)
         ln = cls._filter_name(info.last_name)
         data = {
@@ -212,19 +228,21 @@ class Puppet(CustomPuppetMixin):
         }
         preferences = config["bridge.displayname_preference"]
         name = None
+        quality = 99
         for preference in preferences:
             name = data[preference]
             if name:
                 break
+            quality -= 1
 
         if isinstance(info, User) and info.deleted:
             name = f"Deleted account {info.id}"
+            quality = 99
         elif not name:
             name = str(info.id)
+            quality = 0
 
-        if not enable_format:
-            return name
-        return cls.displayname_template.format_full(name)
+        return (cls.displayname_template.format_full(name) if enable_format else name), quality
 
     async def try_update_info(self, source: 'AbstractUser', info: User) -> None:
         try:
@@ -233,51 +251,67 @@ class Puppet(CustomPuppetMixin):
             source.log.exception(f"Failed to update info of {self.tgid}")
 
     async def update_info(self, source: 'AbstractUser', info: User) -> None:
-        if self.disable_updates:
-            return
         changed = False
         if self.username != info.username:
             self.username = info.username
             changed = True
 
-        try:
-            changed = await self.update_displayname(source, info) or changed
-            if isinstance(info.photo, UserProfilePhoto):
+        if not self.disable_updates:
+            try:
+                changed = await self.update_displayname(source, info) or changed
                 changed = await self.update_avatar(source, info.photo) or changed
-        except Exception:
-            self.log.exception(f"Failed to update info from source {source.tgid}")
+            except Exception:
+                self.log.exception(f"Failed to update info from source {source.tgid}")
 
         self.is_bot = info.bot
 
         if changed:
-            self.save()
+            await self.save()
 
     async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
                                  ) -> bool:
         if self.disable_updates:
             return False
-        allow_source = (source.is_relaybot
-                        or self.displayname_source == source.tgid
-                        # User is not a contact, so there's no custom name
-                        or not info.contact
-                        # No displayname source, so just trust anything
-                        or self.displayname_source is None)
-        if not allow_source:
+        if source.is_relaybot or source.is_bot:
+            allow_because = "user is bot"
+        elif self.displayname_source == source.tgid:
+            allow_because = "user is the primary source"
+        elif not isinstance(info, UpdateUserName) and not info.contact:
+            allow_because = "user is not a contact"
+        elif not self.displayname_source:
+            allow_because = "no primary source set"
+        elif not self.displayname:
+            allow_because = "user has no name"
+        else:
             return False
-        elif isinstance(info, UpdateUserName):
-            info = await source.client.get_entity(PeerUser(self.tgid))
 
-        displayname = self.get_displayname(info)
-        if displayname != self.displayname:
+        if isinstance(info, UpdateUserName):
+            info = await source.client.get_entity(PeerUser(self.tgid))
+        if not info.contact:
+            self.displayname_contact = False
+        elif not self.displayname_contact:
+            if not self.displayname:
+                self.displayname_contact = True
+            else:
+                return False
+
+        displayname, quality = self.get_displayname(info)
+        if displayname != self.displayname and quality >= self.displayname_quality:
+            allow_because = f"{allow_because} and quality {quality} >= {self.displayname_quality}"
+            self.log.debug(f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
+                           f"because {allow_because}) from {self.displayname} to {displayname}")
+            self.log.trace("Displayname source data: %s", info)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            self.displayname_quality = quality
             try:
                 await self.default_mxid_intent.set_displayname(
                     displayname[:config["bridge.displayname_max_length"]])
-            except MatrixRequestError:
+            except MatrixError:
                 self.log.exception("Failed to set displayname")
                 self.displayname = ""
                 self.displayname_source = None
+                self.displayname_quality = 0
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
@@ -289,24 +323,28 @@ class Puppet(CustomPuppetMixin):
         if self.disable_updates:
             return False
 
-        if isinstance(photo, UserProfilePhotoEmpty):
+        if photo is None or isinstance(photo, UserProfilePhotoEmpty):
             photo_id = ""
-        else:
+        elif isinstance(photo, UserProfilePhoto):
             photo_id = str(photo.photo_id)
+        else:
+            self.log.warning(f"Unknown user profile photo type: {type(photo)}")
+            return False
+        if not photo_id and not config["bridge.allow_avatar_remove"]:
+            return False
         if self.photo_id != photo_id:
             if not photo_id:
                 self.photo_id = ""
                 try:
-                    await self.default_mxid_intent.set_avatar_url("")
-                except MatrixRequestError:
+                    await self.default_mxid_intent.set_avatar_url(ContentURI(""))
+                except MatrixError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
                 return True
 
             loc = InputPeerPhotoFileLocation(
                 peer=await self.get_input_entity(source),
-                local_id=photo.photo_big.local_id,
-                volume_id=photo.photo_big.volume_id,
+                photo_id=photo.photo_id,
                 big=True
             )
             file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
@@ -314,12 +352,16 @@ class Puppet(CustomPuppetMixin):
                 self.photo_id = photo_id
                 try:
                     await self.default_mxid_intent.set_avatar_url(file.mxc)
-                except MatrixRequestError:
+                except MatrixError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
                 return True
         return False
 
+    async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
+        portal: p.Portal = p.Portal.get_by_mxid(room_id)
+        return portal and not portal.backfill_lock.locked and portal.peer_type != "user"
+
     # endregion
     # region Getters
 
@@ -342,7 +384,7 @@ class Puppet(CustomPuppetMixin):
         return None
 
     @classmethod
-    def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['Puppet']:
+    def deprecated_sync_get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['Puppet']:
         tgid = cls.get_id_from_mxid(mxid)
         if tgid:
             return cls.get(tgid, create)
@@ -350,7 +392,11 @@ class Puppet(CustomPuppetMixin):
         return None
 
     @classmethod
-    def get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
+    async def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['Puppet']:
+        return cls.deprecated_sync_get_by_mxid(mxid, create)
+
+    @classmethod
+    def deprecated_sync_get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
         if not mxid:
             raise ValueError("Matrix ID can't be empty")
 
@@ -366,9 +412,13 @@ class Puppet(CustomPuppetMixin):
 
         return None
 
+    @classmethod
+    async def get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
+        return cls.deprecated_sync_get_by_custom_mxid(mxid)
+
     @classmethod
     def all_with_custom_mxid(cls) -> Iterable['Puppet']:
-        return (cls.by_custom_mxid[puppet.mxid]
+        return (cls.by_custom_mxid[puppet.custom_mxid]
                 if puppet.custom_mxid in cls.by_custom_mxid
                 else cls.from_db(puppet)
                 for puppet in DBPuppet.all_with_custom_mxid())
@@ -426,8 +476,12 @@ def init(context: 'Context') -> Iterable[Awaitable[Any]]:
     Puppet.displayname_template = SimpleTemplate(config["bridge.displayname_template"],
                                                  "displayname")
 
-    secret = config["bridge.login_shared_secret"]
-    Puppet.login_shared_secret = secret.encode("utf-8") if secret else None
+    Puppet.sync_with_custom_puppets = config["bridge.sync_with_custom_puppets"]
+    Puppet.homeserver_url_map = {server: URL(url) for server, url
+                                 in config["bridge.double_puppet_server_map"].items()}
+    Puppet.allow_discover_url = config["bridge.double_puppet_allow_discovery"]
+    Puppet.login_shared_secret_map = {server: secret.encode("utf-8") for server, secret
+                                      in config["bridge.login_shared_secret_map"].items()}
     Puppet.login_device_name = "Telegram Bridge"
 
     return (puppet.try_start() for puppet in Puppet.all_with_custom_mxid())

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -24,7 +24,8 @@ def log(message, end="\n"):
 
 
 def connect(to):
-    from mautrix.bridge.db import Base, RoomState, UserProfile
+    from mautrix.util.db import Base
+    from mautrix.client.state_store.sqlalchemy import RoomState, UserProfile
     from mautrix_telegram.db import (Portal, Message, UserPortal, User, Contact, Puppet, BotChat,
                                      TelegramFile)
 

mautrix_telegram/sqlstatestore.py

@@ -1,38 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.types import UserID
-from mautrix.bridge.db import SQLStateStore as BaseSQLStateStore
-
-from . import puppet as pu
-
-
-class SQLStateStore(BaseSQLStateStore):
-    def is_registered(self, user_id: UserID) -> bool:
-        puppet = pu.Puppet.get_by_mxid(user_id, create=False)
-        if puppet:
-            return puppet.is_registered
-        custom_puppet = pu.Puppet.get_by_custom_mxid(user_id)
-        if custom_puppet:
-            return True
-        return super().is_registered(user_id)
-
-    def registered(self, user_id: UserID) -> None:
-        puppet = pu.Puppet.get_by_mxid(user_id, create=True)
-        if puppet:
-            puppet.is_registered = True
-            puppet.save()
-        else:
-            super().registered(user_id)

mautrix_telegram/user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,25 +13,34 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import (Awaitable, Dict, List, Iterable, NewType, Optional, Tuple, Any, cast,
+from typing import (Awaitable, Dict, List, Iterable, NamedTuple, Optional, Tuple, Any, cast,
                     TYPE_CHECKING)
+from datetime import datetime, timezone
 import logging
 import asyncio
 
-from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
+from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage,
                                UpdateShortChatMessage, UpdateShortMessage, User as TLUser, Chat,
-                               ChatForbidden)
+                               ChatForbidden, UpdateFolderPeers, UpdatePinnedDialogs,
+                               UpdateNotifySettings, NotifyPeer, InputUserSelf)
+from telethon.tl.custom import Dialog
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.account import UpdateStatusRequest
+from telethon.tl.functions.users import GetUsersRequest
+from telethon.errors import (AuthKeyDuplicatedError, UserDeactivatedError, UserDeactivatedBanError,
+                             SessionRevokedError, UnauthorizedError)
 
 from mautrix.client import Client
-from mautrix.errors import MatrixRequestError
-from mautrix.types import UserID
-from mautrix.bridge import BaseUser
+from mautrix.errors import MatrixRequestError, MNotFound
+from mautrix.types import UserID, RoomID, PushRuleScope, PushRuleKind, PushActionType, RoomTagInfo
+from mautrix.bridge import BaseUser, BridgeState
+from mautrix.util.bridge_state import BridgeStateEvent
+from mautrix.util.logging import TraceLogger
+from mautrix.util.opt_prometheus import Gauge
 
 from .types import TelegramID
-from .db import User as DBUser
+from .db import User as DBUser, Portal as DBPortal, Message as DBMessage
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
@@ -41,11 +50,20 @@ if TYPE_CHECKING:
 
 config: Optional['Config'] = None
 
-SearchResult = NewType('SearchResult', Tuple['pu.Puppet', int])
+SearchResult = NamedTuple('SearchResult', puppet='pu.Puppet', similarity=int)
+
+METRIC_LOGGED_IN = Gauge('bridge_logged_in', 'Users logged into bridge')
+METRIC_CONNECTED = Gauge('bridge_connected', 'Users connected to Telegram')
+
+BridgeState.human_readable_errors.update({
+    "tg-not-connected": "Your Telegram connection failed",
+    "tg-auth-key-duplicated": "The bridge accidentally logged you out",
+    "tg-not-authenticated": "The stored auth token did not work",
+})
 
 
 class User(AbstractUser, BaseUser):
-    log: logging.Logger = logging.getLogger("mau.user")
+    log: TraceLogger = logging.getLogger("mau.user")
     by_mxid: Dict[str, 'User'] = {}
     by_tgid: Dict[int, 'User'] = {}
 
@@ -57,6 +75,7 @@ class User(AbstractUser, BaseUser):
 
     _db_instance: Optional[DBUser]
     _ensure_started_lock: asyncio.Lock
+    _track_connection_task: Optional[asyncio.Task]
 
     def __init__(self, mxid: UserID, tgid: Optional[TelegramID] = None,
                  username: Optional[str] = None, phone: Optional[str] = None,
@@ -64,8 +83,9 @@ class User(AbstractUser, BaseUser):
                  saved_contacts: int = 0, is_bot: bool = False,
                  db_portals: Optional[Iterable[Tuple[TelegramID, TelegramID]]] = None,
                  db_instance: Optional[DBUser] = None) -> None:
-        super().__init__()
+        AbstractUser.__init__(self)
         self.mxid = mxid
+        BaseUser.__init__(self)
         self.tgid = tgid
         self.is_bot = is_bot
         self.username = username
@@ -77,8 +97,8 @@ class User(AbstractUser, BaseUser):
         self.db_portals = db_portals or []
         self._db_instance = db_instance
         self._ensure_started_lock = asyncio.Lock()
-
-        self.command_status = None
+        self._track_connection_task = None
+        self._is_backfilling = False
 
         (self.relaybot_whitelisted,
          self.whitelisted,
@@ -91,8 +111,6 @@ class User(AbstractUser, BaseUser):
         if tgid:
             self.by_tgid[tgid] = self
 
-        self.log = self.log.getChild(self.mxid)
-
     @property
     def name(self) -> str:
         return self.mxid
@@ -150,7 +168,7 @@ class User(AbstractUser, BaseUser):
         return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
                       saved_contacts=self.saved_contacts, portals=self.db_portals)
 
-    def save(self, contacts: bool = False, portals: bool = False) -> None:
+    async def save(self, contacts: bool = False, portals: bool = False) -> None:
         self.db_instance.edit(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
                               saved_contacts=self.saved_contacts)
         if contacts:
@@ -189,23 +207,93 @@ class User(AbstractUser, BaseUser):
             return cast(User, await super().ensure_started(even_if_no_session))
 
     async def start(self, delete_unless_authenticated: bool = False) -> 'User':
-        await super().start()
+        try:
+            await super().start()
+        except AuthKeyDuplicatedError:
+            self.log.warning("Got AuthKeyDuplicatedError in start()")
+            await self.push_bridge_state(BridgeStateEvent.BAD_CREDENTIALS,
+                                         error="tg-auth-key-duplicated")
+            await self.client.disconnect()
+            self.client.session.delete()
+            self.client = None
+            if not delete_unless_authenticated:
+                # The caller wants the client to be connected, so restart the connection.
+                await super().start()
+            return self
+        except Exception:
+            await self.push_bridge_state(BridgeStateEvent.UNKNOWN_ERROR)
+            raise
         if await self.is_logged_in():
             self.log.debug(f"Ensuring post_login() for {self.name}")
-            asyncio.ensure_future(self.post_login(), loop=self.loop)
+            self.loop.create_task(self.post_login())
         elif delete_unless_authenticated:
             self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
             await self.client.disconnect()
+            if self.tgid:
+                await self.push_bridge_state(BridgeStateEvent.BAD_CREDENTIALS,
+                                             error="tg-not-authenticated")
             self.client.session.delete()
         return self
 
+    @property
+    def _is_connected(self) -> bool:
+        return bool(self.client and self.client._sender
+                    and self.client._sender._transport_connected())
+
+    async def _track_connection(self) -> None:
+        self.log.debug("Starting loop to track connection state")
+        while True:
+            await asyncio.sleep(3)
+            connected = self._is_connected
+            self._track_metric(METRIC_CONNECTED, connected)
+            if connected:
+                await self.push_bridge_state(BridgeStateEvent.BACKFILLING if self._is_backfilling
+                                             else BridgeStateEvent.CONNECTED, ttl=3600)
+            else:
+                await self.push_bridge_state(BridgeStateEvent.UNKNOWN_ERROR, ttl=240,
+                                             error="tg-not-connected")
+
+    async def fill_bridge_state(self, state: BridgeState) -> None:
+        await super().fill_bridge_state(state)
+        state.remote_id = str(self.tgid)
+        state.remote_name = self.human_tg_id
+
+    async def get_bridge_states(self) -> List[BridgeState]:
+        if not self.tgid:
+            return []
+        if self._is_connected and await self.is_logged_in():
+            state_event = (BridgeStateEvent.BACKFILLING if self._is_backfilling
+                           else BridgeStateEvent.CONNECTED)
+            ttl = 3600
+        else:
+            state_event = BridgeStateEvent.UNKNOWN_ERROR
+            ttl = 240
+        return [BridgeState(state_event=state_event, ttl=ttl)]
+
+    async def get_puppet(self) -> Optional['pu.Puppet']:
+        if not self.tgid:
+            return None
+        return pu.Puppet.get(self.tgid)
+
+    async def stop(self) -> None:
+        if self._track_connection_task:
+            self._track_connection_task.cancel()
+            self._track_connection_task = None
+        await super().stop()
+        self._track_metric(METRIC_CONNECTED, False)
+
     async def post_login(self, info: TLUser = None, first_login: bool = False) -> None:
+        if config["metrics.enabled"] and not self._track_connection_task:
+            self._track_connection_task = self.loop.create_task(self._track_connection())
+
         try:
             await self.update_info(info)
         except Exception:
             self.log.exception("Failed to update telegram account info")
             return
 
+        self._track_metric(METRIC_LOGGED_IN, True)
+
         try:
             puppet = pu.Puppet.get(self.tgid)
             if puppet.custom_mxid != self.mxid and puppet.can_auto_login(self.mxid):
@@ -216,22 +304,20 @@ class User(AbstractUser, BaseUser):
 
         if not self.is_bot and config["bridge.startup_sync"]:
             try:
+                self._is_backfilling = True
                 await self.sync_dialogs()
                 await self.sync_contacts()
             except Exception:
                 self.log.exception("Failed to run post-login sync")
+            finally:
+                self._is_backfilling = False
 
     async def update(self, update: TypeUpdate) -> bool:
         if not self.is_bot:
             return False
 
         if isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
-            message = update.message
-            if isinstance(message.to_id, PeerUser) and not message.out:
-                portal = po.Portal.get_by_tgid(message.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
-            else:
-                portal = po.Portal.get_by_entity(message.to_id, receiver_id=self.tgid)
+            portal = po.Portal.get_by_entity(update.message.peer_id, receiver_id=self.tgid)
         elif isinstance(update, UpdateShortChatMessage):
             portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
         elif isinstance(update, UpdateShortMessage):
@@ -240,7 +326,7 @@ class User(AbstractUser, BaseUser):
             return False
 
         if portal:
-            self.register_portal(portal)
+            await self.register_portal(portal)
             return False
 
         # Don't bother handling the update
@@ -253,8 +339,22 @@ class User(AbstractUser, BaseUser):
         if not self.is_bot:
             await self.client(UpdateStatusRequest(offline=not online))
 
+    async def get_me(self) -> Optional[TLUser]:
+        try:
+            return (await self.client(GetUsersRequest([InputUserSelf()])))[0]
+        except UnauthorizedError as e:
+            self.log.error(f"Authorization error in get_me(): {e}")
+            await self.push_bridge_state(BridgeStateEvent.BAD_CREDENTIALS, error="tg-auth-error",
+                                         message=str(e), ttl=3600)
+            await self.stop()
+            return None
+
     async def update_info(self, info: TLUser = None) -> None:
-        info = info or await self.client.get_me()
+        if not info:
+            info = await self.get_me()
+            if not info:
+                self.log.warning("get_me() returned None, aborting update_info()")
+                return
         changed = False
         if self.is_bot != info.bot:
             self.is_bot = info.bot
@@ -269,7 +369,7 @@ class User(AbstractUser, BaseUser):
             self.tgid = TelegramID(info.id)
             self.by_tgid[self.tgid] = self
         if changed:
-            self.save()
+            await self.save()
 
     async def log_out(self) -> bool:
         puppet = pu.Puppet.get(self.tgid)
@@ -278,26 +378,31 @@ class User(AbstractUser, BaseUser):
         for _, portal in self.portals.items():
             if not portal or portal.deleted or not portal.mxid or portal.has_bot:
                 continue
-            try:
-                await portal.main_intent.kick_user(portal.mxid, self.mxid,
-                                                   "Logged out of Telegram.")
-            except MatrixRequestError:
-                pass
+            if portal.peer_type == "user":
+                await portal.cleanup_portal("Logged out of Telegram")
+            else:
+                try:
+                    await portal.main_intent.kick_user(portal.mxid, self.mxid,
+                                                       "Logged out of Telegram.")
+                except MatrixRequestError:
+                    pass
         self.portals = {}
         self.contacts = []
-        self.save(portals=True, contacts=True)
+        await self.save(portals=True, contacts=True)
+        await self.push_bridge_state(BridgeStateEvent.LOGGED_OUT)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
             except KeyError:
                 pass
             self.tgid = None
-            self.save()
+            await self.save()
         ok = await self.client.log_out()
-        if not ok:
-            return False
+        self.client.session.delete()
         self.delete()
-        return True
+        await self.stop()
+        self._track_metric(METRIC_LOGGED_IN, False)
+        return ok
 
     def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
                       ) -> List[SearchResult]:
@@ -305,7 +410,7 @@ class User(AbstractUser, BaseUser):
         for contact in self.contacts:
             similarity = contact.similarity(query)
             if similarity >= min_similarity:
-                results.append(SearchResult((contact, similarity)))
+                results.append(SearchResult(contact, similarity))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
@@ -317,7 +422,7 @@ class User(AbstractUser, BaseUser):
         for user in server_results.users:
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
-            results.append(SearchResult((puppet, puppet.similarity(query))))
+            results.append(SearchResult(puppet, puppet.similarity(query)))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
@@ -332,44 +437,158 @@ class User(AbstractUser, BaseUser):
 
         return await self._search_remote(query), True
 
-    async def sync_dialogs(self, synchronous_create: bool = False) -> None:
+    async def get_direct_chats(self) -> Dict[UserID, List[RoomID]]:
+        return {
+            pu.Puppet.get_mxid_from_id(portal.tgid): [portal.mxid]
+            for portal in DBPortal.find_private_chats(self.tgid)
+            if portal.mxid
+        }
+
+    async def _tag_room(self, puppet: pu.Puppet, portal: po.Portal, tag: str, active: bool
+                        ) -> None:
+        if not tag or not portal or not portal.mxid:
+            return
+        tag_info = await puppet.intent.get_room_tag(portal.mxid, tag)
+        if active and tag_info is None:
+            tag_info = RoomTagInfo(order=0.5)
+            tag_info[self.bridge.real_user_content_key] = True
+            await puppet.intent.set_room_tag(portal.mxid, tag, tag_info)
+        elif not active and tag_info and tag_info.get(self.bridge.real_user_content_key, False):
+            await puppet.intent.remove_room_tag(portal.mxid, tag)
+
+    @staticmethod
+    async def _mute_room(puppet: pu.Puppet, portal: po.Portal, mute_until: datetime) -> None:
+        if not config["bridge.mute_bridging"] or not portal or not portal.mxid:
+            return
+        now = datetime.utcnow().replace(tzinfo=timezone.utc)
+        if mute_until is not None and mute_until > now:
+            await puppet.intent.set_push_rule(PushRuleScope.GLOBAL, PushRuleKind.ROOM, portal.mxid,
+                                              actions=[PushActionType.DONT_NOTIFY])
+        else:
+            try:
+                await puppet.intent.remove_push_rule(PushRuleScope.GLOBAL, PushRuleKind.ROOM,
+                                                     portal.mxid)
+            except MNotFound:
+                pass
+
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        for peer in update.folder_peers:
+            portal = po.Portal.get_by_entity(peer.peer, receiver_id=self.tgid, create=False)
+            await self._tag_room(puppet, portal, config["bridge.archive_tag"],
+                                 peer.folder_id == 1)
+
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        # TODO bridge unpinning properly
+        for pinned in update.order:
+            portal = po.Portal.get_by_entity(pinned.peer, receiver_id=self.tgid, create=False)
+            await self._tag_room(puppet, portal, config["bridge.pinned_tag"], True)
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        elif not isinstance(update.peer, NotifyPeer):
+            # TODO handle global notification setting changes?
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        portal = po.Portal.get_by_entity(update.peer.peer, receiver_id=self.tgid, create=False)
+        await self._mute_room(puppet, portal, update.notify_settings.mute_until)
+
+    async def _sync_dialog(self, portal: po.Portal, dialog: Dialog, should_create: bool,
+                           puppet: Optional[pu.Puppet]) -> None:
+        was_created = False
+        if portal.mxid:
+            try:
+                await portal.backfill(self, last_id=dialog.message.id)
+            except Exception:
+                self.log.exception(f"Error while backfilling {portal.tgid_log}")
+            try:
+                await portal.update_matrix_room(self, dialog.entity)
+            except Exception:
+                self.log.exception(f"Error while updating {portal.tgid_log}")
+        elif should_create:
+            try:
+                await portal.create_matrix_room(self, dialog.entity, invites=[self.mxid])
+                was_created = True
+            except Exception:
+                self.log.exception(f"Error while creating {portal.tgid_log}")
+        if portal.mxid and puppet and puppet.is_real_user:
+            tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+            if dialog.unread_count == 0:
+                # This is usually more reliable than finding a specific message
+                # e.g. if the last read message is a service message that isn't in the message db
+                last_read = DBMessage.find_last(portal.mxid, tg_space)
+            else:
+                last_read = DBMessage.get_one_by_tgid(portal.tgid, tg_space,
+                                                      dialog.dialog.read_inbox_max_id)
+            if last_read:
+                await puppet.intent.mark_read(last_read.mx_room, last_read.mxid)
+            if was_created or not config["bridge.tag_only_on_create"]:
+                await self._mute_room(puppet, portal, dialog.dialog.notify_settings.mute_until)
+                await self._tag_room(puppet, portal, config["bridge.pinned_tag"], dialog.pinned)
+                await self._tag_room(puppet, portal, config["bridge.archive_tag"], dialog.archived)
+
+    async def sync_dialogs(self) -> None:
         if self.is_bot:
             return
         creators = []
-        limit = config["bridge.sync_dialog_limit"] or None
-        self.log.debug(f"Syncing dialogs (limit={limit}, synchronous_create={synchronous_create})")
-        async for dialog in self.client.iter_dialogs(limit=limit, ignore_migrated=True,
+        update_limit = config["bridge.sync_update_limit"] or None
+        create_limit = config["bridge.sync_create_limit"]
+        index = 0
+        self.log.debug(f"Syncing dialogs (update_limit={update_limit}, "
+                       f"create_limit={create_limit})")
+        await self.push_bridge_state(BridgeStateEvent.BACKFILLING)
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        dialog: Dialog
+        async for dialog in self.client.iter_dialogs(limit=update_limit, ignore_migrated=True,
                                                      archived=False):
             entity = dialog.entity
             if isinstance(entity, ChatForbidden):
                 self.log.warning(f"Ignoring forbidden chat {entity} while syncing")
+                continue
             elif isinstance(entity, Chat) and (entity.deactivated or entity.left):
                 self.log.warning(f"Ignoring deactivated or left chat {entity} while syncing")
                 continue
             elif isinstance(entity, TLUser) and not config["bridge.sync_direct_chats"]:
+                self.log.trace(f"Ignoring user {entity.id} while syncing")
                 continue
-            portal = po.Portal.get_by_entity(entity)
+            portal = po.Portal.get_by_entity(entity, receiver_id=self.tgid)
             self.portals[portal.tgid_full] = portal
-            creators.append(
-                portal.create_matrix_room(self, entity, invites=[self.mxid],
-                                          synchronous=synchronous_create))
-        self.save(portals=True)
-        await asyncio.gather(*creators, loop=self.loop)
+            coro = self._sync_dialog(portal=portal, dialog=dialog, puppet=puppet,
+                                     should_create=not create_limit or index < create_limit)
+            creators.append(self.loop.create_task(coro))
+            index += 1
+        await self.save(portals=True)
+        await asyncio.gather(*creators)
+        await self.update_direct_chats()
         self.log.debug("Dialog syncing complete")
 
-    def register_portal(self, portal: po.Portal) -> None:
+    async def register_portal(self, portal: po.Portal) -> None:
+        self.log.trace(f"Registering portal {portal.tgid_full}")
         try:
             if self.portals[portal.tgid_full] == portal:
                 return
         except KeyError:
             pass
         self.portals[portal.tgid_full] = portal
-        self.save(portals=True)
+        await self.save(portals=True)
 
-    def unregister_portal(self, portal: po.Portal) -> None:
+    async def unregister_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
+        self.log.trace(f"Unregistering portal {(tgid, tg_receiver)}")
         try:
-            del self.portals[portal.tgid_full]
-            self.save(portals=True)
+            del self.portals[(tgid, tg_receiver)]
+            await self.save(portals=True)
         except KeyError:
             pass
 
@@ -394,13 +613,14 @@ class User(AbstractUser, BaseUser):
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
             self.contacts.append(puppet)
-        self.save(contacts=True)
+        await self.save(contacts=True)
 
     # endregion
     # region Class instance lookup
 
     @classmethod
-    def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['User']:
+    def get_by_mxid(cls, mxid: UserID, create: bool = True, check_db: bool = True
+                    ) -> Optional['User']:
         if not mxid:
             raise ValueError("Matrix ID can't be empty")
 
@@ -409,10 +629,11 @@ class User(AbstractUser, BaseUser):
         except KeyError:
             pass
 
-        user = DBUser.get_by_mxid(mxid)
-        if user:
-            user = cls.from_db(user)
-            return user
+        if check_db:
+            user = DBUser.get_by_mxid(mxid)
+            if user:
+                user = cls.from_db(user)
+                return user
 
         if create:
             user = cls(mxid)
@@ -457,6 +678,7 @@ class User(AbstractUser, BaseUser):
 def init(context: 'Context') -> Iterable[Awaitable['User']]:
     global config
     config = context.config
+    User.bridge = context.bridge
 
     return (User.from_db(db_user).try_ensure_started()
             for db_user in DBUser.all_with_tgid())

mautrix_telegram/util/color_log.py

@@ -13,7 +13,8 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.util.color_log import ColorFormatter as BaseColorFormatter, PREFIX, MXID_COLOR, RESET
+from mautrix.util.logging.color import (ColorFormatter as BaseColorFormatter,
+                                        PREFIX, MXID_COLOR, RESET)
 
 TELETHON_COLOR = PREFIX + "35;1m"  # magenta
 TELETHON_MODULE_COLOR = PREFIX + "35m"

mautrix_telegram/util/file_transfer.py

@@ -18,6 +18,7 @@ from io import BytesIO
 import time
 import logging
 import asyncio
+import tempfile
 
 import magic
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
@@ -30,11 +31,11 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio
 
 from mautrix.appservice import IntentAPI
 
-
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
 from ..util import sane_mimetypes
 from .parallel_file_transfer import parallel_transfer_to_matrix
+from .tgs_converter import convert_tgs_to
 
 try:
     from PIL import Image
@@ -43,14 +44,13 @@ except ImportError:
 
 try:
     from moviepy.editor import VideoFileClip
-    import random
-    import string
-    import os
-    import mimetypes
 except ImportError:
-    VideoFileClip = random = string = os = mimetypes = None
+    VideoFileClip = None
 
-from .tgs_converter import convert_tgs_to
+try:
+    from mautrix.crypto.attachments import encrypt_attachment
+except ImportError:
+    encrypt_attachment = None
 
 log: logging.Logger = logging.getLogger("mau.util")
 
@@ -76,32 +76,23 @@ def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str
         return source_mime, file, None, None
 
 
-def _temp_file_name(ext: str) -> str:
-    return ("/tmp/mxtg-video-"
-            + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
-            + ext)
-
-
 def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str = "png",
                           max_size: Tuple[int, int] = (1024, 720)) -> Tuple[bytes, int, int]:
-    # We don't have any way to read the video from memory, so save it to disk.
-    temp_file = _temp_file_name(video_ext)
-    with open(temp_file, "wb") as file:
+    with tempfile.NamedTemporaryFile(prefix="mxtg_video_", suffix=f".{video_ext}") as file:
+        # We don't have any way to read the video from memory, so save it to disk.
         file.write(data)
 
-    # Read temp file and get frame
-    clip = VideoFileClip(temp_file)
-    frame = clip.get_frame(0)
+        # Read temp file and get frame
+        frame = VideoFileClip(file.name).get_frame(0)
 
     # Convert to png and save to BytesIO
     image = Image.fromarray(frame).convert("RGBA")
+
     thumbnail_file = BytesIO()
     if max_size:
         image.thumbnail(max_size, Image.ANTIALIAS)
     image.save(thumbnail_file, frame_ext)
 
-    os.remove(temp_file)
-
     w, h = image.size
     return thumbnail_file.getvalue(), w, h
 
@@ -111,13 +102,17 @@ def _location_to_id(location: TypeLocation) -> str:
         return f"{location.id}-{location.access_hash}"
     elif isinstance(location, (InputDocumentFileLocation, InputPhotoFileLocation)):
         return f"{location.id}-{location.access_hash}-{location.thumb_size}"
-    elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
+    elif isinstance(location, InputFileLocation):
         return f"{location.volume_id}-{location.local_id}"
+    elif isinstance(location, InputPeerPhotoFileLocation):
+        return str(location.photo_id)
 
 
 async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                       thumbnail_loc: TypeLocation, video: bytes,
-                                       mime: str) -> Optional[DBTelegramFile]:
+                                       thumbnail_loc: TypeLocation, mime_type: str, encrypt: bool,
+                                       video: Optional[bytes], custom_data: Optional[bytes] = None,
+                                       width: Optional[int] = None, height: [int] = None
+                                       ) -> Optional[DBTelegramFile]:
     if not Image or not VideoFileClip:
         return None
 
@@ -125,12 +120,17 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if not loc_id:
         return None
 
+    if custom_data:
+        loc_id += "-mau_custom_thumbnail"
+
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
-    video_ext = sane_mimetypes.guess_extension(mime)
-    if VideoFileClip and video_ext and video:
+    video_ext = sane_mimetypes.guess_extension(mime_type)
+    if custom_data:
+        file = custom_data
+    elif VideoFileClip and video_ext and video:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
         except OSError:
@@ -141,11 +141,18 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
         width, height = None, None
         mime_type = magic.from_buffer(file, mime=True)
 
-    content_uri = await intent.upload_media(file, mime_type)
+    decryption_info = None
+    upload_mime_type = mime_type
+    if encrypt:
+        file, decryption_info = encrypt_attachment(file)
+        upload_mime_type = "application/octet-stream"
+    content_uri = await intent.upload_media(file, upload_mime_type)
+    if decryption_info:
+        decryption_info.url = content_uri
 
     db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
                              was_converted=False, timestamp=int(time.time()), size=len(file),
-                             width=width, height=height)
+                             width=width, height=height, decryption_info=decryption_info)
     try:
         db_file.insert()
     except (IntegrityError, InvalidRequestError) as e:
@@ -161,10 +168,10 @@ TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
 
 
 async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None, *,
                                   is_sticker: bool = False, tgs_convert: Optional[dict] = None,
-                                  filename: Optional[str] = None, parallel_id: Optional[int] = None
-                                  ) -> Optional[DBTelegramFile]:
+                                  filename: Optional[str] = None, encrypt: bool = False,
+                                  parallel_id: Optional[int] = None) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
         return None
@@ -181,22 +188,24 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA
     async with lock:
         return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
                                                        thumbnail, is_sticker, tgs_convert,
-                                                       filename, parallel_id)
+                                                       filename, encrypt, parallel_id)
 
 
 async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                             loc_id: str, location: TypeLocation,
                                             thumbnail: TypeThumbnail, is_sticker: bool,
                                             tgs_convert: Optional[dict], filename: Optional[str],
-                                            parallel_id: Optional[int]
+                                            encrypt: bool, parallel_id: Optional[int]
                                             ) -> Optional[DBTelegramFile]:
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
+    converted_anim = None
+
     if parallel_id and isinstance(location, Document) and (not is_sticker or not tgs_convert):
         db_file = await parallel_transfer_to_matrix(client, intent, loc_id, location, filename,
-                                                    parallel_id)
+                                                    encrypt, parallel_id)
         mime_type = location.mime_type
         file = None
     else:
@@ -213,33 +222,45 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
 
         image_converted = False
         # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
-        if is_sticker and tgs_convert and (mime_type == "application/gzip" or (
-               mime_type == "application/octet-stream"
-               and magic.from_buffer(file).startswith("gzip"))):
-            mime_type, file, width, height = await convert_tgs_to(
-                file, tgs_convert["target"], **tgs_convert["args"])
-            thumbnail = None
+        is_tgs = (mime_type == "application/gzip"
+                  or (mime_type == "application/octet-stream"
+                      and magic.from_buffer(file).startswith("gzip")))
+        if is_sticker and tgs_convert and is_tgs:
+            converted_anim = await convert_tgs_to(file, tgs_convert["target"],
+                                                  **tgs_convert["args"])
+            mime_type = converted_anim.mime
+            file = converted_anim.data
+            width, height = converted_anim.width, converted_anim.height
             image_converted = mime_type != "application/gzip"
-
-        if mime_type == "image/webp":
-            new_mime_type, file, width, height = convert_image(
-                file, source_mime="image/webp", target_type="png",
-                thumbnail_to=(256, 256) if is_sticker else None)
-            image_converted = new_mime_type != mime_type
-            mime_type = new_mime_type
             thumbnail = None
 
-        content_uri = await intent.upload_media(file, mime_type)
+        decryption_info = None
+        upload_mime_type = mime_type
+        if encrypt and encrypt_attachment:
+            file, decryption_info = encrypt_attachment(file)
+            upload_mime_type = "application/octet-stream"
+        content_uri = await intent.upload_media(file, upload_mime_type)
+        if decryption_info:
+            decryption_info.url = content_uri
 
-        db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
+        db_file = DBTelegramFile(id=loc_id, mxc=content_uri, decryption_info=decryption_info,
                                  mime_type=mime_type, was_converted=image_converted,
                                  timestamp=int(time.time()), size=len(file),
                                  width=width, height=height)
     if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
         if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
             thumbnail = thumbnail.location
-        db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
-                                                               mime_type)
+        try:
+            db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail,
+                                                                   video=file, mime_type=mime_type,
+                                                                   encrypt=encrypt)
+        except FileIdInvalidError:
+            log.warning(f"Failed to transfer thumbnail for {thumbnail!s}", exc_info=True)
+    elif converted_anim and converted_anim.thumbnail_data:
+        db_file.thumbnail = await transfer_thumbnail_to_matrix(
+            client, intent, location, video=None, encrypt=encrypt,
+            custom_data=converted_anim.thumbnail_data, mime_type=converted_anim.thumbnail_mime,
+            width=converted_anim.width, height=converted_anim.height)
 
     try:
         db_file.insert()

mautrix_telegram/util/parallel_file_transfer.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, List, AsyncGenerator, Union, Awaitable, DefaultDict, Tuple
+from typing import Optional, List, AsyncGenerator, Union, Awaitable, DefaultDict, Tuple, cast
 from collections import defaultdict
 import hashlib
 import asyncio
@@ -27,19 +27,27 @@ from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLoc
                                InputPhotoFileLocation, InputPeerPhotoFileLocation, TypeInputFile,
                                InputFileBig, InputFile)
 from telethon.tl.functions.auth import ExportAuthorizationRequest, ImportAuthorizationRequest
+from telethon.tl.functions import InvokeWithLayerRequest
 from telethon.tl.functions.upload import (GetFileRequest, SaveFilePartRequest,
                                           SaveBigFilePartRequest)
+from telethon.tl.alltlobjects import LAYER
 from telethon.network import MTProtoSender
 from telethon.crypto import AuthKey
 from telethon import utils, helpers
 
 from mautrix.appservice import IntentAPI
-from mautrix.types import ContentURI
+from mautrix.types import ContentURI, EncryptedFile
+from mautrix.util.logging import TraceLogger
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
 
-log: logging.Logger = logging.getLogger("mau.util")
+try:
+    from mautrix.crypto.attachments import async_encrypt_attachment
+except ImportError:
+    async_encrypt_attachment = None
+
+log: TraceLogger = cast(TraceLogger, logging.getLogger("mau.util"))
 
 TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
                      InputFileLocation, InputPhotoFileLocation]
@@ -97,7 +105,7 @@ class UploadSender:
 
     async def _next(self, data: bytes) -> None:
         self.request.bytes = data
-        log.debug(f"Sending file part {self.request.file_part}/{self.part_count}"
+        log.trace(f"Sending file part {self.request.file_part}/{self.part_count}"
                   f" with {len(data)} bytes")
         await self.sender.send(self.request)
         self.request.file_part += self.stride
@@ -126,7 +134,7 @@ class ParallelTransferrer:
         self.upload_ticker = 0
 
     async def _cleanup(self) -> None:
-        await asyncio.gather(*[sender.disconnect() for sender in self.senders])
+        await asyncio.gather(*(sender.disconnect() for sender in self.senders))
         self.senders = None
 
     @staticmethod
@@ -153,9 +161,9 @@ class ParallelTransferrer:
             await self._create_download_sender(file, 0, part_size, connections * part_size,
                                                get_part_count()),
             *await asyncio.gather(
-                *[self._create_download_sender(file, i, part_size, connections * part_size,
+                *(self._create_download_sender(file, i, part_size, connections * part_size,
                                                get_part_count())
-                  for i in range(1, connections)])
+                  for i in range(1, connections)))
         ]
 
     async def _create_download_sender(self, file: TypeLocation, index: int, part_size: int,
@@ -169,8 +177,8 @@ class ParallelTransferrer:
         self.senders = [
             await self._create_upload_sender(file_id, part_count, big, 0, connections),
             *await asyncio.gather(
-                *[self._create_upload_sender(file_id, part_count, big, i, connections)
-                  for i in range(1, connections)])
+                *(self._create_upload_sender(file_id, part_count, big, i, connections)
+                  for i in range(1, connections)))
         ]
 
     async def _create_upload_sender(self, file_id: int, part_count: int, big: bool, index: int,
@@ -180,16 +188,16 @@ class ParallelTransferrer:
 
     async def _create_sender(self) -> MTProtoSender:
         dc = await self.client._get_dc(self.dc_id)
-        sender = MTProtoSender(self.auth_key, self.loop, loggers=self.client._log)
+        sender = MTProtoSender(self.auth_key, loggers=self.client._log)
         await sender.connect(self.client._connection(dc.ip_address, dc.port, dc.id,
-                                                     loop=self.loop, loggers=self.client._log,
+                                                     loggers=self.client._log,
                                                      proxy=self.client._proxy))
         if not self.auth_key:
             log.debug(f"Exporting auth to DC {self.dc_id}")
             auth = await self.client(ExportAuthorizationRequest(self.dc_id))
-            req = self.client._init_with(ImportAuthorizationRequest(
-                id=auth.id, bytes=auth.bytes
-            ))
+            self.client._init_request.query = ImportAuthorizationRequest(id=auth.id,
+                                                                         bytes=auth.bytes)
+            req = InvokeWithLayerRequest(LAYER, self.client._init_request)
             await sender.send(req)
             self.auth_key = sender.auth_key
         return sender
@@ -231,7 +239,7 @@ class ParallelTransferrer:
                     break
                 yield data
                 part += 1
-                log.debug(f"Part {part} downloaded")
+                log.trace(f"Part {part} downloaded")
 
         log.debug("Parallel download finished, cleaning up connections")
         await self._cleanup()
@@ -242,18 +250,34 @@ parallel_transfer_locks: DefaultDict[int, asyncio.Lock] = defaultdict(lambda: as
 
 async def parallel_transfer_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                       loc_id: str, location: TypeLocation, filename: str,
-                                      parallel_id: int) -> DBTelegramFile:
+                                      encrypt: bool, parallel_id: int) -> DBTelegramFile:
     size = location.size
     mime_type = location.mime_type
     dc_id, location = utils.get_input_location(location)
     # We lock the transfers because telegram has connection count limits
     async with parallel_transfer_locks[parallel_id]:
         downloader = ParallelTransferrer(client, dc_id)
-        content_uri = await intent.upload_media(downloader.download(location, size),
-                                                mime_type=mime_type, filename=filename, size=size)
+        data = downloader.download(location, size)
+        decryption_info = None
+        up_mime_type = mime_type
+        if encrypt and async_encrypt_attachment:
+            async def encrypted(stream):
+                nonlocal decryption_info
+                async for chunk in async_encrypt_attachment(stream):
+                    if isinstance(chunk, EncryptedFile):
+                        decryption_info = chunk
+                    else:
+                        yield chunk
+
+            data = encrypted(data)
+            up_mime_type = "application/octet-stream"
+        content_uri = await intent.upload_media(data, mime_type=up_mime_type, filename=filename,
+                                                size=size if not encrypt else None)
+        if decryption_info:
+            decryption_info.url = content_uri
     return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
                           was_converted=False, timestamp=int(time.time()), size=size,
-                          width=None, height=None)
+                          width=None, height=None, decryption_info=decryption_info)
 
 
 async def _internal_transfer_to_telegram(client: MautrixTelegramClient, response: ClientResponse

mautrix_telegram/util/tgs_converter.py

@@ -21,8 +21,23 @@ import shutil
 import os.path
 import tempfile
 
+from attr import dataclass
+
 log: logging.Logger = logging.getLogger("mau.util.tgs")
-converters: Dict[str, Callable[[bytes, int, int, Any], Awaitable[Tuple[str, bytes]]]] = {}
+
+
+@dataclass
+class ConvertedSticker:
+    mime: str
+    data: bytes
+    thumbnail_mime: Optional[str] = None
+    thumbnail_data: Optional[bytes] = None
+    width: int = 0
+    height: int = 0
+
+
+Converter = Callable[[bytes, int, int, Any], Awaitable[ConvertedSticker]]
+converters: Dict[str, Converter] = {}
 
 
 def abswhich(program: Optional[str]) -> Optional[str]:
@@ -34,7 +49,7 @@ lottieconverter = abswhich("lottieconverter")
 ffmpeg = abswhich("ffmpeg")
 
 if lottieconverter:
-    async def tgs_to_png(file: bytes, width: int, height: int, **_: Any) -> Tuple[str, bytes]:
+    async def tgs_to_png(file: bytes, width: int, height: int, **_: Any) -> ConvertedSticker:
         frame = 1
         proc = await asyncio.create_subprocess_exec(lottieconverter, "-", "-", "png",
                                                     f"{width}x{height}", str(frame),
@@ -42,26 +57,26 @@ if lottieconverter:
                                                     stdin=asyncio.subprocess.PIPE)
         stdout, stderr = await proc.communicate(file)
         if proc.returncode == 0:
-            return "image/png", stdout
+            return ConvertedSticker("image/png", stdout)
         else:
             log.error("lottieconverter error: " + (stderr.decode("utf-8") if stderr is not None
-                      else f"unknown ({proc.returncode})"))
-            return "application/gzip", file
+                                                   else f"unknown ({proc.returncode})"))
+            return ConvertedSticker("application/gzip", file)
 
 
     async def tgs_to_gif(file: bytes, width: int, height: int, background: str = "202020",
-                         **_: Any) -> Tuple[str, bytes]:
+                         **_: Any) -> ConvertedSticker:
         proc = await asyncio.create_subprocess_exec(lottieconverter, "-", "-", "gif",
                                                     f"{width}x{height}", f"0x{background}",
                                                     stdout=asyncio.subprocess.PIPE,
                                                     stdin=asyncio.subprocess.PIPE)
         stdout, stderr = await proc.communicate(file)
         if proc.returncode == 0:
-            return "image/gif", stdout
+            return ConvertedSticker("image/gif", stdout)
         else:
             log.error("lottieconverter error: " + (stderr.decode("utf-8") if stderr is not None
-                      else f"unknown ({proc.returncode})"))
-            return "application/gzip", file
+                                                   else f"unknown ({proc.returncode})"))
+            return ConvertedSticker("application/gzip", file)
 
 
     converters["png"] = tgs_to_png
@@ -69,7 +84,7 @@ if lottieconverter:
 
 if lottieconverter and ffmpeg:
     async def tgs_to_webm(file: bytes, width: int, height: int, fps: int = 30,
-                          **_: Any) -> Tuple[str, bytes]:
+                          **_: Any) -> ConvertedSticker:
         with tempfile.TemporaryDirectory(prefix="tgs_") as tmpdir:
             file_template = tmpdir + "/out_"
             proc = await asyncio.create_subprocess_exec(lottieconverter, "-", file_template,
@@ -78,6 +93,8 @@ if lottieconverter and ffmpeg:
                                                         stdin=asyncio.subprocess.PIPE)
             _, stderr = await proc.communicate(file)
             if proc.returncode == 0:
+                with open(f"{file_template}00.png", "rb") as first_frame_file:
+                    first_frame_data = first_frame_file.read()
                 proc = await asyncio.create_subprocess_exec(ffmpeg, "-hide_banner", "-loglevel",
                                                             "error", "-framerate", str(fps),
                                                             "-pattern_type", "glob", "-i",
@@ -88,25 +105,27 @@ if lottieconverter and ffmpeg:
                                                             stdin=asyncio.subprocess.PIPE)
                 stdout, stderr = await proc.communicate()
                 if proc.returncode == 0:
-                    return "video/webm", stdout
+                    return ConvertedSticker("video/webm", stdout, "image/png", first_frame_data)
                 else:
                     log.error("ffmpeg error: " + (stderr.decode("utf-8") if stderr is not None
-                              else f"unknown ({proc.returncode})"))
+                                                  else f"unknown ({proc.returncode})"))
             else:
                 log.error("lottieconverter error: " + (stderr.decode("utf-8") if stderr is not None
-                          else f"unknown ({proc.returncode})"))
-        return "application/gzip", file
+                                                       else f"unknown ({proc.returncode})"))
+        return ConvertedSticker("application/gzip", file)
 
 
     converters["webm"] = tgs_to_webm
 
 
 async def convert_tgs_to(file: bytes, convert_to: str, width: int, height: int, **kwargs: Any
-                         ) -> Tuple[str, bytes, Optional[int], Optional[int]]:
+                         ) -> ConvertedSticker:
     if convert_to in converters:
         converter = converters[convert_to]
-        mime, out = await converter(file, width, height, **kwargs)
-        return mime, out, width, height
+        converted = await converter(file, width, height, **kwargs)
+        converted.width = width
+        converted.height = height
+        return converted
     elif convert_to != "disable":
         log.warning(f"Unable to convert animated sticker, type {convert_to} not supported")
-    return "application/gzip", file, None, None
+    return ConvertedSticker("application/gzip", file)

mautrix_telegram/web/provisioning/__init__.py

@@ -21,7 +21,10 @@ import json
 from aiohttp import web
 
 from telethon.utils import get_peer_id, resolve_id
-from telethon.tl.types import ChatForbidden, ChannelForbidden, TypeChat
+from telethon.tl.types import ChatForbidden, ChannelForbidden, TypeChat, InputUserSelf
+from telethon.tl.functions.users import GetUsersRequest
+from telethon.errors import (UserDeactivatedError, UserDeactivatedBanError, SessionRevokedError,
+                             UnauthorizedError)
 
 from mautrix.appservice import AppService
 from mautrix.errors import MatrixRequestError, IntentError
@@ -141,6 +144,12 @@ class ProvisioningAPI(AuthAPI):
             return self.get_error_response(403, "not_enough_permissions",
                                            "You do not have the permissions to bridge that room.")
 
+        is_logged_in = user is not None and await user.is_logged_in()
+        acting_user = user if is_logged_in else self.context.bot
+        if not acting_user:
+            return self.get_login_response(status=403, errcode="not_logged_in",
+                                           error="You are not logged in and there is no relay bot.")
+
         portal = Portal.get_by_tgid(tgid, peer_type=peer_type)
         if portal.mxid == room_id:
             return self.get_error_response(200, "bridge_exists",
@@ -157,35 +166,30 @@ class ProvisioningAPI(AuthAPI):
                                                "Telegram chat is already bridged to another "
                                                "Matrix room.")
 
-        is_logged_in = user is not None and await user.is_logged_in()
-        acting_user = user if is_logged_in else self.context.bot
-        if not acting_user:
-            return self.get_login_response(status=403, errcode="not_logged_in",
-                                           error="You are not logged in and there is no relay bot.")
+        async with portal._room_create_lock:
+            entity: Optional[TypeChat] = None
+            try:
+                entity = await acting_user.client.get_entity(portal.peer)
+            except Exception:
+                self.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
 
-        entity: Optional[TypeChat] = None
-        try:
-            entity = await acting_user.client.get_entity(portal.peer)
-        except Exception:
-            self.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
-
-        if not entity or isinstance(entity, (ChatForbidden, ChannelForbidden)):
-            if is_logged_in:
-                return self.get_error_response(403, "user_not_in_chat",
+            if not entity or isinstance(entity, (ChatForbidden, ChannelForbidden)):
+                if is_logged_in:
+                    return self.get_error_response(403, "user_not_in_chat",
+                                                   "Failed to get info of Telegram chat. "
+                                                   "Are you in the chat?")
+                return self.get_error_response(403, "bot_not_in_chat",
                                                "Failed to get info of Telegram chat. "
-                                               "Are you in the chat?")
-            return self.get_error_response(403, "bot_not_in_chat",
-                                           "Failed to get info of Telegram chat. "
-                                           "Is the relay bot in the chat?")
+                                               "Is the relay bot in the chat?")
 
-        direct = False
+            portal.mxid = room_id
+            portal.by_mxid[portal.mxid] = portal
+            (portal.title, portal.about, levels,
+             portal.encrypted) = await get_initial_state(self.az.intent, room_id)
+            portal.photo_id = ""
+            await portal.save()
 
-        portal.mxid = room_id
-        portal.title, portal.about, levels = await get_initial_state(self.az.intent, room_id)
-        portal.photo_id = ""
-        portal.save()
-
-        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
+        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
                               loop=self.loop)
 
         return web.Response(status=202, body="{}")
@@ -216,7 +220,7 @@ class ProvisioningAPI(AuthAPI):
                                            "You do not have the permissions to bridge that room.")
 
         try:
-            title, about, _ = await get_initial_state(self.az.intent, room_id)
+            title, about, _, encrypted = await get_initial_state(self.az.intent, room_id)
         except (MatrixRequestError, IntentError):
             return self.get_error_response(403, "bot_not_in_room",
                                            "The bridge bot is not in the given room.")
@@ -240,11 +244,12 @@ class ProvisioningAPI(AuthAPI):
             "group": "chat",
         }[type]
 
-        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type)
+        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type,
+                        encrypted=encrypted)
         try:
             await portal.create_telegram_chat(user, supergroup=supergroup)
         except ValueError as e:
-            portal.delete()
+            await portal.delete()
             return self.get_error_response(500, "unknown_error", e.args[0])
 
         return web.json_response({
@@ -292,16 +297,17 @@ class ProvisioningAPI(AuthAPI):
 
         user_data = None
         if await user.is_logged_in():
-            me = await user.client.get_me()
-            await user.update_info(me)
-            user_data = {
-                "id": user.tgid,
-                "username": user.username,
-                "first_name": me.first_name,
-                "last_name": me.last_name,
-                "phone": me.phone,
-                "is_bot": user.is_bot,
-            }
+            me = await user.get_me()
+            if me:
+                await user.update_info(me)
+                user_data = {
+                    "id": user.tgid,
+                    "username": user.username,
+                    "first_name": me.first_name,
+                    "last_name": me.last_name,
+                    "phone": me.phone,
+                    "is_bot": user.is_bot,
+                }
         return web.json_response({
             "telegram": user_data,
             "mxid": user.mxid,
@@ -315,9 +321,9 @@ class ProvisioningAPI(AuthAPI):
 
         if not user.is_bot:
             return web.json_response([{
-                "id": get_peer_id(chat),
+                "id": chat.id,
                 "title": chat.title,
-            } async for chat in user.client.get_dialogs(ignore_migrated=True, archived=False)])
+            } async for chat in user.client.iter_dialogs(ignore_migrated=True, archived=False)])
         else:
             return web.json_response([{
                 "id": get_peer_id(chat.peer),
@@ -349,12 +355,13 @@ class ProvisioningAPI(AuthAPI):
         return await self.post_login_password(user, data.get("password", ""))
 
     async def logout(self, request: web.Request) -> web.Response:
-        _, user, err = await self.get_user_request_info(request, expect_logged_in=True,
+        _, user, err = await self.get_user_request_info(request, expect_logged_in=None,
                                                         require_puppeting=False,
                                                         want_data=False)
         if err is not None:
             return err
         await user.log_out()
+        return web.json_response({}, status=200)
 
     async def bridge_info(self, request: web.Request) -> web.Response:
         return web.json_response({
@@ -458,7 +465,7 @@ class ProvisioningAPI(AuthAPI):
                                                 Optional[web.Response]]):
         err = self.check_authorization(request)
         if err is not None:
-            return err
+            return None, None, err
 
         data = None
         if want_data and (request.method == "POST" or request.method == "PUT"):

mautrix_telegram/web/provisioning/spec.yaml

@@ -6,11 +6,11 @@ info:
   description: The provisioning API for Mautrix-Telegram, the Matrix-Telegram puppeting/relaybot bridge.
   license:
     name: AGPLv3
-    url: https://github.com/tulir/mautrix-telegram/blob/master/LICENSE
+    url: https://github.com/mautrix/telegram/blob/master/LICENSE
 
 externalDocs:
-  description: Provisioning API wiki page on GitHub
-  url: https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API
+  description: Provisioning API docs on docs.mau.fi
+  url: https://docs.mau.fi/bridges/python/telegram/provisioning-api.html
 
 basePath: /_matrix/provision/v1
 

optional-requirements.txt

@@ -1,5 +1,30 @@
-cryptg
-Pillow
-moviepy
-prometheus_client
-psycopg2-binary
+# Format: #/name defines a new extras_require group called name
+# Uncommented lines after the group definition insert things into that group.
+
+#/speedups
+cryptg>=0.1,<0.3
+cchardet
+aiodns
+brotli
+
+#/qr_login
+pillow>=4,<9
+qrcode>=6,<7
+
+#/hq_thumbnails
+moviepy>=1,<2
+
+#/metrics
+prometheus_client>=0.6,<0.13
+
+#/postgres
+psycopg2-binary>=2,<3
+asyncpg>=0.20,<0.25
+
+#/sqlite
+aiosqlite>=0.17,<0.18
+
+#/e2be
+python-olm>=3,<4
+pycryptodome>=3,<4
+unpaddedbase64>=1,<2

requirements.txt

@@ -1,9 +1,15 @@
-aiohttp
-mautrix
-ruamel.yaml
-python-magic
-SQLAlchemy
-alembic
-commonmark
-telethon
-telethon-session-sqlalchemy
+SQLAlchemy>=1.2,<1.4
+alembic>=1,<2
+ruamel.yaml>=0.15.35,<0.18
+python-magic>=0.4,<0.5
+commonmark>=0.8,<0.10
+aiohttp>=3,<4
+yarl>=1,<2
+mautrix>=0.11.3,<0.12
+#telethon>=1.22,<1.24
+# Temporary patch for 64-bit IDs until upstream telethon 2.0 is ready
+tulir-telethon==1.24.0a2
+telethon-session-sqlalchemy>=0.2.14,<0.3
+# Temporarily always depend on aiosqlite to prevent breaking old installs
+# Will be removed in v0.12 (after which you need to choose the [sqlite] optional dependency)
+aiosqlite>=0.17,<0.18

setup.py

@@ -3,14 +3,21 @@ import glob
 
 from mautrix_telegram.get_version import git_tag, git_revision, version, linkified_version
 
-extras = {
-    "speedups": ["cryptg>=0.1,<0.3", "cchardet", "aiodns", "Brotli"],
-    "webp_convert": ["Pillow>=4.3.0,<7"],
-    "hq_thumbnails": ["moviepy>=1.0,<2.0"],
-    "metrics": ["prometheus_client>=0.6.0,<0.8.0"],
-    "postgres": ["psycopg2-binary>=2,<3"],
-}
-extras["all"] = list({dep for deps in extras.values() for dep in deps})
+with open("requirements.txt") as reqs:
+    install_requires = reqs.read().splitlines()
+
+with open("optional-requirements.txt") as reqs:
+    extras_require = {}
+    current = []
+    for line in reqs.read().splitlines():
+        if line.startswith("#/"):
+            extras_require[line[2:]] = current = []
+        elif not line or line.startswith("#"):
+            continue
+        else:
+            current.append(line)
+
+extras_require["all"] = list({dep for deps in extras_require.values() for dep in deps})
 
 try:
     long_desc = open("README.md").read()
@@ -29,7 +36,7 @@ linkified_version = {linkified_version!r}
 setuptools.setup(
     name="mautrix-telegram",
     version=version,
-    url="https://github.com/tulir/mautrix-telegram",
+    url="https://github.com/mautrix/telegram",
 
     author="Tulir Asokan",
     author_email="tulir@maunium.net",
@@ -40,19 +47,9 @@ setuptools.setup(
 
     packages=setuptools.find_packages(),
 
-    install_requires=[
-        "aiohttp>=3.0.1,<4",
-        "mautrix>=0.4.0,<0.5",
-        "SQLAlchemy>=1.2.3,<2",
-        "alembic>=1.0.0,<2",
-        "commonmark>=0.8.1,<0.10",
-        "ruamel.yaml>=0.15.35,<0.17",
-        "python-magic>=0.4.15,<0.5",
-        "telethon>=1.10,<1.11",
-        "telethon-session-sqlalchemy>=0.2.14,<0.3",
-    ],
-    extras_require=extras,
-    python_requires="~=3.6",
+    install_requires=install_requires,
+    extras_require=extras_require,
+    python_requires="~=3.7",
 
     setup_requires=["pytest-runner"],
     tests_require=["pytest", "pytest-asyncio", "pytest-mock"],
@@ -64,19 +61,16 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
-    entry_points="""
-        [console_scripts]
-        mautrix-telegram=mautrix_telegram.__main__:main
-    """,
     package_data={"mautrix_telegram": [
         "web/public/*.mako", "web/public/*.png", "web/public/*.css",
+        "example-config.yaml",
     ]},
     data_files=[
-        (".", ["example-config.yaml", "alembic.ini"]),
+        (".", ["alembic.ini", "mautrix_telegram/example-config.yaml"]),
         ("alembic", ["alembic/env.py"]),
         ("alembic/versions", glob.glob("alembic/versions/*.py"))
     ],