Bump version to 0.9.0

If it just kicks the user, logging in again later would cause the
bridge to think there's a portal, but fail to invite the user again.

Fixes #397

.editorconfig

@@ -13,3 +13,6 @@ max_line_length = 99
 
 [*.{yaml,yml,py}]
 indent_style = space
+
+[.gitlab-ci.yml]
+indent_size = 2

.gitignore

@@ -14,4 +14,5 @@ __pycache__
 /registration.yaml
 *.log*
 *.db
+*.pickle
 *.bak

.gitlab-ci.yml

@@ -2,37 +2,47 @@ image: docker:stable
 
 stages:
 - build
-- push
+- manifest
 
 default:
   before_script:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
-build:
+build amd64:
   stage: build
+  tags:
+  - amd64
   script:
   - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  after_script:
+  - |
+    if [ "$CI_COMMIT_BRANCH" = "master" ]; then
+      apk add --update curl
+      rm -rf /var/cache/apk/*
+      curl "$NOVA_ADMIN_API_URL" -H "Content-Type: application/json" -d '{"password":"'"$NOVA_ADMIN_NIGHTLY_PASS"'","bridge":"'$NOVA_BRIDGE_TYPE'","image":"'$CI_REGISTRY_IMAGE':'$CI_COMMIT_SHA'-amd64"}'
+    fi
 
-push latest:
+build arm64:
-  stage: push
+  stage: build
-  only:
+  tags:
-  - master
+  - arm64
-  variables:
-    GIT_STRATEGY: none
   script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
-  - docker push $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
 
-push tag:
+manifest:
-  stage: push
+  stage: manifest
-  variables:
+  before_script:
-    GIT_STRATEGY: none
+  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
-  except:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-  - master
   script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
+  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64

Dockerfile

@@ -1,77 +1,73 @@
-FROM docker.io/alpine:3.10 AS lottieconverter
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.12
 
-WORKDIR /build
+ARG TARGETARCH=amd64
 
-RUN apk add --no-cache git build-base cmake \
+RUN echo $'\
-  && git clone https://github.com/Samsung/rlottie.git \
+@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\n\
-  && cd rlottie \
+@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing\n\
-  && mkdir build \
+@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
-  && cd build \
-  && cmake .. \
-  && make -j2 \
-  && make install \
-  && cd ../..
 
-RUN apk add --no-cache libpng libpng-dev zlib zlib-dev \
+RUN apk add --no-cache \
-  && git clone https://github.com/Eramde/LottieConverter.git \
+      python3 py3-pip py3-setuptools py3-wheel \
-  && cd LottieConverter \
-  && git checkout 543c1d23ac9322f4f03c7fb6612ea7d026d44ac0 \
-  && make
-
-FROM docker.io/alpine:3.11
-
-ENV UID=1337 \
-    GID=1337 \
-    FFMPEG_BINARY=/usr/bin/ffmpeg
-
-COPY --from=lottieconverter /usr/lib/librlottie* /usr/lib/
-COPY --from=lottieconverter /build/LottieConverter/dist/Debug/GNU-Linux/lottieconverter /usr/local/bin/lottieconverter
-
-COPY . /opt/mautrix-telegram
-WORKDIR /opt/mautrix-telegram
-RUN apk add --no-cache --virtual .build-deps \
-      python3-dev \
-      libffi-dev \
-      build-base \
-      git \
-  && apk add --no-cache \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-magic \
       py3-sqlalchemy \
+      py3-telethon-session-sqlalchemy@edge \
+      py3-alembic@edge \
       py3-psycopg2 \
       py3-ruamel.yaml \
+      py3-commonmark@edge \
       # Indirect dependencies
       py3-idna \
-      #commonmark
-        py3-future \
-      #alembic
-        py3-mako \
-        py3-dateutil \
-        py3-markupsafe \
       #moviepy
         py3-decorator \
         py3-tqdm \
         py3-requests \
         #imageio
           py3-numpy \
-      #telethon
+      #py3-telethon@edge \ (outdated)
-        py3-rsa \
+        # Optional for socks proxies
+        py3-pysocks \
         # cryptg
           py3-cffi \
+	  py3-qrcode@edge \
+      py3-brotli \
       # Other dependencies
       ffmpeg \
       ca-certificates \
       su-exec \
       netcat-openbsd \
-      # lottieconverter
+      # encryption
-      zlib libpng \
+      olm-dev \
- && pip3 install .[speedups,hq_thumbnails,metrics] \
+      py3-pycryptodome \
- # pip installs the sources to /usr/lib/python3.8/site-packages, so we don't need them here
+      py3-unpaddedbase64 \
- && rm -rf /opt/mautrix-telegram/mautrix_telegram \
+      py3-future \
+      bash \
+      curl \
+      jq && \
+  curl -sLo yq https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${TARGETARCH} && \
+  chmod +x yq && mv yq /usr/bin/yq
+
+COPY requirements.txt /opt/mautrix-telegram/requirements.txt
+COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
+WORKDIR /opt/mautrix-telegram
+RUN apk add --virtual .build-deps \
+      python3-dev \
+      libffi-dev \
+      build-base \
+ && sed -Ei 's/psycopg2-binary.+//' optional-requirements.txt \
+ && pip3 install -r requirements.txt -r optional-requirements.txt \
  && apk del .build-deps
 
+COPY . /opt/mautrix-telegram
+RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics,e2be] && apk del git \
+  # This doesn't make the image smaller, but it's needed so that the `version` command works properly
+  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
+
 VOLUME /data
+ENV UID=1337 GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
 
 CMD ["/opt/mautrix-telegram/docker-run.sh"]

MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include LICENSE
+include requirements.txt
+include optional-requirements.txt

README.md

@@ -7,9 +7,22 @@
 
 A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
-### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)
+## Sponsors
+* [Joel Lehtonen / Zouppen](https://github.com/zouppen)
 
-### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+### Wiki
+All setup and usage instructions are located in the GitHub
+[wiki](https://github.com/tulir/mautrix-telegram/wiki). Some quick links:
+
+* [Bridge setup](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup)
+  (or [with Docker](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup-with-Docker))
+* Basic usage: [Authentication](https://github.com/tulir/mautrix-telegram/wiki/Authentication),
+  [Creating chats](https://github.com/tulir/mautrix-telegram/wiki/Creating-and-managing-chats),
+  [Relaybot setup](https://github.com/tulir/mautrix-telegram/wiki/Relay-bot)
+
+### Features & Roadmap
+[ROADMAP.md](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+contains a general overview of what is supported by the bridge.
 
 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)
@@ -17,4 +30,4 @@ Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net
 Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
 
 ## Preview
-![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)
+![Preview](preview.png)

ROADMAP.md

@@ -28,7 +28,10 @@
 	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
-  * [ ] Message history
+  * [x] Message history
+    * [x] Manually (`!tg backfill`)
+    * [x] Automatically when creating portal
+    * [x] Automatically for missed messages
   * [x] Avatars
   * [x] Presence
   * [x] Typing notifications
@@ -50,11 +53,11 @@
     * [x] At startup
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [x] Option to use bot to relay messages for unauthenticated Matrix users
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
-  * [x] Option to use own Matrix account for messages sent from other Telegram clients
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
-  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
-  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
+  * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
 
 † Information not automatically sent from source, i.e. implementation may not be possible
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/env.py

@@ -21,7 +21,6 @@ mxtg_config = Config(mxtg_config_path, None, None)
 mxtg_config.load()
 config.set_main_option("sqlalchemy.url", mxtg_config["appservice.database"].replace("%", "%%"))
 
-
 AlchemySessionContainer.create_table_classes(None, "telethon_", Base)
 
 # Interpret the config file for Python logging.
@@ -55,7 +54,8 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True)
+        url=url, target_metadata=target_metadata, literal_binds=True,
+        render_as_batch=True)
 
     with context.begin_transaction():
         context.run_migrations()
@@ -76,7 +76,8 @@ def run_migrations_online():
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
-            target_metadata=target_metadata
+            target_metadata=target_metadata,
+            render_as_batch=True
         )
 
         with context.begin_transaction():

alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py

@@ -0,0 +1,27 @@
+"""Add encrypted field for portals
+
+Revision ID: 24f31fc8a72b
+Revises: a7c04a56041b
+Create Date: 2020-03-28 20:14:29.046699
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "24f31fc8a72b"
+down_revision = "a7c04a56041b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column("encrypted", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column("encrypted")

alembic/versions/3e3745baa458_store_matrix_avatar_url_in_database.py

@@ -0,0 +1,32 @@
+"""Store Matrix avatar URL in database
+
+Revision ID: 3e3745baa458
+Revises: dff56c93da8d
+Create Date: 2020-06-15 14:32:10.454033
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3e3745baa458'
+down_revision = 'dff56c93da8d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('portal', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('avatar_url', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('portal', schema=None) as batch_op:
+        batch_op.drop_column('avatar_url')
+
+    # ### end Alembic commands ###

alembic/versions/888275d58e57_add_double_puppet_base_url_to_puppet_.py

@@ -0,0 +1,30 @@
+"""Add double puppet base URL to puppet table
+
+Revision ID: 888275d58e57
+Revises: a328bf4f0932
+Create Date: 2020-10-14 18:52:00.730666
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '888275d58e57'
+down_revision = 'a328bf4f0932'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('base_url', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('base_url')
+    # ### end Alembic commands ###

alembic/versions/a328bf4f0932_store_encryption_state_event_in_db.py

@@ -0,0 +1,38 @@
+"""Store encryption state event in db
+
+Revision ID: a328bf4f0932
+Revises: ccbaff858240
+Create Date: 2020-07-11 21:31:27.059813
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from mautrix.client.state_store.sqlalchemy import SerializableType
+from mautrix.types import RoomEncryptionStateEventContent
+
+# revision identifiers, used by Alembic.
+revision = 'a328bf4f0932'
+down_revision = 'ccbaff858240'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('encryption',
+                                      SerializableType(RoomEncryptionStateEventContent),
+                                      nullable=True))
+        batch_op.add_column(sa.Column('has_full_member_list', sa.Boolean(), nullable=True))
+        batch_op.add_column(sa.Column('is_encrypted', sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
+        batch_op.drop_column('is_encrypted')
+        batch_op.drop_column('has_full_member_list')
+        batch_op.drop_column('encryption')
+    # ### end Alembic commands ###

alembic/versions/ccbaff858240_switch_to_mautrix_python_crypto.py

@@ -0,0 +1,71 @@
+"""Switch to mautrix-python crypto
+
+Revision ID: ccbaff858240
+Revises: 3e3745baa458
+Create Date: 2020-07-08 19:06:12.588047
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'ccbaff858240'
+down_revision = '3e3745baa458'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('nio_account')
+    op.drop_table('nio_device_key')
+    op.drop_table('nio_outgoing_key_request')
+    op.drop_table('nio_olm_session')
+    op.drop_table('nio_megolm_inbound_session')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('nio_megolm_inbound_session',
+    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('fp_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.Column('forwarded_chains', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('session_id', name='nio_megolm_inbound_session_pkey')
+    )
+    op.create_table('nio_olm_session',
+    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.Column('last_used', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('session_id', name='nio_olm_session_pkey')
+    )
+    op.create_table('nio_outgoing_key_request',
+    sa.Column('request_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('algorithm', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('request_id', name='nio_outgoing_key_request_pkey')
+    )
+    op.create_table('nio_device_key',
+    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('display_name', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('deleted', sa.BOOLEAN(), autoincrement=False, nullable=False),
+    sa.Column('keys', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_device_key_pkey')
+    )
+    op.create_table('nio_account',
+    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('shared', sa.BOOLEAN(), autoincrement=False, nullable=False),
+    sa.Column('sync_token', sa.TEXT(), autoincrement=False, nullable=False),
+    sa.Column('account', postgresql.BYTEA(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_account_pkey')
+    )
+    # ### end Alembic commands ###

alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py

@@ -0,0 +1,26 @@
+"""Add decryption info field for reuploaded telegram files
+
+Revision ID: d3c922a6acd2
+Revises: 24f31fc8a72b
+Create Date: 2020-03-30 20:07:17.340346
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd3c922a6acd2'
+down_revision = '24f31fc8a72b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.add_column(sa.Column("decryption_info", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column("decryption_info")

alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py

@@ -0,0 +1,71 @@
+"""Add matrix-nio state store to main db
+
+Revision ID: dff56c93da8d
+Revises: d3c922a6acd2
+Create Date: 2020-03-31 22:04:04.014048
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'dff56c93da8d'
+down_revision = 'd3c922a6acd2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('nio_account',
+    sa.Column('user_id', sa.String(length=255), nullable=False),
+    sa.Column('device_id', sa.String(length=255), nullable=False),
+    sa.Column('shared', sa.Boolean(), nullable=False),
+    sa.Column('sync_token', sa.Text(), nullable=False),
+    sa.Column('account', sa.LargeBinary(), nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id')
+    )
+    op.create_table('nio_device_key',
+    sa.Column('user_id', sa.String(length=255), nullable=False),
+    sa.Column('device_id', sa.String(length=255), nullable=False),
+    sa.Column('display_name', sa.String(length=255), nullable=False),
+    sa.Column('deleted', sa.Boolean(), nullable=False),
+    sa.Column('keys', sa.PickleType(), nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id')
+    )
+    op.create_table('nio_megolm_inbound_session',
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('sender_key', sa.String(length=255), nullable=False),
+    sa.Column('fp_key', sa.String(length=255), nullable=False),
+    sa.Column('room_id', sa.String(length=255), nullable=False),
+    sa.Column('session', sa.LargeBinary(), nullable=False),
+    sa.Column('forwarded_chains', sa.PickleType(), nullable=False),
+    sa.PrimaryKeyConstraint('session_id')
+    )
+    op.create_table('nio_olm_session',
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('sender_key', sa.String(length=255), nullable=False),
+    sa.Column('session', sa.LargeBinary(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('last_used', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('session_id')
+    )
+    op.create_table('nio_outgoing_key_request',
+    sa.Column('request_id', sa.String(length=255), nullable=False),
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('room_id', sa.String(length=255), nullable=False),
+    sa.Column('algorithm', sa.String(length=255), nullable=False),
+    sa.PrimaryKeyConstraint('request_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('nio_outgoing_key_request')
+    op.drop_table('nio_olm_session')
+    op.drop_table('nio_megolm_inbound_session')
+    op.drop_table('nio_device_key')
+    op.drop_table('nio_account')
+    # ### end Alembic commands ###

docker-run.sh

@@ -13,8 +13,6 @@ sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /d
 if [ -f /data/mx-state.json ]; then
 	ln -s /data/mx-state.json
 fi
-# Check that database is in the right state
-alembic -x config=/data/config.yaml upgrade head
 
 if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
@@ -35,5 +33,8 @@ if [ ! -f /data/registration.yaml ]; then
 	exit
 fi
 
+# Check that database is in the right state
+alembic -x config=/data/config.yaml upgrade head
+
 fixperms
 exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.7.0"
+__version__ = "0.9.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -14,10 +14,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional
-from itertools import chain
 
 from alchemysession import AlchemySessionContainer
 
+from mautrix.types import UserID, RoomID
 from mautrix.bridge import Bridge
 from mautrix.util.db import Base
 
@@ -31,9 +31,8 @@ from .context import Context
 from .db import init as init_db
 from .formatter import init as init_formatter
 from .matrix import MatrixHandler
-from .portal import init as init_portal
+from .portal import Portal, init as init_portal
 from .puppet import Puppet, init as init_puppet
-from .sqlstatestore import SQLStateStore
 from .user import User, init as init_user
 from .version import version, linkified_version
 
@@ -44,6 +43,7 @@ except ImportError:
 
 
 class TelegramBridge(Bridge):
+    module = "mautrix_telegram"
     name = "mautrix-telegram"
     command = "python -m mautrix-telegram"
     description = "A Matrix-Telegram puppeting bridge."
@@ -53,7 +53,6 @@ class TelegramBridge(Bridge):
     markdown_version = linkified_version
     config_class = Config
     matrix_class = MatrixHandler
-    state_store_class = SQLStateStore
 
     config: Config
     session_container: AlchemySessionContainer
@@ -79,13 +78,6 @@ class TelegramBridge(Bridge):
                                    provisioning_api.app)
             context.provisioning_api = provisioning_api
 
-        if self.config["metrics.enabled"]:
-            if prometheus:
-                prometheus.start_http_server(self.config["metrics.listen_port"])
-            else:
-                self.log.warning("Metrics are enabled in the config, "
-                                 "but prometheus_client is not installed.")
-
     def prepare_bridge(self) -> None:
         self.bot = init_bot(self.config)
         context = Context(self.az, self.config, self.loop, self.session_container, self, self.bot)
@@ -96,10 +88,20 @@ class TelegramBridge(Bridge):
         init_abstract_user(context)
         init_formatter(context)
         init_portal(context)
-        puppet_startup = init_puppet(context)
+        self.add_startup_actions(init_puppet(context))
-        user_startup = init_user(context)
+        self.add_startup_actions(init_user(context))
-        bot_startup = [self.bot.start()] if self.bot else []
+        if self.bot:
-        self.startup_actions = chain(puppet_startup, user_startup, bot_startup)
+            self.add_startup_actions(self.bot.start())
+        if self.config["bridge.resend_bridge_info"]:
+            self.add_startup_actions(self.resend_bridge_info())
+
+    async def resend_bridge_info(self) -> None:
+        self.config["bridge.resend_bridge_info"] = False
+        self.config.save()
+        self.log.info("Re-sending bridge info state event to all portals")
+        for portal in Portal.all():
+            await portal.update_bridge_info()
+        self.log.info("Finished re-sending bridge info state events")
 
     def prepare_stop(self) -> None:
         for puppet in Puppet.by_custom_mxid.values():
@@ -109,5 +111,23 @@ class TelegramBridge(Bridge):
             self.manhole.close()
             self.manhole = None
 
+    async def get_user(self, user_id: UserID, create: bool = True) -> User:
+        user = User.get_by_mxid(user_id, create=create)
+        if user:
+            await user.ensure_started()
+        return user
+
+    async def get_portal(self, room_id: RoomID) -> Portal:
+        return Portal.get_by_mxid(room_id)
+
+    async def get_puppet(self, user_id: UserID, create: bool = False) -> Puppet:
+        return await Puppet.get_by_mxid(user_id, create=create)
+
+    async def get_double_puppet(self, user_id: UserID) -> Puppet:
+        return await Puppet.get_by_custom_mxid(user_id)
+
+    def is_bridge_ghost(self, user_id: UserID) -> bool:
+        return bool(Puppet.get_id_from_mxid(user_id))
+
 
 TelegramBridge().run()

mautrix_telegram/abstract_user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -26,15 +26,18 @@ from telethon.network import (ConnectionTcpMTProxyRandomizedIntermediate, Connec
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
     Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdateChatPinnedMessage,
-    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants,
+    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
     UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
     UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
     UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
-    UpdateUserTyping, User, UserStatusOffline, UserStatusOnline)
+    UpdateUserTyping, User, UserStatusOffline, UserStatusOnline, UpdateReadHistoryInbox,
+    UpdateReadChannelInbox, MessageEmpty)
 
 from mautrix.types import UserID, PresenceState
 from mautrix.errors import MatrixError
 from mautrix.appservice import AppService
+from mautrix.util.logging import TraceLogger
+from mautrix.util.opt_prometheus import Histogram, Counter
 from alchemysession import AlchemySessionContainer
 
 from . import portal as po, puppet as pu, __version__
@@ -55,20 +58,16 @@ UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChann
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
-try:
+UPDATE_TIME = Histogram("bridge_telegram_update", "Time spent processing Telegram updates",
-    from prometheus_client import Histogram
+                        ("update_type",))
-
+UPDATE_ERRORS = Counter("bridge_telegram_update_error",
-    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
+                        "Number of fatal errors while handling Telegram updates", ("update_type",))
-                            ["update_type"])
-except ImportError:
-    Histogram = None
-    UPDATE_TIME = None
 
 
 class AbstractUser(ABC):
     session_container: AlchemySessionContainer = None
     loop: asyncio.AbstractEventLoop = None
-    log: logging.Logger
+    log: TraceLogger
     az: AppService
     relaybot: Optional['Bot']
     ignore_incoming_bot_events: bool = True
@@ -97,7 +96,6 @@ class AbstractUser(ABC):
         self.client = None
         self.is_relaybot = False
         self.is_bot = False
-        self.relaybot = None
 
     @property
     def connected(self) -> bool:
@@ -166,6 +164,7 @@ class AbstractUser(ABC):
             request_retries=config["telegram.connection.request_retries"],
             connection=connection,
             proxy=proxy,
+            raise_last_call_error=True,
 
             loop=self.loop,
             base_logger=base_logger
@@ -181,22 +180,23 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     @abstractmethod
-    def register_portal(self, portal: po.Portal) -> None:
+    async def register_portal(self, portal: po.Portal) -> None:
         raise NotImplementedError()
 
     @abstractmethod
-    def unregister_portal(self, portal: po.Portal) -> None:
+    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
         raise NotImplementedError()
 
     async def _update_catch(self, update: TypeUpdate) -> None:
         start_time = time.time()
+        update_type = type(update).__name__
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception(f"Failed to handle Telegram update {update}")
-        if UPDATE_TIME:
+            UPDATE_ERRORS.labels(update_type=update_type).inc()
-            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
+        UPDATE_TIME.labels(update_type=update_type).observe(time.time() - start_time)
 
     @property
     @abstractmethod
@@ -258,8 +258,10 @@ class AbstractUser(ABC):
             await self.update_others_info(update)
         elif isinstance(update, UpdateReadHistoryOutbox):
             await self.update_read_receipt(update)
+        elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
+            await self.update_own_read_receipt(update)
         else:
-            self.log.debug("Unhandled update: %s", update)
+            self.log.trace("Unhandled update: %s", update)
 
     async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
                                                          UpdateChatPinnedMessage]) -> None:
@@ -274,7 +276,7 @@ class AbstractUser(ABC):
     async def update_participants(update: UpdateChatParticipants) -> None:
         portal = po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
         if portal and portal.mxid:
-            await portal.update_telegram_participants(update.participants.participants)
+            await portal.update_power_levels(update.participants.participants)
 
     async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
         if not isinstance(update.peer, PeerUser):
@@ -293,6 +295,32 @@ class AbstractUser(ABC):
         puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
+    async def update_own_read_receipt(self, update: Union[UpdateReadHistoryInbox,
+                                                          UpdateReadChannelInbox]) -> None:
+        puppet = pu.Puppet.get(self.tgid)
+        if not puppet.is_real_user:
+            return
+
+        if isinstance(update, UpdateReadChannelInbox):
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update.peer, PeerChat):
+            portal = po.Portal.get_by_tgid(TelegramID(update.peer.chat_id))
+        elif isinstance(update.peer, PeerUser):
+            portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
+        else:
+            self.log.debug("Unexpected own read receipt peer: %s", update.peer)
+            return
+
+        if not portal or not portal.mxid:
+            return
+
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), tg_space, edit_index=-1)
+        if not message:
+            return
+
+        await puppet.intent.mark_read(portal.mxid, message.mxid)
+
     async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
         # TODO duplication not checked
         portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
@@ -329,12 +357,12 @@ class AbstractUser(ABC):
         if isinstance(update, UpdateUserName):
             puppet.username = update.username
             if await puppet.update_displayname(self, update):
-                puppet.save()
+                await puppet.save()
         elif isinstance(update, UpdateUserPhoto):
             if await puppet.update_avatar(self, update.photo):
-                puppet.save()
+                await puppet.save()
         else:
-            self.log.warning("Unexpected other user info update: %s", update)
+            self.log.warning(f"Unexpected other user info update: {type(update)}")
 
     async def update_status(self, update: UpdateUserStatus) -> None:
         puppet = pu.Puppet.get(TelegramID(update.user_id))
@@ -343,7 +371,7 @@ class AbstractUser(ABC):
         elif isinstance(update.status, UserStatusOffline):
             await puppet.default_mxid_intent.set_presence(PresenceState.OFFLINE)
         else:
-            self.log.warning("Unexpected user status update: %s", update)
+            self.log.warning(f"Unexpected user status update: type({update})")
         return
 
     def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
@@ -360,15 +388,18 @@ class AbstractUser(ABC):
         elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
                                  UpdateEditMessage, UpdateEditChannelMessage)):
             update = update.message
-            if isinstance(update.to_id, PeerUser) and not update.out:
+            if isinstance(update, MessageEmpty):
-                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
+                return update, None, None
-                                               tg_receiver=self.tgid)
+            portal = po.Portal.get_by_entity(update.peer_id, receiver_id=self.tgid)
+            if update.out:
+                sender = pu.Puppet.get(self.tgid)
+            elif isinstance(update.from_id, PeerUser):
+                sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
             else:
-                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
+                sender = None
-            sender = pu.Puppet.get(update.from_id) if update.from_id else None
         else:
-            self.log.warning(
+            self.log.warning("Unexpected message type in User#get_message_details: "
-                f"Unexpected message type in User#get_message_details: {type(update)}")
+                             f"{type(update)}")
             return update, None, None
         return update, sender, portal
 
@@ -422,17 +453,21 @@ class AbstractUser(ABC):
                                f" in unbridged chat {portal.tgid_log}")
                 return
 
-        if self.ignore_incoming_bot_events and self.relaybot and sender.id == self.relaybot.tgid:
+        if ((self.ignore_incoming_bot_events and self.relaybot
-            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
+             and sender and sender.id == self.relaybot.tgid)):
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
             return
 
+        await portal.backfill_lock.wait(update.id)
+
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.debug(f"Ignoring action %s to %s by %d", update.action,
+                self.log.trace(f"Received %s in %s by %d, unregistering portal...",
-                               portal.tgid_log,
+                               update.action, portal.tgid_log, sender.id)
-                               sender.id)
+                await self.unregister_portal(update.action.chat_id, update.action.chat_id)
+                await self.register_portal(portal)
                 return
-            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
+            self.log.trace("Handling action %s to %s by %d", update.action, portal.tgid_log,
                            sender.id)
             return await portal.handle_telegram_action(self, sender, update)
 

mautrix_telegram/bot.py

@@ -108,20 +108,20 @@ class Bot(AbstractUser):
             if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
                 self.remove_chat(TelegramID(chat.id))
 
-        channel_ids = (InputChannel(chat_id, 0)
+        channel_ids = [InputChannel(chat_id, 0)
                        for chat_id, chat_type in self.chats.items()
-                       if chat_type == "channel")
+                       if chat_type == "channel"]
         for channel_id in channel_ids:
             try:
                 await self.client(GetChannelsRequest([channel_id]))
             except (ChannelPrivateError, ChannelInvalidError):
                 self.remove_chat(TelegramID(channel_id.channel_id))
 
-    def register_portal(self, portal: po.Portal) -> None:
+    async def register_portal(self, portal: po.Portal) -> None:
         self.add_chat(portal.tgid, portal.peer_type)
 
-    def unregister_portal(self, portal: po.Portal) -> None:
+    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
-        self.remove_chat(portal.tgid)
+        self.remove_chat(tgid)
 
     def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
         if chat_id not in self.chats:
@@ -147,7 +147,7 @@ class Bot(AbstractUser):
         if self.whitelist_group_admins:
             if isinstance(chat, PeerChannel):
                 p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
+                return isinstance(p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin))
             elif isinstance(chat, PeerChat):
                 chat = await self.client(GetFullChatRequest(chat.chat_id))
                 participants = chat.full_chat.participants.participants
@@ -226,7 +226,7 @@ class Bot(AbstractUser):
 
         return False
 
-    async def handle_command(self, message: Message) -> Optional[bool]:
+    async def handle_command(self, message: Message) -> None:
         def reply(reply_text: str) -> Awaitable[Message]:
             return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)
 
@@ -234,9 +234,8 @@ class Bot(AbstractUser):
 
         if self.match_command(text, "start"):
             pcm = config["bridge.relaybot.private_chat.message"]
-            if not pcm:
+            if pcm:
-                return True
+                await reply(pcm)
-            await reply(pcm)
             return
         elif self.match_command(text, "id"):
             await self.handle_command_id(message, reply)
@@ -246,18 +245,19 @@ class Bot(AbstractUser):
 
         portal = po.Portal.get_by_entity(message.to_id)
 
-        if self.match_command(text, "portal"):
+        is_portal_cmd = self.match_command(text, "portal")
+        is_invite_cmd = self.match_command(text, "invite")
+        if is_portal_cmd or is_invite_cmd:
             if not await self.check_can_use_commands(message, reply):
                 return
-            await self.handle_command_portal(portal, reply)
+            if is_portal_cmd:
-        elif self.match_command(text, "invite"):
+                await self.handle_command_portal(portal, reply)
-            if not await self.check_can_use_commands(message, reply):
+            elif is_invite_cmd:
-                return
+                try:
-            try:
+                    mxid = text[text.index(" ") + 1:]
-                mxid = text[text.index(" ") + 1:]
+                except ValueError:
-            except ValueError:
+                    mxid = ""
-                mxid = ""
+                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
-            await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
 
     def handle_service_message(self, message: MessageService) -> None:
         to_peer = message.to_id
@@ -288,9 +288,10 @@ class Bot(AbstractUser):
 
         is_command = (isinstance(update.message, Message)
                       and update.message.entities and len(update.message.entities) > 0
-                      and isinstance(update.message.entities[0], MessageEntityBotCommand))
+                      and isinstance(update.message.entities[0], MessageEntityBotCommand)
+                      and update.message.entities[0].offset == 0)
         if is_command:
-            return not await self.handle_command(update.message)
+            await self.handle_command(update.message)
         return False
 
     def is_in_chat(self, peer_id) -> bool:

mautrix_telegram/commands/__init__.py

@@ -1,7 +1,7 @@
 from .handler import (command_handler, CommandHandler, CommandProcessor, CommandEvent,
                       SECTION_AUTH, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT,
                       SECTION_MISC, SECTION_ADMIN)
-from . import portal, telegram, clean_rooms, matrix_auth, manhole
+from . import portal, telegram, matrix_auth, manhole
 
 __all__ = ["command_handler", "CommandHandler", "CommandProcessor", "CommandEvent",
            "SECTION_AUTH", "SECTION_MISC", "SECTION_ADMIN", "SECTION_CREATING_PORTALS",

mautrix_telegram/commands/clean_rooms.py

@@ -1,177 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, NamedTuple, Tuple, Union
-
-from mautrix.appservice import IntentAPI
-from mautrix.errors import MatrixRequestError
-from mautrix.types import RoomID, UserID, EventID
-
-from . import command_handler, CommandEvent, SECTION_ADMIN
-from .. import puppet as pu, portal as po
-
-ManagementRoom = NamedTuple('ManagementRoom', room_id=RoomID, user_id=UserID)
-
-
-async def _find_rooms(intent: IntentAPI) -> Tuple[List[ManagementRoom], List[RoomID],
-                                                  List['po.Portal'], List['po.Portal']]:
-    management_rooms: List[ManagementRoom] = []
-    unidentified_rooms: List[RoomID] = []
-    portals: List[po.Portal] = []
-    empty_portals: List[po.Portal] = []
-
-    rooms = await intent.get_joined_rooms()
-    for room_id in rooms:
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
-            try:
-                members = await intent.get_room_members(room_id)
-            except MatrixRequestError:
-                members = []
-            if len(members) == 2:
-                other_member = members[0] if members[0] != intent.mxid else members[1]
-                if pu.Puppet.get_id_from_mxid(other_member):
-                    unidentified_rooms.append(room_id)
-                else:
-                    management_rooms.append(ManagementRoom(room_id, other_member))
-            else:
-                unidentified_rooms.append(room_id)
-        else:
-            members = await portal.get_authenticated_matrix_users()
-            if len(members) == 0:
-                empty_portals.append(portal)
-            else:
-                portals.append(portal)
-
-    return management_rooms, unidentified_rooms, portals, empty_portals
-
-
-@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
-                 help_section=SECTION_ADMIN,
-                 help_text="Clean up unused portal/management rooms.")
-async def clean_rooms(evt: CommandEvent) -> EventID:
-    management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
-
-    reply = ["#### Management rooms (M)"]
-    reply += ([f"{n+1}. [M{n+1}](https://matrix.to/#/{room}) (with {other_member}"
-               for n, (room, other_member) in enumerate(management_rooms)]
-              or ["No management rooms found."])
-    reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(portals)]
-              or ["No active portal rooms found."])
-    reply.append("#### Unidentified rooms (U)")
-    reply += ([f"{n+1}. [U{n+1}](https://matrix.to/#/{room})"
-               for n, room in enumerate(unidentified_rooms)]
-              or ["No unidentified rooms found."])
-    reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(empty_portals)]
-              or ["No inactive portal rooms found."])
-
-    reply += ["#### Usage",
-              ("To clean the recommended set of rooms (unidentified & inactive portals), "
-               "type `$cmdprefix+sp clean-recommended`"),
-              "",
-              ("To clean other groups of rooms, type `$cmdprefix+sp clean-groups <letters>` "
-               "where `letters` are the first letters of the group names (M, A, U, I)"),
-              "",
-              ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
-               "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name. (e.g. `I2-6`)"),
-              "",
-              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
-               "between each use of the commands above.")]
-
-    evt.sender.command_status = {
-        "next": lambda clean_evt: set_rooms_to_clean(clean_evt, management_rooms,
-                                                     unidentified_rooms, portals, empty_portals),
-        "action": "Room cleaning",
-    }
-
-    return await evt.reply("\n".join(reply))
-
-
-async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
-                             unidentified_rooms: List[RoomID], portals: List["po.Portal"],
-                             empty_portals: List["po.Portal"]) -> None:
-    command = evt.args[0]
-    rooms_to_clean: List[Union[po.Portal, RoomID]] = []
-    if command == "clean-recommended":
-        rooms_to_clean += empty_portals
-        rooms_to_clean += unidentified_rooms
-    elif command == "clean-groups":
-        if len(evt.args) < 2:
-            return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1].upper()
-        if "M" in groups_to_clean:
-            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
-        if "A" in groups_to_clean:
-            rooms_to_clean += portals
-        if "U" in groups_to_clean:
-            rooms_to_clean += unidentified_rooms
-        if "I" in groups_to_clean:
-            rooms_to_clean += empty_portals
-    elif command == "clean-range":
-        try:
-            clean_range = evt.args[1]
-            group, clean_range = clean_range[0], clean_range[1:]
-            start, end = clean_range.split("-")
-            start, end = int(start), int(end)
-            if group == "M":
-                group = [room_id for (room_id, user_id) in management_rooms]
-            elif group == "A":
-                group = portals
-            elif group == "U":
-                group = unidentified_rooms
-            elif group == "I":
-                group = empty_portals
-            else:
-                raise ValueError("Unknown group")
-            rooms_to_clean = group[start - 1:end]
-        except (KeyError, ValueError):
-            return await evt.reply(
-                "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
-    else:
-        return await evt.reply(f"Unknown room cleaning action `{command}`. "
-                               "Use `$cmdprefix+sp cancel` to cancel room "
-                               "cleaning.")
-
-    evt.sender.command_status = {
-        "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
-        "action": "Room cleaning",
-    }
-    await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type "
-                    "`$cmdprefix+sp confirm-clean`.")
-
-
-async def execute_room_cleanup(evt, rooms_to_clean: List[Union[po.Portal, RoomID]]) -> None:
-    if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
-        await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
-                        "This might take a while.")
-        cleaned = 0
-        for room in rooms_to_clean:
-            if isinstance(room, po.Portal):
-                await room.cleanup_and_delete()
-                cleaned += 1
-            else:
-                await po.Portal.cleanup_room(evt.az.intent, room, "Room deleted")
-                cleaned += 1
-        evt.sender.command_status = None
-        await evt.reply(f"{cleaned} rooms cleaned up successfully.")
-    else:
-        await evt.reply("Room cleaning cancelled.")

mautrix_telegram/commands/handler.py

@@ -25,11 +25,17 @@ from mautrix.bridge.commands import (HelpSection, CommandEvent as BaseCommandEve
                                      CommandHandlerFunc, command_handler as base_command_handler)
 
 from ..util import format_duration
-from .. import user as u, context as c
+from .. import user as u, context as c, portal as po
+
+
+class HelpCacheKey(NamedTuple):
+    is_management: bool
+    is_portal: bool
+    puppet_whitelisted: bool
+    matrix_puppet_whitelisted: bool
+    is_admin: bool
+    is_logged_in: bool
 
-HelpCacheKey = NamedTuple('HelpCacheKey',
-                          is_management=bool, is_portal=bool, puppet_whitelisted=bool,
-                          matrix_puppet_whitelisted=bool, is_admin=bool, is_logged_in=bool)
 
 SECTION_AUTH = HelpSection("Authentication", 10, "")
 SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
@@ -40,12 +46,13 @@ SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 class CommandEvent(BaseCommandEvent):
     sender: u.User
+    portal: po.Portal
 
     def __init__(self, processor: 'CommandProcessor', room_id: RoomID, event_id: EventID,
                  sender: u.User, command: str, args: List[str], content: MessageEventContent,
-                 is_management: bool, is_portal: bool) -> None:
+                 portal: Optional['po.Portal'], is_management: bool, has_bridge_bot: bool) -> None:
         super().__init__(processor, room_id, event_id, sender, command, args, content,
-                         is_management, is_portal)
+                         portal, is_management, has_bridge_bot)
         self.bridge = processor.bridge
         self.tgbot = processor.tgbot
         self.config = processor.config
@@ -56,19 +63,16 @@ class CommandEvent(BaseCommandEvent):
         return self.sender.is_admin
 
     async def get_help_key(self) -> HelpCacheKey:
-        return HelpCacheKey(self.is_management, self.is_portal, self.sender.puppet_whitelisted,
+        return HelpCacheKey(self.is_management, self.portal is not None,
-                            self.sender.matrix_puppet_whitelisted, self.sender.is_admin,
+                            self.sender.puppet_whitelisted, self.sender.matrix_puppet_whitelisted,
-                            await self.sender.is_logged_in())
+                            self.sender.is_admin, await self.sender.is_logged_in())
 
 
 class CommandHandler(BaseCommandHandler):
     name: str
 
-    management_only: bool
-    needs_auth: bool
     needs_puppeting: bool
     needs_matrix_puppeting: bool
-    needs_admin: bool
 
     def __init__(self, handler: Callable[[CommandEvent], Awaitable[EventID]],
                  management_only: bool, name: str, help_text: str, help_args: str,
@@ -79,25 +83,16 @@ class CommandHandler(BaseCommandHandler):
                          needs_matrix_puppeting=needs_matrix_puppeting, needs_admin=needs_admin)
 
     async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
-        if self.management_only and not evt.is_management:
+        if self.needs_puppeting and not evt.sender.puppet_whitelisted:
-            return (f"`{evt.command}` is a restricted command: "
-                    "you may only run it in management rooms.")
-        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
             return "This command requires puppeting privileges."
         elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
             return "This command requires Matrix puppeting privileges."
-        elif self.needs_admin and not evt.sender.is_admin:
+        return await super().get_permission_error(evt)
-            return "This command requires administrator privileges."
-        elif self.needs_auth and not await evt.sender.is_logged_in():
-            return "This command requires you to be logged in."
-        return None
 
     def has_permission(self, key: HelpCacheKey) -> bool:
-        return ((not self.management_only or key.is_management) and
+        return (super().has_permission(key) and
                 (not self.needs_puppeting or key.puppet_whitelisted) and
-                (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted) and
+                (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted))
-                (not self.needs_admin or key.is_admin) and
-                (not self.needs_auth or key.is_logged_in))
 
 
 def command_handler(_func: Optional[CommandHandlerFunc] = None, *, needs_auth: bool = True,
@@ -115,13 +110,9 @@ def command_handler(_func: Optional[CommandHandlerFunc] = None, *, needs_auth: b
 
 class CommandProcessor(BaseCommandProcessor):
     def __init__(self, context: c.Context) -> None:
-        super().__init__(az=context.az, config=context.config, event_class=CommandEvent,
+        super().__init__(event_class=CommandEvent, bridge=context.bridge)
-                         loop=context.loop, bridge=context.bridge)
         self.tgbot = context.bot
-        self.bridge = context.bridge
-        self.az, self.config, self.loop, self.tgbot = context.core
         self.public_website = context.public_website
-        self.command_prefix = self.config["bridge.command_prefix"]
 
     @staticmethod
     async def _run_handler(handler: Callable[[CommandEvent], Awaitable[Any]], evt: CommandEvent

mautrix_telegram/commands/portal/admin.py

@@ -15,34 +15,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import asyncio
 
-from mautrix.errors import MatrixRequestError
 from mautrix.types import EventID
 
 from ... import portal as po, puppet as pu, user as u
 from .. import command_handler, CommandEvent, SECTION_ADMIN
 
 
-@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
-                 help_section=SECTION_ADMIN,
-                 help_args="<_level_> [_mxid_]",
-                 help_text="Set a temporary power level without affecting Telegram.")
-async def set_power_level(evt: CommandEvent) -> EventID:
-    try:
-        level = int(evt.args[0])
-    except (KeyError, IndexError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp set-pl <level> [mxid]`")
-    except ValueError:
-        return await evt.reply("The level must be an integer.")
-    levels = await evt.az.intent.get_power_levels(evt.room_id)
-    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
-    levels.users[mxid] = level
-    try:
-        return await evt.az.intent.set_power_levels(evt.room_id, levels)
-    except MatrixRequestError:
-        evt.log.exception("Failed to set power level.")
-        return await evt.reply("Failed to set power level.")
-
-
 @command_handler(needs_admin=True, needs_auth=False,
                  help_section=SECTION_ADMIN,
                  help_args="<`portal`|`puppet`|`user`>",
@@ -86,7 +64,7 @@ async def reload_user(evt: CommandEvent) -> EventID:
     user = u.User.get_by_mxid(mxid, create=False)
     if not user:
         return await evt.reply("User not found")
-    puppet = pu.Puppet.get_by_custom_mxid(mxid)
+    puppet = await pu.Puppet.get_by_custom_mxid(mxid)
     if puppet:
         puppet.sync_task.cancel()
     await user.stop()

mautrix_telegram/commands/portal/bridge.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Tuple, Coroutine
+from typing import Optional, Tuple, Awaitable
 import asyncio
 
 from telethon.tl.types import ChatForbidden, ChannelForbidden
@@ -105,18 +105,17 @@ async def bridge(evt: CommandEvent) -> EventID:
 
 
 async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
-                                            ) -> Tuple[
+                                            ) -> Tuple[bool, Optional[Awaitable[None]]]:
-    bool, Optional[Coroutine[None, None, None]]]:
     if not portal.mxid:
         await evt.reply("The portal seems to have lost its Matrix room between you"
                         "calling `$cmdprefix+sp bridge` and this command.\n\n"
                         "Continuing without touching previous Matrix room...")
         return True, None
     elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_portal("Portal deleted (moving to another room)")
+        return True, portal.cleanup_portal("Portal deleted (moving to another room)", delete=False)
     elif evt.args[0] == "unbridge-and-continue":
         return True, portal.cleanup_portal("Room unbridged (portal moving to another room)",
-                                           puppets_only=True)
+                                           puppets_only=True, delete=False)
     else:
         await evt.reply(
             "The chat you were trying to bridge already has a Matrix portal room.\n\n"
@@ -137,6 +136,9 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
         return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
                                "This shouldn't happen unless you're messing with the command "
                                "handler code.")
+
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+
     if "mxid" in status:
         ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
         if not ok:
@@ -154,7 +156,13 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
                                "`$cmdprefix+sp cancel` to cancel.")
 
     evt.sender.command_status = None
-    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+    async with portal._room_create_lock:
+        await _locked_confirm_bridge(evt, portal=portal, room_id=bridge_to_mxid,
+                                     is_logged_in=is_logged_in)
+
+
+async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id: RoomID,
+                                 is_logged_in: bool) -> Optional[EventID]:
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)
@@ -172,14 +180,14 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
         else:
             return await evt.reply("The bot doesn't seem to be in that chat.")
 
-    direct = False
+    portal.mxid = room_id
-
+    portal.by_mxid[portal.mxid] = portal
-    portal.mxid = bridge_to_mxid
+    (portal.title, portal.about, levels,
-    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+     portal.encrypted) = await get_initial_state(evt.az.intent, evt.room_id)
     portal.photo_id = ""
-    portal.save()
+    await portal.save()
 
-    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
+    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
                           loop=evt.loop)
 
     return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -13,9 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable
+from typing import Awaitable, Any
 from io import StringIO
 
+from ruamel.yaml import YAMLError
+
 from mautrix.util.config import yaml
 from mautrix.types import EventID
 
@@ -48,7 +50,11 @@ async def config(evt: CommandEvent) -> None:
         return
 
     key = evt.args[1] if len(evt.args) > 1 else None
-    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    try:
+        value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    except YAMLError as e:
+        await evt.reply(f"Invalid value provided. Values must be valid YAML.\n{e}")
+        return
     if cmd == "set":
         await config_set(evt, portal, key, value)
     elif cmd == "unset":
@@ -57,7 +63,7 @@ async def config(evt: CommandEvent) -> None:
         await config_add_del(evt, portal, key, value, cmd)
     else:
         return
-    portal.save()
+    await portal.save()
 
 
 def config_help(evt: CommandEvent) -> Awaitable[EventID]:
@@ -74,14 +80,11 @@ def config_help(evt: CommandEvent) -> Awaitable[EventID]:
 
 
 def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[EventID]:
-    stream = StringIO()
+    return evt.reply(f"Room-specific config:\n{_str_value(portal.local_config).rstrip()}")
-    yaml.dump(portal.local_config, stream)
-    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
 
 
 def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
-    stream = StringIO()
+    value = _str_value({
-    yaml.dump({
         "bridge_notices": {
             "default": evt.config["bridge.bridge_notices.default"],
             "exceptions": evt.config["bridge.bridge_notices.exceptions"],
@@ -92,15 +95,25 @@ def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
         "emote_format": evt.config["bridge.emote_format"],
         "state_event_formats": evt.config["bridge.state_event_formats"],
         "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
-    }, stream)
+    })
-    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
+    return evt.reply(f"Bridge instance wide config:\n{value.rstrip()}")
 
 
-def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[EventID]:
+def _str_value(value: Any) -> str:
+    stream = StringIO()
+    yaml.dump(value, stream)
+    value_str = stream.getvalue()
+    if "\n" in value_str:
+        return f"\n```yaml\n{value_str}\n```\n"
+    else:
+        return f"`{value_str}`"
+
+
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: Any) -> Awaitable[EventID]:
     if not key or value is None:
         return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
     elif util.recursive_set(portal.local_config, key, value):
-        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
+        return evt.reply(f"Successfully set the value of `{key}` to {_str_value(value)}".rstrip())
     else:
         return evt.reply(f"Failed to set value of `{key}`. "
                          "Does the path contain non-map types?")
@@ -128,11 +141,11 @@ def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, c
         return evt.reply("`{key}` does not seem to be an array.")
     elif cmd == "add":
         if value in arr:
-            return evt.reply(f"The array at `{key}` already contains `{value}`.")
+            return evt.reply(f"The array at `{key}` already contains {_str_value(value)}".rstrip())
         arr.append(value)
-        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
+        return evt.reply(f"Successfully added {_str_value(value)} to the array at `{key}`")
     else:
         if value not in arr:
-            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
+            return evt.reply(f"The array at `{key}` does not contain {_str_value(value)}")
         arr.remove(value)
-        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")
+        return evt.reply(f"Successfully removed {_str_value(value)} from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -25,10 +25,10 @@ from .util import user_has_power_level, get_initial_state
                  help_args="[_type_]",
                  help_text="Create a Telegram chat of the given type for the current Matrix room. "
                            "The type is either `group`, `supergroup` or `channel` (defaults to "
-                           "`group`).")
+                           "`supergroup`).")
 async def create(evt: CommandEvent) -> EventID:
-    type = evt.args[0] if len(evt.args) > 0 else "group"
+    type = evt.args[0] if len(evt.args) > 0 else "supergroup"
-    if type not in {"chat", "group", "supergroup", "channel"}:
+    if type not in ("chat", "group", "supergroup", "channel"):
         return await evt.reply(
             "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
 
@@ -38,7 +38,7 @@ async def create(evt: CommandEvent) -> EventID:
     if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
         return await evt.reply("You do not have the permissions to bridge this room.")
 
-    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    title, about, levels, encrypted = await get_initial_state(evt.az.intent, evt.room_id)
     if not title:
         return await evt.reply("Please set a title before creating a Telegram chat.")
 
@@ -50,11 +50,11 @@ async def create(evt: CommandEvent) -> EventID:
         "group": "chat",
     }[type]
 
-    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type, mxid=evt.room_id,
-                       mxid=evt.room_id, title=title, about=about)
+                       title=title, about=about, encrypted=encrypted)
     try:
         await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
     except ValueError as e:
-        portal.delete()
+        await portal.delete()
         return await evt.reply(e.args[0])
     return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/misc.py

@@ -35,7 +35,7 @@ async def sync_state(evt: CommandEvent) -> EventID:
     elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
         return await evt.reply(f"You do not have the permissions to synchronize this room.")
 
-    await portal.sync_matrix_members()
+    await portal.main_intent.get_joined_members(portal.mxid)
     await evt.reply("Synchronization complete")
 
 

mautrix_telegram/commands/portal/unbridge.py

@@ -22,9 +22,7 @@ from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
 from .util import user_has_power_level
 
 
-async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
+async def _get_portal_and_check_permission(evt: CommandEvent) -> Optional[po.Portal]:
-                                           action: Optional[str] = None
-                                           ) -> Optional[po.Portal]:
     room_id = RoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
 
     portal = po.Portal.get_by_mxid(room_id)
@@ -33,9 +31,14 @@ async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
         await evt.reply(f"{that_this} is not a portal room.")
         return None
 
-    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
+    if portal.peer_type == "user":
-        action = action or f"{permission.replace('_', ' ')}s"
+        if portal.tg_receiver != evt.sender.tgid:
-        await evt.reply(f"You do not have the permissions to {action} that portal.")
+            await evt.reply("You do not have the permissions to unbridge that portal.")
+            return None
+        return portal
+
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+        await evt.reply("You do not have the permissions to unbridge that portal.")
         return None
     return portal
 
@@ -64,7 +67,7 @@ def _get_portal_murder_function(action: str, room_id: str, function: Callable, c
                            "Only works for group chats; to delete a private chat portal, simply "
                            "leave the room.")
 async def delete_portal(evt: CommandEvent) -> Optional[EventID]:
-    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    portal = await _get_portal_and_check_permission(evt)
     if not portal:
         return None
 
@@ -85,7 +88,7 @@ async def delete_portal(evt: CommandEvent) -> Optional[EventID]:
                  help_section=SECTION_PORTAL_MANAGEMENT,
                  help_text="Remove puppets from the current portal room and forget the portal.")
 async def unbridge(evt: CommandEvent) -> Optional[EventID]:
-    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    portal = await _get_portal_and_check_permission(evt)
     if not portal:
         return None
 

mautrix_telegram/commands/portal/util.py

@@ -25,11 +25,12 @@ OptStr = Optional[str]
 
 
 async def get_initial_state(intent: IntentAPI, room_id: RoomID
-                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent]]:
+                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent], bool]:
     state = await intent.get_state(room_id)
     title: OptStr = None
     about: OptStr = None
     levels: Optional[PowerLevelStateEventContent] = None
+    encrypted: bool = False
     for event in state:
         try:
             if event.type == EventType.ROOM_NAME:
@@ -40,10 +41,12 @@ async def get_initial_state(intent: IntentAPI, room_id: RoomID
                 levels = event.content
             elif event.type == EventType.ROOM_CANONICAL_ALIAS:
                 title = title or event.content.canonical_alias
+            elif event.type == EventType.ROOM_ENCRYPTION:
+                encrypted = True
         except KeyError:
             # Some state event probably has empty content
             pass
-    return title, about, levels
+    return title, about, levels, encrypted
 
 
 async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.User,
@@ -55,6 +58,5 @@ async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.Use
         await intent.get_power_levels(room_id)
     except MatrixRequestError:
         return False
-    event_type = EventType.find(f"net.maunium.telegram.{event}")
+    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
-    event_type.t_class = EventType.Class.STATE
+    return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)
-    return intent.state_store.has_power_level(room_id, sender.mxid, event_type)

mautrix_telegram/commands/telegram/auth.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Any, Dict, Optional
 import asyncio
+import io
 
 from telethon.errors import (  # isort: skip
     AccessTokenExpiredError, AccessTokenInvalidError, FirstNameInvalidError, FloodWaitError,
@@ -22,12 +23,23 @@ from telethon.errors import (  # isort: skip
     PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
     PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError,
     PhoneNumberInvalidError)
+from telethon.tl.types import User
 
-from mautrix.types import EventID
+from mautrix.types import (EventID, UserID, MediaMessageEventContent, ImageInfo, MessageType,
+                           TextMessageEventContent)
 
 from ... import user as u
+from ...types import TelegramID
 from ...commands import command_handler, CommandEvent, SECTION_AUTH
-from ...util import format_duration
+from ...util import format_duration as fmt_duration
+
+try:
+    import qrcode
+    import PIL as _
+    from telethon.tl.custom import QRLogin
+except ImportError:
+    qrcode = None
+    QRLogin = None
 
 
 @command_handler(needs_auth=False,
@@ -58,7 +70,7 @@ async def ping_bot(evt: CommandEvent) -> EventID:
                  help_section=SECTION_AUTH,
                  help_args="<_phone_> <_full name_>",
                  help_text="Register to Telegram")
-async def register(evt: CommandEvent) -> Optional[EventID]:
+async def register(evt: CommandEvent) -> EventID:
     if await evt.sender.is_logged_in():
         return await evt.reply("You are already logged in.")
     elif len(evt.args) < 1:
@@ -75,7 +87,8 @@ async def register(evt: CommandEvent) -> Optional[EventID]:
         "action": "Register",
         "full_name": full_name,
     })
-    return None
+    return await evt.reply("By signing up for Telegram, you agree to "
+                           "the terms of service: https://telegram.org/tos")
 
 
 async def enter_code_register(evt: CommandEvent) -> EventID:
@@ -104,18 +117,76 @@ async def enter_code_register(evt: CommandEvent) -> EventID:
                                "Check console for more details.")
 
 
+@command_handler(needs_auth=False, management_only=True, help_section=SECTION_AUTH,
+                 help_text="Log in by scanning a QR code.")
+async def login_qr(evt: CommandEvent) -> EventID:
+    login_as = evt.sender
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        login_as = u.User.get_by_mxid(UserID(evt.args[0]))
+    if not qrcode or not QRLogin:
+        return await evt.reply("This bridge instance does not support logging in with a QR code.")
+    if await login_as.is_logged_in():
+        return await evt.reply(f"You are already logged in as {login_as.human_tg_id}.")
+
+    await login_as.ensure_started(even_if_no_session=True)
+    qr_login = QRLogin(login_as.client, ignored_ids=[])
+    qr_event_id: Optional[EventID] = None
+
+    async def upload_qr() -> None:
+        nonlocal qr_event_id
+        buffer = io.BytesIO()
+        image = qrcode.make(qr_login.url)
+        size = image.pixel_size
+        image.save(buffer, "PNG")
+        qr = buffer.getvalue()
+        mxc = await evt.az.intent.upload_media(qr, "image/png", "login-qr.png", len(qr))
+        content = MediaMessageEventContent(body=qr_login.url, url=mxc, msgtype=MessageType.IMAGE,
+                                           info=ImageInfo(mimetype="image/png", size=len(qr),
+                                                          width=size, height=size))
+        if qr_event_id:
+            content.set_edit(qr_event_id)
+            await evt.az.intent.send_message(evt.room_id, content)
+        else:
+            content.set_reply(evt.event_id)
+            qr_event_id = await evt.az.intent.send_message(evt.room_id, content)
+
+    retries = 4
+    while retries > 0:
+        await qr_login.recreate()
+        await upload_qr()
+        try:
+            user = await qr_login.wait()
+            break
+        except asyncio.TimeoutError:
+            retries -= 1
+        except SessionPasswordNeededError:
+            evt.sender.command_status = {
+                "next": enter_password,
+                "login_as": login_as if login_as != evt.sender else None,
+                "action": "Login (password entry)",
+            }
+            return await evt.reply("Your account has two-factor authentication. "
+                                   "Please send your password here.")
+    else:
+        timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
+        timeout.set_edit(qr_event_id)
+        return await evt.az.intent.send_message(evt.room_id, timeout)
+
+    return await _finish_sign_in(evt, user, login_as=login_as)
+
+
 @command_handler(needs_auth=False, management_only=True,
                  help_section=SECTION_AUTH,
                  help_text="Get instructions on how to log in.")
 async def login(evt: CommandEvent) -> EventID:
     override_sender = False
     if len(evt.args) > 0 and evt.sender.is_admin:
-        evt.sender = await u.User.get_by_mxid(evt.args[0]).ensure_started()
+        evt.sender = await u.User.get_by_mxid(UserID(evt.args[0])).ensure_started()
         override_sender = True
     if await evt.sender.is_logged_in():
         return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 
-    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    allow_matrix_login = evt.config["bridge.allow_matrix_login"]
     if allow_matrix_login and not override_sender:
         evt.sender.command_status = {
             "next": enter_phone_or_token,
@@ -152,21 +223,18 @@ async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[
         ok = True
         return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
     except PhoneNumberAppSignupForbiddenError:
-        return await evt.reply(
+        return await evt.reply("Your phone number does not allow 3rd party apps to sign in.")
-            "Your phone number does not allow 3rd party apps to sign in.")
     except PhoneNumberFloodError:
-        return await evt.reply(
+        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
-            "Your phone number has been temporarily blocked for flooding. "
+                               "The ban is usually applied for around a day.")
-            "The ban is usually applied for around a day.")
     except FloodWaitError as e:
-        return await evt.reply(
+        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
-            "Your phone number has been temporarily blocked for flooding. "
+                               f"Please wait for {fmt_duration(e.seconds)} before trying again.")
-            f"Please wait for {format_duration(e.seconds)} before trying again.")
     except PhoneNumberBannedError:
-        return await  evt.reply("Your phone number has been banned from Telegram.")
+        return await evt.reply("Your phone number has been banned from Telegram.")
     except PhoneNumberUnoccupiedError:
-        return await  evt.reply("That phone number has not been registered. "
+        return await evt.reply("That phone number has not been registered. "
-                                "Please register with `$cmdprefix+sp register <phone>`.")
+                               "Please register with `$cmdprefix+sp register <phone>`.")
     except PhoneNumberInvalidError:
         return await evt.reply("That phone number is not valid.")
     except Exception:
@@ -225,7 +293,8 @@ async def enter_password(evt: CommandEvent) -> Optional[EventID]:
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await _sign_in(evt, password=" ".join(evt.args))
+        await _sign_in(evt, login_as=evt.sender.command_status.get("login_as", None),
+                       password=" ".join(evt.args))
     except AccessTokenInvalidError:
         return await evt.reply("That bot token is not valid.")
     except AccessTokenExpiredError:
@@ -237,20 +306,12 @@ async def enter_password(evt: CommandEvent) -> Optional[EventID]:
     return None
 
 
-async def _sign_in(evt: CommandEvent, **sign_in_info) -> EventID:
+async def _sign_in(evt: CommandEvent, login_as: 'u.User' = None, **sign_in_info) -> EventID:
+    login_as = login_as or evt.sender
     try:
-        await evt.sender.ensure_started(even_if_no_session=True)
+        await login_as.ensure_started(even_if_no_session=True)
-        user = await evt.sender.client.sign_in(**sign_in_info)
+        user = await login_as.client.sign_in(**sign_in_info)
-        existing_user = u.User.get_by_tgid(user.id)
+        await _finish_sign_in(evt, user)
-        if existing_user and existing_user != evt.sender:
-            await existing_user.log_out()
-            await evt.reply(f"[{existing_user.displayname}]"
-                            f"(https://matrix.to/#/{existing_user.mxid})"
-                            " was logged out from the account.")
-        asyncio.ensure_future(evt.sender.post_login(user, first_login=True), loop=evt.loop)
-        evt.sender.command_status = None
-        name = f"@{user.username}" if user.username else f"+{user.phone}"
-        return await evt.reply(f"Successfully logged in as {name}")
     except PhoneCodeExpiredError:
         return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
     except PhoneCodeInvalidError:
@@ -266,6 +327,25 @@ async def _sign_in(evt: CommandEvent, **sign_in_info) -> EventID:
                                "Please send your password here.")
 
 
+async def _finish_sign_in(evt: CommandEvent, user: User, login_as: 'u.User' = None) -> EventID:
+    login_as = login_as or evt.sender
+    existing_user = u.User.get_by_tgid(TelegramID(user.id))
+    if existing_user and existing_user != login_as:
+        await existing_user.log_out()
+        await evt.reply(f"[{existing_user.displayname}]"
+                        f"(https://matrix.to/#/{existing_user.mxid})"
+                        " was logged out from the account.")
+    asyncio.ensure_future(login_as.post_login(user, first_login=True), loop=evt.loop)
+    evt.sender.command_status = None
+    name = f"@{user.username}" if user.username else f"+{user.phone}"
+    if login_as != evt.sender:
+        msg = (f"Successfully logged in [{login_as.mxid}](https://matrix.to/#/{login_as.mxid})"
+               f" as {name}")
+    else:
+        msg = f"Successfully logged in as {name}"
+    return await evt.reply(msg)
+
+
 @command_handler(needs_auth=True,
                  help_section=SECTION_AUTH,
                  help_text="Log out from Telegram.")

mautrix_telegram/commands/telegram/misc.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -20,10 +20,11 @@ import base64
 import re
 
 from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
-                             UserAlreadyParticipantError, ChatIdInvalidError)
+                             UserAlreadyParticipantError, ChatIdInvalidError,
+                             TakeoutInitDelayError, EmoticonInvalidError)
 from telethon.tl.patched import Message
 from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
-                               TypeInputPeer)
+                               TypeInputPeer, InputMediaDice)
 from telethon.tl.types.messages import BotCallbackAnswer
 from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
                                             GetBotCallbackAnswerRequest, SendVoteRequest)
@@ -35,7 +36,8 @@ from ... import puppet as pu, portal as po
 from ...abstract_user import AbstractUser
 from ...db import Message as DBMessage
 from ...types import TelegramID
-from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
+from ...commands import (command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS,
+                         SECTION_PORTAL_MANAGEMENT)
 
 
 @command_handler(needs_auth=False,
@@ -102,7 +104,8 @@ async def pm(evt: CommandEvent) -> EventID:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
 
     try:
-        user = await evt.sender.client.get_entity(evt.args[0])
+        id = "".join(evt.args).translate({ord(c): None for c in "+()- "})
+        user = await evt.sender.client.get_entity(id)
     except ValueError:
         return await evt.reply("Invalid user identifier or user not found.")
 
@@ -162,7 +165,9 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
             try:
                 await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
             except ChatIdInvalidError as e:
-                logging.getLogger("mau.commands").info(updates.stringify())
+                logging.getLogger("mau.commands").trace("ChatIdInvalidError while creating portal "
+                                                        "from !tg join command: %s",
+                                                        updates.stringify())
                 raise e
             return await evt.reply(f"Created room for {portal.title}")
     return None
@@ -180,8 +185,10 @@ async def sync(evt: CommandEvent) -> EventID:
         sync_only = None
 
     if not sync_only or sync_only == "chats":
-        await evt.sender.sync_dialogs(synchronous_create=True)
+        await evt.reply("Synchronizing chats...")
+        await evt.sender.sync_dialogs()
     if not sync_only or sync_only == "contacts":
+        await evt.reply("Synchronizing contacts...")
         await evt.sender.sync_contacts()
     if not sync_only or sync_only == "me":
         await evt.sender.update_info()
@@ -303,3 +310,52 @@ async def vote(evt: CommandEvent) -> EventID:
         return await evt.reply("You passed too many options.")
     # TODO use response
     return await evt.mark_read()
+
+
+@command_handler(help_section=SECTION_MISC, help_args="<_emoji_>",
+                 help_text="Roll a dice (\U0001F3B2), kick a football (\u26BD\uFE0F) or throw a "
+                           "dart (\U0001F3AF) or basketball (\U0001F3C0) on the Telegram servers.")
+async def random(evt: CommandEvent) -> EventID:
+    if not evt.is_portal:
+        return await evt.reply("You can only randomize values in portal rooms")
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    arg = evt.args[0] if len(evt.args) > 0 else "dice"
+    emoticon = {
+        "dart": "\U0001F3AF",
+        "dice": "\U0001F3B2",
+        "ball": "\U0001F3C0",
+        "basketball": "\U0001F3C0",
+        "football": "\u26BD",
+        "soccer": "\u26BD",
+    }.get(arg, arg)
+    try:
+        await evt.sender.client.send_media(await portal.get_input_entity(evt.sender),
+                                           InputMediaDice(emoticon))
+    except EmoticonInvalidError:
+        return await evt.reply("Invalid emoji for randomization")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT, help_args="[_limit_]",
+                 help_text="Backfill messages from Telegram history.")
+async def backfill(evt: CommandEvent) -> None:
+    if not evt.is_portal:
+        await evt.reply("You can only use backfill in portal rooms")
+        return
+    try:
+        limit = int(evt.args[0])
+    except (ValueError, IndexError):
+        limit = -1
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
+        await evt.reply("Backfilling normal groups is disabled in the bridge config")
+        return
+    try:
+        await portal.backfill(evt.sender, limit=limit)
+    except TakeoutInitDelayError:
+        msg = ("Please accept the data export request from a mobile device, "
+               "then re-run the backfill command.")
+        if portal.peer_type == "user":
+            from mautrix.appservice import IntentAPI
+            await portal.main_intent.send_notice(evt.room_id, msg)
+        else:
+            await evt.reply(msg)

mautrix_telegram/config.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,14 +13,14 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Any, Dict, List, NamedTuple
+from typing import Any, List, NamedTuple
 from ruamel.yaml.comments import CommentedMap
 import os
 
 from mautrix.types import UserID
 from mautrix.client import Client
-from mautrix.bridge.config import (BaseBridgeConfig, ConfigUpdateHelper, ForbiddenKey,
+from mautrix.bridge.config import BaseBridgeConfig
-                                   ForbiddenDefault)
+from mautrix.util.config import ForbiddenKey, ForbiddenDefault, ConfigUpdateHelper
 
 Permissions = NamedTuple("Permissions", relaybot=bool, user=bool, puppeting=bool,
                          matrix_puppeting=bool, admin=bool, level=str)
@@ -45,23 +45,20 @@ class Config(BaseBridgeConfig):
         ]
 
     def do_update(self, helper: ConfigUpdateHelper) -> None:
+        super().do_update(helper)
         copy, copy_dict, base = helper
 
-        copy("homeserver.address")
+        copy("homeserver.asmux")
-        copy("homeserver.domain")
-        copy("homeserver.verify_ssl")
 
         if "appservice.protocol" in self and "appservice.address" not in self:
             protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
                                         self["appservice.port"])
             base["appservice.address"] = f"{protocol}://{hostname}:{port}"
-        else:
+        if "appservice.debug" in self and "logging" not in self:
-            copy("appservice.address")
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
-        copy("appservice.hostname")
+            base["logging.root.level"] = level
-        copy("appservice.port")
+            base["logging.loggers.mau.level"] = level
-        copy("appservice.max_body_size")
+            base["logging.loggers.telethon.level"] = level
-
-        copy("appservice.database")
 
         copy("appservice.public.enabled")
         copy("appservice.public.prefix")
@@ -73,16 +70,8 @@ class Config(BaseBridgeConfig):
         if base["appservice.provisioning.shared_secret"] == "generate":
             base["appservice.provisioning.shared_secret"] = self._new_token()
 
-        copy("appservice.id")
-        copy("appservice.bot_username")
-        copy("appservice.bot_displayname")
-        copy("appservice.bot_avatar")
-
         copy("appservice.community_id")
 
-        copy("appservice.as_token")
-        copy("appservice.hs_token")
-
         copy("metrics.enabled")
         copy("metrics.listen_port")
 
@@ -96,12 +85,18 @@ class Config(BaseBridgeConfig):
 
         copy("bridge.displayname_preference")
         copy("bridge.displayname_max_length")
+        copy("bridge.allow_avatar_remove")
 
         copy("bridge.max_initial_member_sync")
         copy("bridge.sync_channel_members")
         copy("bridge.skip_deleted_members")
         copy("bridge.startup_sync")
-        copy("bridge.sync_dialog_limit")
+        if "bridge.sync_dialog_limit" in self:
+            base["bridge.sync_create_limit"] = self["bridge.sync_dialog_limit"]
+            base["bridge.sync_update_limit"] = self["bridge.sync_dialog_limit"]
+        else:
+            copy("bridge.sync_update_limit")
+            copy("bridge.sync_create_limit")
         copy("bridge.sync_direct_chats")
         copy("bridge.max_telegram_delete")
         copy("bridge.sync_matrix_state")
@@ -109,7 +104,15 @@ class Config(BaseBridgeConfig):
         copy("bridge.plaintext_highlights")
         copy("bridge.public_portals")
         copy("bridge.sync_with_custom_puppets")
-        copy("bridge.login_shared_secret")
+        copy("bridge.sync_direct_chat_list")
+        copy("bridge.double_puppet_server_map")
+        copy("bridge.double_puppet_allow_discovery")
+        if "bridge.login_shared_secret" in self:
+            base["bridge.login_shared_secret_map"] = {
+                base["homeserver.domain"]: self["bridge.login_shared_secret"]
+            }
+        else:
+            copy("bridge.login_shared_secret_map")
         copy("bridge.telegram_link_preview")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
@@ -118,6 +121,22 @@ class Config(BaseBridgeConfig):
         copy("bridge.federate_rooms")
         copy("bridge.animated_sticker.target")
         copy("bridge.animated_sticker.args")
+        copy("bridge.encryption.allow")
+        copy("bridge.encryption.default")
+        copy("bridge.encryption.database")
+        copy("bridge.encryption.key_sharing.allow")
+        copy("bridge.encryption.key_sharing.require_cross_signing")
+        copy("bridge.encryption.key_sharing.require_verification")
+        copy("bridge.private_chat_portal_meta")
+        copy("bridge.delivery_receipts")
+        copy("bridge.delivery_error_reports")
+        copy("bridge.resend_bridge_info")
+        copy("bridge.backfill.invite_own_puppet")
+        copy("bridge.backfill.takeout_limit")
+        copy("bridge.backfill.initial_limit")
+        copy("bridge.backfill.missed_limit")
+        copy("bridge.backfill.disable_notifications")
+        copy("bridge.backfill.normal_groups")
 
         copy("bridge.initial_power_level_overrides.group")
         copy("bridge.initial_power_level_overrides.user")
@@ -202,14 +221,6 @@ class Config(BaseBridgeConfig):
         copy("telegram.proxy.username")
         copy("telegram.proxy.password")
 
-        if "appservice.debug" in self and "logging" not in self:
-            level = "DEBUG" if self["appservice.debug"] else "INFO"
-            base["logging.root.level"] = level
-            base["logging.loggers.mau.level"] = level
-            base["logging.loggers.telethon.level"] = level
-        else:
-            copy("logging")
-
     def _get_permissions(self, key: str) -> Permissions:
         level = self["bridge.permissions"].get(key, "")
         admin = level == "admin"
@@ -229,24 +240,3 @@ class Config(BaseBridgeConfig):
             return self._get_permissions(homeserver)
 
         return self._get_permissions("*")
-
-    @property
-    def namespaces(self) -> Dict[str, List[Dict[str, Any]]]:
-        homeserver = self["homeserver.domain"]
-
-        username_format = self["bridge.username_template"].format(userid=".+")
-        alias_format = self["bridge.alias_template"].format(groupname=".+")
-        group_id = ({"group_id": self["appservice.community_id"]}
-                    if self["appservice.community_id"] else {})
-
-        return {
-            "users": [{
-                "exclusive": True,
-                "regex": f"@{username_format}:{homeserver}",
-                **group_id,
-            }],
-            "aliases": [{
-                "exclusive": True,
-                "regex": f"#{alias_format}:{homeserver}",
-            }]
-        }

mautrix_telegram/db/__init__.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from sqlalchemy.engine.base import Engine
 
-from mautrix.bridge.db import UserProfile, RoomState
+from mautrix.client.state_store.sqlalchemy import UserProfile, RoomState
 
 from .bot_chat import BotChat
 from .message import Message
@@ -28,7 +28,4 @@ from .user import User, UserPortal, Contact
 def init(db_engine: Engine) -> None:
     for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
                   RoomState, BotChat):
-        table.db = db_engine
+        table.bind(db_engine)
-        table.t = table.__table__
-        table.c = table.t.c
-        table.column_names = table.c.keys()

mautrix_telegram/db/message.py

@@ -61,6 +61,16 @@ class Message(Base):
         except StopIteration:
             return 0
 
+    @classmethod
+    def find_last(cls, mx_room: RoomID, tg_space: TelegramID) -> Optional['Message']:
+        return cls._one_or_none(cls.db.execute(
+            cls._make_simple_select(cls.c.mx_room == mx_room, cls.c.tg_space == tg_space)
+                .order_by(desc(cls.c.tgid)).limit(1)))
+
+    @classmethod
+    def delete_all(cls, mx_room: RoomID) -> None:
+        cls.db.execute(cls.t.delete().where(cls.c.mx_room == mx_room))
+
     @classmethod
     def get_by_mxid(cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
                     ) -> Optional['Message']:

mautrix_telegram/db/portal.py

@@ -13,11 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Boolean, Text, func
+from sqlalchemy import Column, Integer, String, Boolean, Text, func, sql
 
-from mautrix.types import RoomID
+from mautrix.types import RoomID, ContentURI
 from mautrix.util.db import Base
 
 from ..types import TelegramID
@@ -33,7 +33,9 @@ class Portal(Base):
     megagroup: bool = Column(Boolean)
 
     # Matrix portal information
-    mxid: RoomID = Column(String, unique=True, nullable=True)
+    mxid: Optional[RoomID] = Column(String, unique=True, nullable=True)
+    avatar_url: Optional[ContentURI] = Column(String, nullable=True)
+    encrypted: bool = Column(Boolean, nullable=False, server_default=sql.expression.false())
 
     config: str = Column(Text, nullable=True)
 
@@ -47,6 +49,10 @@ class Portal(Base):
     def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
         return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver)
 
+    @classmethod
+    def find_private_chats(cls, tg_receiver: TelegramID) -> Iterable['Portal']:
+        yield from cls._select_all(cls.c.tg_receiver == tg_receiver, cls.c.peer_type == "user")
+
     @classmethod
     def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
         return cls._select_one_or_none(cls.c.mxid == mxid)
@@ -54,3 +60,7 @@ class Portal(Base):
     @classmethod
     def get_by_username(cls, username: str) -> Optional['Portal']:
         return cls._select_one_or_none(func.lower(cls.c.username) == username)
+
+    @classmethod
+    def all(cls) -> Iterable['Portal']:
+        yield from cls._select_all()

mautrix_telegram/db/puppet.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy import Column, Integer, String, Text, Boolean
 from sqlalchemy.sql import expression, func
 
 from mautrix.types import UserID, SyncToken
@@ -31,6 +31,7 @@ class Puppet(Base):
     custom_mxid: UserID = Column(String, nullable=True)
     access_token: str = Column(String, nullable=True)
     next_batch: SyncToken = Column(String, nullable=True)
+    base_url: str = Column(Text, nullable=True)
     displayname: str = Column(String, nullable=True)
     displayname_source: TelegramID = Column(Integer, nullable=True)
     username: str = Column(String, nullable=True)

mautrix_telegram/db/telegram_file.py

@@ -13,15 +13,37 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from typing import Optional, cast, Dict, Any
 
-from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
+from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
+                        TypeDecorator)
 from sqlalchemy.engine.result import RowProxy
 
-from mautrix.types import ContentURI
+from mautrix.types import ContentURI, EncryptedFile
 from mautrix.util.db import Base
 
 
+class DBEncryptedFile(TypeDecorator):
+    impl = Text
+
+    @property
+    def python_type(self):
+        return EncryptedFile
+
+    def process_bind_param(self, value: EncryptedFile, dialect) -> Optional[str]:
+        if value is not None:
+            return value.json()
+        return None
+
+    def process_result_value(self, value: str, dialect) -> Optional[EncryptedFile]:
+        if value is not None:
+            return EncryptedFile.parse_json(value)
+        return None
+
+    def process_literal_param(self, value, dialect):
+        return value
+
+
 class TelegramFile(Base):
     __tablename__ = "telegram_file"
 
@@ -33,12 +55,13 @@ class TelegramFile(Base):
     size: Optional[int] = Column(Integer, nullable=True)
     width: Optional[int] = Column(Integer, nullable=True)
     height: Optional[int] = Column(Integer, nullable=True)
+    decryption_info: Optional[Dict[str, Any]] = Column(DBEncryptedFile, nullable=True)
     thumbnail_id: str = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
     thumbnail: Optional['TelegramFile'] = None
 
     @classmethod
     def scan(cls, row: RowProxy) -> 'TelegramFile':
-        telegram_file: TelegramFile = super().scan(row)
+        telegram_file = cast(TelegramFile, super().scan(row))
         if isinstance(telegram_file.thumbnail, str):
             telegram_file.thumbnail = cls.get(telegram_file.thumbnail)
         return telegram_file
@@ -52,5 +75,5 @@ class TelegramFile(Base):
             conn.execute(self.t.insert().values(
                 id=self.id, mxc=self.mxc, mime_type=self.mime_type,
                 was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
-                width=self.width, height=self.height,
+                width=self.width, height=self.height, decryption_info=self.decryption_info,
                 thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/example-config.yaml

@@ -7,12 +7,16 @@ homeserver:
     # Whether or not to verify the SSL certificate of the homeserver.
     # Only applies if address starts with https://
     verify_ssl: true
+    asmux: false
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
 appservice:
     # The address that the homeserver can use to connect to this appservice.
     address: http://localhost:29317
+    # When using https:// the TLS certificate and key files for the address.
+    tls_cert: false
+    tls_key: false
 
     # The hostname and port where this appservice should listen.
     hostname: 0.0.0.0
@@ -27,6 +31,8 @@ appservice:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
     database: sqlite:///mautrix-telegram.db
+    # Optional extra arguments for SQLAlchemy's create_engine
+    database_opts: {}
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in
@@ -41,7 +47,7 @@ appservice:
         external: https://example.com/public
 
     # Provisioning API part of the web server for automated portal creation and fetching information.
-    # Used by things like Dimension (https://dimension.t2bot.io/).
+    # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager).
     provisioning:
         # Whether or not the provisioning API should be enabled.
         enabled: true
@@ -62,8 +68,15 @@ appservice:
 
     # Community ID for bridged users (changes registration file) and rooms.
     # Must be created manually.
+    #
+    # Example: "+telegram:example.com". Set to false to disable.
     community_id: false
 
+    # Whether or not to receive ephemeral events via appservice transactions.
+    # Requires MSC2409 support (i.e. Synapse 1.22+).
+    # You should disable bridge -> sync_with_custom_puppets when this is enabled.
+    ephemeral_events: false
+
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
@@ -116,12 +129,16 @@ bridge:
     - phone number
     # Maximum length of displayname
     displayname_max_length: 100
+    # Remove avatars from Telegram ghost users when removed on Telegram. This is disabled by default
+    # as there's no way to determine whether an avatar is removed or just hidden from some users. If
+    # you're on a single-user instance, this should be safe to enable.
+    allow_avatar_remove: false
 
     # Maximum number of members to sync per portal when starting up. Other members will be
     # synced when they send messages. The maximum is 10000, after which the Telegram server
     # will not send any more members.
-    # Defaults to no local limit (-> limited to 10000 by server)
+    # -1 means no limit (which means it's limited to 10000 by the server)
-    max_initial_member_sync: -1
+    max_initial_member_sync: 100
     # Whether or not to sync the member list in channels.
     # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
     # list regardless of this setting.
@@ -133,7 +150,10 @@ bridge:
     startup_sync: true
     # Number of most recently active dialogs to check when syncing chats.
     # Set to 0 to remove limit.
-    sync_dialog_limit: 30
+    sync_update_limit: 0
+    # Number of most recently active dialogs to create portals for when syncing chats.
+    # Set to 0 to remove limit.
+    sync_create_limit: 30
     # Whether or not to sync and create portals for direct chats at startup.
     sync_direct_chats: false
     # The maximum number of simultaneous Telegram deletions to handle.
@@ -142,8 +162,8 @@ bridge:
     # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
     # at startup and when creating a bridge.
     sync_matrix_state: true
-    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
+    # Allow logging in within Matrix. If false, users can only log in using login-qr or the
-    # login website (see appservice.public config section)
+    # out-of-Matrix login website (see appservice.public config section)
     allow_matrix_login: true
     # Whether or not to bridge plaintext highlights.
     # Only enable this if your displayname_template has some static part that the bridge can use to
@@ -151,15 +171,27 @@ bridge:
     plaintext_highlights: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
-    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # Whether or not to use /sync to get presence, read receipts and typing notifications
-    # your own Matrix account as the Matrix puppet for your Telegram account.
+    # when double puppeting is enabled
     sync_with_custom_puppets: true
-    # Shared secret for https://github.com/devture/matrix-synapse-shared-secret-auth
+    # Whether or not to update the m.direct account data event when double puppeting is enabled.
+    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
+    # and is therefore prone to race conditions.
+    sync_direct_chat_list: false
+    # Servers to always allow double puppeting from
+    double_puppet_server_map:
+        example.com: https://example.com
+    # Allow using double puppeting from any server with a valid client .well-known file.
+    double_puppet_allow_discovery: false
+    # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth
     #
     # If set, custom puppets will be enabled automatically for local users
     # instead of users having to find an access token and run `login-matrix`
     # manually.
-    login_shared_secret: null
+    # If using this for other servers than the bridge's server,
+    # you must also set the URL in the double_puppet_server_map.
+    login_shared_secret_map:
+        example.com: foobar
     # Set to false to disable link previews in messages sent to Telegram.
     telegram_link_preview: true
     # Use inline images instead of a separate message for the caption.
@@ -191,6 +223,79 @@ bridge:
             height: 256
             background: "020202"  # only for gif
             fps: 30               # only for webm
+    # End-to-bridge encryption support options. These require matrix-nio to be installed with pip
+    # and login_shared_secret to be configured in order to get a device for the bridge bot.
+    #
+    # Additionally, https://github.com/matrix-org/synapse/pull/5758 is required if using a normal
+    # application service.
+    encryption:
+        # Allow encryption, work in group chat rooms with e2ee enabled
+        allow: false
+        # Default to encryption, force-enable encryption in all portals the bridge creates
+        # This will cause the bridge bot to be in private chats for the encryption to work properly.
+        default: false
+        # Database for the encryption data. Currently only supports Postgres and an in-memory
+        # store that's persisted as a pickle.
+        # If set to `default`, will use the appservice postgres database
+        # or a pickle file if the appservice database is sqlite.
+        #
+        # Format examples:
+        #   Pickle:   pickle:///filename.pickle
+        #   Postgres: postgres://username:password@hostname/dbname
+        database: default
+        # Options for automatic key sharing.
+        key_sharing:
+            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
+            # You must use a client that supports requesting keys from other users to use this feature.
+            allow: false
+            # Require the requesting device to have a valid cross-signing signature?
+            # This doesn't require that the bridge has verified the device, only that the user has verified it.
+            # Not yet implemented.
+            require_cross_signing: false
+            # Require devices to be verified by the bridge?
+            # Verification by the bridge is not yet implemented.
+            require_verification: true
+    # Whether or not to explicitly set the avatar and room name for private
+    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
+    private_chat_portal_meta: false
+    # Whether or not the bridge should send a read receipt from the bridge bot when a message has
+    # been sent to Telegram.
+    delivery_receipts: false
+    # Whether or not delivery errors should be reported as messages in the Matrix room.
+    delivery_error_reports: false
+    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
+    # This field will automatically be changed back to false after it,
+    # except if the config file is not writable.
+    resend_bridge_info: false
+    # Settings for backfilling messages from Telegram.
+    backfill:
+        # Whether or not the Telegram ghosts of logged in Matrix users should be
+        # invited to private chats when backfilling history from Telegram. This is
+        # usually needed to prevent rate limits and to allow timestamp massaging.
+        invite_own_puppet: true
+        # Maximum number of messages to backfill without using a takeout.
+        # The first time a takeout is used, the user has to manually approve it from a different
+        # device. If initial_limit or missed_limit are higher than this value, the bridge will ask
+        # the user to accept the takeout after logging in before syncing any chats.
+        takeout_limit: 100
+        # Maximum number of messages to backfill initially.
+        # Set to 0 to disable backfilling when creating portal, or -1 to disable the limit.
+        #
+        # N.B. Initial backfill will only start after member sync. Make sure your
+        #      max_initial_member_sync is set to a low enough value so it doesn't take forever.
+        initial_limit: 0
+        # Maximum number of messages to backfill if messages were missed while the bridge was
+        # disconnected. Note that this only works for logged in users and only if the chat isn't
+        # older than sync_update_limit
+        # Set to 0 to disable backfilling missed messages.
+        missed_limit: 50
+        # If using double puppeting, should notifications be disabled
+        # while the initial backfill is in progress?
+        disable_notifications: false
+        # Whether or not to enable backfilling in normal groups.
+        # Normal groups have numerous technical problems in Telegram, and backfilling normal groups
+        # will likely cause problems if there are multiple Matrix users in the group.
+        normal_groups: false
 
     # Overrides for base power levels.
     initial_power_level_overrides:
@@ -409,7 +514,7 @@ logging:
         mau:
             level: DEBUG
         telethon:
-            level: DEBUG
+            level: INFO
         aiohttp:
             level: INFO
     root:

mautrix_telegram/formatter/__init__.py

@@ -1,5 +1,4 @@
-from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
+from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram, init_mx
-                          init_mx)
 from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -18,10 +18,12 @@ import re
 import logging
 
 from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
-                               TypeMessageEntity)
+                               TypeMessageEntity, InputMessageEntityMentionName)
 from telethon.helpers import add_surrogate, del_surrogate
+from telethon import TelegramClient
 
 from mautrix.types import RoomID, MessageEventContent
+from mautrix.util.logging import TraceLogger
 
 from ... import puppet as pu
 from ...types import TelegramID
@@ -31,30 +33,19 @@ from .parser import ParsedMessage, parse_html
 if TYPE_CHECKING:
     from ...context import Context
 
-log: logging.Logger = logging.getLogger("mau.fmt.mx")
+log: TraceLogger = logging.getLogger("mau.fmt.mx")
 should_bridge_plaintext_highlights: bool = False
 
 command_regex: Pattern = re.compile(r"^!([A-Za-z0-9@]+)")
 not_command_regex: Pattern = re.compile(r"^\\(![A-Za-z0-9@]+)")
 plain_mention_regex: Optional[Pattern] = None
 
-
-def plain_mention_to_html(match: Match) -> str:
-    puppet = pu.Puppet.find_by_displayname(match.group(2))
-    if puppet:
-        return (f"{match.group(1)}"
-                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                f"{puppet.displayname}"
-                "</a>")
-    return "".join(match.groups())
-
-
 MAX_LENGTH = 4096
 CUTOFF_TEXT = " [message cut]"
 CUT_MAX_LENGTH = MAX_LENGTH - len(CUTOFF_TEXT)
 
 
-def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+def _cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
     if len(message) > MAX_LENGTH:
         message = message[0:CUT_MAX_LENGTH] + CUTOFF_TEXT
         new_entities = []
@@ -73,23 +64,6 @@ class FormatError(Exception):
     pass
 
 
-def matrix_to_telegram(html: str) -> ParsedMessage:
-    try:
-        html = command_regex.sub(r"<command>\1</command>", html)
-        html = html.replace("\t", " " * 4)
-        html = not_command_regex.sub(r"\1", html)
-        if should_bridge_plaintext_highlights:
-            html = plain_mention_regex.sub(plain_mention_to_html, html)
-
-        text, entities = parse_html(add_surrogate(html))
-        text = del_surrogate(text.strip())
-        text, entities = cut_long_message(text, entities)
-
-        return text, entities
-    except Exception as e:
-        raise FormatError(f"Failed to convert Matrix format: {html}") from e
-
-
 def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
                              room_id: Optional[RoomID] = None) -> Optional[TelegramID]:
     event_id = content.get_reply_to()
@@ -103,19 +77,61 @@ def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
     return None
 
 
-def matrix_text_to_telegram(text: str) -> ParsedMessage:
+async def matrix_to_telegram(client: TelegramClient, *, text: Optional[str] = None,
+                             html: Optional[str] = None) -> ParsedMessage:
+    if html is not None:
+        text, entities = _matrix_html_to_telegram(html)
+    elif text is not None:
+        text, entities = _matrix_text_to_telegram(text)
+    else:
+        raise ValueError("text or html must be provided to convert formatting")
+    await _fix_name_mentions(client, entities)
+    return text, entities
+
+
+def _matrix_html_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(_plain_mention_to_html, html)
+
+        text, entities = parse_html(add_surrogate(html))
+        text = del_surrogate(text.strip())
+        text, entities = _cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def _matrix_text_to_telegram(text: str) -> ParsedMessage:
     text = command_regex.sub(r"/\1", text)
     text = text.replace("\t", " " * 4)
     text = not_command_regex.sub(r"\1", text)
     if should_bridge_plaintext_highlights:
-        entities, pmr_replacer = plain_mention_to_text()
+        entities, pmr_replacer = _plain_mention_to_text()
         text = plain_mention_regex.sub(pmr_replacer, text)
     else:
         entities = []
     return text, entities
 
 
-def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
+async def _fix_name_mentions(client: TelegramClient, entities: List[TypeMessageEntity]) -> None:
+    for index in reversed(range(len(entities))):
+        entity = entities[index]
+        if isinstance(entity, (MessageEntityMentionName, InputMessageEntityMentionName)):
+            try:
+                user = await client.get_input_entity(entity.user_id)
+            except (ValueError, TypeError) as e:
+                log.trace(f"Dropping mention of {entity.user_id}: {e}")
+                del entities[index]
+            else:
+                entities[index] = InputMessageEntityMentionName(entity.offset, entity.length, user)
+
+
+def _plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
     entities = []
 
     def replacer(match: Match) -> str:
@@ -136,6 +152,16 @@ def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match],
     return entities, replacer
 
 
+def _plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
 def init_mx(context: "Context") -> None:
     global plain_mention_regex, should_bridge_plaintext_highlights
     config = context.config

mautrix_telegram/formatter/from_matrix/parser.py

@@ -48,7 +48,7 @@ class MatrixParser(BaseMatrixParser[TelegramMessage]):
 
     @classmethod
     def user_pill_to_fstring(cls, msg: TelegramMessage, user_id: UserID) -> TelegramMessage:
-        user = (pu.Puppet.get_by_mxid(user_id)
+        user = (pu.Puppet.deprecated_sync_get_by_mxid(user_id)
                 or u.User.get_by_mxid(user_id, create=False))
         if not user:
             return msg

mautrix_telegram/formatter/from_telegram.py

@@ -22,7 +22,7 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, M
                                MessageEntityEmail, MessageEntityTextUrl, MessageEntityBold,
                                MessageEntityItalic, MessageEntityCode, MessageEntityPre,
                                MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
-                               MessageEntityPhone, TypeMessageEntity, PeerChannel,
+                               MessageEntityPhone, TypeMessageEntity, PeerChannel, PeerChat,
                                MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
                                MessageEntityUnderline, PeerUser)
 from telethon.tl.custom import Message
@@ -45,13 +45,13 @@ log: logging.Logger = logging.getLogger("mau.fmt.tg")
 
 
 def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Optional[RelatesTo]:
-    if evt.reply_to_msg_id:
+    if evt.reply_to:
-        space = (evt.to_id.channel_id
+        space = (evt.peer_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+                 if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to_msg_id), space)
+        msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
         if msg:
-            return RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
+            return RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
     return None
 
 
@@ -61,15 +61,15 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
         content.format = Format.HTML
         content.formatted_body = escape(content.body)
     fwd_from_html, fwd_from_text = None, None
-    if fwd_from.from_id:
+    if isinstance(fwd_from.from_id, PeerUser):
-        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id.user_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
             fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
                              f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
-            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
+            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id.user_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
                 fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
@@ -77,14 +77,16 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
 
         if not fwd_from_text:
             try:
-                user = await source.client.get_entity(PeerUser(fwd_from.from_id))
+                user = await source.client.get_entity(fwd_from.from_id)
                 if user:
                     fwd_from_text = pu.Puppet.get_displayname(user, False)
                     fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
             except (ValueError, RPCError):
                 fwd_from_text = fwd_from_html = "unknown user"
-    elif fwd_from.channel_id:
+    elif isinstance(fwd_from.from_id, (PeerChannel, PeerChat)):
-        portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
+        from_id = (fwd_from.from_id.chat_id if isinstance(fwd_from.from_id, PeerChat)
+                   else fwd_from.from_id.channel_id)
+        portal = po.Portal.get_by_tgid(TelegramID(from_id))
         if portal:
             fwd_from_text = portal.title
             if portal.alias:
@@ -94,7 +96,7 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
                 fwd_from_html = f"channel <b>{escape(fwd_from_text)}</b>"
         else:
             try:
-                channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
+                channel = await source.client.get_entity(fwd_from.from_id)
                 if channel:
                     fwd_from_text = f"channel {channel.title}"
                     fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
@@ -116,21 +118,21 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
 
 async def _add_reply_header(source: 'AbstractUser', content: TextMessageEventContent, evt: Message,
                             main_intent: IntentAPI):
-    space = (evt.to_id.channel_id
+    space = (evt.peer_id.channel_id
-             if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+             if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to_msg_id), space)
+    msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
     if not msg:
         return
 
-    content.relates_to = RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
+    content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
 
     try:
         event: MessageEvent = await main_intent.get_event(msg.mx_room, msg.mxid)
         if isinstance(event.content, TextMessageEventContent):
             event.content.trim_reply_fallback()
-        puppet = pu.Puppet.get_by_mxid(event.sender, create=False)
+        puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
         content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
     except MatrixRequestError:
         log.exception("Failed to get event to add reply fallback")
@@ -162,7 +164,7 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
     if evt.fwd_from:
         await _add_forward_header(source, content, evt.fwd_from)
 
-    if evt.reply_to_msg_id and not no_reply_fallback:
+    if evt.reply_to and not no_reply_fallback:
         await _add_reply_header(source, content, evt, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:

mautrix_telegram/matrix.py

@@ -20,7 +20,8 @@ from mautrix.types import (Event, EventType, RoomID, UserID, EventID, ReceiptEve
                            ReceiptEventContent, PresenceEvent, PresenceState, TypingEvent,
                            MessageEvent, StateEvent, RedactionEvent, RoomNameStateEventContent,
                            RoomAvatarStateEventContent, RoomTopicStateEventContent,
-                           MemberStateEventContent)
+                           MemberStateEventContent, EncryptedEvent, TextMessageEventContent,
+                           MessageType)
 from mautrix.errors import MatrixError
 
 from . import user as u, portal as po, puppet as pu, commands as com
@@ -29,14 +30,6 @@ if TYPE_CHECKING:
     from .context import Context
     from .bot import Bot
 
-try:
-    from prometheus_client import Histogram
-
-    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events", ["event_type"])
-except ImportError:
-    Histogram = None
-    EVENT_TIME = None
-
 RoomMetaStateEventContent = Union[RoomNameStateEventContent, RoomAvatarStateEventContent,
                                   RoomTopicStateEventContent]
 
@@ -47,24 +40,20 @@ class MatrixHandler(BaseMatrixHandler):
     previously_typing: Dict[RoomID, Set[UserID]]
 
     def __init__(self, context: 'Context') -> None:
-        super(MatrixHandler, self).__init__(context.az, context.config, loop=context.loop,
+        prefix, suffix = context.config["bridge.username_template"].format(userid=":").split(":")
-                                            command_processor=com.CommandProcessor(context))
+        homeserver = context.config["homeserver.domain"]
+        self.user_id_prefix = f"@{prefix}"
+        self.user_id_suffix = f"{suffix}:{homeserver}"
+
+        super().__init__(command_processor=com.CommandProcessor(context), bridge=context.bridge)
+
         self.bot = context.bot
         self.previously_typing = {}
 
-    async def get_user(self, user_id: UserID) -> 'u.User':
-        return await u.User.get_by_mxid(user_id).ensure_started()
-
-    async def get_portal(self, room_id: RoomID) -> 'po.Portal':
-        return po.Portal.get_by_mxid(room_id)
-
-    async def get_puppet(self, user_id: UserID) -> 'pu.Puppet':
-        return pu.Puppet.get_by_mxid(user_id)
-
     async def handle_puppet_invite(self, room_id: RoomID, puppet: pu.Puppet, inviter: u.User,
                                    event_id: EventID) -> None:
         intent = puppet.default_mxid_intent
-        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}")
+        self.log.debug(f"{inviter.mxid} invited puppet for {puppet.tgid} to {room_id}")
         if not await inviter.is_logged_in():
             await intent.error_and_leave(
                 room_id, text="Please log in before inviting Telegram puppets.")
@@ -79,11 +68,12 @@ class MatrixHandler(BaseMatrixHandler):
             await intent.join_room(room_id)
             return
         try:
-            members = await self.az.intent.get_room_members(room_id)
+            members = await intent.get_room_members(room_id)
         except MatrixError:
-            members = []
+            self.log.exception(f"Failed to get members after joining {room_id} as {intent.mxid}")
+            return
         if self.az.bot_mxid not in members:
-            if len(members) > 1:
+            if len(members) > 2:
                 await intent.error_and_leave(room_id, text=None, html=(
                     f"Please invite "
                     f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
@@ -104,9 +94,23 @@ class MatrixHandler(BaseMatrixHandler):
                 except MatrixError:
                     pass
             portal.mxid = room_id
-            portal.save()
+            e2be_ok = None
-            inviter.register_portal(portal)
+            if self.config["bridge.encryption.default"] and self.e2ee:
-            await intent.send_notice(room_id, "Portal to private chat created.")
+                e2be_ok = await portal.enable_dm_encryption()
+            await portal.save()
+            await inviter.register_portal(portal)
+            if e2be_ok is True:
+                evt_type, content = await self.e2ee.encrypt(
+                    room_id, EventType.ROOM_MESSAGE,
+                    TextMessageEventContent(msgtype=MessageType.NOTICE,
+                                            body="Portal to private chat created and end-to-bridge"
+                                                 " encryption enabled."))
+                await intent.send_message_event(room_id, evt_type, content)
+            else:
+                message = "Portal to private chat created."
+                if e2be_ok is False:
+                    message += "\n\nWarning: Failed to enable end-to-bridge encryption"
+                await intent.send_notice(room_id, message)
         else:
             await intent.join_room(room_id)
             await intent.send_notice(room_id, "This puppet will remain inactive until a "
@@ -156,7 +160,7 @@ class MatrixHandler(BaseMatrixHandler):
                                                "messages for unauthenticated users.")
             return
 
-        self.log.debug(f"{user} joined {room_id}")
+        self.log.debug(f"{user.mxid} joined {room_id}")
         if await user.is_logged_in() or portal.has_bot:
             await portal.join_matrix(user, event_id)
 
@@ -194,7 +198,7 @@ class MatrixHandler(BaseMatrixHandler):
             return
         await sender.ensure_started()
 
-        puppet = pu.Puppet.get_by_mxid(user_id)
+        puppet = await pu.Puppet.get_by_mxid(user_id)
         if puppet:
             if ban:
                 await portal.ban_matrix(puppet, sender)
@@ -246,7 +250,7 @@ class MatrixHandler(BaseMatrixHandler):
         if not portal:
             return
 
-        await portal.handle_matrix_deletion(sender, evt.redacts)
+        await portal.handle_matrix_deletion(sender, evt.redacts, evt.event_id)
 
     @staticmethod
     async def handle_power_levels(evt: StateEvent) -> None:
@@ -254,11 +258,12 @@ class MatrixHandler(BaseMatrixHandler):
         sender = await u.User.get_by_mxid(evt.sender).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
             await portal.handle_matrix_power_levels(sender, evt.content.users,
-                                                    evt.unsigned.prev_content.users)
+                                                    evt.unsigned.prev_content.users,
+                                                    evt.event_id)
 
     @staticmethod
     async def handle_room_meta(evt_type: EventType, room_id: RoomID, sender_mxid: UserID,
-                               content: RoomMetaStateEventContent) -> None:
+                               content: RoomMetaStateEventContent, event_id: EventID) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
@@ -269,27 +274,29 @@ class MatrixHandler(BaseMatrixHandler):
             }[evt_type]
             if not isinstance(content, content_type):
                 return
-            await handler(sender, content[content_key])
+            await handler(sender, content[content_key], event_id)
 
     @staticmethod
     async def handle_room_pin(room_id: RoomID, sender_mxid: UserID,
-                              new_events: Set[str], old_events: Set[str]) -> None:
+                              new_events: Set[str], old_events: Set[str],
+                              event_id: EventID) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
             events = new_events - old_events
             if len(events) > 0:
                 # New event pinned, set that as pinned in Telegram.
-                await portal.handle_matrix_pin(sender, EventID(events.pop()))
+                await portal.handle_matrix_pin(sender, EventID(events.pop()), event_id)
             elif len(new_events) == 0:
                 # All pinned events removed, remove pinned event in Telegram.
-                await portal.handle_matrix_pin(sender, None)
+                await portal.handle_matrix_pin(sender, None, event_id)
 
     @staticmethod
-    async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID) -> None:
+    async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID,
+                                  event_id: EventID) -> None:
         portal = po.Portal.get_by_mxid(room_id)
         if portal:
-            await portal.handle_matrix_upgrade(sender, new_room_id)
+            await portal.handle_matrix_upgrade(sender, new_room_id, event_id)
 
     async def handle_member_info_change(self, room_id: RoomID, user_id: UserID,
                                         profile: MemberStateEventContent,
@@ -321,17 +328,15 @@ class MatrixHandler(BaseMatrixHandler):
             return
 
         for user_id, event_id in receipts:
-            user = await u.User.get_by_mxid(user_id).ensure_started()
+            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
-            if not await user.is_logged_in():
+            if user and await user.is_logged_in():
-                continue
+                await portal.mark_read(user, event_id)
-            await portal.mark_read(user, event_id)
 
     @staticmethod
     async def handle_presence(user_id: UserID, presence: PresenceState) -> None:
-        user = await u.User.get_by_mxid(user_id).ensure_started()
+        user = u.User.get_by_mxid(user_id, check_db=False, create=False)
-        if not await user.is_logged_in():
+        if user and await user.is_logged_in():
-            return
+            await user.set_presence(presence == PresenceState.ONLINE)
-        await user.set_presence(presence == PresenceState.ONLINE)
 
     async def handle_typing(self, room_id: RoomID, now_typing: Set[UserID]) -> None:
         portal = po.Portal.get_by_mxid(room_id)
@@ -346,17 +351,22 @@ class MatrixHandler(BaseMatrixHandler):
             if is_typing and was_typing:
                 continue
 
-            user = await u.User.get_by_mxid(user_id).ensure_started()
+            user = u.User.get_by_mxid(user_id, check_db=False, create=False)
-            if not await user.is_logged_in():
+            if user and await user.is_logged_in():
-                continue
+                await portal.set_typing(user, is_typing)
-
-            await portal.set_typing(user, is_typing)
 
         self.previously_typing[room_id] = now_typing
 
     def filter_matrix_event(self, evt: Event) -> bool:
-        if not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent)):
+        if isinstance(evt, (TypingEvent, ReceiptEvent, PresenceEvent)):
+            return False
+        elif not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent, EncryptedEvent)):
             return True
+        if evt.content.get(self.az.real_user_content_key, False):
+            puppet = pu.Puppet.deprecated_sync_get_by_custom_mxid(evt.sender)
+            if puppet:
+                self.log.debug("Ignoring puppet-sent event %s", evt.event_id)
+                return True
         return evt.sender and (evt.sender == self.az.bot_mxid
                                or pu.Puppet.get_id_from_mxid(evt.sender) is not None)
 
@@ -377,17 +387,16 @@ class MatrixHandler(BaseMatrixHandler):
         if evt.type == EventType.ROOM_POWER_LEVELS:
             await self.handle_power_levels(evt)
         elif evt.type in (EventType.ROOM_NAME, EventType.ROOM_AVATAR, EventType.ROOM_TOPIC):
-            await self.handle_room_meta(evt.type, evt.room_id, evt.sender, evt.content)
+            await self.handle_room_meta(evt.type, evt.room_id, evt.sender, evt.content,
+                                        evt.event_id)
         elif evt.type == EventType.ROOM_PINNED_EVENTS:
             new_events = set(evt.content.pinned)
             try:
                 old_events = set(evt.unsigned.prev_content.pinned)
             except (KeyError, ValueError, TypeError, AttributeError):
                 old_events = set()
-            await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events)
+            await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events,
+                                       evt.event_id)
         elif evt.type == EventType.ROOM_TOMBSTONE:
-            await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room)
+            await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room,
-
+                                           evt.event_id)
-    async def log_event_handle_duration(self, evt: Event, duration: float) -> None:
-        if EVENT_TIME:
-            EVENT_TIME.labels(event_type=str(evt.type)).observe(duration)

mautrix_telegram/portal/__init__.pyi

@@ -1,8 +1,8 @@
 from typing import Union
 from .base import BasePortal
-from .portal_matrix import PortalMatrix
+from .matrix import PortalMatrix
-from .portal_metadata import PortalMetadata
+from .metadata import PortalMetadata
-from .portal_telegram import PortalTelegram
+from .telegram import PortalTelegram
 from ..context import Context
 
 Portal = Union[BasePortal, PortalMatrix, PortalMetadata, PortalTelegram]

mautrix_telegram/portal/base.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
+from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, Set, Iterable, TYPE_CHECKING
 from abc import ABC, abstractmethod
 import asyncio
 import logging
@@ -30,12 +30,16 @@ from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, ChatInviteE
 
 from mautrix.errors import MatrixRequestError, IntentError
 from mautrix.appservice import AppService, IntentAPI
-from mautrix.types import RoomID, RoomAlias, UserID, EventType, PowerLevelStateEventContent
+from mautrix.types import (RoomID, RoomAlias, UserID, EventID, EventType, MessageEventContent,
+                           PowerLevelStateEventContent, ContentURI)
 from mautrix.util.simple_template import SimpleTemplate
+from mautrix.util.simple_lock import SimpleLock
+from mautrix.util.logging import TraceLogger
+from mautrix.bridge import BasePortal as MautrixBasePortal
 
 from ..types import TelegramID
 from ..context import Context
-from ..db import Portal as DBPortal
+from ..db import Portal as DBPortal, Message as DBMessage
 from .. import puppet as p, user as u, util
 from .deduplication import PortalDedup
 from .send_lock import PortalSendLock
@@ -44,6 +48,7 @@ if TYPE_CHECKING:
     from ..bot import Bot
     from ..abstract_user import AbstractUser
     from ..config import Config
+    from ..matrix import MatrixHandler
     from . import Portal
 
 TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
@@ -53,11 +58,12 @@ InviteList = Union[UserID, List[UserID]]
 config: Optional['Config'] = None
 
 
-class BasePortal(ABC):
+class BasePortal(MautrixBasePortal, ABC):
-    base_log: logging.Logger = logging.getLogger("mau.portal")
+    base_log: TraceLogger = logging.getLogger("mau.portal")
     az: AppService = None
     bot: 'Bot' = None
     loop: asyncio.AbstractEventLoop = None
+    matrix: 'MatrixHandler' = None
 
     # Config cache
     filter_mode: str = None
@@ -67,6 +73,7 @@ class BasePortal(ABC):
     sync_channel_members: bool = True
     sync_matrix_state: bool = True
     public_portals: bool = False
+    private_chat_portal_meta: bool = False
 
     alias_template: SimpleTemplate[str]
     hs_domain: str
@@ -85,8 +92,13 @@ class BasePortal(ABC):
     about: Optional[str]
     photo_id: Optional[str]
     local_config: Dict[str, Any]
+    avatar_url: Optional[ContentURI]
+    encrypted: bool
     deleted: bool
-    log: logging.Logger
+    backfill_lock: SimpleLock
+    backfill_method_lock: asyncio.Lock
+    backfill_leave: Optional[Set[IntentAPI]]
+    log: TraceLogger
 
     alias: Optional[RoomAlias]
 
@@ -95,12 +107,14 @@ class BasePortal(ABC):
 
     _db_instance: DBPortal
     _main_intent: Optional[IntentAPI]
+    _room_create_lock: asyncio.Lock
 
     def __init__(self, tgid: TelegramID, peer_type: str, tg_receiver: Optional[TelegramID] = None,
                  mxid: Optional[RoomID] = None, username: Optional[str] = None,
                  megagroup: Optional[bool] = False, title: Optional[str] = None,
                  about: Optional[str] = None, photo_id: Optional[str] = None,
-                 local_config: Optional[str] = None, db_instance: DBPortal = None) -> None:
+                 local_config: Optional[str] = None, avatar_url: Optional[ContentURI] = None,
+                 encrypted: Optional[bool] = False, db_instance: DBPortal = None) -> None:
         self.mxid = mxid
         self.tgid = tgid
         self.tg_receiver = tg_receiver or tgid
@@ -111,10 +125,16 @@ class BasePortal(ABC):
         self.about = about
         self.photo_id = photo_id
         self.local_config = json.loads(local_config or "{}")
+        self.avatar_url = avatar_url
+        self.encrypted = encrypted
         self._db_instance = db_instance
         self._main_intent = None
         self.deleted = False
         self.log = self.base_log.getChild(self.tgid_log if self.tgid else self.mxid)
+        self.backfill_lock = SimpleLock("Waiting for backfilling to finish before handling %s",
+                                        log=self.log)
+        self.backfill_method_lock = asyncio.Lock()
+        self.backfill_leave = None
 
         self.dedup = PortalDedup(self)
         self.send_lock = PortalSendLock()
@@ -124,7 +144,7 @@ class BasePortal(ABC):
         if mxid:
             self.by_mxid[mxid] = self
 
-    # region Propegrties
+    # region Properties
 
     @property
     def tgid_full(self) -> Tuple[TelegramID, TelegramID]:
@@ -136,6 +156,10 @@ class BasePortal(ABC):
             return str(self.tgid)
         return f"{self.tg_receiver}<->{self.tgid}"
 
+    @property
+    def name(self) -> str:
+        return self.title
+
     @property
     def alias(self) -> Optional[RoomAlias]:
         if not self.username:
@@ -194,9 +218,8 @@ class BasePortal(ABC):
     def _get_largest_photo_size(photo: Union[Photo, Document]
                                 ) -> Tuple[Optional[InputPhotoFileLocation],
                                            Optional[TypePhotoSize]]:
-        if not photo:
+        if not photo or isinstance(photo, PhotoEmpty) or (isinstance(photo, Document)
-            return None, None
+                                                          and not photo.thumbs):
-        if isinstance(photo, Document) and not photo.thumbs:
             return None, None
 
         largest = max(photo.thumbs if isinstance(photo, Document) else photo.sizes,
@@ -220,9 +243,8 @@ class BasePortal(ABC):
             await self.main_intent.get_power_levels(self.mxid)
         except MatrixRequestError:
             return False
-        evt_type = EventType.find(f"net.maunium.telegram.{event}")
+        evt_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
-        evt_type.t_class = EventType.Class.STATE
+        return await self.main_intent.state_store.has_power_level(self.mxid, user.mxid, evt_type)
-        return self.main_intent.state_store.has_power_level(self.mxid, user.mxid, event=evt_type)
 
     def get_input_entity(self, user: 'AbstractUser'
                          ) -> Awaitable[Union[TypeInputPeer, TypeInputChannel]]:
@@ -233,8 +255,7 @@ class BasePortal(ABC):
             return await user.client.get_entity(self.peer)
         except ValueError:
             if user.is_bot:
-                self.log.warning(f"Could not find entity with bot {user.tgid}. "
+                self.log.warning(f"Could not find entity with bot {user.tgid}. Failing...")
-                                 "Failing...")
                 raise
             self.log.warning(f"Could not find entity with user {user.tgid}. "
                              "falling back to get_dialogs.")
@@ -256,60 +277,32 @@ class BasePortal(ABC):
     # endregion
     # region Matrix room cleanup
 
-    async def get_authenticated_matrix_users(self) -> List['u.User']:
+    async def get_authenticated_matrix_users(self) -> List[UserID]:
         try:
             members = await self.main_intent.get_room_members(self.mxid)
         except MatrixRequestError:
             return []
-        authenticated: List[u.User] = []
+        authenticated: List[UserID] = []
         has_bot = self.has_bot
-        for member_str in members:
+        for member in members:
-            member = UserID(member_str)
+            if p.Puppet.get_id_from_mxid(member) or member == self.az.bot_mxid:
-            if p.Puppet.get_id_from_mxid(member) or member == self.main_intent.mxid:
                 continue
             user = await u.User.get_by_mxid(member).ensure_started()
             authenticated_through_bot = has_bot and user.relaybot_whitelisted
             if authenticated_through_bot or await user.has_full_access(allow_bot=True):
-                authenticated.append(user)
+                authenticated.append(user.mxid)
         return authenticated
 
-    @staticmethod
+    async def cleanup_portal(self, message: str, puppets_only: bool = False, delete: bool = True
-    async def cleanup_room(intent: IntentAPI, room_id: RoomID, message: str,
+                             ) -> None:
-                           puppets_only: bool = False) -> None:
-        try:
-            members = await intent.get_room_members(room_id)
-        except MatrixRequestError:
-            members = []
-        for user in members:
-            puppet = p.Puppet.get_by_mxid(UserID(user), create=False)
-            if user != intent.mxid and (not puppets_only or puppet):
-                try:
-                    if puppet:
-                        await puppet.default_mxid_intent.leave_room(room_id)
-                    else:
-                        await intent.kick_user(room_id, user, message)
-                except (MatrixRequestError, IntentError):
-                    pass
-        try:
-            await intent.leave_room(room_id)
-        except (MatrixRequestError, IntentError):
-            self.log.warning("Failed to leave room when cleaning up room", exc_info=True)
-
-    async def cleanup_portal(self, message: str, puppets_only: bool = False) -> None:
         if self.username:
             try:
                 await self.main_intent.remove_room_alias(self.alias_localpart)
             except (MatrixRequestError, IntentError):
                 self.log.warning("Failed to remove alias when cleaning up room", exc_info=True)
         await self.cleanup_room(self.main_intent, self.mxid, message, puppets_only)
-
+        if delete:
-    async def unbridge(self) -> None:
+            await self.delete()
-        await self.cleanup_portal("Room unbridged", puppets_only=True)
-        self.delete()
-
-    async def cleanup_and_delete(self) -> None:
-        await self.cleanup_portal("Portal deleted")
-        self.delete()
 
     # endregion
     # region Database conversion
@@ -324,14 +317,19 @@ class BasePortal(ABC):
         return DBPortal(tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
                         mxid=self.mxid, username=self.username, megagroup=self.megagroup,
                         title=self.title, about=self.about, photo_id=self.photo_id,
-                        config=json.dumps(self.local_config))
+                        config=json.dumps(self.local_config), avatar_url=self.avatar_url,
+                        encrypted=self.encrypted)
 
-    def save(self) -> None:
+    async def save(self) -> None:
         self.db_instance.edit(mxid=self.mxid, username=self.username, title=self.title,
                               about=self.about, photo_id=self.photo_id, megagroup=self.megagroup,
-                              config=json.dumps(self.local_config))
+                              config=json.dumps(self.local_config), avatar_url=self.avatar_url,
+                              encrypted=self.encrypted)
 
-    def delete(self) -> None:
+    async def delete(self) -> None:
+        self.delete_sync()
+
+    def delete_sync(self) -> None:
         try:
             del self.by_tgid[self.tgid_full]
         except KeyError:
@@ -342,19 +340,29 @@ class BasePortal(ABC):
             pass
         if self._db_instance:
             self._db_instance.delete()
+        DBMessage.delete_all(self.mxid)
         self.deleted = True
 
     @classmethod
     def from_db(cls, db_portal: DBPortal) -> 'Portal':
         return cls(tgid=db_portal.tgid, tg_receiver=db_portal.tg_receiver,
-                   peer_type=db_portal.peer_type, mxid=db_portal.mxid,
+                   peer_type=db_portal.peer_type, mxid=db_portal.mxid, username=db_portal.username,
-                   username=db_portal.username, megagroup=db_portal.megagroup,
+                   megagroup=db_portal.megagroup, title=db_portal.title, about=db_portal.about,
-                   title=db_portal.title, about=db_portal.about, photo_id=db_portal.photo_id,
+                   photo_id=db_portal.photo_id, local_config=db_portal.config,
-                   local_config=db_portal.config, db_instance=db_portal)
+                   avatar_url=db_portal.avatar_url, encrypted=db_portal.encrypted,
+                   db_instance=db_portal)
 
     # endregion
     # region Class instance lookup
 
+    @classmethod
+    def all(cls) -> Iterable['Portal']:
+        for db_portal in DBPortal.all():
+            try:
+                yield cls.by_tgid[(db_portal.tgid, db_portal.tg_receiver)]
+            except KeyError:
+                yield cls.from_db(db_portal)
+
     @classmethod
     def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
         try:
@@ -392,6 +400,8 @@ class BasePortal(ABC):
     @classmethod
     def get_by_tgid(cls, tgid: TelegramID, tg_receiver: Optional[TelegramID] = None,
                     peer_type: str = None) -> Optional['Portal']:
+        if peer_type == "user" and tg_receiver is None:
+            raise ValueError("tg_receiver is required when peer_type is \"user\"")
         tg_receiver = tg_receiver or tgid
         tgid_full = (tgid, tg_receiver)
         try:
@@ -486,9 +496,24 @@ class BasePortal(ABC):
     def _migrate_and_save_telegram(self, new_id: TelegramID) -> None:
         pass
 
+    @abstractmethod
+    async def update_bridge_info(self) -> None:
+        pass
+
     @abstractmethod
     def handle_matrix_power_levels(self, sender: 'u.User', new_levels: Dict[UserID, int],
-                                   old_levels: Dict[UserID, int]) -> Awaitable[None]:
+                                   old_levels: Dict[UserID, int], event_id: Optional[EventID]
+                                   ) -> Awaitable[None]:
+        pass
+
+    @abstractmethod
+    def backfill(self, source: 'AbstractUser', is_initial: bool = False,
+                 limit: Optional[int] = None, last_id: Optional[int] = None) -> Awaitable[None]:
+        pass
+
+    @abstractmethod
+    async def _send_delivery_receipt(self, event_id: EventID, room_id: Optional[RoomID] = None
+                                     ) -> None:
         pass
 
     # endregion
@@ -497,10 +522,13 @@ class BasePortal(ABC):
 def init(context: Context) -> None:
     global config
     BasePortal.az, config, BasePortal.loop, BasePortal.bot = context.core
+    BasePortal.matrix = context.mx
+    MautrixBasePortal.bridge = context.bridge
     BasePortal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
     BasePortal.sync_channel_members = config["bridge.sync_channel_members"]
     BasePortal.sync_matrix_state = config["bridge.sync_matrix_state"]
     BasePortal.public_portals = config["bridge.public_portals"]
+    BasePortal.private_chat_portal_meta = config["bridge.private_chat_portal_meta"]
     BasePortal.filter_mode = config["bridge.filter.mode"]
     BasePortal.filter_list = config["bridge.filter.list"]
     BasePortal.hs_domain = config["homeserver.domain"]

mautrix_telegram/portal/deduplication.py

@@ -50,7 +50,7 @@ class PortalDedup:
 
     @property
     def _always_force_hash(self) -> bool:
-        return self._portal.peer_type != 'channel'
+        return self._portal.peer_type == 'chat'
 
     @staticmethod
     def _hash_event(event: TypeMessage) -> str:
@@ -69,7 +69,7 @@ class PortalDedup:
                     hash_content += {
                         MessageMediaContact: lambda media: [media.user_id],
                         MessageMediaDocument: lambda media: [media.document.id],
-                        MessageMediaPhoto: lambda media: [media.photo.id],
+                        MessageMediaPhoto: lambda media: [media.photo.id if media.photo else 0],
                         MessageMediaGeo: lambda media: [media.geo.long, media.geo.lat],
                     }[type(event.media)](event.media)
                 except KeyError:

mautrix_telegram/portal/matrix.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
+from typing import Awaitable, Dict, Optional, Union, Any, TYPE_CHECKING
 from html import escape as escape_html
 from string import Template
 from abc import ABC
@@ -25,18 +25,18 @@ from telethon.tl.functions.messages import (EditChatPhotoRequest, EditChatTitleR
                                             EditChatAboutRequest)
 from telethon.tl.functions.channels import EditPhotoRequest, EditTitleRequest, JoinChannelRequest
 from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError,
-                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError)
+                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError,
+                             RPCError)
 from telethon.tl.patched import Message, MessageService
-from telethon.tl.types import (
+from telethon.tl.types import (DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
-    DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
+                               InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
-    InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
+                               SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer,
-    SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer, TypeMessageEntity,
+                               UpdateNewMessage, InputMediaUploadedDocument,
-    UpdateNewMessage, InputMediaUploadedDocument, InputMediaUploadedPhoto)
+                               InputMediaUploadedPhoto)
 
 from mautrix.types import (EventID, RoomID, UserID, ContentURI, MessageType, MessageEventContent,
                            TextMessageEventContent, MediaMessageEventContent, Format,
-                           LocationMessageEventContent)
+                           LocationMessageEventContent, ImageInfo, VideoInfo)
-from mautrix.bridge import BasePortal as MautrixBasePortal
 
 from ..types import TelegramID
 from ..db import Message as DBMessage
@@ -50,12 +50,17 @@ if TYPE_CHECKING:
     from ..tgclient import MautrixTelegramClient
     from ..config import Config
 
+try:
+    from mautrix.crypto.attachments import decrypt_attachment
+except ImportError:
+    decrypt_attachment = None
+
 TypeMessage = Union[Message, MessageService]
 
 config: Optional['Config'] = None
 
 
-class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
+class PortalMatrix(BasePortal, ABC):
     async def _get_state_change_message(self, event: str, user: 'u.User', **kwargs: Any
                                         ) -> Optional[str]:
         tpl = self.get_config(f"state_event_formats.{event}")
@@ -82,9 +87,9 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             message = await self._get_state_change_message(event, user, **kwargs)
             if not message:
                 return
-            response = await self.bot.client.send_message(
+            message, entities = await formatter.matrix_to_telegram(self.bot.client, html=message)
-                self.peer, message,
+            response = await self.bot.client.send_message(self.peer, message,
-                parse_mode=self._matrix_event_to_entities)
+                                                          formatting_entities=entities)
             space = self.tgid if self.peer_type == "channel" else self.bot.tgid
             self.dedup.check(response, (event_id, space))
 
@@ -117,7 +122,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
         if user.tgid == source.tgid:
             return None
         if self.peer_type == "user" and user.tgid == self.tgid:
-            self.delete()
+            await self.delete()
             return None
         if isinstance(user, u.User) and await user.needs_relaybot(self):
             if not self.bot:
@@ -147,7 +152,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
 
         if self.peer_type == "user":
             await self.main_intent.leave_room(self.mxid)
-            self.delete()
+            await self.delete()
             try:
                 del self.by_tgid[self.tgid_full]
                 del self.by_mxid[self.mxid]
@@ -209,52 +214,53 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
         elif content.msgtype == MessageType.EMOTE:
             await self._apply_emote_format(sender, content)
 
-    @staticmethod
-    def _matrix_event_to_entities(event: Union[str, MessageEventContent]
-                                  ) -> Tuple[str, Optional[List[TypeMessageEntity]]]:
-        try:
-            if isinstance(event, str):
-                message, entities = formatter.matrix_to_telegram(event)
-            elif isinstance(event, TextMessageEventContent) and event.format == Format.HTML:
-                message, entities = formatter.matrix_to_telegram(event.formatted_body)
-            else:
-                message, entities = formatter.matrix_text_to_telegram(event.body)
-        except KeyError:
-            message, entities = None, None
-        return message, entities
-
     async def _handle_matrix_text(self, sender_id: TelegramID, event_id: EventID,
                                   space: TelegramID, client: 'MautrixTelegramClient',
                                   content: TextMessageEventContent, reply_to: TelegramID) -> None:
+        message, entities = await formatter.matrix_to_telegram(client, text=content.body,
+                                                               html=content.formatted(Format.HTML))
         async with self.send_lock(sender_id):
             lp = self.get_config("telegram_link_preview")
             if content.get_edit():
                 orig_msg = DBMessage.get_by_mxid(content.get_edit(), self.mxid, space)
                 if orig_msg:
-                    response = await client.edit_message(self.peer, orig_msg.tgid, content,
+                    response = await client.edit_message(self.peer, orig_msg.tgid, message,
-                                                         parse_mode=self._matrix_event_to_entities,
+                                                         formatting_entities=entities,
                                                          link_preview=lp)
                     self._add_telegram_message_to_db(event_id, space, -1, response)
                     return
-            response = await client.send_message(self.peer, content, reply_to=reply_to,
+            response = await client.send_message(self.peer, message, reply_to=reply_to,
-                                                 parse_mode=self._matrix_event_to_entities,
+                                                 formatting_entities=entities,
                                                  link_preview=lp)
             self._add_telegram_message_to_db(event_id, space, 0, response)
+        await self._send_delivery_receipt(event_id)
 
     async def _handle_matrix_file(self, sender_id: TelegramID, event_id: EventID,
                                   space: TelegramID, client: 'MautrixTelegramClient',
                                   content: MediaMessageEventContent, reply_to: TelegramID,
                                   caption: TextMessageEventContent = None) -> None:
         mime = content.info.mimetype
-        w, h = content.info.width, content.info.height
+        if isinstance(content.info, (ImageInfo, VideoInfo)):
+            w, h = content.info.width, content.info.height
+        else:
+            w = h = None
         file_name = content["net.maunium.telegram.internal.filename"]
         max_image_size = config["bridge.image_as_file_size"] * 1000 ** 2
 
-        if config["bridge.parallel_file_transfer"]:
+        if config["bridge.parallel_file_transfer"] and content.url:
             file_handle, file_size = await parallel_transfer_to_telegram(client, self.main_intent,
                                                                          content.url, sender_id)
         else:
-            file = await self.main_intent.download_media(content.url)
+            if content.file:
+                if not decrypt_attachment:
+                    self.log.warning(f"Can't bridge encrypted media event {event_id}:"
+                                     " matrix-nio not installed")
+                    return
+                file = await self.main_intent.download_media(content.file.url)
+                file = decrypt_attachment(file, content.file.key.key,
+                                          content.file.hashes.get("sha256"), content.file.iv)
+            else:
+                file = await self.main_intent.download_media(content.url)
 
             if content.msgtype == MessageType.STICKER:
                 if mime != "image/gif":
@@ -279,20 +285,23 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             media = InputMediaUploadedDocument(file=file_handle, attributes=attributes,
                                                mime_type=mime or "application/octet-stream")
 
-        caption, entities = self._matrix_event_to_entities(caption) if caption else (None, None)
+        capt, entities = (await formatter.matrix_to_telegram(client, text=caption.body,
+                                                             html=caption.formatted(Format.HTML))
+                          if caption else (None, None))
 
         async with self.send_lock(sender_id):
-            if await self._matrix_document_edit(client, content, space, caption, media, event_id):
+            if await self._matrix_document_edit(client, content, space, capt, media, event_id):
                 return
             try:
                 response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption, entities=entities)
+                                                   caption=capt, entities=entities)
             except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
                 media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
                                                    attributes=attributes)
                 response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption, entities=entities)
+                                                   caption=capt, entities=entities)
             self._add_telegram_message_to_db(event_id, space, 0, response)
+        await self._send_delivery_receipt(event_id)
 
     async def _matrix_document_edit(self, client: 'MautrixTelegramClient',
                                     content: MessageEventContent, space: TelegramID,
@@ -303,6 +312,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
                 response = await client.edit_message(self.peer, orig_msg.tgid,
                                                      caption, file=media)
                 self._add_telegram_message_to_db(event_id, space, -1, response)
+                await self._send_delivery_receipt(event_id)
                 return True
         return False
 
@@ -316,7 +326,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
         except (KeyError, ValueError):
             self.log.exception("Failed to parse location")
             return None
-        caption, entities = self._matrix_event_to_entities(content)
+        caption, entities = await formatter.matrix_to_telegram(client, text=content.body)
         media = MessageMediaGeo(geo=GeoPoint(lat, long, access_hash=0))
 
         async with self.send_lock(sender_id):
@@ -325,10 +335,11 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             response = await client.send_media(self.peer, media, reply_to=reply_to,
                                                caption=caption, entities=entities)
             self._add_telegram_message_to_db(event_id, space, 0, response)
+        await self._send_delivery_receipt(event_id)
 
     def _add_telegram_message_to_db(self, event_id: EventID, space: TelegramID,
                                     edit_index: int, response: TypeMessage) -> None:
-        self.log.debug("Handled Matrix message: %s", response)
+        self.log.trace("Handled Matrix message: %s", response)
         self.dedup.check(response, (event_id, space), force_hash=edit_index != 0)
         if edit_index < 0:
             prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1)
@@ -340,17 +351,27 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             mxid=event_id,
             edit_index=edit_index).insert()
 
+    async def _send_bridge_error(self, msg: str) -> None:
+        if config["bridge.delivery_error_reports"]:
+            await self._send_message(self.main_intent,
+                                     TextMessageEventContent(msgtype=MessageType.NOTICE, body=msg))
+
     async def handle_matrix_message(self, sender: 'u.User', content: MessageEventContent,
                                     event_id: EventID) -> None:
+        try:
+            await self._handle_matrix_message(sender, content, event_id)
+        except RPCError as e:
+            if config["bridge.delivery_error_reports"]:
+                await self._send_bridge_error(
+                    f"\u26a0 Your message may not have been bridged: {e}")
+            raise
+
+    async def _handle_matrix_message(self, sender: 'u.User', content: MessageEventContent,
+                                     event_id: EventID) -> None:
         if not content.body or not content.msgtype:
             self.log.debug(f"Ignoring message {event_id} in {self.mxid} without body or msgtype")
             return
 
-        puppet = p.Puppet.get_by_custom_mxid(sender.mxid)
-        if puppet and content.get("net.maunium.telegram.puppet", False):
-            self.log.debug("Ignoring puppet-sent message by confirmed puppet user %s", sender.mxid)
-            return
-
         logged_in = not await sender.needs_relaybot(self)
         client = sender.client if logged_in else self.bot.client
         sender_id = sender.tgid if logged_in else self.bot.tgid
@@ -389,10 +410,10 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             await self._handle_matrix_file(sender_id, event_id, space, client, content, reply_to,
                                            caption_content)
         else:
-            self.log.debug(f"Unhandled Matrix event: {content}")
+            self.log.trace("Unhandled Matrix event: %s", content)
 
-    async def handle_matrix_pin(self, sender: 'u.User',
+    async def handle_matrix_pin(self, sender: 'u.User', pinned_message: Optional[EventID],
-                                pinned_message: Optional[EventID]) -> None:
+                                pin_event_id: EventID) -> None:
         if self.peer_type != "chat" and self.peer_type != "channel":
             return
         try:
@@ -405,10 +426,12 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
                     self.log.warning(f"Could not find pinned {pinned_message} in {self.mxid}")
                     return
                 await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=message.tgid))
+            await self._send_delivery_receipt(pin_event_id)
         except ChatNotModifiedError:
             pass
 
-    async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID) -> None:
+    async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID,
+                                     redaction_event_id: EventID) -> None:
         real_deleter = deleter if not await deleter.needs_relaybot(self) else self.bot
         space = self.tgid if self.peer_type == "channel" else real_deleter.tgid
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
@@ -416,6 +439,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             return
         if message.edit_index == 0:
             await real_deleter.client.delete_messages(self.peer, [message.tgid])
+            await self._send_delivery_receipt(redaction_event_id)
         else:
             self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
 
@@ -430,7 +454,8 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
                                        pin_messages=moderator, add_admins=admin)
 
     async def handle_matrix_power_levels(self, sender: 'u.User', new_users: Dict[UserID, int],
-                                         old_users: Dict[UserID, int]) -> None:
+                                         old_users: Dict[UserID, int], event_id: Optional[EventID]
+                                         ) -> None:
         # TODO handle all power level changes and bridge exact admin rights to supergroups/channels
         for user, level in new_users.items():
             if not user or user == self.main_intent.mxid or user == sender.mxid:
@@ -446,15 +471,16 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             if user not in old_users or level != old_users[user]:
                 await self._update_telegram_power_level(sender, user_id, level)
 
-    async def handle_matrix_about(self, sender: 'u.User', about: str) -> None:
+    async def handle_matrix_about(self, sender: 'u.User', about: str, event_id: EventID) -> None:
         if self.peer_type not in ("chat", "channel"):
             return
         peer = await self.get_input_entity(sender)
         await sender.client(EditChatAboutRequest(peer=peer, about=about))
         self.about = about
-        self.save()
+        await self.save()
+        await self._send_delivery_receipt(event_id)
 
-    async def handle_matrix_title(self, sender: 'u.User', title: str) -> None:
+    async def handle_matrix_title(self, sender: 'u.User', title: str, event_id: EventID) -> None:
         if self.peer_type not in ("chat", "channel"):
             return
 
@@ -465,13 +491,19 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             response = await sender.client(EditTitleRequest(channel=channel, title=title))
         self.dedup.register_outgoing_actions(response)
         self.title = title
-        self.save()
+        await self.save()
+        await self._send_delivery_receipt(event_id)
+        await self.update_bridge_info()
 
-    async def handle_matrix_avatar(self, sender: 'u.User', url: ContentURI) -> None:
+    async def handle_matrix_avatar(self, sender: 'u.User', url: ContentURI, event_id: EventID
+                                   ) -> None:
         if self.peer_type not in ("chat", "channel"):
             # Invalid peer type
             return
+        elif self.avatar_url == url:
+            return
 
+        self.avatar_url = url
         file = await self.main_intent.download_media(url)
         mime = magic.from_buffer(file, mime=True)
         ext = sane_mimetypes.guess_extension(mime)
@@ -491,10 +523,13 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             if is_photo_update:
                 loc, size = self._get_largest_photo_size(update.message.action.photo)
                 self.photo_id = f"{size.location.volume_id}-{size.location.local_id}"
-                self.save()
+                await self.save()
                 break
+        await self._send_delivery_receipt(event_id)
+        await self.update_bridge_info()
 
-    async def handle_matrix_upgrade(self, sender: UserID, new_room: RoomID) -> None:
+    async def handle_matrix_upgrade(self, sender: UserID, new_room: RoomID, event_id: EventID
+                                    ) -> None:
         _, server = self.main_intent.parse_user_id(sender)
         old_room = self.mxid
         self.migrate_and_save_matrix(new_room)
@@ -521,6 +556,7 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
             return
         await self.update_matrix_room(user, entity, direct=self.peer_type == "user")
         self.log.info(f"{sender} upgraded room from {old_room} to {self.mxid}")
+        await self._send_delivery_receipt(event_id, room_id=old_room)
 
     def migrate_and_save_matrix(self, new_id: RoomID) -> None:
         try:
@@ -531,6 +567,16 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
         self.db_instance.edit(mxid=self.mxid)
         self.by_mxid[self.mxid] = self
 
+    async def enable_dm_encryption(self) -> bool:
+        ok = await super().enable_dm_encryption()
+        if ok:
+            try:
+                puppet = p.Puppet.get(self.tgid)
+                await self.main_intent.set_room_name(self.mxid, puppet.displayname)
+            except Exception:
+                self.log.warning(f"Failed to set room name", exc_info=True)
+        return ok
+
 
 def init(context: Context) -> None:
     global config

mautrix_telegram/portal/metadata.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Optional, Tuple, Union, Callable, TYPE_CHECKING
+from typing import List, Optional, Iterable, Union, Dict, Any, TYPE_CHECKING
 from abc import ABC
 import asyncio
 
@@ -26,12 +26,13 @@ from telethon.tl.types import (
     Channel, ChatBannedRights, ChannelParticipantsRecent, ChannelParticipantsSearch, ChatPhoto,
     PhotoEmpty, InputChannel, InputUser, ChatPhotoEmpty, PeerUser, Photo, TypeChat, TypeInputPeer,
     TypeUser, User, InputPeerPhotoFileLocation, ChatParticipantAdmin, ChannelParticipantAdmin,
-    ChatParticipantCreator, ChannelParticipantCreator)
+    ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty)
 
 from mautrix.errors import MForbidden
-from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership, Member,
+from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership,
-                           PowerLevelStateEventContent, RoomAlias)
+                           PowerLevelStateEventContent, RoomTopicStateEventContent,
-from mautrix.appservice import IntentAPI
+                           RoomNameStateEventContent, RoomAvatarStateEventContent,
+                           StateEventContent, EventID)
 
 from ..types import TelegramID
 from ..context import Context
@@ -44,6 +45,9 @@ if TYPE_CHECKING:
 
 config: Optional['Config'] = None
 
+StateBridge = EventType.find("m.bridge", EventType.Class.STATE)
+StateHalfShotBridge = EventType.find("uk.half-shot.bridge", EventType.Class.STATE)
+
 
 class PortalMetadata(BasePortal, ABC):
     _room_create_lock: asyncio.Lock
@@ -93,7 +97,7 @@ class PortalMetadata(BasePortal, ABC):
             pass
         try:
             existing = self.by_tgid[(new_id, new_id)]
-            existing.delete()
+            existing.delete_sync()
         except KeyError:
             pass
         self.db_instance.edit(tgid=new_id, tg_receiver=new_id, peer_type=self.peer_type)
@@ -110,7 +114,7 @@ class PortalMetadata(BasePortal, ABC):
         await source.client(
             UpdateUsernameRequest(await self.get_input_entity(source), username))
         if await self._update_username(username):
-            self.save()
+            await self.save()
 
     async def create_telegram_chat(self, source: 'u.User', supergroup: bool = False) -> None:
         if not self.mxid:
@@ -155,7 +159,7 @@ class PortalMetadata(BasePortal, ABC):
         if levels.get_user_level(self.main_intent.mxid) == 100:
             levels = self._get_base_power_levels(levels, entity)
             await self.main_intent.set_power_levels(self.mxid, levels)
-        await self.handle_matrix_power_levels(source, levels.users, {})
+        await self.handle_matrix_power_levels(source, levels.users, {}, None)
 
     async def invite_telegram(self, source: 'u.User',
                               puppet: Union[p.Puppet, 'AbstractUser']) -> None:
@@ -168,17 +172,6 @@ class PortalMetadata(BasePortal, ABC):
         elif not self.bot or self.tg_receiver != self.bot.tgid:
             raise ValueError("Invalid peer type for Telegram user invite")
 
-    async def sync_matrix_members(self) -> None:
-        resp = await self.main_intent.get_room_joined_memberships(self.mxid)
-        members = resp["joined"]
-        for mxid, info in members.items():
-            member = Member(membership=Membership.JOIN)
-            if "display_name" in info:
-                member.displayname = info["display_name"]
-            if "avatar_url" in info:
-                member.avatar_url = info["avatar_url"]
-            self.az.state_store.set_member(self.mxid, mxid, member)
-
     # endregion
     # region Telegram -> Matrix
 
@@ -192,38 +185,53 @@ class PortalMetadata(BasePortal, ABC):
     async def update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
                                  direct: bool = None, puppet: p.Puppet = None,
                                  levels: PowerLevelStateEventContent = None,
-                                 users: List[User] = None,
+                                 users: List[User] = None) -> None:
-                                 participants: List[TypeParticipant] = None) -> None:
         if direct is None:
             direct = self.peer_type == "user"
         try:
-            await self._update_matrix_room(user, entity, direct, puppet, levels, users,
+            await self._update_matrix_room(user, entity, direct, puppet, levels, users)
-                                           participants)
         except Exception:
             self.log.exception("Fatal error updating Matrix room")
 
     async def _update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
                                   direct: bool, puppet: p.Puppet = None,
                                   levels: PowerLevelStateEventContent = None,
-                                  users: List[User] = None,
+                                  users: List[User] = None) -> None:
-                                  participants: List[TypeParticipant] = None) -> None:
         if not direct:
             await self.update_info(user, entity)
-            if not users or not participants:
+            if not users:
-                users, participants = await self._get_users(user, entity)
+                users = await self._get_users(user, entity)
             await self._sync_telegram_users(user, users)
-            await self.update_telegram_participants(participants, levels)
+            await self.update_power_levels(users, levels)
         else:
             if not puppet:
                 puppet = p.Puppet.get(self.tgid)
             await puppet.update_info(user, entity)
             await puppet.intent_for(self).join_room(self.mxid)
-        if self.sync_matrix_state:
+            if self.encrypted or self.private_chat_portal_meta:
-            await self.sync_matrix_members()
+                # The bridge bot needs to join for e2ee, but that messes up the default name
+                # generation. If/when canonical DMs happen, this might not be necessary anymore.
+                changed = await self._update_title(puppet.displayname)
+                changed = await self._update_avatar(user, entity.photo) or changed
+                if changed:
+                    await self.save()
+                    await self.update_bridge_info()
 
-    async def create_matrix_room(self, user: 'AbstractUser', entity: TypeChat = None,
+            puppet = await p.Puppet.get_by_custom_mxid(user.mxid)
-                                 invites: InviteList = None, update_if_exists: bool = True,
+            if puppet:
-                                 synchronous: bool = False) -> Optional[str]:
+                try:
+                    did_join = await puppet.intent.ensure_joined(self.mxid)
+                    if isinstance(user, u.User) and did_join and self.peer_type == "user":
+                        await user.update_direct_chats({self.main_intent.mxid: [self.mxid]})
+                except Exception:
+                    self.log.exception("Failed to ensure %s is joined to portal", user.mxid)
+
+        if self.sync_matrix_state:
+            await self.main_intent.get_joined_members(self.mxid)
+
+    async def create_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User] = None,
+                                 invites: InviteList = None, update_if_exists: bool = True
+                                 ) -> Optional[RoomID]:
         if self.mxid:
             if update_if_exists:
                 if not entity:
@@ -233,10 +241,7 @@ class PortalMetadata(BasePortal, ABC):
                         self.log.exception(f"Failed to get entity through {user.tgid} for update")
                         return self.mxid
                 update = self.update_matrix_room(user, entity, self.peer_type == "user")
-                if synchronous:
+                self.loop.create_task(update)
-                    await update
-                else:
-                    asyncio.ensure_future(update, loop=self.loop)
                 await self.invite_to_matrix(invites or [])
             return self.mxid
         async with self._room_create_lock:
@@ -245,19 +250,62 @@ class PortalMetadata(BasePortal, ABC):
             except Exception:
                 self.log.exception("Fatal error creating Matrix room")
 
-    async def _create_matrix_room(self, user: 'AbstractUser', entity: TypeChat, invites: InviteList
+    @property
-                                  ) -> Optional[RoomID]:
+    def bridge_info_state_key(self) -> str:
-        direct = self.peer_type == "user"
+        return f"net.maunium.telegram://telegram/{self.tgid}"
 
+    @property
+    def bridge_info(self) -> Dict[str, Any]:
+        info = {
+            "bridgebot": self.az.bot_mxid,
+            "creator": self.main_intent.mxid,
+            "protocol": {
+                "id": "telegram",
+                "displayname": "Telegram",
+                "avatar_url": config["appservice.bot_avatar"],
+                "external_url": "https://telegram.org",
+            },
+            "channel": {
+                "id": str(self.tgid),
+                "displayname": self.title,
+                "avatar_url": self.avatar_url,
+            }
+        }
+        if self.username:
+            info["channel"]["external_url"] = f"https://t.me/{self.username}"
+        elif self.peer_type == "user":
+            puppet = p.Puppet.get(self.tgid)
+            if puppet and puppet.username:
+                info["channel"]["external_url"] = f"https://t.me/{puppet.username}"
+        return info
+
+    async def update_bridge_info(self) -> None:
+        if not self.mxid:
+            self.log.debug("Not updating bridge info: no Matrix room created")
+            return
+        try:
+            self.log.debug("Updating bridge info...")
+            await self.main_intent.send_state_event(self.mxid, StateBridge,
+                                                    self.bridge_info, self.bridge_info_state_key)
+            # TODO remove this once https://github.com/matrix-org/matrix-doc/pull/2346 is in spec
+            await self.main_intent.send_state_event(self.mxid, StateHalfShotBridge,
+                                                    self.bridge_info, self.bridge_info_state_key)
+        except Exception:
+            self.log.warning("Failed to update bridge info", exc_info=True)
+
+    async def _create_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
+                                  invites: InviteList) -> Optional[RoomID]:
         if self.mxid:
             return self.mxid
-
+        elif not self.allow_bridging:
-        if not self.allow_bridging:
             return None
 
+        direct = self.peer_type == "user"
+        invites = invites or []
+
         if not entity:
             entity = await self.get_entity(user)
-            self.log.debug(f"Fetched data: {entity}")
+            self.log.trace("Fetched data: %s", entity)
 
         self.log.debug("Creating room")
 
@@ -271,6 +319,8 @@ class PortalMetadata(BasePortal, ABC):
             self.about = "Your Telegram cloud storage chat"
 
         puppet = p.Puppet.get(self.tgid) if direct else None
+        if puppet:
+            await puppet.update_info(user, entity)
         self._main_intent = puppet.intent_for(self) if direct else self.az.intent
 
         if self.peer_type == "channel":
@@ -290,24 +340,44 @@ class PortalMetadata(BasePortal, ABC):
             await self.main_intent.remove_room_alias(alias)
 
         power_levels = self._get_base_power_levels(entity=entity)
-        users = participants = None
+        users = None
         if not direct:
-            users, participants = await self._get_users(user, entity)
+            users = await self._get_users(user, entity)
             if self.has_bot:
                 extra_invites = config["bridge.relaybot.group_chat_invite"]
                 invites += extra_invites
                 for invite in extra_invites:
                     power_levels.users.setdefault(invite, 100)
-            self._participants_to_power_levels(participants, power_levels)
+            await self._participants_to_power_levels(users, power_levels)
         elif self.bot and self.tg_receiver == self.bot.tgid:
             invites = config["bridge.relaybot.private_chat.invite"]
             for invite in invites:
                 power_levels.users.setdefault(invite, 100)
             self.title = puppet.displayname
+
         initial_state = [{
             "type": EventType.ROOM_POWER_LEVELS.serialize(),
             "content": power_levels.serialize(),
+        }, {
+            "type": str(StateBridge),
+            "state_key": self.bridge_info_state_key,
+            "content": self.bridge_info,
+        }, {
+            # TODO remove this once https://github.com/matrix-org/matrix-doc/pull/2346 is in spec
+            "type": str(StateHalfShotBridge),
+            "state_key": self.bridge_info_state_key,
+            "content": self.bridge_info,
         }]
+        if config["bridge.encryption.default"] and self.matrix.e2ee:
+            self.encrypted = True
+            initial_state.append({
+                "type": "m.room.encryption",
+                "content": {"algorithm": "m.megolm.v1.aes-sha2"},
+            })
+            if direct:
+                invites.append(self.az.bot_mxid)
+        if direct and (self.encrypted or self.private_chat_portal_meta):
+            self.title = puppet.displayname
         if config["appservice.community_id"]:
             initial_state.append({
                 "type": "m.room.related_groups",
@@ -317,22 +387,40 @@ class PortalMetadata(BasePortal, ABC):
         if not config["bridge.federate_rooms"]:
             creation_content["m.federate"] = False
 
-        room_id = await self.main_intent.create_room(alias_localpart=alias, preset=preset,
+        with self.backfill_lock:
-                                                     is_direct=direct, invitees=invites or [],
+            room_id = await self.main_intent.create_room(alias_localpart=alias, preset=preset,
-                                                     name=self.title, topic=self.about,
+                                                         is_direct=direct, invitees=invites or [],
-                                                     initial_state=initial_state,
+                                                         name=self.title, topic=self.about,
-                                                     creation_content=creation_content)
+                                                         initial_state=initial_state,
-        if not room_id:
+                                                         creation_content=creation_content)
-            raise Exception(f"Failed to create room")
+            if not room_id:
+                raise Exception(f"Failed to create room")
 
-        self.mxid = RoomID(room_id)
+            if self.encrypted and self.matrix.e2ee and direct:
-        self.by_mxid[self.mxid] = self
+                try:
-        self.save()
+                    await self.az.intent.ensure_joined(room_id)
-        self.az.state_store.set_power_levels(self.mxid, power_levels)
+                except Exception:
-        user.register_portal(self)
+                    self.log.warning(f"Failed to add bridge bot to new private chat {room_id}")
-        asyncio.ensure_future(self.update_matrix_room(user, entity, direct, puppet,
+
-                                                      levels=power_levels, users=users,
+            self.mxid = room_id
-                                                      participants=participants), loop=self.loop)
+            self.by_mxid[self.mxid] = self
+            await self.save()
+            await self.az.state_store.set_power_levels(self.mxid, power_levels)
+            await user.register_portal(self)
+
+            update_room = self.loop.create_task(self.update_matrix_room(
+                user, entity, direct, puppet,
+                levels=power_levels, users=users))
+
+            if config["bridge.backfill.initial_limit"] > 0:
+                self.log.debug("Initial backfill is enabled. Waiting for room members to sync "
+                               "and then starting backfill")
+                await update_room
+
+                try:
+                    await self.backfill(user, is_initial=True)
+                except Exception:
+                    self.log.exception("Failed to backfill new portal")
 
         return self.mxid
 
@@ -362,7 +450,7 @@ class PortalMetadata(BasePortal, ABC):
             levels.kick = overrides.get("kick", 50)
             levels.redact = overrides.get("redact", 50)
             levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
-            levels.events[EventType.ROOM_ENCRYPTED] = 99
+            levels.events[EventType.ROOM_ENCRYPTION] = 99
             levels.events[EventType.ROOM_TOMBSTONE] = 99
             levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
             levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0
@@ -405,24 +493,28 @@ class PortalMetadata(BasePortal, ABC):
             return True
         return False
 
-    def _participants_to_power_levels(self, participants: List[TypeParticipant],
+    async def _participants_to_power_levels(self, users: List[Union[TypeUser, TypeParticipant]],
-                                      levels: PowerLevelStateEventContent) -> bool:
+                                            levels: PowerLevelStateEventContent) -> bool:
         bot_level = levels.get_user_level(self.main_intent.mxid)
         if bot_level < levels.get_event_level(EventType.ROOM_POWER_LEVELS):
             return False
         changed = False
         admin_power_level = min(75 if self.peer_type == "channel" else 50, bot_level)
-        if levels.events[EventType.ROOM_POWER_LEVELS] != admin_power_level:
+        if levels.get_event_level(EventType.ROOM_POWER_LEVELS) != admin_power_level:
             changed = True
             levels.events[EventType.ROOM_POWER_LEVELS] = admin_power_level
 
-        for participant in participants:
+        for user in users:
+            # The User objects we get from TelegramClient.get_participants have a custom
+            # participant property
+            participant = getattr(user, "participant", user)
+
             puppet = p.Puppet.get(TelegramID(participant.user_id))
             user = u.User.get_by_tgid(TelegramID(participant.user_id))
             new_level = self._get_level_from_participant(participant)
 
             if user:
-                user.register_portal(self)
+                await user.register_portal(self)
                 changed = self._participant_to_power_levels(levels, user, new_level,
                                                             bot_level) or changed
 
@@ -431,72 +523,88 @@ class PortalMetadata(BasePortal, ABC):
                                                             bot_level) or changed
         return changed
 
-    async def update_telegram_participants(self, participants: List[TypeParticipant],
+    async def update_power_levels(self, users: List[Union[TypeUser, TypeParticipant]],
-                                           levels: PowerLevelStateEventContent = None) -> None:
+                                  levels: PowerLevelStateEventContent = None) -> None:
         if not levels:
             levels = await self.main_intent.get_power_levels(self.mxid)
-        if self._participants_to_power_levels(participants, levels):
+        if await self._participants_to_power_levels(users, levels):
             await self.main_intent.set_power_levels(self.mxid, levels)
 
-    def _add_bot_chat(self, bot: User) -> None:
+    async def _add_bot_chat(self, bot: User) -> None:
         if self.bot and bot.id == self.bot.tgid:
             self.bot.add_chat(self.tgid, self.peer_type)
             return
 
         user = u.User.get_by_tgid(TelegramID(bot.id))
         if user and user.is_bot:
-            user.register_portal(self)
+            await user.register_portal(self)
 
     async def _sync_telegram_users(self, source: 'AbstractUser', users: List[User]) -> None:
         allowed_tgids = set()
         skip_deleted = config["bridge.skip_deleted_members"]
         for entity in users:
-            if skip_deleted and entity.deleted:
-                continue
             puppet = p.Puppet.get(TelegramID(entity.id))
             if entity.bot:
-                self._add_bot_chat(entity)
+                await self._add_bot_chat(entity)
             allowed_tgids.add(entity.id)
-            await puppet.intent_for(self).ensure_joined(self.mxid)
+
             await puppet.update_info(source, entity)
+            if skip_deleted and entity.deleted:
+                continue
+
+            await puppet.intent_for(self).ensure_joined(self.mxid)
 
             user = u.User.get_by_tgid(TelegramID(entity.id))
             if user:
                 await self.invite_to_matrix(user.mxid)
 
+                puppet = await p.Puppet.get_by_custom_mxid(user.mxid)
+                if puppet:
+                    try:
+                        await puppet.intent.ensure_joined(self.mxid)
+                    except Exception:
+                        self.log.exception("Failed to ensure %s is joined to portal", user.mxid)
+
         # We can't trust the member list if any of the following cases is true:
         #  * There are close to 10 000 users, because Telegram might not be sending all members.
         #  * The member sync count is limited, because then we might ignore some members.
         #  * It's a channel, because non-admins don't have access to the member list.
-        trust_member_list = (len(allowed_tgids) < 9900
+        trust_member_list = ((len(allowed_tgids) < 9900
-                             and self.max_initial_member_sync == -1
+                              if self.max_initial_member_sync < 0
+                              else len(allowed_tgids) < self.max_initial_member_sync - 10)
                              and (self.megagroup or self.peer_type != "channel"))
-        if trust_member_list:
+        if not trust_member_list:
-            joined_mxids = await self.main_intent.get_room_members(self.mxid)
+            return
-            for user_mxid in joined_mxids:
-                if user_mxid == self.az.bot_mxid:
-                    continue
-                puppet_id = p.Puppet.get_id_from_mxid(user_mxid)
-                if puppet_id and puppet_id not in allowed_tgids:
-                    if self.bot and puppet_id == self.bot.tgid:
-                        self.bot.remove_chat(self.tgid)
-                    try:
-                        await self.main_intent.kick_user(self.mxid, user_mxid,
-                                                         "User had left this Telegram chat.")
-                    except MForbidden:
-                        pass
-                    continue
-                mx_user = u.User.get_by_mxid(user_mxid, create=False)
-                if mx_user and mx_user.is_bot and mx_user.tgid not in allowed_tgids:
-                    mx_user.unregister_portal(self)
 
-                if mx_user and not self.has_bot and mx_user.tgid not in allowed_tgids:
+        for user_mxid in await self.main_intent.get_room_members(self.mxid):
+            if user_mxid == self.az.bot_mxid:
+                continue
+
+            puppet_id = p.Puppet.get_id_from_mxid(user_mxid)
+            if puppet_id:
+                if puppet_id in allowed_tgids:
+                    continue
+                if self.bot and puppet_id == self.bot.tgid:
+                    self.bot.remove_chat(self.tgid)
+                try:
+                    await self.main_intent.kick_user(self.mxid, user_mxid,
+                                                     "User had left this Telegram chat.")
+                except MForbidden:
+                    pass
+                continue
+
+            mx_user = u.User.get_by_mxid(user_mxid, create=False)
+            if mx_user:
+                if mx_user.tgid in allowed_tgids:
+                    continue
+                if mx_user.is_bot:
+                    await mx_user.unregister_portal(*self.tgid_full)
+                if not self.has_bot:
                     try:
                         await self.main_intent.kick_user(self.mxid, mx_user.mxid,
                                                          "You had left this Telegram chat.")
                     except MForbidden:
                         pass
-                    continue
 
     async def _add_telegram_user(self, user_id: TelegramID, source: Optional['AbstractUser'] = None
                                  ) -> None:
@@ -508,7 +616,7 @@ class PortalMetadata(BasePortal, ABC):
 
         user = u.User.get_by_tgid(user_id)
         if user:
-            user.register_portal(self)
+            await user.register_portal(self)
             await self.invite_to_matrix(user.mxid)
 
     async def _delete_telegram_user(self, user_id: TelegramID, sender: p.Puppet) -> None:
@@ -525,7 +633,7 @@ class PortalMetadata(BasePortal, ABC):
         else:
             await puppet.intent_for(self).leave_room(self.mxid)
         if user:
-            user.unregister_portal(self)
+            await user.unregister_portal(*self.tgid_full)
             if sender.tgid != puppet.tgid:
                 try:
                     await sender.intent_for(self).kick_user(self.mxid, puppet.mxid)
@@ -542,12 +650,12 @@ class PortalMetadata(BasePortal, ABC):
             self.log.warning("Called update_info() for direct chat portal")
             return
 
+        changed = False
         self.log.debug("Updating info")
         try:
             if not entity:
                 entity = await self.get_entity(user)
-                self.log.debug(f"Fetched data: {entity}")
+                self.log.trace("Fetched data: %s", entity)
-            changed = False
 
             if self.peer_type == "channel":
                 changed = self.megagroup != entity.megagroup or changed
@@ -565,7 +673,8 @@ class PortalMetadata(BasePortal, ABC):
             self.log.exception(f"Failed to update info from source {user.tgid}")
 
         if changed:
-            self.save()
+            await self.save()
+            await self.update_bridge_info()
 
     async def _update_username(self, username: str, save: bool = False) -> bool:
         if self.username == username:
@@ -582,18 +691,21 @@ class PortalMetadata(BasePortal, ABC):
             await self.main_intent.set_join_rule(self.mxid, "invite")
 
         if save:
-            self.save()
+            await self.save()
         return True
 
-    async def _try_use_intent(self, sender: Optional['p.Puppet'],
+    async def _try_set_state(self, sender: Optional['p.Puppet'], evt_type: EventType,
-                              action: Callable[[IntentAPI], None]) -> None:
+                             content: StateEventContent) -> None:
         if sender:
             try:
-                await action(sender.intent_for(self))
+                intent = sender.intent_for(self)
+                if sender.is_real_user:
+                    content[self.az.real_user_content_key] = True
+                await intent.send_state_event(self.mxid, evt_type, content)
             except MForbidden:
-                await action(self.main_intent)
+                await self.main_intent.send_state_event(self.mxid, evt_type, content)
         else:
-            await action(self.main_intent)
+            await self.main_intent.send_state_event(self.mxid, evt_type, content)
 
     async def _update_about(self, about: str, sender: Optional['p.Puppet'] = None,
                             save: bool = False) -> bool:
@@ -601,10 +713,10 @@ class PortalMetadata(BasePortal, ABC):
             return False
 
         self.about = about
-        await self._try_use_intent(sender,
+        await self._try_set_state(sender, EventType.ROOM_TOPIC,
-                                   lambda intent: intent.set_room_topic(self.mxid, self.about))
+                                  RoomTopicStateEventContent(topic=self.about))
         if save:
-            self.save()
+            await self.save()
         return True
 
     async def _update_title(self, title: str, sender: Optional['p.Puppet'] = None,
@@ -613,95 +725,125 @@ class PortalMetadata(BasePortal, ABC):
             return False
 
         self.title = title
-        await self._try_use_intent(sender,
+        await self._try_set_state(sender, EventType.ROOM_NAME,
-                                   lambda intent: intent.set_room_name(self.mxid, self.title))
+                                  RoomNameStateEventContent(name=self.title))
         if save:
-            self.save()
+            await self.save()
         return True
 
     async def _update_avatar(self, user: 'AbstractUser', photo: TypeChatPhoto,
                              sender: Optional['p.Puppet'] = None, save: bool = False) -> bool:
-        if isinstance(photo, ChatPhoto):
+        if isinstance(photo, (ChatPhoto, UserProfilePhoto)):
             loc = InputPeerPhotoFileLocation(
                 peer=await self.get_input_entity(user),
                 local_id=photo.photo_big.local_id,
                 volume_id=photo.photo_big.volume_id,
                 big=True
             )
-            photo_id = f"{loc.volume_id}-{loc.local_id}"
+            photo_id = (f"{loc.volume_id}-{loc.local_id}" if isinstance(photo, ChatPhoto)
+                        else photo.photo_id)
         elif isinstance(photo, Photo):
             loc, largest = self._get_largest_photo_size(photo)
             photo_id = f"{largest.location.volume_id}-{largest.location.local_id}"
-        elif isinstance(photo, (ChatPhotoEmpty, PhotoEmpty)):
+        elif isinstance(photo, (UserProfilePhotoEmpty, ChatPhotoEmpty, PhotoEmpty, type(None))):
             photo_id = ""
             loc = None
         else:
             raise ValueError(f"Unknown photo type {type(photo)}")
+        if self.peer_type == "user" and not photo_id and not config["bridge.allow_avatar_remove"]:
+            return False
         if self.photo_id != photo_id:
             if not photo_id:
-                await self._try_use_intent(sender,
+                await self._try_set_state(sender, EventType.ROOM_AVATAR,
-                                           lambda intent: intent.set_room_avatar(self.mxid, None))
+                                          RoomAvatarStateEventContent(url=None))
                 self.photo_id = ""
+                self.avatar_url = None
                 if save:
-                    self.save()
+                    await self.save()
                 return True
             file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
             if file:
-                await self._try_use_intent(sender, lambda intent: intent.set_room_avatar(self.mxid,
+                await self._try_set_state(sender, EventType.ROOM_AVATAR,
-                                                                                         file.mxc))
+                                          RoomAvatarStateEventContent(url=file.mxc))
                 self.photo_id = photo_id
+                self.avatar_url = file.mxc
                 if save:
-                    self.save()
+                    await self.save()
                 return True
         return False
 
+    @staticmethod
+    def _filter_participants(users: List[TypeUser], participants: List[TypeParticipant]
+                             ) -> Iterable[TypeUser]:
+        participant_map = {part.user_id: part for part in participants}
+        for user in users:
+            try:
+                user.participant = participant_map[user.id]
+            except KeyError:
+                pass
+            else:
+                yield user
+
+    async def _get_channel_users(self, user: 'AbstractUser', entity: InputChannel, limit: int
+                                 ) -> List[TypeUser]:
+        if 0 < limit <= 200:
+            response = await user.client(GetParticipantsRequest(
+                entity, ChannelParticipantsRecent(), offset=0, limit=limit, hash=0))
+            return list(self._filter_participants(response.users, response.participants))
+        elif limit > 200 or limit == -1:
+            users: List[TypeUser] = []
+            offset = 0
+            remaining_quota = limit if limit > 0 else 1000000
+            query = (ChannelParticipantsSearch("") if limit == -1
+                     else ChannelParticipantsRecent())
+            while True:
+                if remaining_quota <= 0:
+                    break
+                response = await user.client(GetParticipantsRequest(
+                    entity, query, offset=offset, limit=min(remaining_quota, 200), hash=0))
+                if not response.users:
+                    break
+                users += self._filter_participants(response.users, response.participants)
+                offset += len(response.participants)
+                remaining_quota -= len(response.participants)
+            return users
+
     async def _get_users(self, user: 'AbstractUser',
                          entity: Union[TypeInputPeer, InputUser, TypeChat, TypeUser, InputChannel]
-                         ) -> Tuple[List[TypeUser], List[TypeParticipant]]:
+                         ) -> List[TypeUser]:
-        # TODO replace with client.get_participants
         if self.peer_type == "chat":
             chat = await user.client(GetFullChatRequest(chat_id=self.tgid))
-            return chat.users, chat.full_chat.participants.participants
+            return list(self._filter_participants(chat.users,
+                                                  chat.full_chat.participants.participants))
         elif self.peer_type == "channel":
             if not self.megagroup and not self.sync_channel_members:
-                return [], []
+                return []
 
             limit = self.max_initial_member_sync
             if limit == 0:
-                return [], []
+                return []
 
             try:
-                if 0 < limit <= 200:
+                return await self._get_channel_users(user, entity, limit)
-                    response = await user.client(GetParticipantsRequest(
-                        entity, ChannelParticipantsRecent(), offset=0, limit=limit, hash=0))
-                    return response.users, response.participants
-                elif limit > 200 or limit == -1:
-                    users: List[TypeUser] = []
-                    participants: List[TypeParticipant] = []
-                    offset = 0
-                    remaining_quota = limit if limit > 0 else 1000000
-                    query = (ChannelParticipantsSearch("") if limit == -1
-                             else ChannelParticipantsRecent())
-                    while True:
-                        if remaining_quota <= 0:
-                            break
-                        response = await user.client(GetParticipantsRequest(
-                            entity, query, offset=offset, limit=min(remaining_quota, 100), hash=0))
-                        if not response.users:
-                            break
-                        participants += response.participants
-                        users += response.users
-                        offset += len(response.participants)
-                        remaining_quota -= len(response.participants)
-                    return users, participants
             except ChatAdminRequiredError:
-                return [], []
+                return []
         elif self.peer_type == "user":
-            return [entity], []
+            return [entity]
-        return [], []
+        else:
+            raise RuntimeError(f"Unexpected peer type {self.peer_type}")
 
     # endregion
 
+    async def _send_delivery_receipt(self, event_id: EventID, room_id: Optional[RoomID] = None
+                                     ) -> None:
+        # TODO maybe check if the bot is in the room rather than assuming based on self.encrypted
+        if event_id and config["bridge.delivery_receipts"] and (self.encrypted
+                                                                or self.peer_type != "user"):
+            try:
+                await self.az.intent.mark_read(room_id or self.mxid, event_id)
+            except Exception:
+                self.log.exception("Failed to send delivery receipt for %s", event_id)
+
 
 def init(context: Context) -> None:
     global config

mautrix_telegram/portal/telegram.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,13 +14,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Awaitable, Dict, List, Optional, Tuple, Union, NamedTuple, TYPE_CHECKING
-from html import escape as escape_html
 from abc import ABC
 import random
 import mimetypes
 import codecs
 import unicodedata
 import base64
+import asyncio
 
 from sqlalchemy.exc import IntegrityError
 
@@ -30,21 +30,23 @@ from telethon.tl.types import (
     MessageMediaPoll, MessageActionChannelCreate, MessageActionChatAddUser,
     MessageActionChatCreate, MessageActionChatDeletePhoto, MessageActionChatDeleteUser,
     MessageActionChatEditPhoto, MessageActionChatEditTitle, MessageActionChatJoinedByLink,
-    MessageActionChatMigrateTo, MessageActionPinMessage, MessageActionGameScore,
+    MessageActionChatMigrateTo, MessageActionGameScore, MessageMediaDocument, MessageMediaGeo,
-    MessageMediaDocument, MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported,
+    MessageMediaPhoto, MessageMediaDice, MessageMediaGame, MessageMediaUnsupported, PeerUser,
-    MessageMediaGame, PeerUser, PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant,
+    PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant, TypeDocumentAttribute,
-    TypeDocumentAttribute, TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping,
+    TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping, UpdateUserTyping,
-    UpdateUserTyping, MessageEntityPre, ChatPhotoEmpty)
+    MessageEntityPre, ChatPhotoEmpty)
 
 from mautrix.appservice import IntentAPI
 from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType,
                            EventType, MediaMessageEventContent, TextMessageEventContent,
                            LocationMessageEventContent, Format)
+from mautrix.bridge import NotificationDisabler
 
 from ..types import TelegramID
 from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
 from ..util import sane_mimetypes
 from ..context import Context
+from ..tgclient import TelegramClient
 from .. import puppet as p, user as u, formatter, util
 from .base import BasePortal
 
@@ -72,36 +74,62 @@ class PortalTelegram(BasePortal, ABC):
             return f"https://t.me/c/{self.tgid}/{evt.id}"
         return None
 
+    async def _expire_telegram_photo(self, intent: IntentAPI, event_id: EventID, ttl: int) -> None:
+        try:
+            content = TextMessageEventContent(msgtype=MessageType.NOTICE, body="Photo has expired")
+            content.set_edit(event_id)
+            await asyncio.sleep(ttl)
+            await self._send_message(intent, content)
+        except Exception:
+            self.log.warning("Failed to expire Telegram photo %s", event_id, exc_info=True)
+
     async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
-                                    relates_to: Dict = None) -> Optional[EventID]:
+                                    relates_to: RelatesTo = None) -> Optional[EventID]:
-        loc, largest_size = self._get_largest_photo_size(evt.media.photo)
+        media: MessageMediaPhoto = evt.media
-        file = await util.transfer_file_to_matrix(source.client, intent, loc)
+        if media.photo is None and media.ttl_seconds:
+            return await self._send_message(intent, TextMessageEventContent(
+                msgtype=MessageType.NOTICE, body="Photo has expired"))
+        loc, largest_size = self._get_largest_photo_size(media.photo)
+        if loc is None:
+            content = TextMessageEventContent(msgtype=MessageType.TEXT,
+                                              body="Failed to bridge image",
+                                              external_url=self._get_external_url(evt))
+            return await self._send_message(intent, content, timestamp=evt.date)
+        file = await util.transfer_file_to_matrix(source.client, intent, loc,
+                                                  encrypt=self.encrypted)
         if not file:
             return None
-        if self.get_config("inline_images") and (evt.message
+        if self.get_config("inline_images") and (evt.message or evt.fwd_from or evt.reply_to):
-                                                 or evt.fwd_from or evt.reply_to_msg_id):
             content = await formatter.telegram_to_matrix(
                 evt, source, self.main_intent,
                 prefix_html=f"<img src='{file.mxc}' alt='Inline Telegram photo'/><br/>",
                 prefix_text="Inline image: ")
             content.external_url = self._get_external_url(evt)
             await intent.set_typing(self.mxid, is_typing=False)
-            return await intent.send_message(self.mxid, content, timestamp=evt.date)
+            return await self._send_message(intent, content, timestamp=evt.date)
         info = ImageInfo(
             height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type,
             size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize))
                   else largest_size.size))
-        name = f"image{sane_mimetypes.guess_extension(file.mime_type)}"
+        ext = sane_mimetypes.guess_extension(file.mime_type)
+        name = f"disappearing_image{ext}" if media.ttl_seconds else f"image{ext}"
         await intent.set_typing(self.mxid, is_typing=False)
-        content = MediaMessageEventContent(url=file.mxc, msgtype=MessageType.IMAGE, info=info,
+        content = MediaMessageEventContent(msgtype=MessageType.IMAGE, info=info,
                                            body=name, relates_to=relates_to,
                                            external_url=self._get_external_url(evt))
-        result = await intent.send_message(self.mxid, content, timestamp=evt.date)
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+        result = await self._send_message(intent, content, timestamp=evt.date)
+        if media.ttl_seconds:
+            self.loop.create_task(self._expire_telegram_photo(intent, result,
+                                                              media.ttl_seconds))
         if evt.message:
             caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
                                                                  no_reply_fallback=True)
             caption_content.external_url = content.external_url
-            result = await intent.send_message(self.mxid, caption_content, timestamp=evt.date)
+            result = await self._send_message(intent, caption_content, timestamp=evt.date)
         return result
 
     @staticmethod
@@ -122,7 +150,7 @@ class PortalTelegram(BasePortal, ABC):
     def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: DocAttrs,
                                       thumb_size: TypePhotoSize) -> Tuple[ImageInfo, str]:
         document = evt.media.document
-        name = evt.message or attrs.name
+        name = attrs.name
         if attrs.is_sticker:
             alt = attrs.sticker_alt
             if len(alt) > 0:
@@ -134,6 +162,8 @@ class PortalTelegram(BasePortal, ABC):
         generic_types = ("text/plain", "application/octet-stream")
         if file.mime_type in generic_types and document.mime_type not in generic_types:
             mime_type = document.mime_type or file.mime_type
+        elif file.mime_type == 'application/ogg':
+            mime_type = 'audio/ogg'
         else:
             mime_type = file.mime_type or document.mime_type
         info = ImageInfo(size=file.size, mimetype=mime_type)
@@ -146,11 +176,21 @@ class PortalTelegram(BasePortal, ABC):
             info.width, info.height = attrs.width, attrs.height
 
         if file.thumbnail:
-            info.thumbnail_url = file.thumbnail.mxc
+            if file.thumbnail.decryption_info:
+                info.thumbnail_file = file.thumbnail.decryption_info
+            else:
+                info.thumbnail_url = file.thumbnail.mxc
             info.thumbnail_info = ThumbnailInfo(mimetype=file.thumbnail.mime_type,
                                                 height=file.thumbnail.height or thumb_size.h,
                                                 width=file.thumbnail.width or thumb_size.w,
                                                 size=file.thumbnail.size)
+        else:
+            # This is a hack for bad clients like Riot iOS that require a thumbnail
+            if file.decryption_info:
+                info.thumbnail_file = file.decryption_info
+            else:
+                info.thumbnail_url = file.mxc
+            info.thumbnail_info = ImageInfo.deserialize(info.serialize())
 
         return info, name
 
@@ -164,6 +204,7 @@ class PortalTelegram(BasePortal, ABC):
         if document.size > config["bridge.max_document_size"] * 1000 ** 2:
             name = attrs.name or ""
             caption = f"\n{evt.message}" if evt.message else ""
+            # TODO encrypt
             return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
 
         thumb_loc, thumb_size = self._get_largest_photo_size(document)
@@ -175,7 +216,8 @@ class PortalTelegram(BasePortal, ABC):
         file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
                                                   is_sticker=attrs.is_sticker,
                                                   tgs_convert=config["bridge.animated_sticker"],
-                                                  filename=attrs.name, parallel_id=parallel_id)
+                                                  filename=attrs.name, parallel_id=parallel_id,
+                                                  encrypt=self.encrypted)
         if not file:
             return None
 
@@ -188,46 +230,57 @@ class PortalTelegram(BasePortal, ABC):
         if attrs.is_sticker and file.mime_type.startswith("image/"):
             event_type = EventType.STICKER
         content = MediaMessageEventContent(
-            body=name or "unnamed file", info=info, url=file.mxc, relates_to=relates_to,
+            body=name or "unnamed file", info=info, relates_to=relates_to,
             external_url=self._get_external_url(evt),
             msgtype={
                 "video/": MessageType.VIDEO,
                 "audio/": MessageType.AUDIO,
                 "image/": MessageType.IMAGE,
             }.get(info.mimetype[:6], MessageType.FILE))
-        return await intent.send_message_event(self.mxid, event_type, content, timestamp=evt.date)
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+        res = await self._send_message(intent, content, event_type=event_type, timestamp=evt.date)
+        if evt.message:
+            caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                                 no_reply_fallback=True)
+            caption_content.external_url = content.external_url
+            res = await self._send_message(intent, caption_content, timestamp=evt.date)
+        return res
 
-    def handle_telegram_location(self, _: 'AbstractUser', intent: IntentAPI, evt: Message,
+    def handle_telegram_location(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
-                                 relates_to: dict = None) -> Awaitable[EventID]:
+                                 relates_to: RelatesTo = None) -> Awaitable[EventID]:
         long = evt.media.geo.long
         lat = evt.media.geo.lat
         long_char = "E" if long > 0 else "W"
         lat_char = "N" if lat > 0 else "S"
+        geo = f"{round(lat, 6)},{round(long, 6)}"
 
-        body = f"{round(lat, 5)}° {lat_char}, {round(long, 5)}° {long_char}"
+        body = f"{round(abs(lat), 4)}° {lat_char}, {round(abs(long), 4)}° {long_char}"
-        url = f"https://maps.google.com/?q={lat},{long}"
+        url = f"https://maps.google.com/?q={geo}"
 
         content = LocationMessageEventContent(
-            msgtype=MessageType.LOCATION, geo_uri=f"geo:{lat},{long}",
+            msgtype=MessageType.LOCATION, geo_uri=f"geo:{geo}",
             body=f"Location: {body}\n{url}",
             relates_to=relates_to, external_url=self._get_external_url(evt))
         content["format"] = str(Format.HTML)
         content["formatted_body"] = f"Location: <a href='{url}'>{body}</a>"
 
-        return intent.send_message(self.mxid, content, timestamp=evt.date)
+        return self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_text(self, source: 'AbstractUser', intent: IntentAPI, is_bot: bool,
                                    evt: Message) -> EventID:
-        self.log.debug(f"Sending {evt.message} to {self.mxid} by {intent.mxid}")
+        self.log.trace(f"Sending {evt.message} to {self.mxid} by {intent.mxid}")
         content = await formatter.telegram_to_matrix(evt, source, self.main_intent)
         content.external_url = self._get_external_url(evt)
         if is_bot and self.get_config("bot_messages_as_notices"):
             content.msgtype = MessageType.NOTICE
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
-                                          evt: Message, relates_to: dict = None) -> EventID:
+                                          evt: Message, relates_to: RelatesTo = None) -> EventID:
         override_text = ("This message is not supported on your version of Mautrix-Telegram. "
                          "Please check https://github.com/tulir/mautrix-telegram or ask your "
                          "bridge administrator about possible updates.")
@@ -237,7 +290,7 @@ class PortalTelegram(BasePortal, ABC):
         content.external_url = self._get_external_url(evt)
         content["net.maunium.telegram.unsupported"] = True
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_poll(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                    relates_to: RelatesTo) -> EventID:
@@ -263,11 +316,28 @@ class PortalTelegram(BasePortal, ABC):
             relates_to=relates_to, external_url=self._get_external_url(evt))
 
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
+
+    async def handle_telegram_dice(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
+                                   relates_to: RelatesTo) -> EventID:
+        emoji_text = {
+            "\U0001F3AF": " Dart throw",
+            "\U0001F3B2": " Dice roll",
+            "\U0001F3C0": " Basketball throw",
+            "\u26BD": " Football kick"
+        }
+        roll: MessageMediaDice = evt.media
+        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {roll.value}"
+        content = TextMessageEventContent(msgtype=MessageType.TEXT, format=Format.HTML, body=text,
+                                          formatted_body=f"<h4>{text}</h4>", relates_to=relates_to,
+                                          external_url=self._get_external_url(evt))
+        content["net.maunium.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     @staticmethod
     def _int_to_bytes(i: int) -> bytes:
-        hex_value = "{0:010x}".format(i)
+        hex_value = "{0:010x}".format(i).encode("utf-8")
         return codecs.decode(hex_value, "hex_codec")
 
     def _encode_msgid(self, source: 'AbstractUser', evt: Message) -> str:
@@ -305,11 +375,12 @@ class PortalTelegram(BasePortal, ABC):
         content["net.maunium.telegram.game"] = play_id
 
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_edit(self, source: 'AbstractUser', sender: p.Puppet, evt: Message
                                    ) -> None:
         if not self.mxid:
+            self.log.trace("Ignoring edit to %d as chat has no Matrix room", evt.id)
             return
         elif hasattr(evt, "media") and isinstance(evt.media, MessageMediaGame):
             self.log.debug("Ignoring game message edit event")
@@ -349,21 +420,126 @@ class PortalTelegram(BasePortal, ABC):
 
         intent = sender.intent_for(self) if sender else self.main_intent
         await intent.set_typing(self.mxid, is_typing=False)
-        event_id = await intent.send_message(self.mxid, content)
+        event_id = await self._send_message(intent, content)
 
         prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
         DBMessage(mxid=event_id, mx_room=self.mxid, tg_space=tg_space, tgid=TelegramID(evt.id),
                   edit_index=prev_edit_msg.edit_index + 1).insert()
         DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=event_id)
 
+    @property
+    def _takeout_options(self) -> Dict[str, Union[bool, int]]:
+        return {
+            "files": True,
+            "megagroups": self.megagroup,
+            "chats": self.peer_type == "chat",
+            "users": self.peer_type == "user",
+            "channels": (self.peer_type == "channel" and not self.megagroup),
+            "max_file_size": min(config["bridge.max_document_size"], 2000) * 1024 * 1024
+        }
+
+    async def backfill(self, source: 'u.User', is_initial: bool = False,
+                       limit: Optional[int] = None, last_id: Optional[int] = None) -> None:
+        async with self.backfill_method_lock:
+            await self._locked_backfill(source, is_initial, limit, last_id)
+
+    async def _locked_backfill(self, source: 'u.User', is_initial: bool = False,
+                               limit: Optional[int] = None, last_id: Optional[int] = None) -> None:
+        limit = limit or (config["bridge.backfill.initial_limit"] if is_initial
+                          else config["bridge.backfill.missed_limit"])
+        if limit == 0:
+            return
+        if not config["bridge.backfill.normal_groups"] and self.peer_type == "chat":
+            return
+        last = DBMessage.find_last(self.mxid, (source.tgid if self.peer_type != "channel"
+                                               else self.tgid))
+        min_id = last.tgid if last else 0
+        if last_id is None:
+            messages = await source.client.get_messages(self.peer, limit=1)
+            if not messages:
+                # The chat seems empty
+                return
+            last_id = messages[0].id
+        if last_id <= min_id:
+            # Nothing to backfill
+            return
+        if limit < 0:
+            limit = last_id - min_id
+            self.log.debug(f"Backfilling approximately {last_id - min_id} messages "
+                           f"through {source.mxid}")
+        elif self.peer_type == "channel":
+            # This is a channel or supergroup, so we'll backfill messages based on the ID.
+            # There are some cases, such as deleted messages, where this may backfill less
+            # messages than the limit.
+            min_id = max(last_id - limit, min_id)
+            self.log.debug(f"Backfilling messages after ID {min_id} (last message: {last_id}) "
+                           f"through {source.mxid}")
+        else:
+            # Private chats and normal groups don't have their own message ID namespace,
+            # which means we'll have to fetch messages a different way.
+            # The _backfill_messages method will detect min_id=None and not use reverse=True
+            min_id = None
+            self.log.debug(f"Backfilling up to {limit} messages through {source.mxid}")
+        with self.backfill_lock:
+            await self._backfill(source, min_id, limit)
+
+    async def _backfill(self, source: 'u.User', min_id: Optional[int], limit: int) -> None:
+        self.backfill_leave = set()
+        if ((self.peer_type == "user" and self.tgid != source.tgid
+             and config["bridge.backfill.invite_own_puppet"])):
+            self.log.debug("Adding %s's default puppet to room for backfilling", source.mxid)
+            sender = p.Puppet.get(source.tgid)
+            await self.main_intent.invite_user(self.mxid, sender.default_mxid)
+            await sender.default_mxid_intent.join_room_by_id(self.mxid)
+            self.backfill_leave.add(sender.default_mxid_intent)
+
+        client = source.client
+        async with NotificationDisabler(self.mxid, source):
+            if limit > config["bridge.backfill.takeout_limit"]:
+                self.log.debug(f"Opening takeout client for {source.tgid}")
+                async with client.takeout(**self._takeout_options) as takeout:
+                    count = await self._backfill_messages(source, min_id, limit, takeout)
+            else:
+                count = await self._backfill_messages(source, min_id, limit, client)
+
+        for intent in self.backfill_leave:
+            self.log.trace("Leaving room with %s post-backfill", intent.mxid)
+            await intent.leave_room(self.mxid)
+        self.backfill_leave = None
+        self.log.info("Backfilled %d messages through %s", count, source.mxid)
+
+    async def _backfill_messages(self, source: 'AbstractUser', min_id: Optional[int], limit: int,
+                                 client: TelegramClient) -> int:
+        count = 0
+        entity = await self.get_input_entity(source)
+        if min_id is not None:
+            self.log.debug(f"Iterating all messages starting with {min_id} (approx: {limit})")
+            messages = client.iter_messages(entity, reverse=True, min_id=min_id)
+            async for message in messages:
+                sender = (p.Puppet.get(message.from_id.user_id)
+                          if isinstance(message.from_id, PeerUser) else None)
+                # TODO handle service messages?
+                await self.handle_telegram_message(source, sender, message)
+                count += 1
+        else:
+            self.log.debug(f"Fetching up to {limit} most recent messages")
+            messages = await client.get_messages(entity, limit=limit)
+            for message in reversed(messages):
+                sender = (p.Puppet.get(TelegramID(message.from_id.user_id))
+                          if isinstance(message.from_id, PeerUser) else None)
+                await self.handle_telegram_message(source, sender, message)
+                count += 1
+        return count
+
     async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
                                       evt: Message) -> None:
         if not self.mxid:
+            self.log.trace("Got telegram message %d, but no room exists, creating...", evt.id)
             await self.create_matrix_room(source, invites=[source.mxid], update_if_exists=False)
 
-        if (self.peer_type == "user" and sender.tgid == self.tg_receiver
+        if (self.peer_type == "user" and sender and sender.tgid == self.tg_receiver
-            and not sender.is_real_user and not self.az.state_store.is_joined(self.mxid,
+            and not sender.is_real_user and not await self.az.state_store.is_joined(self.mxid,
-                                                                              sender.mxid)):
+                                                                                    sender.mxid)):
             self.log.debug(f"Ignoring private chat message {evt.id}@{source.tgid} as receiver does"
                            " not have matrix puppeting and their default puppet isn't in the room")
             return
@@ -383,15 +559,17 @@ class PortalTelegram(BasePortal, ABC):
                               tg_space=tg_space, edit_index=0).insert()
                 return
 
-        if self.dedup.pre_db_check and self.peer_type == "channel":
+        if self.backfill_lock.locked or (self.dedup.pre_db_check and self.peer_type == "channel"):
             msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
             if msg:
-                self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
+                self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already "
                                f"handled into {msg.mxid}. This duplicate was catched in the db "
-                               "check. If you get this message often, consider increasing"
+                               "check. If you get this message often, consider increasing "
                                "bridge.deduplication.cache_queue_length in the config.")
                 return
 
+        self.log.trace("Handling Telegram message %s", evt)
+
         if sender and not sender.displayname:
             self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a "
                            "displayname, updating info...")
@@ -399,10 +577,18 @@ class PortalTelegram(BasePortal, ABC):
             await sender.update_info(source, entity)
 
         allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
-                         MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
+                         MessageMediaGame, MessageMediaDice, MessageMediaPoll,
+                         MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
-        intent = sender.intent_for(self) if sender else self.main_intent
+        if sender:
+            intent = sender.intent_for(self)
+            if ((self.backfill_lock.locked and intent != sender.default_mxid_intent
+                 and config["bridge.backfill.invite_own_puppet"])):
+                intent = sender.default_mxid_intent
+                self.backfill_leave.add(intent)
+        else:
+            intent = self.main_intent
         if not media and evt.message:
             is_bot = sender.is_bot if sender else False
             event_id = await self.handle_telegram_text(source, intent, is_bot, evt)
@@ -412,12 +598,13 @@ class PortalTelegram(BasePortal, ABC):
                 MessageMediaDocument: self.handle_telegram_document,
                 MessageMediaGeo: self.handle_telegram_location,
                 MessageMediaPoll: self.handle_telegram_poll,
+                MessageMediaDice: self.handle_telegram_dice,
                 MessageMediaUnsupported: self.handle_telegram_unsupported,
                 MessageMediaGame: self.handle_telegram_game,
             }[type(media)](source, intent, evt,
                            relates_to=formatter.telegram_reply_to_matrix(evt, source))
         else:
-            self.log.debug("Unhandled Telegram message: %s", evt)
+            self.log.debug("Unhandled Telegram message %d", evt.id)
             return
 
         if not event_id:
@@ -434,7 +621,7 @@ class PortalTelegram(BasePortal, ABC):
             await intent.redact(self.mxid, event_id)
             return
 
-        self.log.debug("Handled Telegram message: %s", evt)
+        self.log.debug("Handled telegram message %d -> %s", evt.id, event_id)
         try:
             DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=event_id,
                       tg_space=tg_space, edit_index=0).insert()
@@ -445,6 +632,7 @@ class PortalTelegram(BasePortal, ABC):
                                "dedup cache queue. You can try enabling bridge.deduplication."
                                "pre_db_check in the config.")
             await intent.redact(self.mxid, event_id)
+        await self._send_delivery_receipt(event_id)
 
     async def _create_room_on_action(self, source: 'AbstractUser',
                                      action: TypeMessageAction) -> bool:
@@ -468,10 +656,13 @@ class PortalTelegram(BasePortal, ABC):
             return
         if isinstance(action, MessageActionChatEditTitle):
             await self._update_title(action.title, sender=sender, save=True)
+            await self.update_bridge_info()
         elif isinstance(action, MessageActionChatEditPhoto):
             await self._update_avatar(source, action.photo, sender=sender, save=True)
+            await self.update_bridge_info()
         elif isinstance(action, MessageActionChatDeletePhoto):
             await self._update_avatar(source, ChatPhotoEmpty(), sender=sender, save=True)
+            await self.update_bridge_info()
         elif isinstance(action, MessageActionChatAddUser):
             for user_id in action.users:
                 await self._add_telegram_user(TelegramID(user_id), source)
@@ -482,13 +673,15 @@ class PortalTelegram(BasePortal, ABC):
         elif isinstance(action, MessageActionChatMigrateTo):
             self.peer_type = "channel"
             self._migrate_and_save_telegram(TelegramID(action.channel_id))
+            # TODO encrypt
             await sender.intent_for(self).send_emote(self.mxid,
                                                      "upgraded this group to a supergroup.")
+            await self.update_bridge_info()
         elif isinstance(action, MessageActionGameScore):
             # TODO handle game score
             pass
         else:
-            self.log.debug("Unhandled Telegram action in %s: %s", self.title, action)
+            self.log.trace("Unhandled Telegram action in %s: %s", self.title, action)
 
     async def set_telegram_admin(self, user_id: TelegramID) -> None:
         puppet = p.Puppet.get(user_id)
@@ -502,7 +695,7 @@ class PortalTelegram(BasePortal, ABC):
         await self.main_intent.set_power_levels(self.mxid, levels)
 
     async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
-        tg_space = receiver  if self.peer_type != "channel" else self.tgid
+        tg_space = receiver if self.peer_type != "channel" else self.tgid
         message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
         if message:
             await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
@@ -521,3 +714,5 @@ class PortalTelegram(BasePortal, ABC):
 def init(context: Context) -> None:
     global config
     config = context.config
+    NotificationDisabler.puppet_cls = p.Puppet
+    NotificationDisabler.config_enabled = config["bridge.backfill.disable_notifications"]

mautrix_telegram/puppet.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -21,11 +21,12 @@ import logging
 
 from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
                                InputPeerPhotoFileLocation, UserProfilePhotoEmpty, TypeInputUser)
+from yarl import URL
 
 from mautrix.appservice import AppService, IntentAPI
 from mautrix.errors import MatrixRequestError
-from mautrix.bridge import CustomPuppetMixin
+from mautrix.bridge import BasePuppet
-from mautrix.types import UserID, SyncToken
+from mautrix.types import UserID, SyncToken, RoomID
 from mautrix.util.simple_template import SimpleTemplate
 
 from .types import TelegramID
@@ -41,7 +42,7 @@ if TYPE_CHECKING:
 config: Optional['Config'] = None
 
 
-class Puppet(CustomPuppetMixin):
+class Puppet(BasePuppet):
     log: logging.Logger = logging.getLogger("mau.puppet")
     az: AppService
     mx: 'MatrixHandler'
@@ -57,6 +58,7 @@ class Puppet(CustomPuppetMixin):
     access_token: Optional[str]
     custom_mxid: Optional[UserID]
     _next_batch: Optional[SyncToken]
+    base_url: Optional[URL]
     default_mxid: UserID
 
     username: Optional[str]
@@ -79,6 +81,7 @@ class Puppet(CustomPuppetMixin):
                  access_token: Optional[str] = None,
                  custom_mxid: Optional[UserID] = None,
                  next_batch: Optional[SyncToken] = None,
+                 base_url: Optional[str] = None,
                  username: Optional[str] = None,
                  displayname: Optional[str] = None,
                  displayname_source: Optional[TelegramID] = None,
@@ -91,6 +94,7 @@ class Puppet(CustomPuppetMixin):
         self.access_token = access_token
         self.custom_mxid = custom_mxid
         self._next_batch = next_batch
+        self.base_url = URL(base_url) if base_url else None
         self.default_mxid = self.get_mxid_from_id(self.id)
 
         self.username = username
@@ -161,20 +165,20 @@ class Puppet(CustomPuppetMixin):
                     custom_mxid=self.custom_mxid, username=self.username, is_bot=self.is_bot,
                     displayname=self.displayname, displayname_source=self.displayname_source,
                     photo_id=self.photo_id, matrix_registered=self.is_registered,
-                    disable_updates=self.disable_updates)
+                    disable_updates=self.disable_updates, base_url=self.base_url)
 
     def new_db_instance(self) -> DBPuppet:
         return DBPuppet(id=self.id, **self._fields)
 
-    def save(self) -> None:
+    async def save(self) -> None:
         self.db_instance.edit(**self._fields)
 
     @classmethod
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
-                      db_puppet.next_batch, db_puppet.username, db_puppet.displayname,
+                      db_puppet.next_batch, db_puppet.base_url, db_puppet.username,
-                      db_puppet.displayname_source, db_puppet.photo_id, db_puppet.is_bot,
+                      db_puppet.displayname, db_puppet.displayname_source, db_puppet.photo_id,
-                      db_puppet.matrix_registered, db_puppet.disable_updates,
+                      db_puppet.is_bot, db_puppet.matrix_registered, db_puppet.disable_updates,
                       db_instance=db_puppet)
 
     # endregion
@@ -233,42 +237,45 @@ class Puppet(CustomPuppetMixin):
             source.log.exception(f"Failed to update info of {self.tgid}")
 
     async def update_info(self, source: 'AbstractUser', info: User) -> None:
-        if self.disable_updates:
-            return
         changed = False
         if self.username != info.username:
             self.username = info.username
             changed = True
 
-        try:
+        if not self.disable_updates:
-            changed = await self.update_displayname(source, info) or changed
+            try:
-            if isinstance(info.photo, UserProfilePhoto):
+                changed = await self.update_displayname(source, info) or changed
                 changed = await self.update_avatar(source, info.photo) or changed
-        except Exception:
+            except Exception:
-            self.log.exception(f"Failed to update info from source {source.tgid}")
+                self.log.exception(f"Failed to update info from source {source.tgid}")
 
         self.is_bot = info.bot
 
         if changed:
-            self.save()
+            await self.save()
 
     async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
                                  ) -> bool:
         if self.disable_updates:
             return False
-        allow_source = (source.is_relaybot
+        if source.is_relaybot or source.is_bot:
-                        or self.displayname_source == source.tgid
+            allow_because = "user is bot"
-                        # User is not a contact, so there's no custom name
+        elif self.displayname_source == source.tgid:
-                        or not info.contact
+            allow_because = "user is the primary source"
-                        # No displayname source, so just trust anything
+        elif not isinstance(info, UpdateUserName) and not info.contact:
-                        or self.displayname_source is None)
+            allow_because = "user is not a contact"
-        if not allow_source:
+        elif self.displayname_source is None:
+            allow_because = "no primary source set"
+        else:
             return False
-        elif isinstance(info, UpdateUserName):
+
+        if isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
+            self.log.debug(f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
+                           f"because {allow_because}) from {self.displayname} to {displayname}")
             self.displayname = displayname
             self.displayname_source = source.tgid
             try:
@@ -289,10 +296,15 @@ class Puppet(CustomPuppetMixin):
         if self.disable_updates:
             return False
 
-        if isinstance(photo, UserProfilePhotoEmpty):
+        if photo is None or isinstance(photo, UserProfilePhotoEmpty):
             photo_id = ""
-        else:
+        elif isinstance(photo, UserProfilePhoto):
             photo_id = str(photo.photo_id)
+        else:
+            self.log.warning(f"Unknown user profile photo type: {type(photo)}")
+            return False
+        if not photo_id and not config["bridge.allow_avatar_remove"]:
+            return False
         if self.photo_id != photo_id:
             if not photo_id:
                 self.photo_id = ""
@@ -320,6 +332,10 @@ class Puppet(CustomPuppetMixin):
                 return True
         return False
 
+    def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
+        portal: p.Portal = p.Portal.get_by_mxid(room_id)
+        return portal and not portal.backfill_lock.locked and portal.peer_type != "user"
+
     # endregion
     # region Getters
 
@@ -342,7 +358,7 @@ class Puppet(CustomPuppetMixin):
         return None
 
     @classmethod
-    def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['Puppet']:
+    def deprecated_sync_get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['Puppet']:
         tgid = cls.get_id_from_mxid(mxid)
         if tgid:
             return cls.get(tgid, create)
@@ -350,7 +366,11 @@ class Puppet(CustomPuppetMixin):
         return None
 
     @classmethod
-    def get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
+    async def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['Puppet']:
+        return cls.deprecated_sync_get_by_mxid(mxid, create)
+
+    @classmethod
+    def deprecated_sync_get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
         if not mxid:
             raise ValueError("Matrix ID can't be empty")
 
@@ -366,9 +386,13 @@ class Puppet(CustomPuppetMixin):
 
         return None
 
+    @classmethod
+    async def get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
+        return cls.deprecated_sync_get_by_custom_mxid(mxid)
+
     @classmethod
     def all_with_custom_mxid(cls) -> Iterable['Puppet']:
-        return (cls.by_custom_mxid[puppet.mxid]
+        return (cls.by_custom_mxid[puppet.custom_mxid]
                 if puppet.custom_mxid in cls.by_custom_mxid
                 else cls.from_db(puppet)
                 for puppet in DBPuppet.all_with_custom_mxid())
@@ -426,8 +450,12 @@ def init(context: 'Context') -> Iterable[Awaitable[Any]]:
     Puppet.displayname_template = SimpleTemplate(config["bridge.displayname_template"],
                                                  "displayname")
 
-    secret = config["bridge.login_shared_secret"]
+    Puppet.sync_with_custom_puppets = config["bridge.sync_with_custom_puppets"]
-    Puppet.login_shared_secret = secret.encode("utf-8") if secret else None
+    Puppet.homeserver_url_map = {server: URL(url) for server, url
+                                 in config["bridge.double_puppet_server_map"].items()}
+    Puppet.allow_discover_url = config["bridge.double_puppet_allow_discovery"]
+    Puppet.login_shared_secret_map = {server: secret.encode("utf-8") for server, secret
+                                      in config["bridge.login_shared_secret_map"].items()}
     Puppet.login_device_name = "Telegram Bridge"
 
     return (puppet.try_start() for puppet in Puppet.all_with_custom_mxid())

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -24,7 +24,8 @@ def log(message, end="\n"):
 
 
 def connect(to):
-    from mautrix.bridge.db import Base, RoomState, UserProfile
+    from mautrix.util.db import Base
+    from mautrix.client.state_store.sqlalchemy import RoomState, UserProfile
     from mautrix_telegram.db import (Portal, Message, UserPortal, User, Contact, Puppet, BotChat,
                                      TelegramFile)
 

mautrix_telegram/sqlstatestore.py

@@ -1,38 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.types import UserID
-from mautrix.bridge.db import SQLStateStore as BaseSQLStateStore
-
-from . import puppet as pu
-
-
-class SQLStateStore(BaseSQLStateStore):
-    def is_registered(self, user_id: UserID) -> bool:
-        puppet = pu.Puppet.get_by_mxid(user_id, create=False)
-        if puppet:
-            return puppet.is_registered
-        custom_puppet = pu.Puppet.get_by_custom_mxid(user_id)
-        if custom_puppet:
-            return True
-        return super().is_registered(user_id)
-
-    def registered(self, user_id: UserID) -> None:
-        puppet = pu.Puppet.get_by_mxid(user_id, create=True)
-        if puppet:
-            puppet.is_registered = True
-            puppet.save()
-        else:
-            super().registered(user_id)

mautrix_telegram/user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,25 +13,29 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import (Awaitable, Dict, List, Iterable, NewType, Optional, Tuple, Any, cast,
+from typing import (Awaitable, Dict, List, Iterable, NamedTuple, Optional, Tuple, Any, cast,
                     TYPE_CHECKING)
+from collections import defaultdict
 import logging
 import asyncio
 
 from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
                                UpdateShortChatMessage, UpdateShortMessage, User as TLUser, Chat,
                                ChatForbidden)
+from telethon.tl.custom import Dialog
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.account import UpdateStatusRequest
 
 from mautrix.client import Client
 from mautrix.errors import MatrixRequestError
-from mautrix.types import UserID
+from mautrix.types import UserID, RoomID
 from mautrix.bridge import BaseUser
+from mautrix.util.logging import TraceLogger
+from mautrix.util.opt_prometheus import Gauge
 
 from .types import TelegramID
-from .db import User as DBUser
+from .db import User as DBUser, Portal as DBPortal
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
@@ -41,11 +45,14 @@ if TYPE_CHECKING:
 
 config: Optional['Config'] = None
 
-SearchResult = NewType('SearchResult', Tuple['pu.Puppet', int])
+SearchResult = NamedTuple('SearchResult', puppet='pu.Puppet', similarity=int)
+
+METRIC_LOGGED_IN = Gauge('bridge_logged_in', 'Users logged into bridge')
+METRIC_CONNECTED = Gauge('bridge_connected', 'Users connected to Telegram')
 
 
 class User(AbstractUser, BaseUser):
-    log: logging.Logger = logging.getLogger("mau.user")
+    log: TraceLogger = logging.getLogger("mau.user")
     by_mxid: Dict[str, 'User'] = {}
     by_tgid: Dict[int, 'User'] = {}
 
@@ -57,6 +64,7 @@ class User(AbstractUser, BaseUser):
 
     _db_instance: Optional[DBUser]
     _ensure_started_lock: asyncio.Lock
+    _track_connection_task: Optional[asyncio.Task]
 
     def __init__(self, mxid: UserID, tgid: Optional[TelegramID] = None,
                  username: Optional[str] = None, phone: Optional[str] = None,
@@ -77,6 +85,9 @@ class User(AbstractUser, BaseUser):
         self.db_portals = db_portals or []
         self._db_instance = db_instance
         self._ensure_started_lock = asyncio.Lock()
+        self.dm_update_lock = asyncio.Lock()
+        self._metric_value = defaultdict(lambda: False)
+        self._track_connection_task = None
 
         self.command_status = None
 
@@ -150,7 +161,7 @@ class User(AbstractUser, BaseUser):
         return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
                       saved_contacts=self.saved_contacts, portals=self.db_portals)
 
-    def save(self, contacts: bool = False, portals: bool = False) -> None:
+    async def save(self, contacts: bool = False, portals: bool = False) -> None:
         self.db_instance.edit(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
                               saved_contacts=self.saved_contacts)
         if contacts:
@@ -192,20 +203,40 @@ class User(AbstractUser, BaseUser):
         await super().start()
         if await self.is_logged_in():
             self.log.debug(f"Ensuring post_login() for {self.name}")
-            asyncio.ensure_future(self.post_login(), loop=self.loop)
+            self.loop.create_task(self.post_login())
         elif delete_unless_authenticated:
             self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
             await self.client.disconnect()
             self.client.session.delete()
         return self
 
+    async def _track_connection(self) -> None:
+        self.log.debug("Starting loop to track connection state")
+        while True:
+            await asyncio.sleep(3)
+            connected = bool(self.client._sender._transport_connected
+                             if self.client and self.client._sender else False)
+            self._track_metric(METRIC_CONNECTED, connected)
+
+    async def stop(self) -> None:
+        await super().stop()
+        if self._track_connection_task:
+            self._track_connection_task.cancel()
+            self._track_connection_task = None
+        self._track_metric(METRIC_CONNECTED, False)
+
     async def post_login(self, info: TLUser = None, first_login: bool = False) -> None:
+        if config["metrics.enabled"] and not self._track_connection_task:
+            self._track_connection_task = self.loop.create_task(self._track_connection())
+
         try:
             await self.update_info(info)
         except Exception:
             self.log.exception("Failed to update telegram account info")
             return
 
+        self._track_metric(METRIC_LOGGED_IN, True)
+
         try:
             puppet = pu.Puppet.get(self.tgid)
             if puppet.custom_mxid != self.mxid and puppet.can_auto_login(self.mxid):
@@ -226,12 +257,7 @@ class User(AbstractUser, BaseUser):
             return False
 
         if isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
-            message = update.message
+            portal = po.Portal.get_by_entity(update.message.peer_id, receiver_id=self.tgid)
-            if isinstance(message.to_id, PeerUser) and not message.out:
-                portal = po.Portal.get_by_tgid(message.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
-            else:
-                portal = po.Portal.get_by_entity(message.to_id, receiver_id=self.tgid)
         elif isinstance(update, UpdateShortChatMessage):
             portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
         elif isinstance(update, UpdateShortMessage):
@@ -240,7 +266,7 @@ class User(AbstractUser, BaseUser):
             return False
 
         if portal:
-            self.register_portal(portal)
+            await self.register_portal(portal)
             return False
 
         # Don't bother handling the update
@@ -269,7 +295,7 @@ class User(AbstractUser, BaseUser):
             self.tgid = TelegramID(info.id)
             self.by_tgid[self.tgid] = self
         if changed:
-            self.save()
+            await self.save()
 
     async def log_out(self) -> bool:
         puppet = pu.Puppet.get(self.tgid)
@@ -278,25 +304,30 @@ class User(AbstractUser, BaseUser):
         for _, portal in self.portals.items():
             if not portal or portal.deleted or not portal.mxid or portal.has_bot:
                 continue
-            try:
+            if portal.peer_type == "user":
-                await portal.main_intent.kick_user(portal.mxid, self.mxid,
+                await portal.cleanup_portal("Logged out of Telegram")
-                                                   "Logged out of Telegram.")
+            else:
-            except MatrixRequestError:
+                try:
-                pass
+                    await portal.main_intent.kick_user(portal.mxid, self.mxid,
+                                                       "Logged out of Telegram.")
+                except MatrixRequestError:
+                    pass
         self.portals = {}
         self.contacts = []
-        self.save(portals=True, contacts=True)
+        await self.save(portals=True, contacts=True)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
             except KeyError:
                 pass
             self.tgid = None
-            self.save()
+            await self.save()
         ok = await self.client.log_out()
         if not ok:
             return False
         self.delete()
+        await self.stop()
+        self._track_metric(METRIC_LOGGED_IN, False)
         return True
 
     def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
@@ -305,7 +336,7 @@ class User(AbstractUser, BaseUser):
         for contact in self.contacts:
             similarity = contact.similarity(query)
             if similarity >= min_similarity:
-                results.append(SearchResult((contact, similarity)))
+                results.append(SearchResult(contact, similarity))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
@@ -317,7 +348,7 @@ class User(AbstractUser, BaseUser):
         for user in server_results.users:
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
-            results.append(SearchResult((puppet, puppet.similarity(query))))
+            results.append(SearchResult(puppet, puppet.similarity(query)))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
@@ -332,44 +363,75 @@ class User(AbstractUser, BaseUser):
 
         return await self._search_remote(query), True
 
-    async def sync_dialogs(self, synchronous_create: bool = False) -> None:
+    async def _catch(self, action: str, task: asyncio.Task) -> None:
+        try:
+            await task
+        except Exception:
+            self.log.exception(f"Error while {action}")
+
+    async def get_direct_chats(self) -> Dict[UserID, List[RoomID]]:
+        return {
+            pu.Puppet.get_mxid_from_id(portal.tgid): [portal.mxid]
+            for portal in DBPortal.find_private_chats(self.tgid)
+            if portal.mxid
+        }
+
+    async def sync_dialogs(self) -> None:
         if self.is_bot:
             return
         creators = []
-        limit = config["bridge.sync_dialog_limit"] or None
+        update_limit = config["bridge.sync_update_limit"] or None
-        self.log.debug(f"Syncing dialogs (limit={limit}, synchronous_create={synchronous_create})")
+        create_limit = config["bridge.sync_create_limit"]
-        async for dialog in self.client.iter_dialogs(limit=limit, ignore_migrated=True,
+        index = 0
+        self.log.debug(f"Syncing dialogs (update_limit={update_limit}, "
+                       f"create_limit={create_limit})")
+        dialog: Dialog
+        async for dialog in self.client.iter_dialogs(limit=update_limit, ignore_migrated=True,
                                                      archived=False):
             entity = dialog.entity
             if isinstance(entity, ChatForbidden):
                 self.log.warning(f"Ignoring forbidden chat {entity} while syncing")
+                continue
             elif isinstance(entity, Chat) and (entity.deactivated or entity.left):
                 self.log.warning(f"Ignoring deactivated or left chat {entity} while syncing")
                 continue
             elif isinstance(entity, TLUser) and not config["bridge.sync_direct_chats"]:
+                self.log.trace(f"Ignoring user {entity.id} while syncing")
                 continue
-            portal = po.Portal.get_by_entity(entity)
+            portal = po.Portal.get_by_entity(entity, receiver_id=self.tgid)
             self.portals[portal.tgid_full] = portal
-            creators.append(
+            if portal.mxid:
-                portal.create_matrix_room(self, entity, invites=[self.mxid],
+                update_task = portal.update_matrix_room(self, entity)
-                                          synchronous=synchronous_create))
+                backfill_task = portal.backfill(self, last_id=dialog.message.id)
-        self.save(portals=True)
+                creators.append(self._catch(f"updating {portal.tgid_log}",
-        await asyncio.gather(*creators, loop=self.loop)
+                                            self.loop.create_task(update_task)))
+                creators.append(self._catch(f"backfilling {portal.tgid_log}",
+                                            self.loop.create_task(backfill_task)))
+            elif not create_limit or index < create_limit:
+                create_task = portal.create_matrix_room(self, entity, invites=[self.mxid])
+                creators.append(self._catch(f"creating {portal.tgid_log}",
+                                            self.loop.create_task(create_task)))
+            index += 1
+        await self.save(portals=True)
+        await asyncio.gather(*creators)
+        await self.update_direct_chats()
         self.log.debug("Dialog syncing complete")
 
-    def register_portal(self, portal: po.Portal) -> None:
+    async def register_portal(self, portal: po.Portal) -> None:
+        self.log.trace(f"Registering portal {portal.tgid_full}")
         try:
             if self.portals[portal.tgid_full] == portal:
                 return
         except KeyError:
             pass
         self.portals[portal.tgid_full] = portal
-        self.save(portals=True)
+        await self.save(portals=True)
 
-    def unregister_portal(self, portal: po.Portal) -> None:
+    async def unregister_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
+        self.log.trace(f"Unregistering portal {(tgid, tg_receiver)}")
         try:
-            del self.portals[portal.tgid_full]
+            del self.portals[(tgid, tg_receiver)]
-            self.save(portals=True)
+            await self.save(portals=True)
         except KeyError:
             pass
 
@@ -394,13 +456,14 @@ class User(AbstractUser, BaseUser):
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
             self.contacts.append(puppet)
-        self.save(contacts=True)
+        await self.save(contacts=True)
 
     # endregion
     # region Class instance lookup
 
     @classmethod
-    def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Optional['User']:
+    def get_by_mxid(cls, mxid: UserID, create: bool = True, check_db: bool = True
+                    ) -> Optional['User']:
         if not mxid:
             raise ValueError("Matrix ID can't be empty")
 
@@ -409,10 +472,11 @@ class User(AbstractUser, BaseUser):
         except KeyError:
             pass
 
-        user = DBUser.get_by_mxid(mxid)
+        if check_db:
-        if user:
+            user = DBUser.get_by_mxid(mxid)
-            user = cls.from_db(user)
+            if user:
-            return user
+                user = cls.from_db(user)
+                return user
 
         if create:
             user = cls(mxid)
@@ -457,6 +521,7 @@ class User(AbstractUser, BaseUser):
 def init(context: 'Context') -> Iterable[Awaitable['User']]:
     global config
     config = context.config
+    User.bridge = context.bridge
 
     return (User.from_db(db_user).try_ensure_started()
             for db_user in DBUser.all_with_tgid())

mautrix_telegram/util/color_log.py

@@ -13,7 +13,8 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.util.color_log import ColorFormatter as BaseColorFormatter, PREFIX, MXID_COLOR, RESET
+from mautrix.util.logging.color import (ColorFormatter as BaseColorFormatter,
+                                        PREFIX, MXID_COLOR, RESET)
 
 TELETHON_COLOR = PREFIX + "35;1m"  # magenta
 TELETHON_MODULE_COLOR = PREFIX + "35m"

mautrix_telegram/util/file_transfer.py

@@ -18,6 +18,7 @@ from io import BytesIO
 import time
 import logging
 import asyncio
+import tempfile
 
 import magic
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
@@ -29,12 +30,13 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio
                              SecurityError, FileIdInvalidError)
 
 from mautrix.appservice import IntentAPI
-
+from mautrix.util.network_retry import call_with_net_retry
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
 from ..util import sane_mimetypes
 from .parallel_file_transfer import parallel_transfer_to_matrix
+from .tgs_converter import convert_tgs_to
 
 try:
     from PIL import Image
@@ -43,14 +45,13 @@ except ImportError:
 
 try:
     from moviepy.editor import VideoFileClip
-    import random
-    import string
-    import os
-    import mimetypes
 except ImportError:
-    VideoFileClip = random = string = os = mimetypes = None
+    VideoFileClip = None
 
-from .tgs_converter import convert_tgs_to
+try:
+    from mautrix.crypto.attachments import encrypt_attachment
+except ImportError:
+    encrypt_attachment = None
 
 log: logging.Logger = logging.getLogger("mau.util")
 
@@ -76,32 +77,23 @@ def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str
         return source_mime, file, None, None
 
 
-def _temp_file_name(ext: str) -> str:
-    return ("/tmp/mxtg-video-"
-            + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
-            + ext)
-
-
 def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str = "png",
                           max_size: Tuple[int, int] = (1024, 720)) -> Tuple[bytes, int, int]:
-    # We don't have any way to read the video from memory, so save it to disk.
+    with tempfile.NamedTemporaryFile(prefix="mxtg_video_", suffix=f".{video_ext}") as file:
-    temp_file = _temp_file_name(video_ext)
+        # We don't have any way to read the video from memory, so save it to disk.
-    with open(temp_file, "wb") as file:
         file.write(data)
 
-    # Read temp file and get frame
+        # Read temp file and get frame
-    clip = VideoFileClip(temp_file)
+        frame = VideoFileClip(file.name).get_frame(0)
-    frame = clip.get_frame(0)
 
     # Convert to png and save to BytesIO
     image = Image.fromarray(frame).convert("RGBA")
+
     thumbnail_file = BytesIO()
     if max_size:
         image.thumbnail(max_size, Image.ANTIALIAS)
     image.save(thumbnail_file, frame_ext)
 
-    os.remove(temp_file)
-
     w, h = image.size
     return thumbnail_file.getvalue(), w, h
 
@@ -116,8 +108,10 @@ def _location_to_id(location: TypeLocation) -> str:
 
 
 async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                       thumbnail_loc: TypeLocation, video: bytes,
+                                       thumbnail_loc: TypeLocation, mime_type: str, encrypt: bool,
-                                       mime: str) -> Optional[DBTelegramFile]:
+                                       video: Optional[bytes], custom_data: Optional[bytes] = None,
+                                       width: Optional[int] = None, height: [int] = None
+                                       ) -> Optional[DBTelegramFile]:
     if not Image or not VideoFileClip:
         return None
 
@@ -125,12 +119,17 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if not loc_id:
         return None
 
+    if custom_data:
+        loc_id += "-mau_custom_thumbnail"
+
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
-    video_ext = sane_mimetypes.guess_extension(mime)
+    video_ext = sane_mimetypes.guess_extension(mime_type)
-    if VideoFileClip and video_ext and video:
+    if custom_data:
+        file = custom_data
+    elif VideoFileClip and video_ext and video:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
         except OSError:
@@ -141,11 +140,19 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
         width, height = None, None
         mime_type = magic.from_buffer(file, mime=True)
 
-    content_uri = await intent.upload_media(file, mime_type)
+    decryption_info = None
+    upload_mime_type = mime_type
+    if encrypt:
+        file, decryption_info = encrypt_attachment(file)
+        upload_mime_type = "application/octet-stream"
+    content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
+                                            _action="upload media")
+    if decryption_info:
+        decryption_info.url = content_uri
 
     db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
                              was_converted=False, timestamp=int(time.time()), size=len(file),
-                             width=width, height=height)
+                             width=width, height=height, decryption_info=decryption_info)
     try:
         db_file.insert()
     except (IntegrityError, InvalidRequestError) as e:
@@ -161,10 +168,10 @@ TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
 
 
 async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None, *,
                                   is_sticker: bool = False, tgs_convert: Optional[dict] = None,
-                                  filename: Optional[str] = None, parallel_id: Optional[int] = None
+                                  filename: Optional[str] = None, encrypt: bool = False,
-                                  ) -> Optional[DBTelegramFile]:
+                                  parallel_id: Optional[int] = None) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
         return None
@@ -181,22 +188,24 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA
     async with lock:
         return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
                                                        thumbnail, is_sticker, tgs_convert,
-                                                       filename, parallel_id)
+                                                       filename, encrypt, parallel_id)
 
 
 async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                             loc_id: str, location: TypeLocation,
                                             thumbnail: TypeThumbnail, is_sticker: bool,
                                             tgs_convert: Optional[dict], filename: Optional[str],
-                                            parallel_id: Optional[int]
+                                            encrypt: bool, parallel_id: Optional[int]
                                             ) -> Optional[DBTelegramFile]:
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
+    converted_anim = None
+
     if parallel_id and isinstance(location, Document) and (not is_sticker or not tgs_convert):
         db_file = await parallel_transfer_to_matrix(client, intent, loc_id, location, filename,
-                                                    parallel_id)
+                                                    encrypt, parallel_id)
         mime_type = location.mime_type
         file = None
     else:
@@ -213,13 +222,17 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
 
         image_converted = False
         # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
-        if is_sticker and tgs_convert and (mime_type == "application/gzip" or (
+        is_tgs = (mime_type == "application/gzip" or (mime_type == "application/octet-stream"
-               mime_type == "application/octet-stream"
+                                                      and magic.from_buffer(file).startswith(
-               and magic.from_buffer(file).startswith("gzip"))):
+                "gzip")))
-            mime_type, file, width, height = await convert_tgs_to(
+        if is_sticker and tgs_convert and is_tgs:
-                file, tgs_convert["target"], **tgs_convert["args"])
+            converted_anim = await convert_tgs_to(file, tgs_convert["target"],
-            thumbnail = None
+                                                  **tgs_convert["args"])
+            mime_type = converted_anim.mime
+            file = converted_anim.data
+            width, height = converted_anim.width, converted_anim.height
             image_converted = mime_type != "application/gzip"
+            thumbnail = None
 
         if mime_type == "image/webp":
             new_mime_type, file, width, height = convert_image(
@@ -229,17 +242,34 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
             mime_type = new_mime_type
             thumbnail = None
 
-        content_uri = await intent.upload_media(file, mime_type)
+        decryption_info = None
+        upload_mime_type = mime_type
+        if encrypt and encrypt_attachment:
+            file, decryption_info = encrypt_attachment(file)
+            upload_mime_type = "application/octet-stream"
+        content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
+                                                _action="upload media")
+        if decryption_info:
+            decryption_info.url = content_uri
 
-        db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
+        db_file = DBTelegramFile(id=loc_id, mxc=content_uri, decryption_info=decryption_info,
                                  mime_type=mime_type, was_converted=image_converted,
                                  timestamp=int(time.time()), size=len(file),
                                  width=width, height=height)
     if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
         if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
             thumbnail = thumbnail.location
-        db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
+        try:
-                                                               mime_type)
+            db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail,
+                                                                   video=file, mime_type=mime_type,
+                                                                   encrypt=encrypt)
+        except FileIdInvalidError:
+            log.warning(f"Failed to transfer thumbnail for {thumbnail!s}", exc_info=True)
+    elif converted_anim and converted_anim.thumbnail_data:
+        db_file.thumbnail = await transfer_thumbnail_to_matrix(
+            client, intent, location, video=None, encrypt=encrypt,
+            custom_data=converted_anim.thumbnail_data, mime_type=converted_anim.thumbnail_mime,
+            width=converted_anim.width, height=converted_anim.height)
 
     try:
         db_file.insert()

mautrix_telegram/util/parallel_file_transfer.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, List, AsyncGenerator, Union, Awaitable, DefaultDict, Tuple
+from typing import Optional, List, AsyncGenerator, Union, Awaitable, DefaultDict, Tuple, cast
 from collections import defaultdict
 import hashlib
 import asyncio
@@ -34,12 +34,18 @@ from telethon.crypto import AuthKey
 from telethon import utils, helpers
 
 from mautrix.appservice import IntentAPI
-from mautrix.types import ContentURI
+from mautrix.types import ContentURI, EncryptedFile
+from mautrix.util.logging import TraceLogger
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
 
-log: logging.Logger = logging.getLogger("mau.util")
+try:
+    from mautrix.crypto.attachments import async_encrypt_attachment
+except ImportError:
+    async_encrypt_attachment = None
+
+log: TraceLogger = cast(TraceLogger, logging.getLogger("mau.util"))
 
 TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
                      InputFileLocation, InputPhotoFileLocation]
@@ -97,7 +103,7 @@ class UploadSender:
 
     async def _next(self, data: bytes) -> None:
         self.request.bytes = data
-        log.debug(f"Sending file part {self.request.file_part}/{self.part_count}"
+        log.trace(f"Sending file part {self.request.file_part}/{self.part_count}"
                   f" with {len(data)} bytes")
         await self.sender.send(self.request)
         self.request.file_part += self.stride
@@ -180,9 +186,9 @@ class ParallelTransferrer:
 
     async def _create_sender(self) -> MTProtoSender:
         dc = await self.client._get_dc(self.dc_id)
-        sender = MTProtoSender(self.auth_key, self.loop, loggers=self.client._log)
+        sender = MTProtoSender(self.auth_key, loggers=self.client._log)
         await sender.connect(self.client._connection(dc.ip_address, dc.port, dc.id,
-                                                     loop=self.loop, loggers=self.client._log,
+                                                     loggers=self.client._log,
                                                      proxy=self.client._proxy))
         if not self.auth_key:
             log.debug(f"Exporting auth to DC {self.dc_id}")
@@ -231,7 +237,7 @@ class ParallelTransferrer:
                     break
                 yield data
                 part += 1
-                log.debug(f"Part {part} downloaded")
+                log.trace(f"Part {part} downloaded")
 
         log.debug("Parallel download finished, cleaning up connections")
         await self._cleanup()
@@ -242,18 +248,34 @@ parallel_transfer_locks: DefaultDict[int, asyncio.Lock] = defaultdict(lambda: as
 
 async def parallel_transfer_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                       loc_id: str, location: TypeLocation, filename: str,
-                                      parallel_id: int) -> DBTelegramFile:
+                                      encrypt: bool, parallel_id: int) -> DBTelegramFile:
     size = location.size
     mime_type = location.mime_type
     dc_id, location = utils.get_input_location(location)
     # We lock the transfers because telegram has connection count limits
     async with parallel_transfer_locks[parallel_id]:
         downloader = ParallelTransferrer(client, dc_id)
-        content_uri = await intent.upload_media(downloader.download(location, size),
+        data = downloader.download(location, size)
-                                                mime_type=mime_type, filename=filename, size=size)
+        decryption_info = None
+        up_mime_type = mime_type
+        if encrypt and async_encrypt_attachment:
+            async def encrypted(stream):
+                nonlocal decryption_info
+                async for chunk in async_encrypt_attachment(stream):
+                    if isinstance(chunk, EncryptedFile):
+                        decryption_info = chunk
+                    else:
+                        yield chunk
+
+            data = encrypted(data)
+            up_mime_type = "application/octet-stream"
+        content_uri = await intent.upload_media(data, mime_type=up_mime_type, filename=filename,
+                                                size=size if not encrypt else None)
+        if decryption_info:
+            decryption_info.url = content_uri
     return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
                           was_converted=False, timestamp=int(time.time()), size=size,
-                          width=None, height=None)
+                          width=None, height=None, decryption_info=decryption_info)
 
 
 async def _internal_transfer_to_telegram(client: MautrixTelegramClient, response: ClientResponse

mautrix_telegram/util/tgs_converter.py

@@ -21,8 +21,23 @@ import shutil
 import os.path
 import tempfile
 
+from attr import dataclass
+
 log: logging.Logger = logging.getLogger("mau.util.tgs")
-converters: Dict[str, Callable[[bytes, int, int, Any], Awaitable[Tuple[str, bytes]]]] = {}
+
+
+@dataclass
+class ConvertedSticker:
+    mime: str
+    data: bytes
+    thumbnail_mime: Optional[str] = None
+    thumbnail_data: Optional[bytes] = None
+    width: int = 0
+    height: int = 0
+
+
+Converter = Callable[[bytes, int, int, Any], Awaitable[ConvertedSticker]]
+converters: Dict[str, Converter] = {}
 
 
 def abswhich(program: Optional[str]) -> Optional[str]:
@@ -34,7 +49,7 @@ lottieconverter = abswhich("lottieconverter")
 ffmpeg = abswhich("ffmpeg")
 
 if lottieconverter:
-    async def tgs_to_png(file: bytes, width: int, height: int, **_: Any) -> Tuple[str, bytes]:
+    async def tgs_to_png(file: bytes, width: int, height: int, **_: Any) -> ConvertedSticker:
         frame = 1
         proc = await asyncio.create_subprocess_exec(lottieconverter, "-", "-", "png",
                                                     f"{width}x{height}", str(frame),
@@ -42,26 +57,26 @@ if lottieconverter:
                                                     stdin=asyncio.subprocess.PIPE)
         stdout, stderr = await proc.communicate(file)
         if proc.returncode == 0:
-            return "image/png", stdout
+            return ConvertedSticker("image/png", stdout)
         else:
             log.error("lottieconverter error: " + (stderr.decode("utf-8") if stderr is not None
-                      else f"unknown ({proc.returncode})"))
+                                                   else f"unknown ({proc.returncode})"))
-            return "application/gzip", file
+            return ConvertedSticker("application/gzip", file)
 
 
     async def tgs_to_gif(file: bytes, width: int, height: int, background: str = "202020",
-                         **_: Any) -> Tuple[str, bytes]:
+                         **_: Any) -> ConvertedSticker:
         proc = await asyncio.create_subprocess_exec(lottieconverter, "-", "-", "gif",
                                                     f"{width}x{height}", f"0x{background}",
                                                     stdout=asyncio.subprocess.PIPE,
                                                     stdin=asyncio.subprocess.PIPE)
         stdout, stderr = await proc.communicate(file)
         if proc.returncode == 0:
-            return "image/gif", stdout
+            return ConvertedSticker("image/gif", stdout)
         else:
             log.error("lottieconverter error: " + (stderr.decode("utf-8") if stderr is not None
-                      else f"unknown ({proc.returncode})"))
+                                                   else f"unknown ({proc.returncode})"))
-            return "application/gzip", file
+            return ConvertedSticker("application/gzip", file)
 
 
     converters["png"] = tgs_to_png
@@ -69,7 +84,7 @@ if lottieconverter:
 
 if lottieconverter and ffmpeg:
     async def tgs_to_webm(file: bytes, width: int, height: int, fps: int = 30,
-                          **_: Any) -> Tuple[str, bytes]:
+                          **_: Any) -> ConvertedSticker:
         with tempfile.TemporaryDirectory(prefix="tgs_") as tmpdir:
             file_template = tmpdir + "/out_"
             proc = await asyncio.create_subprocess_exec(lottieconverter, "-", file_template,
@@ -78,6 +93,8 @@ if lottieconverter and ffmpeg:
                                                         stdin=asyncio.subprocess.PIPE)
             _, stderr = await proc.communicate(file)
             if proc.returncode == 0:
+                with open(f"{file_template}00.png", "rb") as first_frame_file:
+                    first_frame_data = first_frame_file.read()
                 proc = await asyncio.create_subprocess_exec(ffmpeg, "-hide_banner", "-loglevel",
                                                             "error", "-framerate", str(fps),
                                                             "-pattern_type", "glob", "-i",
@@ -88,25 +105,27 @@ if lottieconverter and ffmpeg:
                                                             stdin=asyncio.subprocess.PIPE)
                 stdout, stderr = await proc.communicate()
                 if proc.returncode == 0:
-                    return "video/webm", stdout
+                    return ConvertedSticker("video/webm", stdout, "image/png", first_frame_data)
                 else:
                     log.error("ffmpeg error: " + (stderr.decode("utf-8") if stderr is not None
-                              else f"unknown ({proc.returncode})"))
+                                                  else f"unknown ({proc.returncode})"))
             else:
                 log.error("lottieconverter error: " + (stderr.decode("utf-8") if stderr is not None
-                          else f"unknown ({proc.returncode})"))
+                                                       else f"unknown ({proc.returncode})"))
-        return "application/gzip", file
+        return ConvertedSticker("application/gzip", file)
 
 
     converters["webm"] = tgs_to_webm
 
 
 async def convert_tgs_to(file: bytes, convert_to: str, width: int, height: int, **kwargs: Any
-                         ) -> Tuple[str, bytes, Optional[int], Optional[int]]:
+                         ) -> ConvertedSticker:
     if convert_to in converters:
         converter = converters[convert_to]
-        mime, out = await converter(file, width, height, **kwargs)
+        converted = await converter(file, width, height, **kwargs)
-        return mime, out, width, height
+        converted.width = width
+        converted.height = height
+        return converted
     elif convert_to != "disable":
         log.warning(f"Unable to convert animated sticker, type {convert_to} not supported")
-    return "application/gzip", file, None, None
+    return ConvertedSticker("application/gzip", file)

mautrix_telegram/web/provisioning/__init__.py

@@ -141,6 +141,12 @@ class ProvisioningAPI(AuthAPI):
             return self.get_error_response(403, "not_enough_permissions",
                                            "You do not have the permissions to bridge that room.")
 
+        is_logged_in = user is not None and await user.is_logged_in()
+        acting_user = user if is_logged_in else self.context.bot
+        if not acting_user:
+            return self.get_login_response(status=403, errcode="not_logged_in",
+                                           error="You are not logged in and there is no relay bot.")
+
         portal = Portal.get_by_tgid(tgid, peer_type=peer_type)
         if portal.mxid == room_id:
             return self.get_error_response(200, "bridge_exists",
@@ -157,35 +163,30 @@ class ProvisioningAPI(AuthAPI):
                                                "Telegram chat is already bridged to another "
                                                "Matrix room.")
 
-        is_logged_in = user is not None and await user.is_logged_in()
+        async with portal._room_create_lock:
-        acting_user = user if is_logged_in else self.context.bot
+            entity: Optional[TypeChat] = None
-        if not acting_user:
+            try:
-            return self.get_login_response(status=403, errcode="not_logged_in",
+                entity = await acting_user.client.get_entity(portal.peer)
-                                           error="You are not logged in and there is no relay bot.")
+            except Exception:
+                self.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
 
-        entity: Optional[TypeChat] = None
+            if not entity or isinstance(entity, (ChatForbidden, ChannelForbidden)):
-        try:
+                if is_logged_in:
-            entity = await acting_user.client.get_entity(portal.peer)
+                    return self.get_error_response(403, "user_not_in_chat",
-        except Exception:
+                                                   "Failed to get info of Telegram chat. "
-            self.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+                                                   "Are you in the chat?")
-
+                return self.get_error_response(403, "bot_not_in_chat",
-        if not entity or isinstance(entity, (ChatForbidden, ChannelForbidden)):
-            if is_logged_in:
-                return self.get_error_response(403, "user_not_in_chat",
                                                "Failed to get info of Telegram chat. "
-                                               "Are you in the chat?")
+                                               "Is the relay bot in the chat?")
-            return self.get_error_response(403, "bot_not_in_chat",
-                                           "Failed to get info of Telegram chat. "
-                                           "Is the relay bot in the chat?")
 
-        direct = False
+            portal.mxid = room_id
+            portal.by_mxid[portal.mxid] = portal
+            (portal.title, portal.about, levels,
+             portal.encrypted) = await get_initial_state(self.az.intent, room_id)
+            portal.photo_id = ""
+            await portal.save()
 
-        portal.mxid = room_id
+        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
-        portal.title, portal.about, levels = await get_initial_state(self.az.intent, room_id)
-        portal.photo_id = ""
-        portal.save()
-
-        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
                               loop=self.loop)
 
         return web.Response(status=202, body="{}")
@@ -216,7 +217,7 @@ class ProvisioningAPI(AuthAPI):
                                            "You do not have the permissions to bridge that room.")
 
         try:
-            title, about, _ = await get_initial_state(self.az.intent, room_id)
+            title, about, _, encrypted = await get_initial_state(self.az.intent, room_id)
         except (MatrixRequestError, IntentError):
             return self.get_error_response(403, "bot_not_in_room",
                                            "The bridge bot is not in the given room.")
@@ -240,11 +241,12 @@ class ProvisioningAPI(AuthAPI):
             "group": "chat",
         }[type]
 
-        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type)
+        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type,
+                        encrypted=encrypted)
         try:
             await portal.create_telegram_chat(user, supergroup=supergroup)
         except ValueError as e:
-            portal.delete()
+            await portal.delete()
             return self.get_error_response(500, "unknown_error", e.args[0])
 
         return web.json_response({
@@ -315,9 +317,9 @@ class ProvisioningAPI(AuthAPI):
 
         if not user.is_bot:
             return web.json_response([{
-                "id": get_peer_id(chat),
+                "id": chat.id,
                 "title": chat.title,
-            } async for chat in user.client.get_dialogs(ignore_migrated=True, archived=False)])
+            } async for chat in user.client.iter_dialogs(ignore_migrated=True, archived=False)])
         else:
             return web.json_response([{
                 "id": get_peer_id(chat.peer),
@@ -355,6 +357,7 @@ class ProvisioningAPI(AuthAPI):
         if err is not None:
             return err
         await user.log_out()
+        return web.json_response({}, status=200)
 
     async def bridge_info(self, request: web.Request) -> web.Response:
         return web.json_response({

optional-requirements.txt

@@ -1,5 +1,30 @@
-cryptg
+# Format: #/name defines a new extras_require group called name
-Pillow
+# Uncommented lines after the group definition insert things into that group.
-moviepy
+
-prometheus_client
+#/speedups
-psycopg2-binary
+cryptg>=0.1,<0.3
+cchardet
+aiodns
+brotli
+
+#/webp_convert
+pillow>=4,<8
+
+#/qr_login
+pillow>=4,<8
+qrcode>=6,<7
+
+#/hq_thumbnails
+moviepy>=1,<2
+
+#/metrics
+prometheus_client>=0.6,<0.9
+
+#/postgres
+psycopg2-binary>=2,<3
+
+#/e2be
+asyncpg>=0.20,<0.22
+python-olm>=3,<4
+pycryptodome>=3,<4
+unpaddedbase64>=1,<2

requirements.txt

@@ -1,9 +1,10 @@
-aiohttp
+SQLAlchemy>=1.2,<2
-mautrix
+alembic>=1,<2
-ruamel.yaml
+ruamel.yaml>=0.15.35,<0.17
-python-magic
+python-magic>=0.4,<0.5
-SQLAlchemy
+commonmark>=0.8,<0.10
-alembic
+aiohttp>=3,<4
-commonmark
+yarl>=1,<2
-telethon
+mautrix>=0.8.3,<0.9
-telethon-session-sqlalchemy
+telethon>=1.17,<1.18
+telethon-session-sqlalchemy>=0.2.14,<0.3

setup.py

@@ -3,14 +3,21 @@ import glob
 
 from mautrix_telegram.get_version import git_tag, git_revision, version, linkified_version
 
-extras = {
+with open("requirements.txt") as reqs:
-    "speedups": ["cryptg>=0.1,<0.3", "cchardet", "aiodns", "Brotli"],
+    install_requires = reqs.read().splitlines()
-    "webp_convert": ["Pillow>=4.3.0,<7"],
+
-    "hq_thumbnails": ["moviepy>=1.0,<2.0"],
+with open("optional-requirements.txt") as reqs:
-    "metrics": ["prometheus_client>=0.6.0,<0.8.0"],
+    extras_require = {}
-    "postgres": ["psycopg2-binary>=2,<3"],
+    current = []
-}
+    for line in reqs.read().splitlines():
-extras["all"] = list({dep for deps in extras.values() for dep in deps})
+        if line.startswith("#/"):
+            extras_require[line[2:]] = current = []
+        elif not line or line.startswith("#"):
+            continue
+        else:
+            current.append(line)
+
+extras_require["all"] = list({dep for deps in extras_require.values() for dep in deps})
 
 try:
     long_desc = open("README.md").read()
@@ -40,18 +47,8 @@ setuptools.setup(
 
     packages=setuptools.find_packages(),
 
-    install_requires=[
+    install_requires=install_requires,
-        "aiohttp>=3.0.1,<4",
+    extras_require=extras_require,
-        "mautrix>=0.4.0,<0.5",
-        "SQLAlchemy>=1.2.3,<2",
-        "alembic>=1.0.0,<2",
-        "commonmark>=0.8.1,<0.10",
-        "ruamel.yaml>=0.15.35,<0.17",
-        "python-magic>=0.4.15,<0.5",
-        "telethon>=1.10,<1.11",
-        "telethon-session-sqlalchemy>=0.2.14,<0.3",
-    ],
-    extras_require=extras,
     python_requires="~=3.6",
 
     setup_requires=["pytest-runner"],
@@ -64,19 +61,16 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
-    entry_points="""
-        [console_scripts]
-        mautrix-telegram=mautrix_telegram.__main__:main
-    """,
     package_data={"mautrix_telegram": [
         "web/public/*.mako", "web/public/*.png", "web/public/*.css",
+        "example-config.yaml",
     ]},
     data_files=[
-        (".", ["example-config.yaml", "alembic.ini"]),
+        (".", ["alembic.ini", "mautrix_telegram/example-config.yaml"]),
         ("alembic", ["alembic/env.py"]),
         ("alembic/versions", glob.glob("alembic/versions/*.py"))
     ],

tests/commands/test_handler.py

@@ -26,7 +26,7 @@ def context(request: FixtureRequest) -> Context:
     """
     # Config(path, registration_path, base_path)
     config = getattr(request.cls, 'config', Config("", "", ""))
-    return Context(az=Mock(), config=config, loop=Mock(), session_container=Mock(), bot=Mock())
+    return Context(az=Mock(), config=config, loop=Mock(), session_container=Mock(), bridge=Mock(), bot=Mock())
 
 
 @pytest.fixture
@@ -52,6 +52,7 @@ class TestCommandEvent:
             sender=u.User(UserID("@sender:example.org")),
             command="help",
             args=[],
+            content=Mock(),
             is_management=True,
             is_portal=False,
         )
@@ -107,6 +108,7 @@ class TestCommandEvent:
             sender=u.User(UserID("@sender:example.org")),
             command="help",
             args=[],
+            content=Mock(),
             is_management=False,
             is_portal=False,
         )
@@ -133,6 +135,7 @@ class TestCommandEvent:
             sender=u.User(UserID("@sender:example.org")),
             command="help",
             args=[],
+            content=Mock(),
             is_management=True,
             is_portal=False,
         )
@@ -209,6 +212,7 @@ class TestCommandHandler:
             sender=sender,
             command=command,
             args=[],
+            content=Mock(),
             is_management=False,
             is_portal=boolean,
         )
@@ -271,6 +275,7 @@ class TestCommandHandler:
             sender=sender,
             command=command,
             args=[],
+            content=Mock(),
             is_management=is_management,
             is_portal=boolean,
         )
@@ -307,6 +312,7 @@ class TestCommandProcessor:
             sender=sender,
             command="hElp",
             args=[],
+            content=Mock(),
             is_management=boolean2[0],
             is_portal=boolean2[1])
 
@@ -333,6 +339,7 @@ class TestCommandProcessor:
             sender=sender,
             command="foo",
             args=[],
+            content=Mock(),
             is_management=boolean2[0],
             is_portal=boolean2[1],
         )
@@ -361,6 +368,7 @@ class TestCommandProcessor:
             sender=sender,  # u.User
             command="foo",
             args=[],
+            content=Mock(),
             is_management=boolean2[0],
             is_portal=boolean2[1]
         )