Bump version to 0.8.0rc1

Do not crash in _create_matrix_room if `invites` is `None`

.editorconfig

@@ -13,3 +13,6 @@ max_line_length = 99
 
 [*.{yaml,yml,py}]
 indent_style = space
+
+[.gitlab-ci.yml]
+indent_size = 2

.gitlab-ci.yml

@@ -2,37 +2,40 @@ image: docker:stable
 
 stages:
 - build
-- push
+- manifest
 
 default:
   before_script:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
-build:
+build amd64:
   stage: build
+  tags:
+  - amd64
   script:
   - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
 
-push latest:
-  stage: push
-  only:
-  - master
-  variables:
-    GIT_STRATEGY: none
+build arm64:
+  stage: build
+  tags:
+  - arm64
   script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
-  - docker push $CI_REGISTRY_IMAGE:latest
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
 
-push tag:
-  stage: push
-  variables:
-    GIT_STRATEGY: none
-  except:
-  - master
+manifest:
+  stage: manifest
+  before_script:
+  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
   script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - if [ $CI_COMMIT_BRANCH == "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
+  - if [ $CI_COMMIT_BRANCH != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64

Dockerfile

@@ -1,55 +1,21 @@
-FROM docker.io/alpine:3.10 AS lottieconverter
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.11
 
-WORKDIR /build
+RUN echo "@edge_main http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories
+RUN echo "@edge_testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
+RUN echo "@edge_community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories
 
-RUN apk add --no-cache git build-base cmake \
-  && git clone https://github.com/Samsung/rlottie.git \
-  && cd rlottie \
-  && mkdir build \
-  && cd build \
-  && cmake .. \
-  && make -j2 \
-  && make install \
-  && cd ../..
-
-RUN apk add --no-cache libpng libpng-dev zlib zlib-dev \
-  && git clone https://github.com/Eramde/LottieConverter.git \
-  && cd LottieConverter \
-  && git checkout 543c1d23ac9322f4f03c7fb6612ea7d026d44ac0 \
-  && make
-
-FROM docker.io/alpine:3.11
-
-ENV UID=1337 \
-    GID=1337 \
-    FFMPEG_BINARY=/usr/bin/ffmpeg
-
-COPY --from=lottieconverter /usr/lib/librlottie* /usr/lib/
-COPY --from=lottieconverter /build/LottieConverter/dist/Debug/GNU-Linux/lottieconverter /usr/local/bin/lottieconverter
-
-COPY . /opt/mautrix-telegram
-WORKDIR /opt/mautrix-telegram
-RUN apk add --no-cache --virtual .build-deps \
-      python3-dev \
-      libffi-dev \
-      build-base \
-      git \
-  && apk add --no-cache \
+RUN apk add --no-cache \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-magic \
       py3-sqlalchemy \
+      py3-alembic@edge_testing \
       py3-psycopg2 \
       py3-ruamel.yaml \
+      py3-commonmark@edge_testing \
       # Indirect dependencies
       py3-idna \
-      #commonmark
-        py3-future \
-      #alembic
-        py3-mako \
-        py3-dateutil \
-        py3-markupsafe \
       #moviepy
         py3-decorator \
         py3-tqdm \
@@ -58,20 +24,48 @@ RUN apk add --no-cache --virtual .build-deps \
           py3-numpy \
       #telethon
         py3-rsa \
+        # Optional for socks proxies
+        py3-pysocks \
         # cryptg
           py3-cffi \
+      py3-brotli \
       # Other dependencies
       ffmpeg \
       ca-certificates \
       su-exec \
       netcat-openbsd \
-      # lottieconverter
-      zlib libpng \
- && pip3 install .[speedups,hq_thumbnails,metrics] \
- # pip installs the sources to /usr/lib/python3.8/site-packages, so we don't need them here
- && rm -rf /opt/mautrix-telegram/mautrix_telegram \
+      # olm
+      olm-dev@edge_community \
+      # matrix-nio?
+      py3-future \
+      py3-atomicwrites \
+      py3-pycryptodome@edge_main \
+      py3-peewee@edge_community \
+      py3-pyrsistent@edge_community \
+      py3-jsonschema \
+      py3-aiofiles \
+      py3-cachetools@edge_community \
+      py3-prometheus-client@edge_community \
+      py3-unpaddedbase64 \
+      py3-pyaes@edge_testing \
+      py3-logbook@edge_testing
+
+COPY requirements.txt /opt/mautrix-telegram/requirements.txt
+COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
+WORKDIR /opt/mautrix-telegram
+RUN apk add --virtual .build-deps \
+      python3-dev \
+      libffi-dev \
+      build-base \
+ && sed -Ei 's/psycopg2-binary.+//' optional-requirements.txt \
+ && pip3 install -r requirements.txt -r optional-requirements.txt \
  && apk del .build-deps
 
+COPY . /opt/mautrix-telegram
+RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics,e2be] && apk del git
+
 VOLUME /data
+ENV UID=1337 GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
 
 CMD ["/opt/mautrix-telegram/docker-run.sh"]

ROADMAP.md

@@ -6,9 +6,9 @@
   * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
-  * [x] Typing notifications
-  * [x] Read receipts
-  * [x] Pinning messages
+  * [x] Typing notifications*
+  * [x] Read receipts*
+  * [x] Pinning messages*
   * [x] Power level
     * [x] Normal chats
       * [ ] Non-hardcoded PL requirements
@@ -56,5 +56,6 @@
   * [ ] ‡ Secret chats (not yet supported by Telethon)
   * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
+\* Requires [double puppeting](https://github.com/tulir/mautrix-telegram/wiki/Authentication#replacing-telegram-accounts-matrix-puppet-with-matrix-account) to be enabled  
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py

@@ -0,0 +1,27 @@
+"""Add encrypted field for portals
+
+Revision ID: 24f31fc8a72b
+Revises: a7c04a56041b
+Create Date: 2020-03-28 20:14:29.046699
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "24f31fc8a72b"
+down_revision = "a7c04a56041b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column("encrypted", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column("encrypted")

alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py

@@ -0,0 +1,26 @@
+"""Add decryption info field for reuploaded telegram files
+
+Revision ID: d3c922a6acd2
+Revises: 24f31fc8a72b
+Create Date: 2020-03-30 20:07:17.340346
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd3c922a6acd2'
+down_revision = '24f31fc8a72b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.add_column(sa.Column("decryption_info", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column("decryption_info")

alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py

@@ -0,0 +1,71 @@
+"""Add matrix-nio state store to main db
+
+Revision ID: dff56c93da8d
+Revises: d3c922a6acd2
+Create Date: 2020-03-31 22:04:04.014048
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'dff56c93da8d'
+down_revision = 'd3c922a6acd2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('nio_account',
+    sa.Column('user_id', sa.String(length=255), nullable=False),
+    sa.Column('device_id', sa.String(length=255), nullable=False),
+    sa.Column('shared', sa.Boolean(), nullable=False),
+    sa.Column('sync_token', sa.Text(), nullable=False),
+    sa.Column('account', sa.LargeBinary(), nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id')
+    )
+    op.create_table('nio_device_key',
+    sa.Column('user_id', sa.String(length=255), nullable=False),
+    sa.Column('device_id', sa.String(length=255), nullable=False),
+    sa.Column('display_name', sa.String(length=255), nullable=False),
+    sa.Column('deleted', sa.Boolean(), nullable=False),
+    sa.Column('keys', sa.PickleType(), nullable=False),
+    sa.PrimaryKeyConstraint('user_id', 'device_id')
+    )
+    op.create_table('nio_megolm_inbound_session',
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('sender_key', sa.String(length=255), nullable=False),
+    sa.Column('fp_key', sa.String(length=255), nullable=False),
+    sa.Column('room_id', sa.String(length=255), nullable=False),
+    sa.Column('session', sa.LargeBinary(), nullable=False),
+    sa.Column('forwarded_chains', sa.PickleType(), nullable=False),
+    sa.PrimaryKeyConstraint('session_id')
+    )
+    op.create_table('nio_olm_session',
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('sender_key', sa.String(length=255), nullable=False),
+    sa.Column('session', sa.LargeBinary(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('last_used', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('session_id')
+    )
+    op.create_table('nio_outgoing_key_request',
+    sa.Column('request_id', sa.String(length=255), nullable=False),
+    sa.Column('session_id', sa.String(length=255), nullable=False),
+    sa.Column('room_id', sa.String(length=255), nullable=False),
+    sa.Column('algorithm', sa.String(length=255), nullable=False),
+    sa.PrimaryKeyConstraint('request_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('nio_outgoing_key_request')
+    op.drop_table('nio_olm_session')
+    op.drop_table('nio_megolm_inbound_session')
+    op.drop_table('nio_device_key')
+    op.drop_table('nio_account')
+    # ### end Alembic commands ###

docker-run.sh

@@ -13,11 +13,9 @@ sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /d
 if [ -f /data/mx-state.json ]; then
 	ln -s /data/mx-state.json
 fi
-# Check that database is in the right state
-alembic -x config=/data/config.yaml upgrade head
 
 if [ ! -f /data/config.yaml ]; then
-	cp example-config.yaml /data/config.yaml
+	cp mautrix_telegram/example-config.yaml /data/config.yaml
 	echo "Didn't find a config file."
 	echo "Copied default config file to /data/config.yaml"
 	echo "Modify that config file to your liking."
@@ -35,5 +33,8 @@ if [ ! -f /data/registration.yaml ]; then
 	exit
 fi
 
+# Check that database is in the right state
+alembic -x config=/data/config.yaml upgrade head
+
 fixperms
 exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.7.1rc1"
+__version__ = "0.8.0rc1"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -44,6 +44,7 @@ except ImportError:
 
 
 class TelegramBridge(Bridge):
+    module = "mautrix_telegram"
     name = "mautrix-telegram"
     command = "python -m mautrix-telegram"
     description = "A Matrix-Telegram puppeting bridge."

mautrix_telegram/abstract_user.py

@@ -97,7 +97,6 @@ class AbstractUser(ABC):
         self.client = None
         self.is_relaybot = False
         self.is_bot = False
-        self.relaybot = None
 
     @property
     def connected(self) -> bool:
@@ -422,8 +421,9 @@ class AbstractUser(ABC):
                                f" in unbridged chat {portal.tgid_log}")
                 return
 
-        if self.ignore_incoming_bot_events and self.relaybot and sender.id == self.relaybot.tgid:
-            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
+        if ((self.ignore_incoming_bot_events and self.relaybot
+             and sender and sender.id == self.relaybot.tgid)):
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
             return
 
         if isinstance(update, MessageService):

mautrix_telegram/commands/portal/create_chat.py

@@ -25,10 +25,10 @@ from .util import user_has_power_level, get_initial_state
                  help_args="[_type_]",
                  help_text="Create a Telegram chat of the given type for the current Matrix room. "
                            "The type is either `group`, `supergroup` or `channel` (defaults to "
-                           "`group`).")
+                           "`supergroup`).")
 async def create(evt: CommandEvent) -> EventID:
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
+    type = evt.args[0] if len(evt.args) > 0 else "supergroup"
+    if type not in ("chat", "group", "supergroup", "channel"):
         return await evt.reply(
             "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
 

mautrix_telegram/commands/telegram/misc.py

@@ -20,10 +20,11 @@ import base64
 import re
 
 from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
-                             UserAlreadyParticipantError, ChatIdInvalidError)
+                             UserAlreadyParticipantError, ChatIdInvalidError,
+                             TakeoutInitDelayError, EmoticonInvalidError)
 from telethon.tl.patched import Message
 from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
-                               TypeInputPeer)
+                               TypeInputPeer, InputMediaDice)
 from telethon.tl.types.messages import BotCallbackAnswer
 from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
                                             GetBotCallbackAnswerRequest, SendVoteRequest)
@@ -35,7 +36,8 @@ from ... import puppet as pu, portal as po
 from ...abstract_user import AbstractUser
 from ...db import Message as DBMessage
 from ...types import TelegramID
-from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
+from ...commands import (command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS,
+                         SECTION_PORTAL_MANAGEMENT)
 
 
 @command_handler(needs_auth=False,
@@ -102,7 +104,8 @@ async def pm(evt: CommandEvent) -> EventID:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
 
     try:
-        user = await evt.sender.client.get_entity(evt.args[0])
+        id = "".join(evt.args).translate({ord(c): None for c in "+()- "})
+        user = await evt.sender.client.get_entity(id)
     except ValueError:
         return await evt.reply("Invalid user identifier or user not found.")
 
@@ -303,3 +306,39 @@ async def vote(evt: CommandEvent) -> EventID:
         return await evt.reply("You passed too many options.")
     # TODO use response
     return await evt.mark_read()
+
+
+@command_handler(help_section=SECTION_MISC, help_args="<_emoji_>",
+                 help_text="Roll a dice (\U0001F3B2) or throw a dart (\U0001F3AF) "
+                           "on the Telegram servers.")
+async def random(evt: CommandEvent) -> EventID:
+    if not evt.is_portal:
+        return await evt.reply("You can only roll dice in portal rooms")
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    arg = evt.args[0] if len(evt.args) > 0 else "dice"
+    emoticon = {
+        "dart": "\U0001F3AF",
+        "dice": "\U0001F3B2",
+    }.get(arg, arg)
+    try:
+        await evt.sender.client.send_media(await portal.get_input_entity(evt.sender),
+                                           InputMediaDice(emoticon))
+    except EmoticonInvalidError:
+        return await evt.reply("Invalid emoji for randomization")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="<_number of messages_> [--takeout]",
+                 help_text="Backfill messages from Telegram history.")
+async def backfill(evt: CommandEvent) -> None:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    try:
+        await portal.backfill(evt.sender)
+    except TakeoutInitDelayError:
+        msg = ("Please accept the data export request from a mobile device, "
+               "then re-run the backfill command.")
+        if portal.peer_type == "user":
+            from mautrix.appservice import IntentAPI
+            await portal.main_intent.send_notice(evt.room_id, msg)
+        else:
+            await evt.reply(msg)

mautrix_telegram/config.py

@@ -57,6 +57,9 @@ class Config(BaseBridgeConfig):
             base["appservice.address"] = f"{protocol}://{hostname}:{port}"
         else:
             copy("appservice.address")
+
+        copy("appservice.tls_cert")
+        copy("appservice.tls_key")
         copy("appservice.hostname")
         copy("appservice.port")
         copy("appservice.max_body_size")
@@ -118,6 +121,8 @@ class Config(BaseBridgeConfig):
         copy("bridge.federate_rooms")
         copy("bridge.animated_sticker.target")
         copy("bridge.animated_sticker.args")
+        copy("bridge.encryption.allow")
+        copy("bridge.encryption.default")
 
         copy("bridge.initial_power_level_overrides.group")
         copy("bridge.initial_power_level_overrides.user")

mautrix_telegram/db/__init__.py

@@ -24,6 +24,11 @@ from .puppet import Puppet
 from .telegram_file import TelegramFile
 from .user import User, UserPortal, Contact
 
+try:
+    from mautrix.bridge.db.nio_state_store import init as init_nio_db
+except ImportError:
+    init_nio_db = None
+
 
 def init(db_engine: Engine) -> None:
     for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
@@ -32,3 +37,5 @@ def init(db_engine: Engine) -> None:
         table.t = table.__table__
         table.c = table.t.c
         table.column_names = table.c.keys()
+    if init_nio_db:
+        init_nio_db(db_engine)

mautrix_telegram/db/message.py

@@ -61,6 +61,16 @@ class Message(Base):
         except StopIteration:
             return 0
 
+    @classmethod
+    def find_last(cls, mx_room: RoomID, tg_space: TelegramID) -> Optional['Message']:
+        return cls._one_or_none(cls.db.execute(
+            cls._make_simple_select(cls.c.mx_room == mx_room, cls.c.tg_space == tg_space)
+                .order_by(desc(cls.c.tgid)).limit(1)))
+
+    @classmethod
+    def delete_all(cls, mx_room: RoomID) -> None:
+        cls.db.execute(cls.t.delete().where(cls.c.mx_room == mx_room))
+
     @classmethod
     def get_by_mxid(cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
                     ) -> Optional['Message']:

mautrix_telegram/db/portal.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional
 
-from sqlalchemy import Column, Integer, String, Boolean, Text, func
+from sqlalchemy import Column, Integer, String, Boolean, Text, func, sql
 
 from mautrix.types import RoomID
 from mautrix.util.db import Base
@@ -34,6 +34,7 @@ class Portal(Base):
 
     # Matrix portal information
     mxid: RoomID = Column(String, unique=True, nullable=True)
+    encrypted: bool = Column(Boolean, nullable=False, server_default=sql.expression.false())
 
     config: str = Column(Text, nullable=True)
 

mautrix_telegram/db/telegram_file.py

@@ -13,15 +13,37 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from typing import Optional, cast, Dict, Any
 
-from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
+from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
+                        TypeDecorator)
 from sqlalchemy.engine.result import RowProxy
 
-from mautrix.types import ContentURI
+from mautrix.types import ContentURI, EncryptedFile
 from mautrix.util.db import Base
 
 
+class DBEncryptedFile(TypeDecorator):
+    impl = Text
+
+    @property
+    def python_type(self):
+        return EncryptedFile
+
+    def process_bind_param(self, value: EncryptedFile, dialect) -> Optional[str]:
+        if value is not None:
+            return value.json()
+        return None
+
+    def process_result_value(self, value: str, dialect) -> Optional[EncryptedFile]:
+        if value is not None:
+            return EncryptedFile.parse_json(value)
+        return None
+
+    def process_literal_param(self, value, dialect):
+        return value
+
+
 class TelegramFile(Base):
     __tablename__ = "telegram_file"
 
@@ -33,12 +55,13 @@ class TelegramFile(Base):
     size: Optional[int] = Column(Integer, nullable=True)
     width: Optional[int] = Column(Integer, nullable=True)
     height: Optional[int] = Column(Integer, nullable=True)
+    decryption_info: Optional[Dict[str, Any]] = Column(DBEncryptedFile, nullable=True)
     thumbnail_id: str = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
     thumbnail: Optional['TelegramFile'] = None
 
     @classmethod
     def scan(cls, row: RowProxy) -> 'TelegramFile':
-        telegram_file: TelegramFile = super().scan(row)
+        telegram_file = cast(TelegramFile, super().scan(row))
         if isinstance(telegram_file.thumbnail, str):
             telegram_file.thumbnail = cls.get(telegram_file.thumbnail)
         return telegram_file
@@ -52,5 +75,5 @@ class TelegramFile(Base):
             conn.execute(self.t.insert().values(
                 id=self.id, mxc=self.mxc, mime_type=self.mime_type,
                 was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
-                width=self.width, height=self.height,
+                width=self.width, height=self.height, decryption_info=self.decryption_info,
                 thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/example-config.yaml

@@ -13,6 +13,9 @@ homeserver:
 appservice:
     # The address that the homeserver can use to connect to this appservice.
     address: http://localhost:29317
+    # When using https:// the TLS certificate and key files for the address.
+    tls_cert: false
+    tls_key: false
 
     # The hostname and port where this appservice should listen.
     hostname: 0.0.0.0
@@ -62,6 +65,8 @@ appservice:
 
     # Community ID for bridged users (changes registration file) and rooms.
     # Must be created manually.
+    #
+    # Example: "+telegram:example.com". Set to false to disable.
     community_id: false
 
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
@@ -191,6 +196,17 @@ bridge:
             height: 256
             background: "020202"  # only for gif
             fps: 30               # only for webm
+    # End-to-bridge encryption support options. These require matrix-nio to be installed with pip
+    # and login_shared_secret to be configured in order to get a device for the bridge bot.
+    #
+    # Additionally, https://github.com/matrix-org/synapse/pull/5758 is required if using a normal
+    # application service.
+    encryption:
+        # Allow encryption, work in group chat rooms with e2ee enabled
+        allow: false
+        # Default to encryption, force-enable encryption in all portals the bridge creates
+        # This will cause the bridge bot to be in private chats for the encryption to work properly.
+        default: false
 
     # Overrides for base power levels.
     initial_power_level_overrides:
@@ -409,7 +425,7 @@ logging:
         mau:
             level: DEBUG
         telethon:
-            level: DEBUG
+            level: INFO
         aiohttp:
             level: INFO
     root:

mautrix_telegram/matrix.py

@@ -13,14 +13,15 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, Set, Tuple, Union, Iterable, TYPE_CHECKING
+from typing import Dict, Set, Tuple, Union, Iterable, List, TYPE_CHECKING
 
 from mautrix.bridge import BaseMatrixHandler
 from mautrix.types import (Event, EventType, RoomID, UserID, EventID, ReceiptEvent, ReceiptType,
                            ReceiptEventContent, PresenceEvent, PresenceState, TypingEvent,
                            MessageEvent, StateEvent, RedactionEvent, RoomNameStateEventContent,
                            RoomAvatarStateEventContent, RoomTopicStateEventContent,
-                           MemberStateEventContent)
+                           MemberStateEventContent, EncryptedEvent, TextMessageEventContent,
+                           MessageType)
 from mautrix.errors import MatrixError
 
 from . import user as u, portal as po, puppet as pu, commands as com
@@ -47,8 +48,15 @@ class MatrixHandler(BaseMatrixHandler):
     previously_typing: Dict[RoomID, Set[UserID]]
 
     def __init__(self, context: 'Context') -> None:
+        prefix, suffix = context.config["bridge.username_template"].format(userid=":").split(":")
+        homeserver = context.config["homeserver.domain"]
+        self.user_id_prefix = f"@{prefix}"
+        self.user_id_suffix = f"{suffix}:{homeserver}"
+
         super(MatrixHandler, self).__init__(context.az, context.config, loop=context.loop,
-                                            command_processor=com.CommandProcessor(context))
+                                            command_processor=com.CommandProcessor(context),
+                                            bridge=context.bridge)
+
         self.bot = context.bot
         self.previously_typing = {}
 
@@ -104,14 +112,38 @@ class MatrixHandler(BaseMatrixHandler):
                 except MatrixError:
                     pass
             portal.mxid = room_id
+            e2be_ok = None
+            if self.config["bridge.encryption.default"] and self.e2ee:
+                e2be_ok = await self.enable_dm_encryption(portal, members=members)
             portal.save()
             inviter.register_portal(portal)
-            await intent.send_notice(room_id, "Portal to private chat created.")
+            if e2be_ok is True:
+                evt_type, content = await self.e2ee.encrypt(
+                    room_id, EventType.ROOM_MESSAGE,
+                    TextMessageEventContent(msgtype=MessageType.NOTICE,
+                                            body="Portal to private chat created and end-to-bridge"
+                                                 " encryption enabled."))
+                await intent.send_message_event(room_id, evt_type, content)
+            else:
+                message = "Portal to private chat created."
+                if e2be_ok is False:
+                    message += "\n\nWarning: Failed to enable end-to-bridge encryption"
+                await intent.send_notice(room_id, message)
         else:
             await intent.join_room(room_id)
             await intent.send_notice(room_id, "This puppet will remain inactive until a "
                                               "Telegram chat is created for this room.")
 
+    async def enable_dm_encryption(self, portal: po.Portal, members: List[UserID]) -> bool:
+        ok = await super().enable_dm_encryption(portal, members)
+        if ok:
+            try:
+                puppet = pu.Puppet.get(portal.tgid)
+                await portal.main_intent.set_room_name(portal.mxid, puppet.displayname)
+            except Exception:
+                self.log.warning(f"Failed to set room name for {portal.mxid}", exc_info=True)
+        return ok
+
     async def send_welcome_message(self, room_id: RoomID, inviter: 'u.User') -> None:
         try:
             is_management = len(await self.az.intent.get_room_members(room_id)) == 2
@@ -156,7 +188,7 @@ class MatrixHandler(BaseMatrixHandler):
                                                "messages for unauthenticated users.")
             return
 
-        self.log.debug(f"{user} joined {room_id}")
+        self.log.debug(f"{user.mxid} joined {room_id}")
         if await user.is_logged_in() or portal.has_bot:
             await portal.join_matrix(user, event_id)
 
@@ -355,7 +387,7 @@ class MatrixHandler(BaseMatrixHandler):
         self.previously_typing[room_id] = now_typing
 
     def filter_matrix_event(self, evt: Event) -> bool:
-        if not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent)):
+        if not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent, EncryptedEvent)):
             return True
         return evt.sender and (evt.sender == self.az.bot_mxid
                                or pu.Puppet.get_id_from_mxid(evt.sender) is not None)
@@ -387,6 +419,11 @@ class MatrixHandler(BaseMatrixHandler):
             await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events)
         elif evt.type == EventType.ROOM_TOMBSTONE:
             await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room)
+        elif evt.type == EventType.ROOM_ENCRYPTION:
+            portal = po.Portal.get_by_mxid(evt.room_id)
+            if portal:
+                portal.encrypted = True
+                portal.save()
 
     async def log_event_handle_duration(self, evt: Event, duration: float) -> None:
         if EVENT_TIME:

mautrix_telegram/portal/__init__.pyi

@@ -1,8 +1,8 @@
 from typing import Union
 from .base import BasePortal
-from .portal_matrix import PortalMatrix
-from .portal_metadata import PortalMetadata
-from .portal_telegram import PortalTelegram
+from .matrix import PortalMatrix
+from .metadata import PortalMetadata
+from .telegram import PortalTelegram
 from ..context import Context
 
 Portal = Union[BasePortal, PortalMatrix, PortalMetadata, PortalTelegram]

mautrix_telegram/portal/base.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
+from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, Set, TYPE_CHECKING
 from abc import ABC, abstractmethod
 import asyncio
 import logging
@@ -35,7 +35,7 @@ from mautrix.util.simple_template import SimpleTemplate
 
 from ..types import TelegramID
 from ..context import Context
-from ..db import Portal as DBPortal
+from ..db import Portal as DBPortal, Message as DBMessage
 from .. import puppet as p, user as u, util
 from .deduplication import PortalDedup
 from .send_lock import PortalSendLock
@@ -44,6 +44,7 @@ if TYPE_CHECKING:
     from ..bot import Bot
     from ..abstract_user import AbstractUser
     from ..config import Config
+    from ..matrix import MatrixHandler
     from . import Portal
 
 TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
@@ -58,6 +59,7 @@ class BasePortal(ABC):
     az: AppService = None
     bot: 'Bot' = None
     loop: asyncio.AbstractEventLoop = None
+    matrix: 'MatrixHandler' = None
 
     # Config cache
     filter_mode: str = None
@@ -85,7 +87,10 @@ class BasePortal(ABC):
     about: Optional[str]
     photo_id: Optional[str]
     local_config: Dict[str, Any]
+    encrypted: bool
     deleted: bool
+    backfilling: bool
+    backfill_leave: Optional[Set[IntentAPI]]
     log: logging.Logger
 
     alias: Optional[RoomAlias]
@@ -100,7 +105,8 @@ class BasePortal(ABC):
                  mxid: Optional[RoomID] = None, username: Optional[str] = None,
                  megagroup: Optional[bool] = False, title: Optional[str] = None,
                  about: Optional[str] = None, photo_id: Optional[str] = None,
-                 local_config: Optional[str] = None, db_instance: DBPortal = None) -> None:
+                 local_config: Optional[str] = None, encrypted: Optional[bool] = False,
+                 db_instance: DBPortal = None) -> None:
         self.mxid = mxid
         self.tgid = tgid
         self.tg_receiver = tg_receiver or tgid
@@ -111,10 +117,13 @@ class BasePortal(ABC):
         self.about = about
         self.photo_id = photo_id
         self.local_config = json.loads(local_config or "{}")
+        self.encrypted = encrypted
         self._db_instance = db_instance
         self._main_intent = None
         self.deleted = False
         self.log = self.base_log.getChild(self.tgid_log if self.tgid else self.mxid)
+        self.backfilling = False
+        self.backfill_leave = None
 
         self.dedup = PortalDedup(self)
         self.send_lock = PortalSendLock()
@@ -273,8 +282,8 @@ class BasePortal(ABC):
                 authenticated.append(user)
         return authenticated
 
-    @staticmethod
-    async def cleanup_room(intent: IntentAPI, room_id: RoomID, message: str,
+    @classmethod
+    async def cleanup_room(cls, intent: IntentAPI, room_id: RoomID, message: str,
                            puppets_only: bool = False) -> None:
         try:
             members = await intent.get_room_members(room_id)
@@ -293,7 +302,7 @@ class BasePortal(ABC):
         try:
             await intent.leave_room(room_id)
         except (MatrixRequestError, IntentError):
-            self.log.warning("Failed to leave room when cleaning up room", exc_info=True)
+            cls.log.warning(f"Failed to leave room {room_id} when cleaning up room", exc_info=True)
 
     async def cleanup_portal(self, message: str, puppets_only: bool = False) -> None:
         if self.username:
@@ -324,12 +333,12 @@ class BasePortal(ABC):
         return DBPortal(tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
                         mxid=self.mxid, username=self.username, megagroup=self.megagroup,
                         title=self.title, about=self.about, photo_id=self.photo_id,
-                        config=json.dumps(self.local_config))
+                        config=json.dumps(self.local_config), encrypted=self.encrypted)
 
     def save(self) -> None:
         self.db_instance.edit(mxid=self.mxid, username=self.username, title=self.title,
                               about=self.about, photo_id=self.photo_id, megagroup=self.megagroup,
-                              config=json.dumps(self.local_config))
+                              config=json.dumps(self.local_config), encrypted=self.encrypted)
 
     def delete(self) -> None:
         try:
@@ -342,15 +351,16 @@ class BasePortal(ABC):
             pass
         if self._db_instance:
             self._db_instance.delete()
+        DBMessage.delete_all(self.mxid)
         self.deleted = True
 
     @classmethod
     def from_db(cls, db_portal: DBPortal) -> 'Portal':
         return cls(tgid=db_portal.tgid, tg_receiver=db_portal.tg_receiver,
-                   peer_type=db_portal.peer_type, mxid=db_portal.mxid,
-                   username=db_portal.username, megagroup=db_portal.megagroup,
-                   title=db_portal.title, about=db_portal.about, photo_id=db_portal.photo_id,
-                   local_config=db_portal.config, db_instance=db_portal)
+                   peer_type=db_portal.peer_type, mxid=db_portal.mxid, username=db_portal.username,
+                   megagroup=db_portal.megagroup, title=db_portal.title, about=db_portal.about,
+                   photo_id=db_portal.photo_id, local_config=db_portal.config,
+                   encrypted=db_portal.encrypted, db_instance=db_portal)
 
     # endregion
     # region Class instance lookup
@@ -491,12 +501,17 @@ class BasePortal(ABC):
                                    old_levels: Dict[UserID, int]) -> Awaitable[None]:
         pass
 
+    @abstractmethod
+    def backfill(self, source: 'AbstractUser') -> Awaitable[None]:
+        pass
+
     # endregion
 
 
 def init(context: Context) -> None:
     global config
     BasePortal.az, config, BasePortal.loop, BasePortal.bot = context.core
+    BasePortal.matrix = context.mx
     BasePortal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
     BasePortal.sync_channel_members = config["bridge.sync_channel_members"]
     BasePortal.sync_matrix_state = config["bridge.sync_matrix_state"]

mautrix_telegram/portal/matrix.py

@@ -50,6 +50,11 @@ if TYPE_CHECKING:
     from ..tgclient import MautrixTelegramClient
     from ..config import Config
 
+try:
+    from nio.crypto import decrypt_attachment
+except ImportError:
+    decrypt_attachment = None
+
 TypeMessage = Union[Message, MessageService]
 
 config: Optional['Config'] = None
@@ -250,11 +255,20 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
         file_name = content["net.maunium.telegram.internal.filename"]
         max_image_size = config["bridge.image_as_file_size"] * 1000 ** 2
 
-        if config["bridge.parallel_file_transfer"]:
+        if config["bridge.parallel_file_transfer"] and content.url:
             file_handle, file_size = await parallel_transfer_to_telegram(client, self.main_intent,
                                                                          content.url, sender_id)
         else:
-            file = await self.main_intent.download_media(content.url)
+            if content.file:
+                if not decrypt_attachment:
+                    self.log.warning(f"Can't bridge encrypted media event {event_id}:"
+                                     " matrix-nio not installed")
+                    return
+                file = await self.main_intent.download_media(content.file.url)
+                file = decrypt_attachment(file, content.file.key.key,
+                                          content.file.hashes.get("sha256"), content.file.iv)
+            else:
+                file = await self.main_intent.download_media(content.url)
 
             if content.msgtype == MessageType.STICKER:
                 if mime != "image/gif":

mautrix_telegram/portal/metadata.py

@@ -30,7 +30,7 @@ from telethon.tl.types import (
 
 from mautrix.errors import MForbidden
 from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership, Member,
-                           PowerLevelStateEventContent, RoomAlias)
+                           PowerLevelStateEventContent)
 from mautrix.appservice import IntentAPI
 
 from ..types import TelegramID
@@ -249,6 +249,9 @@ class PortalMetadata(BasePortal, ABC):
                                   ) -> Optional[RoomID]:
         direct = self.peer_type == "user"
 
+        if invites is None:
+            invites = []
+
         if self.mxid:
             return self.mxid
 
@@ -308,6 +311,17 @@ class PortalMetadata(BasePortal, ABC):
             "type": EventType.ROOM_POWER_LEVELS.serialize(),
             "content": power_levels.serialize(),
         }]
+        if config["bridge.encryption.default"] and self.matrix.e2ee:
+            self.encrypted = True
+            initial_state.append({
+                "type": "m.room.encryption",
+                "content": {"algorithm": "m.megolm.v1.aes-sha2"},
+            })
+            if direct:
+                invites.append(self.az.bot_mxid)
+                # The bridge bot needs to join for e2ee, but that messes up the default name
+                # generation. If/when canonical DMs happen, this might not be necessary anymore.
+                self.title = puppet.displayname
         if config["appservice.community_id"]:
             initial_state.append({
                 "type": "m.room.related_groups",
@@ -325,6 +339,16 @@ class PortalMetadata(BasePortal, ABC):
         if not room_id:
             raise Exception(f"Failed to create room")
 
+        if self.encrypted and self.matrix.e2ee:
+            members = [self.main_intent.mxid]
+            if direct:
+                try:
+                    await self.az.intent.join_room_by_id(room_id)
+                    members += [self.az.intent.mxid]
+                except Exception:
+                    self.log.warning(f"Failed to add bridge bot to new private chat {room_id}")
+            await self.matrix.e2ee.add_room(room_id, members=members, encrypted=True)
+
         self.mxid = RoomID(room_id)
         self.by_mxid[self.mxid] = self
         self.save()
@@ -362,7 +386,7 @@ class PortalMetadata(BasePortal, ABC):
             levels.kick = overrides.get("kick", 50)
             levels.redact = overrides.get("redact", 50)
             levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
-            levels.events[EventType.ROOM_ENCRYPTED] = 99
+            levels.events[EventType.ROOM_ENCRYPTION] = 99
             levels.events[EventType.ROOM_TOMBSTONE] = 99
             levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
             levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0
@@ -412,7 +436,7 @@ class PortalMetadata(BasePortal, ABC):
             return False
         changed = False
         admin_power_level = min(75 if self.peer_type == "channel" else 50, bot_level)
-        if levels.events[EventType.ROOM_POWER_LEVELS] != admin_power_level:
+        if levels.get_event_level(EventType.ROOM_POWER_LEVELS) != admin_power_level:
             changed = True
             levels.events[EventType.ROOM_POWER_LEVELS] = admin_power_level
 

mautrix_telegram/portal/telegram.py

@@ -14,7 +14,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Awaitable, Dict, List, Optional, Tuple, Union, NamedTuple, TYPE_CHECKING
-from html import escape as escape_html
 from abc import ABC
 import random
 import mimetypes
@@ -30,16 +29,16 @@ from telethon.tl.types import (
     MessageMediaPoll, MessageActionChannelCreate, MessageActionChatAddUser,
     MessageActionChatCreate, MessageActionChatDeletePhoto, MessageActionChatDeleteUser,
     MessageActionChatEditPhoto, MessageActionChatEditTitle, MessageActionChatJoinedByLink,
-    MessageActionChatMigrateTo, MessageActionPinMessage, MessageActionGameScore,
-    MessageMediaDocument, MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported,
-    MessageMediaGame, PeerUser, PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant,
-    TypeDocumentAttribute, TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping,
-    UpdateUserTyping, MessageEntityPre, ChatPhotoEmpty)
+    MessageActionChatMigrateTo, MessageActionGameScore, MessageMediaDocument, MessageMediaGeo,
+    MessageMediaPhoto, MessageMediaDice, MessageMediaGame, MessageMediaUnsupported, PeerUser,
+    PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant, TypeDocumentAttribute,
+    TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping, UpdateUserTyping,
+    MessageEntityPre, ChatPhotoEmpty)
 
 from mautrix.appservice import IntentAPI
 from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType,
                            EventType, MediaMessageEventContent, TextMessageEventContent,
-                           LocationMessageEventContent, Format)
+                           LocationMessageEventContent, Format, MessageEventContent)
 
 from ..types import TelegramID
 from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
@@ -72,10 +71,19 @@ class PortalTelegram(BasePortal, ABC):
             return f"https://t.me/c/{self.tgid}/{evt.id}"
         return None
 
+    async def _send_message(self, intent: IntentAPI, content: MessageEventContent,
+                            event_type: EventType = EventType.ROOM_MESSAGE, **kwargs) -> EventID:
+        if self.encrypted and self.matrix.e2ee:
+            if intent.api.is_real_user:
+                content[intent.api.real_user_content_key] = True
+            event_type, content = await self.matrix.e2ee.encrypt(self.mxid, event_type, content)
+        return await intent.send_message_event(self.mxid, event_type, content, **kwargs)
+
     async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                     relates_to: Dict = None) -> Optional[EventID]:
         loc, largest_size = self._get_largest_photo_size(evt.media.photo)
-        file = await util.transfer_file_to_matrix(source.client, intent, loc)
+        file = await util.transfer_file_to_matrix(source.client, intent, loc,
+                                                  encrypt=self.encrypted)
         if not file:
             return None
         if self.get_config("inline_images") and (evt.message
@@ -86,22 +94,26 @@ class PortalTelegram(BasePortal, ABC):
                 prefix_text="Inline image: ")
             content.external_url = self._get_external_url(evt)
             await intent.set_typing(self.mxid, is_typing=False)
-            return await intent.send_message(self.mxid, content, timestamp=evt.date)
+            return await self._send_message(intent, content, timestamp=evt.date)
         info = ImageInfo(
             height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type,
             size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize))
                   else largest_size.size))
         name = f"image{sane_mimetypes.guess_extension(file.mime_type)}"
         await intent.set_typing(self.mxid, is_typing=False)
-        content = MediaMessageEventContent(url=file.mxc, msgtype=MessageType.IMAGE, info=info,
+        content = MediaMessageEventContent(msgtype=MessageType.IMAGE, info=info,
                                            body=name, relates_to=relates_to,
                                            external_url=self._get_external_url(evt))
-        result = await intent.send_message(self.mxid, content, timestamp=evt.date)
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+        result = await self._send_message(intent, content, timestamp=evt.date)
         if evt.message:
             caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
                                                                  no_reply_fallback=True)
             caption_content.external_url = content.external_url
-            result = await intent.send_message(self.mxid, caption_content, timestamp=evt.date)
+            result = await self._send_message(intent, caption_content, timestamp=evt.date)
         return result
 
     @staticmethod
@@ -134,6 +146,8 @@ class PortalTelegram(BasePortal, ABC):
         generic_types = ("text/plain", "application/octet-stream")
         if file.mime_type in generic_types and document.mime_type not in generic_types:
             mime_type = document.mime_type or file.mime_type
+        elif file.mime_type == 'application/ogg':
+            mime_type = 'audio/ogg'
         else:
             mime_type = file.mime_type or document.mime_type
         info = ImageInfo(size=file.size, mimetype=mime_type)
@@ -146,11 +160,21 @@ class PortalTelegram(BasePortal, ABC):
             info.width, info.height = attrs.width, attrs.height
 
         if file.thumbnail:
-            info.thumbnail_url = file.thumbnail.mxc
+            if file.thumbnail.decryption_info:
+                info.thumbnail_file = file.thumbnail.decryption_info
+            else:
+                info.thumbnail_url = file.thumbnail.mxc
             info.thumbnail_info = ThumbnailInfo(mimetype=file.thumbnail.mime_type,
                                                 height=file.thumbnail.height or thumb_size.h,
                                                 width=file.thumbnail.width or thumb_size.w,
                                                 size=file.thumbnail.size)
+        else:
+            # This is a hack for bad clients like Riot iOS that require a thumbnail
+            if file.decryption_info:
+                info.thumbnail_file = file.decryption_info
+            else:
+                info.thumbnail_url = file.mxc
+            info.thumbnail_info = ImageInfo.deserialize(info.serialize())
 
         return info, name
 
@@ -164,6 +188,7 @@ class PortalTelegram(BasePortal, ABC):
         if document.size > config["bridge.max_document_size"] * 1000 ** 2:
             name = attrs.name or ""
             caption = f"\n{evt.message}" if evt.message else ""
+            # TODO encrypt
             return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
 
         thumb_loc, thumb_size = self._get_largest_photo_size(document)
@@ -175,7 +200,8 @@ class PortalTelegram(BasePortal, ABC):
         file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
                                                   is_sticker=attrs.is_sticker,
                                                   tgs_convert=config["bridge.animated_sticker"],
-                                                  filename=attrs.name, parallel_id=parallel_id)
+                                                  filename=attrs.name, parallel_id=parallel_id,
+                                                  encrypt=self.encrypted)
         if not file:
             return None
 
@@ -188,14 +214,18 @@ class PortalTelegram(BasePortal, ABC):
         if attrs.is_sticker and file.mime_type.startswith("image/"):
             event_type = EventType.STICKER
         content = MediaMessageEventContent(
-            body=name or "unnamed file", info=info, url=file.mxc, relates_to=relates_to,
+            body=name or "unnamed file", info=info, relates_to=relates_to,
             external_url=self._get_external_url(evt),
             msgtype={
                 "video/": MessageType.VIDEO,
                 "audio/": MessageType.AUDIO,
                 "image/": MessageType.IMAGE,
             }.get(info.mimetype[:6], MessageType.FILE))
-        return await intent.send_message_event(self.mxid, event_type, content, timestamp=evt.date)
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+        return await self._send_message(intent, content, event_type=event_type, timestamp=evt.date)
 
     def handle_telegram_location(self, _: 'AbstractUser', intent: IntentAPI, evt: Message,
                                  relates_to: dict = None) -> Awaitable[EventID]:
@@ -214,7 +244,7 @@ class PortalTelegram(BasePortal, ABC):
         content["format"] = str(Format.HTML)
         content["formatted_body"] = f"Location: <a href='{url}'>{body}</a>"
 
-        return intent.send_message(self.mxid, content, timestamp=evt.date)
+        return self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_text(self, source: 'AbstractUser', intent: IntentAPI, is_bot: bool,
                                    evt: Message) -> EventID:
@@ -224,7 +254,7 @@ class PortalTelegram(BasePortal, ABC):
         if is_bot and self.get_config("bot_messages_as_notices"):
             content.msgtype = MessageType.NOTICE
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
                                           evt: Message, relates_to: dict = None) -> EventID:
@@ -237,7 +267,7 @@ class PortalTelegram(BasePortal, ABC):
         content.external_url = self._get_external_url(evt)
         content["net.maunium.telegram.unsupported"] = True
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_poll(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                    relates_to: RelatesTo) -> EventID:
@@ -263,7 +293,22 @@ class PortalTelegram(BasePortal, ABC):
             relates_to=relates_to, external_url=self._get_external_url(evt))
 
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
+
+    async def handle_telegram_dice(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
+                                   relates_to: RelatesTo) -> EventID:
+        emoji_text = {
+            "\U0001F3AF": " Dart throw",
+            "\U0001F3B2": " Dice roll",
+        }
+        roll: MessageMediaDice = evt.media
+        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {roll.value}"
+        content = TextMessageEventContent(msgtype=MessageType.TEXT, format=Format.HTML, body=text,
+                                          formatted_body=f"<h4>{text}</h4>", relates_to=relates_to,
+                                          external_url=self._get_external_url(evt))
+        content["net.maunium.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     @staticmethod
     def _int_to_bytes(i: int) -> bytes:
@@ -305,7 +350,7 @@ class PortalTelegram(BasePortal, ABC):
         content["net.maunium.telegram.game"] = play_id
 
         await intent.set_typing(self.mxid, is_typing=False)
-        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        return await self._send_message(intent, content, timestamp=evt.date)
 
     async def handle_telegram_edit(self, source: 'AbstractUser', sender: p.Puppet, evt: Message
                                    ) -> None:
@@ -349,13 +394,43 @@ class PortalTelegram(BasePortal, ABC):
 
         intent = sender.intent_for(self) if sender else self.main_intent
         await intent.set_typing(self.mxid, is_typing=False)
-        event_id = await intent.send_message(self.mxid, content)
+        event_id = await self._send_message(intent, content)
 
         prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
         DBMessage(mxid=event_id, mx_room=self.mxid, tg_space=tg_space, tgid=TelegramID(evt.id),
                   edit_index=prev_edit_msg.edit_index + 1).insert()
         DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=event_id)
 
+    async def backfill(self, source: 'AbstractUser') -> None:
+        last = DBMessage.find_last(self.mxid, (source.tgid if self.peer_type != "channel"
+                                               else self.tgid))
+        min_id = last.tgid if last else 0
+        self.backfilling = True
+        self.backfill_leave = set()
+        if self.peer_type == "user":
+            sender = p.Puppet.get(source.tgid)
+            await self.main_intent.invite_user(self.mxid, sender.default_mxid)
+            await sender.default_mxid_intent.join_room_by_id(self.mxid)
+            self.backfill_leave.add(sender.default_mxid_intent)
+        max_file_size = min(config["bridge.max_document_size"], 1500) * 1024 * 1024
+        async with source.client.takeout(files=True, megagroups=self.megagroup,
+                                         chats=self.peer_type == "chat",
+                                         users=self.peer_type == "user",
+                                         channels=(self.peer_type == "channel"
+                                                   and not self.megagroup),
+                                         max_file_size=max_file_size
+                                         ) as takeout_client:
+            async for message in takeout_client.iter_messages(await self.get_input_entity(source),
+                                                              reverse=True, min_id=min_id):
+                sender = p.Puppet.get(message.sender_id)
+                # if isinstance(message, MessageService):
+                #    await self.handle_telegram_action(source, sender, message)
+                await self.handle_telegram_message(source, sender, message)
+        for intent in self.backfill_leave:
+            await intent.leave_room(self.mxid)
+        self.backfilling = False
+        self.backfill_leave = None
+
     async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
                                       evt: Message) -> None:
         if not self.mxid:
@@ -383,7 +458,7 @@ class PortalTelegram(BasePortal, ABC):
                               tg_space=tg_space, edit_index=0).insert()
                 return
 
-        if self.dedup.pre_db_check and self.peer_type == "channel":
+        if self.backfilling or (self.dedup.pre_db_check and self.peer_type == "channel"):
             msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
             if msg:
                 self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
@@ -399,10 +474,17 @@ class PortalTelegram(BasePortal, ABC):
             await sender.update_info(source, entity)
 
         allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
-                         MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
+                         MessageMediaGame, MessageMediaDice, MessageMediaPoll,
+                         MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
-        intent = sender.intent_for(self) if sender else self.main_intent
+        if sender:
+            intent = sender.intent_for(self)
+            if self.backfilling and intent != sender.default_mxid_intent:
+                intent = sender.default_mxid_intent
+                self.backfill_leave.add(intent)
+        else:
+            intent = self.main_intent
         if not media and evt.message:
             is_bot = sender.is_bot if sender else False
             event_id = await self.handle_telegram_text(source, intent, is_bot, evt)
@@ -412,6 +494,7 @@ class PortalTelegram(BasePortal, ABC):
                 MessageMediaDocument: self.handle_telegram_document,
                 MessageMediaGeo: self.handle_telegram_location,
                 MessageMediaPoll: self.handle_telegram_poll,
+                MessageMediaDice: self.handle_telegram_dice,
                 MessageMediaUnsupported: self.handle_telegram_unsupported,
                 MessageMediaGame: self.handle_telegram_game,
             }[type(media)](source, intent, evt,
@@ -482,6 +565,7 @@ class PortalTelegram(BasePortal, ABC):
         elif isinstance(action, MessageActionChatMigrateTo):
             self.peer_type = "channel"
             self._migrate_and_save_telegram(TelegramID(action.channel_id))
+            # TODO encrypt
             await sender.intent_for(self).send_emote(self.mxid,
                                                      "upgraded this group to a supergroup.")
         elif isinstance(action, MessageActionGameScore):
@@ -502,7 +586,7 @@ class PortalTelegram(BasePortal, ABC):
         await self.main_intent.set_power_levels(self.mxid, levels)
 
     async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
-        tg_space = receiver  if self.peer_type != "channel" else self.tgid
+        tg_space = receiver if self.peer_type != "channel" else self.tgid
         message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
         if message:
             await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])

mautrix_telegram/puppet.py

@@ -25,7 +25,7 @@ from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser,
 from mautrix.appservice import AppService, IntentAPI
 from mautrix.errors import MatrixRequestError
 from mautrix.bridge import CustomPuppetMixin
-from mautrix.types import UserID, SyncToken
+from mautrix.types import UserID, SyncToken, RoomID
 from mautrix.util.simple_template import SimpleTemplate
 
 from .types import TelegramID
@@ -320,6 +320,10 @@ class Puppet(CustomPuppetMixin):
                 return True
         return False
 
+    def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
+        portal: p.Portal = p.Portal.get_by_mxid(room_id)
+        return portal and not portal.backfilling and portal.peer_type != "user"
+
     # endregion
     # region Getters
 
@@ -368,7 +372,7 @@ class Puppet(CustomPuppetMixin):
 
     @classmethod
     def all_with_custom_mxid(cls) -> Iterable['Puppet']:
-        return (cls.by_custom_mxid[puppet.mxid]
+        return (cls.by_custom_mxid[puppet.custom_mxid]
                 if puppet.custom_mxid in cls.by_custom_mxid
                 else cls.from_db(puppet)
                 for puppet in DBPuppet.all_with_custom_mxid())

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -24,7 +24,8 @@ def log(message, end="\n"):
 
 
 def connect(to):
-    from mautrix.bridge.db import Base, RoomState, UserProfile
+    from mautrix.util.db import Base
+    from mautrix.bridge.db import RoomState, UserProfile
     from mautrix_telegram.db import (Portal, Message, UserPortal, User, Contact, Puppet, BotChat,
                                      TelegramFile)
 

mautrix_telegram/util/file_transfer.py

@@ -29,12 +29,13 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio
                              SecurityError, FileIdInvalidError)
 
 from mautrix.appservice import IntentAPI
-
+from mautrix.types import EncryptedFile
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
 from ..util import sane_mimetypes
 from .parallel_file_transfer import parallel_transfer_to_matrix
+from .tgs_converter import convert_tgs_to
 
 try:
     from PIL import Image
@@ -50,7 +51,10 @@ try:
 except ImportError:
     VideoFileClip = random = string = os = mimetypes = None
 
-from .tgs_converter import convert_tgs_to
+try:
+    from nio.crypto import encrypt_attachment
+except ImportError:
+    encrypt_attachment = None
 
 log: logging.Logger = logging.getLogger("mau.util")
 
@@ -116,8 +120,8 @@ def _location_to_id(location: TypeLocation) -> str:
 
 
 async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                       thumbnail_loc: TypeLocation, video: bytes,
-                                       mime: str) -> Optional[DBTelegramFile]:
+                                       thumbnail_loc: TypeLocation, video: bytes, mime: str,
+                                       encrypt: bool) -> Optional[DBTelegramFile]:
     if not Image or not VideoFileClip:
         return None
 
@@ -141,11 +145,19 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
         width, height = None, None
         mime_type = magic.from_buffer(file, mime=True)
 
-    content_uri = await intent.upload_media(file, mime_type)
+    decryption_info = None
+    upload_mime_type = mime_type
+    if encrypt:
+        file, decryption_info_dict = encrypt_attachment(file)
+        decryption_info = EncryptedFile.deserialize(decryption_info_dict)
+        upload_mime_type = "application/octet-stream"
+    content_uri = await intent.upload_media(file, upload_mime_type)
+    if decryption_info:
+        decryption_info.url = content_uri
 
     db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
                              was_converted=False, timestamp=int(time.time()), size=len(file),
-                             width=width, height=height)
+                             width=width, height=height, decryption_info=decryption_info)
     try:
         db_file.insert()
     except (IntegrityError, InvalidRequestError) as e:
@@ -161,10 +173,10 @@ TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
 
 
 async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None, *,
                                   is_sticker: bool = False, tgs_convert: Optional[dict] = None,
-                                  filename: Optional[str] = None, parallel_id: Optional[int] = None
-                                  ) -> Optional[DBTelegramFile]:
+                                  filename: Optional[str] = None, encrypt: bool = False,
+                                  parallel_id: Optional[int] = None) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
         return None
@@ -181,14 +193,14 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA
     async with lock:
         return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
                                                        thumbnail, is_sticker, tgs_convert,
-                                                       filename, parallel_id)
+                                                       filename, encrypt, parallel_id)
 
 
 async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                             loc_id: str, location: TypeLocation,
                                             thumbnail: TypeThumbnail, is_sticker: bool,
                                             tgs_convert: Optional[dict], filename: Optional[str],
-                                            parallel_id: Optional[int]
+                                            encrypt: bool, parallel_id: Optional[int]
                                             ) -> Optional[DBTelegramFile]:
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
@@ -196,7 +208,7 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
 
     if parallel_id and isinstance(location, Document) and (not is_sticker or not tgs_convert):
         db_file = await parallel_transfer_to_matrix(client, intent, loc_id, location, filename,
-                                                    parallel_id)
+                                                    encrypt, parallel_id)
         mime_type = location.mime_type
         file = None
     else:
@@ -214,8 +226,8 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
         image_converted = False
         # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
         if is_sticker and tgs_convert and (mime_type == "application/gzip" or (
-               mime_type == "application/octet-stream"
-               and magic.from_buffer(file).startswith("gzip"))):
+            mime_type == "application/octet-stream"
+            and magic.from_buffer(file).startswith("gzip"))):
             mime_type, file, width, height = await convert_tgs_to(
                 file, tgs_convert["target"], **tgs_convert["args"])
             thumbnail = None
@@ -229,17 +241,28 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
             mime_type = new_mime_type
             thumbnail = None
 
-        content_uri = await intent.upload_media(file, mime_type)
+        decryption_info = None
+        upload_mime_type = mime_type
+        if encrypt and encrypt_attachment:
+            file, decryption_info_dict = encrypt_attachment(file)
+            decryption_info = EncryptedFile.deserialize(decryption_info_dict)
+            upload_mime_type = "application/octet-stream"
+        content_uri = await intent.upload_media(file, upload_mime_type)
+        if decryption_info:
+            decryption_info.url = content_uri
 
-        db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
+        db_file = DBTelegramFile(id=loc_id, mxc=content_uri, decryption_info=decryption_info,
                                  mime_type=mime_type, was_converted=image_converted,
                                  timestamp=int(time.time()), size=len(file),
                                  width=width, height=height)
     if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
         if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
             thumbnail = thumbnail.location
-        db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
-                                                               mime_type)
+        try:
+            db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
+                                                                   mime_type, encrypt)
+        except FileIdInvalidError:
+            log.warning(f"Failed to transfer thumbnail for {thumbnail!s}", exc_info=True)
 
     try:
         db_file.insert()

mautrix_telegram/util/parallel_file_transfer.py

@@ -34,11 +34,16 @@ from telethon.crypto import AuthKey
 from telethon import utils, helpers
 
 from mautrix.appservice import IntentAPI
-from mautrix.types import ContentURI
+from mautrix.types import ContentURI, EncryptedFile
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
 
+try:
+    from nio.crypto import async_encrypt_attachment
+except ImportError:
+    async_encrypt_attachment = None
+
 log: logging.Logger = logging.getLogger("mau.util")
 
 TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
@@ -242,18 +247,34 @@ parallel_transfer_locks: DefaultDict[int, asyncio.Lock] = defaultdict(lambda: as
 
 async def parallel_transfer_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                       loc_id: str, location: TypeLocation, filename: str,
-                                      parallel_id: int) -> DBTelegramFile:
+                                      encrypt: bool, parallel_id: int) -> DBTelegramFile:
     size = location.size
     mime_type = location.mime_type
     dc_id, location = utils.get_input_location(location)
     # We lock the transfers because telegram has connection count limits
     async with parallel_transfer_locks[parallel_id]:
         downloader = ParallelTransferrer(client, dc_id)
-        content_uri = await intent.upload_media(downloader.download(location, size),
-                                                mime_type=mime_type, filename=filename, size=size)
+        data = downloader.download(location, size)
+        decryption_info = None
+        up_mime_type = mime_type
+        if encrypt and async_encrypt_attachment:
+            async def encrypted(stream):
+                nonlocal decryption_info
+                async for chunk in async_encrypt_attachment(stream):
+                    if isinstance(chunk, dict):
+                        decryption_info = EncryptedFile.deserialize(chunk)
+                    else:
+                        yield chunk
+
+            data = encrypted(data)
+            up_mime_type = "application/octet-stream"
+        content_uri = await intent.upload_media(data, mime_type=up_mime_type, filename=filename,
+                                                size=size if not encrypt else None)
+        if decryption_info:
+            decryption_info.url = content_uri
     return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
                           was_converted=False, timestamp=int(time.time()), size=size,
-                          width=None, height=None)
+                          width=None, height=None, decryption_info=decryption_info)
 
 
 async def _internal_transfer_to_telegram(client: MautrixTelegramClient, response: ClientResponse

mautrix_telegram/web/provisioning/__init__.py

@@ -315,9 +315,9 @@ class ProvisioningAPI(AuthAPI):
 
         if not user.is_bot:
             return web.json_response([{
-                "id": get_peer_id(chat),
+                "id": chat.id,
                 "title": chat.title,
-            } async for chat in user.client.get_dialogs(ignore_migrated=True, archived=False)])
+            } async for chat in user.client.iter_dialogs(ignore_migrated=True, archived=False)])
         else:
             return web.json_response([{
                 "id": get_peer_id(chat.peer),
@@ -355,6 +355,7 @@ class ProvisioningAPI(AuthAPI):
         if err is not None:
             return err
         await user.log_out()
+        return web.json_response({}, status=200)
 
     async def bridge_info(self, request: web.Request) -> web.Response:
         return web.json_response({

optional-requirements.txt

@@ -1,5 +1,23 @@
-cryptg
-Pillow
-moviepy
-prometheus_client
-psycopg2-binary
+# Format: #/name defines a new extras_require group called name
+# Uncommented lines after the group definition insert things into that group.
+
+#/speedups
+cryptg>=0.1,<0.3
+cchardet
+aiodns
+brotli
+
+#/webp_convert
+pillow>=4.3,<8
+
+#/hq_thumbnails
+moviepy>=1,<2
+
+#/metrics
+prometheus_client>=0.6,<0.8
+
+#/postgres
+psycopg2-binary>=2,<3
+
+#/e2be
+matrix-nio[e2e]>=0.9,<0.11

requirements.txt

@@ -1,9 +1,9 @@
-aiohttp
-mautrix
-ruamel.yaml
-python-magic
-SQLAlchemy
-alembic
-commonmark
-telethon
-telethon-session-sqlalchemy
+SQLAlchemy>=1.2,<2
+alembic>=1,<2
+ruamel.yaml>=0.15.35,<0.17
+python-magic>=0.4,<0.5
+commonmark>=0.8,<0.10
+aiohttp>=3,<4
+mautrix==0.5.0.beta13
+telethon>=1.13,<1.14
+telethon-session-sqlalchemy>=0.2.14,<0.3

setup.py

@@ -3,14 +3,21 @@ import glob
 
 from mautrix_telegram.get_version import git_tag, git_revision, version, linkified_version
 
-extras = {
-    "speedups": ["cryptg>=0.1,<0.3", "cchardet", "aiodns", "Brotli"],
-    "webp_convert": ["Pillow>=4.3.0,<7"],
-    "hq_thumbnails": ["moviepy>=1.0,<2.0"],
-    "metrics": ["prometheus_client>=0.6.0,<0.8.0"],
-    "postgres": ["psycopg2-binary>=2,<3"],
-}
-extras["all"] = list({dep for deps in extras.values() for dep in deps})
+with open("requirements.txt") as reqs:
+    install_requires = reqs.read().splitlines()
+
+with open("optional-requirements.txt") as reqs:
+    extras_require = {}
+    current = []
+    for line in reqs.read().splitlines():
+        if line.startswith("#/"):
+            extras_require[line[2:]] = current = []
+        elif not line or line.startswith("#"):
+            continue
+        else:
+            current.append(line)
+
+extras_require["all"] = list({dep for deps in extras_require.values() for dep in deps})
 
 try:
     long_desc = open("README.md").read()
@@ -40,18 +47,8 @@ setuptools.setup(
 
     packages=setuptools.find_packages(),
 
-    install_requires=[
-        "aiohttp>=3.0.1,<4",
-        "mautrix>=0.4.0,<0.5",
-        "SQLAlchemy>=1.2.3,<2",
-        "alembic>=1.0.0,<2",
-        "commonmark>=0.8.1,<0.10",
-        "ruamel.yaml>=0.15.35,<0.17",
-        "python-magic>=0.4.15,<0.5",
-        "telethon>=1.10,<1.11",
-        "telethon-session-sqlalchemy>=0.2.14,<0.3",
-    ],
-    extras_require=extras,
+    install_requires=install_requires,
+    extras_require=extras_require,
     python_requires="~=3.6",
 
     setup_requires=["pytest-runner"],
@@ -74,9 +71,10 @@ setuptools.setup(
     """,
     package_data={"mautrix_telegram": [
         "web/public/*.mako", "web/public/*.png", "web/public/*.css",
+        "example-config.yaml",
     ]},
     data_files=[
-        (".", ["example-config.yaml", "alembic.ini"]),
+        (".", ["alembic.ini"]),
         ("alembic", ["alembic/env.py"]),
         ("alembic/versions", glob.glob("alembic/versions/*.py"))
     ],