Bump version to 0.12.2

Fixes #867

.github/ISSUE_TEMPLATE/bug.md

@@ -0,0 +1,7 @@
+---
+name: Bug report
+about: If something is definitely wrong in the bridge (rather than just a setup issue),
+  file a bug report. Remember to include relevant logs.
+labels: bug
+
+---

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,7 @@
+contact_links:
+  - name: Troubleshooting docs & FAQ
+    url: https://docs.mau.fi/bridges/general/troubleshooting.html
+    about: Check this first if you're having problems setting up the bridge.
+  - name: Support room
+    url: https://matrix.to/#/#telegram:maunium.net
+    about: For setup issues not answered by the troubleshooting docs, ask in the Matrix room.

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,6 @@
+---
+name: Enhancement request
+about: Submit a feature request or other suggestion
+labels: enhancement
+
+---

.gitignore

@@ -14,5 +14,8 @@ __pycache__
 /registration.yaml
 *.log*
 *.db
-*.pickle
+/*.pickle
 *.bak
+/*.session
+/*.session-journal
+/*.json

.gitlab-ci.yml

@@ -1,66 +1,3 @@
-image: docker:stable
-
-stages:
-- build
-- manifest
-
-default:
-  before_script:
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-
-build amd64:
-  stage: build
-  tags:
-  - amd64
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  after_script:
-  - |
-    if [[ "$CI_COMMIT_BRANCH" == "master" && "$CI_JOB_STATUS" == "success" ]]; then
-      apk add --update curl jq
-      rm -rf /var/cache/apk/*
-
-      jq -n '
-         {
-           password:  env.BEEPER_DEV_ADMIN_NIGHTLY_PASS,
-           bridge:    env.BEEPER_BRIDGE_TYPE,
-           image:     "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
-           channel:   "STABLE"
-         }
-        ' | curl "$BEEPER_DEV_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
-
-      jq -n '
-         {
-           password:    env.BEEPER_PROD_ADMIN_NIGHTLY_PASS,
-           bridge:      env.BEEPER_BRIDGE_TYPE,
-           image:       "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
-           channel:     "INTERNAL",
-           deployNext:  true
-         }
-         ' | curl "$BEEPER_PROD_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
-    fi
-
-build arm64:
-  stage: build
-  tags:
-  - arm64
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-
-manifest:
-  stage: manifest
-  before_script:
-  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
-  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+include:
+- project: 'mautrix/ci'
+  file: '/python.yml'

CHANGELOG.md

@@ -1,3 +1,119 @@
+# v0.12.2 (2022-11-26)
+
+### Added
+* Added built-in custom emoji packs to allow reacting with any standard unicode
+  emoji from Matrix (note that only premium users can use custom emojis).
+* Added infinite backfill using [MSC2716].
+  * The new system includes a backwards compatibility mechanism which uses the
+    old method of just sending events to the room. By default, MSC2716 is not
+    enabled and the legacy method will be used.
+
+### Improved
+* Redacting reactions on Matrix no longer removes the user's other reactions to
+  the same message (premium users can have up to 3 reactions per message).
+* Changes to default user permissions on Telegram are now bridged.
+* Added database index to make reaction polling more efficient
+  (thanks to [@AndrewFerr] in [#862]).
+
+### Fixed
+* Fixed provisioning API not working with URL-encoded parameters.
+
+[MSC2716]: https://github.com/matrix-org/matrix-spec-proposals/pull/2716
+[@AndrewFerr]: https://github.com/AndrewFerr
+[#862]: https://github.com/mautrix/telegram/pull/862
+
+# v0.12.1 (2022-09-26)
+
+### Added
+* Support for custom emojis in reactions.
+  * Like other bridges with custom emoji reactions, they're bridged as `mxc://`
+    URIs, so client support is required to render them properly.
+
+### Improved
+* The bridge will now poll for reactions to 20 most recent messages when
+  receiving a read receipt. This works around Telegram's bad protocol that
+  doesn't notify clients on reactions to other users' messages.
+* The docker image now has an option to bypass the startup script by setting
+  the `MAUTRIX_DIRECT_STARTUP` environment variable. Additionally, it will
+  refuse to run as a non-root user if that variable is not set (and print an
+  error message suggesting to either set the variable or use a custom command).
+* Moved environment variable overrides for config fields to mautrix-python.
+  The new system also allows loading JSON values to enable overriding maps like
+  `login_shared_secret_map`.
+
+### Fixed
+* `ChatParticipantsForbidden` is handled properly when syncing non-supergroup
+  info.
+* Fixed some bugs with file transfers when using SQLite.
+* Fixed error when attempting to log in again after logging out.
+* Fixed QR login not working.
+* Fixed error syncing chats if bridging a message had previously been
+  interrupted.
+
+# v0.12.0 (2022-08-26)
+
+**N.B.** This release requires a homeserver with Matrix v1.1 support, which
+bumps up the minimum homeserver versions to Synapse 1.54 and Dendrite 0.8.7.
+Minimum Conduit version remains at 0.4.0.
+
+### Added
+* Added provisioning API for resolving Telegram identifiers (like usernames).
+* Added support for bridging Telegram custom emojis to Matrix.
+* Added option to not bridge chats with lots of members.
+* Added option to include captions in the same message as the media to
+  implement [MSC2530]. Sending captions the same way is also supported and
+  enabled by default.
+* Added commands to kick or ban relaybot users from Telegram.
+* Added support for Telegram's disappearing messages.
+* Added support for bridging forwarded messages as forwards on Telegram.
+  * Forwarding is not allowed in relay mode as the bot wouldn't be able to
+    specify who sent the message.
+  * Matrix doesn't have real forwarding (there's no forwarding metadata), so
+    only messages bridged from Telegram can be forwarded.
+  * Double puppeted messages from Telegram currently can't be forwarded without
+    removing the `fi.mau.double_puppet_source` key from the content.
+  * If forwarding fails (e.g. due to it being blocked in the source chat), the
+    bridge will automatically fall back to sending it as a normal new message.
+* Added options to make encryption more secure.
+  * The `encryption` -> `verification_levels` config options can be used to
+    make the bridge require encrypted messages to come from cross-signed
+    devices, with trust-on-first-use validation of the cross-signing master
+    key.
+  * The `encryption` -> `require` option can be used to make the bridge ignore
+    any unencrypted messages.
+  * Key rotation settings can be configured with the `encryption` -> `rotation`
+    config.
+
+### Improved
+* Improved handling the bridge user leaving chats on Telegram, and new users
+  being added on Telegram.
+* Improved animated sticker conversion options: added support for animated webp
+  and added option to convert video stickers (webm) to the specified image
+  format.
+* Audio and video metadata is now bridged properly to Telegram.
+* Added database index on Telegram usernames (used when bridging username
+  @-mentions in messages).
+* Changed `/login/send_code` provisioning API to return a proper error when the
+  phone number is not registered on Telegram.
+  * The same login code can be used for registering an account, but registering
+    is not currently supported in the provisioning API.
+* Removed `plaintext_highlights` config option (the code using it was already
+  removed in v0.11.0).
+* Enabled appservice ephemeral events by default for new installations.
+  * Existing bridges can turn it on by enabling `ephemeral_events` and disabling
+    `sync_with_custom_puppets` in the config, then regenerating the registration
+    file.
+* Updated to API layer 144 so that Telegram would send new message types like
+  premium stickers to the bridge.
+* Updated Docker image to Alpine 3.16 and made it smaller.
+
+### Fixed
+* Fixed command prefix in game and poll messages (thanks to [@cynhr] in [#804]).
+
+[MSC2530]: https://github.com/matrix-org/matrix-spec-proposals/pull/2530
+[@cynhr]: https://github.com/cynhr
+[#804]: https://github.com/mautrix/telegram/pull/804
+
 # v0.11.3 (2022-04-17)
 
 **N.B.** This release drops support for old homeservers which don't support the

Dockerfile

@@ -1,22 +1,23 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.15
-
-ARG TARGETARCH=amd64
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.16
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
-      py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-magic \
       py3-ruamel.yaml \
       py3-commonmark \
-      py3-prometheus-client \
+      py3-phonenumbers \
+      py3-mako \
+      #py3-prometheus-client \ (pulls in twisted unnecessarily)
       # Indirect dependencies
       py3-idna \
+      py3-rsa \
       #moviepy
         py3-decorator \
         py3-tqdm \
         py3-requests \
+        #py3-proglog \
         #imageio
           py3-numpy \
       #py3-telethon \ (outdated)
@@ -25,7 +26,7 @@ RUN apk add --no-cache \
         py3-pyaes \
         # cryptg
           py3-cffi \
-	  py3-qrcode \
+          py3-qrcode \
       py3-brotli \
       # Other dependencies
       ffmpeg \
@@ -46,13 +47,15 @@ COPY requirements.txt /opt/mautrix-telegram/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
 WORKDIR /opt/mautrix-telegram
 RUN apk add --virtual .build-deps python3-dev libffi-dev build-base \
- && pip3 install -r requirements.txt -r optional-requirements.txt \
- && apk del .build-deps
+ && pip3 install /cryptg-*.whl \
+ && pip3 install --no-cache-dir -r requirements.txt -r optional-requirements.txt \
+ && apk del .build-deps \
+ && rm -f /cryptg-*.whl
 
 COPY . /opt/mautrix-telegram
-RUN apk add git && pip3 install .[all] && apk del git \
+RUN apk add git && pip3 install --no-cache-dir .[all] && apk del git \
   # This doesn't make the image smaller, but it's needed so that the `version` command works properly
-  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
+  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram .git build
 
 VOLUME /data
 ENV UID=1337 GID=1337 \

README.md

@@ -11,13 +11,13 @@ A Matrix-Telegram hybrid puppeting/relaybot bridge.
 ## Sponsors
 * [Joel Lehtonen / Zouppen](https://github.com/zouppen)
 
-### Documentation
+## Documentation
 All setup and usage instructions are located on
 [docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
 Some quick links:
 
-* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
-  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
+* [Bridge setup](https://docs.mau.fi/bridges/python/setup.html?bridge=telegram)
+  (or [with Docker](https://docs.mau.fi/bridges/general/docker-setup.html?bridge=telegram))
 * Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
   [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
   [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)

ROADMAP.md

@@ -24,6 +24,7 @@
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
   * [ ] Advanced message content/media
+    * [x] Custom emojis
     * [x] Polls
 	* [x] Games
 	* [ ] Buttons
@@ -54,7 +55,7 @@
   * [x] Automatic portal creation
     * [x] At startup
     * [x] When receiving invite or message
-  * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
+  * [x] Portal creation by inviting Matrix puppet of Telegram user to new room
   * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
   * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
   * [ ] ‡ Calls (hard, not yet supported by Telethon)

docker-run.sh

@@ -1,4 +1,18 @@
 #!/bin/sh
+if [ ! -z "$MAUTRIX_DIRECT_STARTUP" ]; then
+	if [ $(id -u) == 0 ]; then
+		echo "|------------------------------------------|"
+		echo "| Warning: running bridge unsafely as root |"
+		echo "|------------------------------------------|"
+	fi
+	exec python3 -m mautrix_telegram -c /data/config.yaml
+elif [ $(id -u) != 0 ]; then
+	echo "The startup script must run as root. It will use su-exec to drop permissions before running the bridge."
+	echo "To bypass the startup script, either set the `MAUTRIX_DIRECT_STARTUP` environment variable,"
+	echo "or just use `python3 -m mautrix_telegram -c /data/config.yaml` as the run command."
+	echo "Note that the config and registration will not be auto-generated when bypassing the startup script."
+	exit 1
+fi
 
 # Define functions.
 function fixperms {

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.11.3"
+__version__ = "0.12.2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -103,7 +103,9 @@ class TelegramBridge(Bridge):
     def prepare_stop(self) -> None:
         for puppet in Puppet.by_custom_mxid.values():
             puppet.stop()
-        self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
+        self.add_shutdown_actions(user.stop() for user in User.by_tgid.values())
+        if self.bot:
+            self.add_shutdown_actions(self.bot.stop())
 
     async def get_user(self, user_id: UserID, create: bool = True) -> User | None:
         user = await User.get_by_mxid(user_id, create=create)

mautrix_telegram/abstract_user.py

@@ -22,6 +22,7 @@ import logging
 import platform
 import time
 
+from telethon.errors import UnauthorizedError
 from telethon.network import (
     Connection,
     ConnectionTcpFull,
@@ -38,7 +39,9 @@ from telethon.tl.types import (
     PeerChat,
     PeerUser,
     TypeUpdate,
+    UpdateChannel,
     UpdateChannelUserTyping,
+    UpdateChatDefaultBannedRights,
     UpdateChatParticipantAdmin,
     UpdateChatParticipants,
     UpdateChatUserTyping,
@@ -57,6 +60,7 @@ from telethon.tl.types import (
     UpdateReadChannelInbox,
     UpdateReadHistoryInbox,
     UpdateReadHistoryOutbox,
+    UpdateShort,
     UpdateShortChatMessage,
     UpdateShortMessage,
     UpdateUserName,
@@ -223,11 +227,27 @@ class AbstractUser(ABC):
             connection=connection,
             proxy=proxy,
             raise_last_call_error=True,
+            catch_up=self.config["telegram.catch_up"],
+            sequential_updates=self.config["telegram.sequential_updates"],
             loop=self.loop,
             base_logger=base_logger,
+            update_error_callback=self._telethon_update_error_callback,
         )
         self.client.add_event_handler(self._update_catch)
 
+    async def _telethon_update_error_callback(self, err: Exception) -> None:
+        if self.config["telegram.exit_on_update_error"]:
+            self.log.critical(f"Stopping due to update handling error {type(err).__name__}")
+            self.bridge.manual_stop(50)
+        else:
+            if isinstance(err, UnauthorizedError):
+                self.log.warning("Not recreating Telethon update loop")
+                return
+            self.log.info("Recreating Telethon update loop in 60 seconds")
+            await asyncio.sleep(60)
+            self.log.debug("Now recreating Telethon update loop")
+            self.client._updates_handle = self.loop.create_task(self.client._update_loop())
+
     @abstractmethod
     async def update(self, update: TypeUpdate) -> bool:
         return False
@@ -282,21 +302,24 @@ class AbstractUser(ABC):
     async def ensure_started(self, even_if_no_session=False) -> AbstractUser:
         if self.connected:
             return self
-        if even_if_no_session or await PgSession.has(self.mxid):
+        session_exists = await PgSession.has(self.mxid)
+        if even_if_no_session or session_exists:
             self.log.debug(
-                "Starting client due to ensure_started"
-                f"(even_if_no_session={even_if_no_session})"
+                f"Starting client due to ensure_started({even_if_no_session=}, {session_exists=})"
             )
             await self.start(delete_unless_authenticated=not even_if_no_session)
         return self
 
     async def stop(self) -> None:
-        await self.client.disconnect()
-        self.client = None
+        if self.client:
+            await self.client.disconnect()
+            self.client = None
 
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
+        if isinstance(update, UpdateShort):
+            update = update.update
         asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
         if isinstance(
             update,
@@ -324,6 +347,8 @@ class AbstractUser(ABC):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
+        elif isinstance(update, UpdateChatDefaultBannedRights):
+            await self.update_default_banned_rights(update)
         elif isinstance(update, (UpdatePinnedMessages, UpdatePinnedChannelMessages)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
@@ -338,6 +363,8 @@ class AbstractUser(ABC):
             await self.update_pinned_dialogs(update)
         elif isinstance(update, UpdateNotifySettings):
             await self.update_notify_settings(update)
+        elif isinstance(update, UpdateChannel):
+            await self.update_channel(update)
         else:
             self.log.trace("Unhandled update: %s", update)
 
@@ -368,6 +395,12 @@ class AbstractUser(ABC):
         if portal and portal.mxid:
             await portal.update_power_levels(update.participants.participants)
 
+    @staticmethod
+    async def update_default_banned_rights(update: UpdateChatDefaultBannedRights) -> None:
+        portal = await po.Portal.get_by_entity(update.peer)
+        if portal and portal.mxid:
+            await portal.update_default_banned_rights(update.default_banned_rights)
+
     async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
         if not isinstance(update.peer, PeerUser):
             self.log.debug("Unexpected read receipt peer: %s", update.peer)
@@ -396,6 +429,7 @@ class AbstractUser(ABC):
         if not puppet.is_real_user:
             return
 
+        self.log.debug("Handling own read receipt: %s", update)
         if isinstance(update, UpdateReadChannelInbox):
             portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
         elif isinstance(update.peer, PeerChat):
@@ -409,6 +443,7 @@ class AbstractUser(ABC):
             return
 
         if not portal or not portal.mxid:
+            self.log.debug(f"Dropping own read receipt in unknown chat ({update.peer})")
             return
 
         tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
@@ -416,6 +451,9 @@ class AbstractUser(ABC):
             TelegramID(update.max_id), tg_space, edit_index=-1
         )
         if not message:
+            self.log.debug(
+                f"Dropping own read receipt: unknown message {update.max_id}@{tg_space}"
+            )
             return
 
         await puppet.intent.mark_read(portal.mxid, message.mxid)
@@ -491,9 +529,7 @@ class AbstractUser(ABC):
         self, update: UpdateMessage
     ) -> tuple[UpdateMessageContent, pu.Puppet | None, po.Portal | None]:
         if isinstance(update, UpdateShortChatMessage):
-            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
-            if not portal:
-                self.log.warning(f"Received message in chat with unknown type {update.chat_id}")
+            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
             sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id))
         elif isinstance(update, UpdateShortMessage):
             portal = await po.Portal.get_by_tgid(
@@ -517,6 +553,8 @@ class AbstractUser(ABC):
                 sender = await pu.Puppet.get_by_tgid(self.tgid)
             elif isinstance(update.from_id, (PeerUser, PeerChannel)):
                 sender = await pu.Puppet.get_by_peer(update.from_id)
+            elif isinstance(update.peer_id, PeerUser):
+                sender = await pu.Puppet.get_by_peer(update.peer_id)
             else:
                 sender = None
         else:
@@ -568,6 +606,39 @@ class AbstractUser(ABC):
             return
         await portal.handle_telegram_reactions(self, TelegramID(update.msg_id), update.reactions)
 
+    async def update_channel(self, update: UpdateChannel) -> None:
+        portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        if not portal:
+            return
+        if getattr(update, "mau_telethon_is_leave", False):
+            self.log.debug("UpdateChannel has mau_telethon_is_leave, leaving portal")
+            await portal.delete_telegram_user(self.tgid, sender=None)
+        elif chan := getattr(update, "mau_channel", None):
+            if not portal.mxid:
+                asyncio.create_task(self._delayed_create_channel(chan))
+            else:
+                self.log.debug("Updating channel info with data fetched by Telethon")
+                await portal.update_info(self, chan)
+                await portal.invite_to_matrix(self.mxid)
+
+    async def _delayed_create_channel(self, chan: Channel) -> None:
+        self.log.debug("Waiting 5 seconds before handling UpdateChannel for non-existent portal")
+        await asyncio.sleep(5)
+        portal = await po.Portal.get_by_tgid(TelegramID(chan.id))
+        if portal.mxid:
+            self.log.debug(
+                "Portal started existing after waiting 5 seconds, dropping UpdateChannel"
+            )
+            return
+        else:
+            self.log.info(
+                "Creating Matrix room with data fetched by Telethon due to UpdateChannel"
+            )
+            await portal.create_matrix_room(self, chan, invites=[self.mxid])
+
+    async def _check_server_notice_edit(self, message: Message) -> None:
+        pass
+
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = await self.get_message_details(original_update)
         if not portal:
@@ -601,7 +672,7 @@ class AbstractUser(ABC):
 
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.trace(
+                self.log.debug(
                     "Received %s in %s by %d, unregistering portal...",
                     update.action,
                     portal.tgid_log,
@@ -619,6 +690,8 @@ class AbstractUser(ABC):
             return await portal.handle_telegram_action(self, sender, update)
 
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
+            if sender and sender.tgid == 777000:
+                await self._check_server_notice_edit(update)
             return await portal.handle_telegram_edit(self, sender, update)
         return await portal.handle_telegram_message(self, sender, update)
 

mautrix_telegram/bot.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,8 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Tuple
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Awaitable, Callable, Literal
 import logging
+import time
 
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
@@ -26,6 +29,7 @@ from telethon.tl.types import (
     ChatForbidden,
     ChatParticipantAdmin,
     ChatParticipantCreator,
+    ChatParticipantsForbidden,
     InputChannel,
     InputUser,
     MessageActionChatAddUser,
@@ -35,31 +39,64 @@ from telethon.tl.types import (
     PeerChannel,
     PeerChat,
     PeerUser,
+    TypeChannelParticipant,
+    TypeChatParticipant,
+    TypeInputPeer,
     TypePeer,
     UpdateNewChannelMessage,
     UpdateNewMessage,
     User,
 )
+from telethon.utils import add_surrogate, del_surrogate
 
-from mautrix.types import UserID
+from mautrix.errors import MBadState, MForbidden
+from mautrix.types import RoomID, UserID
 
 from . import portal as po, puppet as pu, user as u
 from .abstract_user import AbstractUser
-from .db import BotChat
+from .db import BotChat, Message as DBMessage
 from .types import TelegramID
 
+if TYPE_CHECKING:
+    from asyncio import Future
+
 ReplyFunc = Callable[[str], Awaitable[Message]]
+BanFunc = Callable[[RoomID, UserID, str], Awaitable[None]]
+TelegramAdminPermission = Literal[
+    "change_info",
+    "post_messages",
+    "edit_messages",
+    "delete_messages",
+    "ban_users",
+    "invite_users",
+    "pin_messages",
+    "add_admins",
+    "anonymous",
+    "manage_call",
+    "other",
+]
 
 
 class Bot(AbstractUser):
     log: logging.Logger = logging.getLogger("mau.user.bot")
 
     token: str
-    chats: Dict[int, str]
-    tg_whitelist: List[int]
+    chats: dict[int, str]
+    tg_whitelist: list[int]
     whitelist_group_admins: bool
-    _me_info: Optional[User]
-    _me_mxid: Optional[UserID]
+    _me_info: User | None
+    _me_mxid: UserID | None
+    _admin_cache: dict[
+        tuple[int, int],
+        tuple[ChatParticipantAdmin | ChatParticipantCreator | None, float],
+    ]
+    _login_wait_fut: Future | None
+    required_permissions: dict[str, TelegramAdminPermission] = {
+        "portal": None,
+        "invite": "invite_users",
+        "mxban": "ban_users",
+        "mxkick": "ban_users",
+    }
 
     def __init__(self, token: str) -> None:
         super().__init__()
@@ -73,14 +110,16 @@ class Bot(AbstractUser):
         self.is_relaybot = True
         self.is_bot = True
         self.chats = {}
+        self._admin_cache = {}
         self.tg_whitelist = []
         self.whitelist_group_admins = (
             self.config["bridge.relaybot.whitelist_group_admins"] or False
         )
         self._me_info = None
         self._me_mxid = None
+        self._login_wait_fut = self.loop.create_future()
 
-    async def get_me(self, use_cache: bool = True) -> Tuple[User, UserID]:
+    async def get_me(self, use_cache: bool = True) -> tuple[User, UserID]:
         if not use_cache or not self._me_mxid:
             self._me_info = await self.client.get_me()
             self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
@@ -98,7 +137,7 @@ class Bot(AbstractUser):
             if isinstance(user_id, int):
                 self.tg_whitelist.append(user_id)
 
-    async def start(self, delete_unless_authenticated: bool = False) -> "Bot":
+    async def start(self, delete_unless_authenticated: bool = False) -> Bot:
         self.chats = {chat.id: chat.type for chat in await BotChat.all()}
         await super().start(delete_unless_authenticated)
         if not await self.is_logged_in():
@@ -112,6 +151,9 @@ class Bot(AbstractUser):
         self.tgid = TelegramID(info.id)
         self.tg_username = info.username
         self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
+        if self._login_wait_fut:
+            self._login_wait_fut.set_result(None)
+            self._login_wait_fut = None
 
         chat_ids = [chat_id for chat_id, chat_type in self.chats.items() if chat_type == "chat"]
         response = await self.client(GetChatsRequest(chat_ids))
@@ -148,7 +190,46 @@ class Bot(AbstractUser):
             pass
         await BotChat.delete_by_id(chat_id)
 
-    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
+    async def _get_admin_participant(
+        self, chat: TypePeer | TypeInputPeer, tgid: TelegramID
+    ) -> TypeChatParticipant | TypeChannelParticipant | None:
+        chan_id = chat.channel_id if isinstance(chat, PeerChannel) else chat.chat_id
+        try:
+            cached, created = self._admin_cache[chan_id, tgid]
+            if created + 60 < time.time():
+                return cached
+        except KeyError:
+            pass
+        if isinstance(chat, PeerChannel):
+            p = await self.client(GetParticipantRequest(chat, tgid))
+            pcp = p.participant
+            self._admin_cache[chat.channel_id, tgid] = (pcp, time.time())
+            return pcp
+        elif isinstance(chat, PeerChat):
+            chat = await self.client(GetFullChatRequest(chat.chat_id))
+            if isinstance(chat.full_chat.participants, ChatParticipantsForbidden):
+                return None
+            participants = chat.full_chat.participants.participants
+            for p in participants:
+                self._admin_cache[chat.channel_id, tgid] = (p, time.time())
+                if p.user_id == tgid:
+                    return p
+        return None
+
+    @staticmethod
+    def _has_participant_permission(
+        pcp: TypeChatParticipant | TypeChannelParticipant | None,
+        permission: TelegramAdminPermission | None,
+    ) -> bool:
+        if isinstance(pcp, (ChannelParticipantCreator, ChannelParticipantAdmin)):
+            return permission is None or getattr(pcp.admin_rights, permission, False)
+        elif isinstance(pcp, (ChatParticipantCreator, ChatParticipantAdmin)):
+            return True
+        return False
+
+    async def _can_use_commands(
+        self, chat: TypePeer, tgid: TelegramID, permission: TelegramAdminPermission | None = None
+    ) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
@@ -158,22 +239,20 @@ class Bot(AbstractUser):
             return True
 
         if self.whitelist_group_admins:
-            if isinstance(chat, PeerChannel):
-                p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(
-                    p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin)
-                )
-            elif isinstance(chat, PeerChat):
-                chat = await self.client(GetFullChatRequest(chat.chat_id))
-                participants = chat.full_chat.participants.participants
-                for p in participants:
-                    if p.user_id == tgid:
-                        return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+            pcp = await self._get_admin_participant(chat, tgid)
+            return self._has_participant_permission(pcp, permission)
         return False
 
-    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
-        # FIXME event.from_id is not int
-        if not await self._can_use_commands(event.to_id, TelegramID(event.from_id)):
+    async def check_can_use_command(self, event: Message, reply: ReplyFunc, command: str) -> bool:
+        if command not in self.required_permissions:
+            # Unknown command
+            return False
+        elif not isinstance(event.from_id, PeerUser):
+            await reply("Channels can't use commands")
+            return False
+        elif not await self._can_use_commands(
+            event.to_id, TelegramID(event.from_id.user_id), self.required_permissions[command]
+        ):
             await reply("You do not have the permission to use that command.")
             return False
         return True
@@ -193,6 +272,8 @@ class Bot(AbstractUser):
                 )
             else:
                 return await reply("Portal is not public. Use `/invite <mxid>` to get an invite.")
+        else:
+            return await reply("Couldn't create portal room")
 
     async def handle_command_invite(
         self, portal: po.Portal, reply: ReplyFunc, mxid_input: UserID
@@ -213,9 +294,59 @@ class Bot(AbstractUser):
                 f"Just invite [{displayname}](tg://user?id={user.tgid})"
             )
         else:
-            await portal.invite_to_matrix(user.mxid)
+            try:
+                await portal.invite_to_matrix(user.mxid)
+            except MBadState:
+                try:
+                    await portal.main_intent.unban_user(
+                        portal.mxid, user.mxid, reason="Invited from Telegram"
+                    )
+                except Exception:
+                    return await reply(f"Failed to unban `{user.mxid}` from the portal.")
+                await portal.invite_to_matrix(user.mxid)
+                return await reply(f"Unbanned and invited `{user.mxid}` to the portal.")
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
+    async def handle_command_ban(
+        self,
+        message: Message,
+        portal: po.Portal,
+        reply: ReplyFunc,
+        reason: str,
+        action: Literal["kick", "ban"] = "ban",
+    ) -> Message:
+        if not message.reply_to:
+            return await reply("You must reply to a relaybot message when using that command")
+        reply_to_id = TelegramID(message.reply_to.reply_to_msg_id)
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        msg = await DBMessage.get_one_by_tgid(reply_to_id, tg_space)
+        if not msg or msg.sender != self.tgid or not msg.sender_mxid:
+            return await reply("Target message is not a relayed message")
+        puppet = await pu.Puppet.get_by_peer(message.from_id)
+        actioned = "Banned" if action == "ban" else "Kicked"
+        try:
+            intent = puppet.intent_for(portal)
+            func: BanFunc = intent.ban_user if action == "ban" else intent.kick_user
+            await func(portal.mxid, msg.sender_mxid, reason)
+        except MForbidden as e:
+            self.log.warning(
+                f"Failed to {action} {msg.sender_mxid} from {portal.mxid} as {puppet.mxid}: {e}, "
+                f"falling back to bridge bot"
+            )
+            reason_prefix = f"{actioned} by {puppet.displayname or puppet.tgid}"
+            reason = f"{reason_prefix}: {reason}" if reason else reason_prefix
+            try:
+                func: BanFunc = (
+                    self.az.intent.ban_user if action == "ban" else self.az.intent.kick_user
+                )
+                await func(portal.mxid, msg.sender_mxid, reason)
+            except MForbidden as e:
+                self.log.warning(
+                    f"Failed to {action} {msg.sender_mxid} from {portal.mxid} as bridge bot: {e}"
+                )
+                return await reply(f"Failed to {action} `{msg.sender_mxid}`")
+        return await reply(f"Successfully {actioned.lower()} `{msg.sender_mxid}`")
+
     @staticmethod
     def handle_command_id(message: Message, reply: ReplyFunc) -> Awaitable[Message]:
         # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
@@ -233,53 +364,46 @@ class Bot(AbstractUser):
         else:
             return reply("Failed to find chat ID.")
 
-    def match_command(self, text: str, command: str) -> bool:
-        text = text.lower()
-        command = f"/{command.lower()}"
-        command_targeted = f"{command}@{self.tg_username.lower()}"
+    def parse_command(self, message: Message) -> tuple[str | None, str | None]:
+        if not message.entities or len(message.entities) < 1 or not message.message:
+            return None, None
+        cmd_entity = message.entities[0]
+        if not isinstance(cmd_entity, MessageEntityBotCommand) or cmd_entity.offset != 0:
+            return None, None
+        surrogated_text = add_surrogate(message.message)
+        command: str = del_surrogate(surrogated_text[: cmd_entity.length]).lower()
+        rest_of_message: str = ""
+        if len(surrogated_text) > cmd_entity.length + 1:
+            rest_of_message: str = del_surrogate(surrogated_text[cmd_entity.length + 1 :])
+        command, *target = command.split("@", 1)
+        if not command.startswith("/"):
+            return None, None
+        elif target and target[0] != self.tg_username.lower():
+            return None, None
+        return command[1:], rest_of_message
 
-        is_plain_command = text == command or text == command_targeted
-        if is_plain_command:
-            return True
-
-        is_arg_command = text.startswith(command + " ") or text.startswith(command_targeted + " ")
-        if is_arg_command:
-            return True
-
-        return False
-
-    async def handle_command(self, message: Message) -> None:
+    async def handle_command(self, message: Message, command: str, args: str) -> None:
         def reply(reply_text: str) -> Awaitable[Message]:
             return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)
 
-        text = message.message
-
-        if self.match_command(text, "start"):
+        if command == "start":
             pcm = self.config["bridge.relaybot.private_chat.message"]
             if pcm:
                 await reply(pcm)
-            return
-        elif self.match_command(text, "id"):
+        elif command == "id":
             await self.handle_command_id(message, reply)
-            return
-        elif message.is_private:
-            return
-
-        portal = await po.Portal.get_by_entity(message.to_id)
-
-        is_portal_cmd = self.match_command(text, "portal")
-        is_invite_cmd = self.match_command(text, "invite")
-        if is_portal_cmd or is_invite_cmd:
-            if not await self.check_can_use_commands(message, reply):
+        elif not message.is_private:
+            if not await self.check_can_use_command(message, reply, command):
                 return
-            if is_portal_cmd:
+            portal = await po.Portal.get_by_entity(message.to_id)
+            if command == "portal":
                 await self.handle_command_portal(portal, reply)
-            elif is_invite_cmd:
-                try:
-                    mxid = text[text.index(" ") + 1 :]
-                except ValueError:
-                    mxid = ""
-                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
+            elif command == "invite":
+                await self.handle_command_invite(portal, reply, mxid_input=UserID(args))
+            elif command == "mxban":
+                await self.handle_command_ban(message, portal, reply, reason=args)
+            elif command == "mxkick":
+                await self.handle_command_ban(message, portal, reply, reason=args, action="kick")
 
     async def handle_service_message(self, message: MessageService) -> None:
         to_peer = message.to_id
@@ -302,21 +426,18 @@ class Bot(AbstractUser):
             await self.add_chat(TelegramID(action.channel_id), "channel")
 
     async def update(self, update) -> bool:
+        if self._login_wait_fut:
+            await self._login_wait_fut
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
             return False
         if isinstance(update.message, MessageService):
             await self.handle_service_message(update.message)
             return False
 
-        is_command = (
-            isinstance(update.message, Message)
-            and update.message.entities
-            and len(update.message.entities) > 0
-            and isinstance(update.message.entities[0], MessageEntityBotCommand)
-            and update.message.entities[0].offset == 0
-        )
-        if is_command:
-            await self.handle_command(update.message)
+        if isinstance(update.message, Message):
+            command, args = self.parse_command(update.message)
+            if command:
+                await self.handle_command(update.message, command, args)
         return False
 
     def is_in_chat(self, peer_id) -> bool:

mautrix_telegram/commands/portal/config.py

@@ -29,6 +29,7 @@ from .. import SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
 
 @command_handler(
     needs_auth=False,
+    needs_puppeting=False,
     help_section=SECTION_PORTAL_MANAGEMENT,
     help_text="View or change per-portal settings.",
     help_args="<`help`|_subcommand_> [...]",
@@ -98,7 +99,7 @@ def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
                 "exceptions": evt.config["bridge.bridge_notices.exceptions"],
             },
             "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
-            "inline_images": evt.config["bridge.inline_images"],
+            "caption_in_message": evt.config["bridge.caption_in_message"],
             "message_formats": evt.config["bridge.message_formats"],
             "emote_format": evt.config["bridge.emote_format"],
             "state_event_formats": evt.config["bridge.state_event_formats"],

mautrix_telegram/commands/portal/create_chat.py

@@ -81,4 +81,3 @@ async def create(evt: CommandEvent) -> EventID:
     except ValueError as e:
         await portal.delete()
         return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/util.py

@@ -68,5 +68,5 @@ async def user_has_power_level(
         await intent.get_power_levels(room_id)
     except MatrixRequestError:
         return False
-    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
+    event_type = EventType.find(f"fi.mau.telegram.{event}", t_class=EventType.Class.STATE)
     return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)

mautrix_telegram/commands/telegram/auth.py

@@ -38,6 +38,7 @@ from telethon.errors import (
 from telethon.tl.types import User
 
 from mautrix.client import Client
+from mautrix.errors import MForbidden
 from mautrix.types import (
     EventID,
     ImageInfo,
@@ -215,6 +216,10 @@ async def login_qr(evt: CommandEvent) -> EventID:
             return await evt.reply(
                 "Your account has two-factor authentication. Please send your password here."
             )
+        try:
+            await evt.main_intent.redact(evt.room_id, qr_event_id, reason="QR code scanned")
+        except Exception:
+            pass
     else:
         timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
         timeout.set_edit(qr_event_id)
@@ -377,6 +382,7 @@ async def enter_password(evt: CommandEvent) -> EventID | None:
             "This bridge instance does not allow in-Matrix login. "
             "Please use `$cmdprefix+sp login` to get login instructions"
         )
+    await evt.redact()
     try:
         await _sign_in(
             evt,

mautrix_telegram/commands/telegram/misc.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 from typing import cast
 import base64
 import codecs
+import math
 import re
 
 from aiohttp import ClientSession, InvalidURL
@@ -66,6 +67,7 @@ from ...types import TelegramID
 
 @command_handler(
     needs_auth=False,
+    needs_puppeting=False,
     help_section=SECTION_MISC,
     help_args="<_caption_>",
     help_text="Set a caption for the next image you send",
@@ -233,7 +235,10 @@ async def join(evt: CommandEvent) -> EventID | None:
                     updates.stringify(),
                 )
                 raise e
-            return await evt.reply(f"Created room for {portal.title}")
+            if portal.mxid:
+                return await evt.reply(f"Created room for {portal.title}")
+            else:
+                return await evt.reply(f"Couldn't create room for {portal.title}")
     return None
 
 
@@ -423,6 +428,9 @@ async def backfill(evt: CommandEvent) -> None:
     if not evt.is_portal:
         await evt.reply("You can only use backfill in portal rooms")
         return
+    elif not evt.config["bridge.backfill.enable"]:
+        await evt.reply("Backfilling is disabled in the bridge config")
+        return
     try:
         limit = int(evt.args[0])
     except (ValueError, IndexError):
@@ -431,16 +439,14 @@ async def backfill(evt: CommandEvent) -> None:
     if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
         await evt.reply("Backfilling normal groups is disabled in the bridge config")
         return
-    try:
-        await portal.backfill(evt.sender, limit=limit)
-    except TakeoutInitDelayError:
-        msg = (
-            "Please accept the data export request from a mobile device, "
-            "then re-run the backfill command."
-        )
-        if portal.peer_type == "user":
-            from mautrix.appservice import IntentAPI
-
-            await portal.main_intent.send_notice(evt.room_id, msg)
-        else:
-            await evt.reply(msg)
+    if portal.backfill_msc2716:
+        messages_per_batch = evt.config["bridge.backfill.incremental.messages_per_batch"]
+        batches = math.ceil(limit / messages_per_batch)
+        rounded = ""
+        if batches * messages_per_batch != limit:
+            rounded = f" (rounded message limit to {batches}*{messages_per_batch})"
+        await portal.enqueue_backfill(evt.sender, priority=0, max_batches=batches)
+        await evt.reply(f"Backfill queued{rounded}")
+    else:
+        output = await portal.forward_backfill(evt.sender, initial=False, override_limit=limit)
+        await evt.reply(output)

mautrix_telegram/config.py

@@ -35,12 +35,6 @@ Permissions = NamedTuple(
 
 
 class Config(BaseBridgeConfig):
-    def __getitem__(self, key: str) -> Any:
-        try:
-            return os.environ[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
-        except KeyError:
-            return super().__getitem__(key)
-
     @property
     def forbidden_defaults(self) -> List[ForbiddenDefault]:
         return [
@@ -63,8 +57,6 @@ class Config(BaseBridgeConfig):
         super().do_update(helper)
         copy, copy_dict, base = helper
 
-        copy("homeserver.asmux")
-
         if "appservice.protocol" in self and "appservice.address" not in self:
             protocol, hostname, port = (
                 self["appservice.protocol"],
@@ -111,6 +103,7 @@ class Config(BaseBridgeConfig):
         copy("bridge.allow_avatar_remove")
 
         copy("bridge.max_initial_member_sync")
+        copy("bridge.max_member_count")
         copy("bridge.sync_channel_members")
         copy("bridge.skip_deleted_members")
         copy("bridge.startup_sync")
@@ -120,16 +113,17 @@ class Config(BaseBridgeConfig):
         else:
             copy("bridge.sync_update_limit")
             copy("bridge.sync_create_limit")
+        copy("bridge.sync_deferred_create_all")
         copy("bridge.sync_direct_chats")
         copy("bridge.max_telegram_delete")
         copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
-        copy("bridge.plaintext_highlights")
         copy("bridge.public_portals")
         copy("bridge.sync_with_custom_puppets")
         copy("bridge.sync_direct_chat_list")
         copy("bridge.double_puppet_server_map")
         copy("bridge.double_puppet_allow_discovery")
+        copy("bridge.create_group_on_invite")
         if "bridge.login_shared_secret" in self:
             base["bridge.login_shared_secret_map"] = {
                 base["homeserver.domain"]: self["bridge.login_shared_secret"]
@@ -138,24 +132,25 @@ class Config(BaseBridgeConfig):
             copy("bridge.login_shared_secret_map")
         copy("bridge.telegram_link_preview")
         copy("bridge.invite_link_resolve")
-        copy("bridge.inline_images")
+        copy("bridge.caption_in_message")
         copy("bridge.image_as_file_size")
         copy("bridge.image_as_file_pixels")
         copy("bridge.parallel_file_transfer")
         copy("bridge.federate_rooms")
+        copy("bridge.always_custom_emoji_reaction")
         copy("bridge.animated_sticker.target")
+        copy("bridge.animated_sticker.convert_from_webm")
         copy("bridge.animated_sticker.args.width")
         copy("bridge.animated_sticker.args.height")
         copy("bridge.animated_sticker.args.fps")
-        copy("bridge.encryption.allow")
-        copy("bridge.encryption.default")
-        copy("bridge.encryption.database")
-        copy("bridge.encryption.key_sharing.allow")
-        copy("bridge.encryption.key_sharing.require_cross_signing")
-        copy("bridge.encryption.key_sharing.require_verification")
+        copy("bridge.animated_emoji.target")
+        copy("bridge.animated_emoji.args.width")
+        copy("bridge.animated_emoji.args.height")
+        copy("bridge.animated_emoji.args.fps")
         copy("bridge.private_chat_portal_meta")
         copy("bridge.delivery_receipts")
         copy("bridge.delivery_error_reports")
+        copy("bridge.message_status_events")
         copy("bridge.resend_bridge_info")
         copy("bridge.mute_bridging")
         copy("bridge.pinned_tag")
@@ -164,12 +159,19 @@ class Config(BaseBridgeConfig):
         copy("bridge.bridge_matrix_leave")
         copy("bridge.kick_on_logout")
         copy("bridge.always_read_joined_telegram_notice")
-        copy("bridge.backfill.invite_own_puppet")
-        copy("bridge.backfill.takeout_limit")
-        copy("bridge.backfill.initial_limit")
-        copy("bridge.backfill.missed_limit")
-        copy("bridge.backfill.disable_notifications")
+        copy("bridge.backfill.enable")
+        copy("bridge.backfill.msc2716")
+        copy("bridge.backfill.double_puppet_backfill")
         copy("bridge.backfill.normal_groups")
+        copy("bridge.backfill.unread_hours_threshold")
+        copy("bridge.backfill.forward.initial_limit")
+        copy("bridge.backfill.forward.sync_limit")
+        copy("bridge.backfill.incremental.messages_per_batch")
+        copy("bridge.backfill.incremental.post_batch_delay")
+        copy("bridge.backfill.incremental.max_batches.user")
+        copy("bridge.backfill.incremental.max_batches.normal_group")
+        copy("bridge.backfill.incremental.max_batches.supergroup")
+        copy("bridge.backfill.incremental.max_batches.channel")
 
         copy("bridge.initial_power_level_overrides.group")
         copy("bridge.initial_power_level_overrides.user")
@@ -228,6 +230,10 @@ class Config(BaseBridgeConfig):
         copy("telegram.api_hash")
         copy("telegram.bot_token")
 
+        copy("telegram.catch_up")
+        copy("telegram.sequential_updates")
+        copy("telegram.exit_on_update_error")
+
         copy("telegram.connection.timeout")
         copy("telegram.connection.retries")
         copy("telegram.connection.retry_delay")

mautrix_telegram/db/__init__.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from mautrix.util.async_db import Database
 
+from .backfill_queue import Backfill, BackfillType
 from .bot_chat import BotChat
 from .disappearing_message import DisappearingMessage
 from .message import Message
@@ -38,6 +39,7 @@ def init(db: Database) -> None:
         BotChat,
         PgSession,
         DisappearingMessage,
+        Backfill,
     ):
         table.db = db
 
@@ -54,4 +56,5 @@ __all__ = [
     "BotChat",
     "PgSession",
     "DisappearingMessage",
+    "Backfill",
 ]

mautrix_telegram/db/backfill_queue.py

@@ -0,0 +1,235 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan, Sumner Evans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar
+from datetime import datetime, timedelta
+from enum import Enum
+import json
+
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.types import UserID
+from mautrix.util.async_db import Database
+
+from ..types import TelegramID
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+class BackfillType(Enum):
+    HISTORICAL = "historical"
+    SYNC_DIALOG = "sync_dialog"
+
+
+@dataclass
+class Backfill:
+    db: ClassVar[Database] = fake_db
+
+    queue_id: int | None
+    user_mxid: UserID
+    priority: int
+    type: BackfillType
+    portal_tgid: TelegramID
+    portal_tg_receiver: TelegramID
+    anchor_msg_id: TelegramID | None
+    extra_data: dict[str, Any]
+    messages_per_batch: int
+    post_batch_delay: int
+    max_batches: int
+    dispatch_time: datetime | None
+    completed_at: datetime | None
+    cooldown_timeout: datetime | None
+
+    @staticmethod
+    def new(
+        user_mxid: UserID,
+        priority: int,
+        type: BackfillType,
+        portal_tgid: TelegramID,
+        portal_tg_receiver: TelegramID,
+        messages_per_batch: int,
+        anchor_msg_id: TelegramID | None = None,
+        extra_data: dict[str, Any] | None = None,
+        post_batch_delay: int = 0,
+        max_batches: int = -1,
+    ) -> "Backfill":
+        return Backfill(
+            queue_id=None,
+            user_mxid=user_mxid,
+            priority=priority,
+            type=type,
+            portal_tgid=portal_tgid,
+            portal_tg_receiver=portal_tg_receiver,
+            anchor_msg_id=anchor_msg_id,
+            extra_data=extra_data or {},
+            messages_per_batch=messages_per_batch,
+            post_batch_delay=post_batch_delay,
+            max_batches=max_batches,
+            dispatch_time=None,
+            completed_at=None,
+            cooldown_timeout=None,
+        )
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> Backfill | None:
+        if row is None:
+            return None
+        data = {**row}
+        type = BackfillType(data.pop("type"))
+        extra_data = json.loads(data.pop("extra_data", None) or "{}")
+        return cls(**data, type=type, extra_data=extra_data)
+
+    columns = [
+        "user_mxid",
+        "priority",
+        "type",
+        "portal_tgid",
+        "portal_tg_receiver",
+        "anchor_msg_id",
+        "extra_data",
+        "messages_per_batch",
+        "post_batch_delay",
+        "max_batches",
+        "dispatch_time",
+        "completed_at",
+        "cooldown_timeout",
+    ]
+    columns_str = ",".join(columns)
+
+    @classmethod
+    async def get_next(cls, user_mxid: UserID) -> Backfill | None:
+        q = f"""
+        SELECT queue_id, {cls.columns_str}
+        FROM backfill_queue
+        WHERE user_mxid=$1
+            AND (
+                dispatch_time IS NULL
+                OR (
+                    dispatch_time < $2
+                    AND completed_at IS NULL
+                )
+            )
+            AND (
+                cooldown_timeout IS NULL
+                OR cooldown_timeout < current_timestamp
+            )
+        ORDER BY priority, queue_id
+        LIMIT 1
+        """
+        return cls._from_row(
+            await cls.db.fetchrow(q, user_mxid, datetime.now() - timedelta(minutes=15))
+        )
+
+    @classmethod
+    async def delete_existing(
+        cls,
+        user_mxid: UserID,
+        portal_tgid: int,
+        portal_tg_receiver: int,
+        type: BackfillType,
+    ) -> Backfill | None:
+        q = f"""
+        WITH deleted_entries AS (
+            DELETE FROM backfill_queue
+            WHERE user_mxid=$1
+              AND portal_tgid=$2
+              AND portal_tg_receiver=$3
+              AND type=$4
+              AND dispatch_time IS NULL
+              AND completed_at IS NULL
+            RETURNING 1
+        )
+        WITH dispatched_entries AS (
+            SELECT 1 FROM backfill_queue
+            WHERE user_mxid=$1
+              AND portal_tgid=$2
+              AND portal_tg_receiver=$3
+              AND type=$4
+              AND dispatch_time IS NOT NULL
+              AND completed_at IS NULL
+        )
+        """
+        return cls._from_row(
+            await cls.db.fetchrow(q, user_mxid, portal_tgid, portal_tg_receiver, type.value)
+        )
+
+    @classmethod
+    async def delete_all(cls, user_mxid: UserID) -> None:
+        await cls.db.execute("DELETE FROM backfill_queue WHERE user_mxid=$1", user_mxid)
+
+    @classmethod
+    async def delete_for_portal(cls, tgid: int, tg_receiver: int) -> None:
+        q = "DELETE FROM backfill_queue WHERE portal_tgid=$1 AND portal_tg_receiver=$2"
+        await cls.db.execute(q, tgid, tg_receiver)
+
+    async def insert(self) -> list[Backfill]:
+        delete_q = f"""
+        DELETE FROM backfill_queue
+        WHERE user_mxid=$1
+          AND portal_tgid=$2
+          AND portal_tg_receiver=$3
+          AND type=$4
+          AND dispatch_time IS NULL
+          AND completed_at IS NULL
+        RETURNING {self.columns_str}
+        """
+        q = f"""
+        INSERT INTO backfill_queue ({self.columns_str})
+        VALUES ({','.join(f'${i+1}' for i in range(len(self.columns)))})
+        RETURNING queue_id
+        """
+        async with self.db.acquire() as conn, conn.transaction():
+            deleted_rows = await conn.fetch(
+                delete_q,
+                self.user_mxid,
+                self.portal_tgid,
+                self.portal_tg_receiver,
+                self.type.value,
+            )
+            self.queue_id = await conn.fetchval(
+                q,
+                self.user_mxid,
+                self.priority,
+                self.type.value,
+                self.portal_tgid,
+                self.portal_tg_receiver,
+                self.anchor_msg_id,
+                json.dumps(self.extra_data) if self.extra_data else None,
+                self.messages_per_batch,
+                self.post_batch_delay,
+                self.max_batches,
+                self.dispatch_time,
+                self.completed_at,
+                self.cooldown_timeout,
+            )
+        return [self._from_row(row) for row in deleted_rows]
+
+    async def mark_dispatched(self) -> None:
+        q = "UPDATE backfill_queue SET dispatch_time=$1 WHERE queue_id=$2"
+        await self.db.execute(q, datetime.now(), self.queue_id)
+
+    async def mark_done(self) -> None:
+        q = "UPDATE backfill_queue SET completed_at=$1 WHERE queue_id=$2"
+        await self.db.execute(q, datetime.now(), self.queue_id)
+
+    async def set_cooldown_timeout(self, timeout: int) -> None:
+        """
+        Set the backfill request to cooldown for ``timeout`` seconds.
+        """
+        q = "UPDATE backfill_queue SET cooldown_timeout=$1 WHERE queue_id=$2"
+        await self.db.execute(q, datetime.now() + timedelta(seconds=timeout), self.queue_id)

mautrix_telegram/db/message.py

@@ -19,8 +19,9 @@ from typing import TYPE_CHECKING, ClassVar
 
 from asyncpg import Record
 from attr import dataclass
+import attr
 
-from mautrix.types import EventID, RoomID
+from mautrix.types import EventID, RoomID, UserID
 from mautrix.util.async_db import Database, Scheme
 
 from ..types import TelegramID
@@ -39,6 +40,8 @@ class Message:
     edit_index: int
     redacted: bool = False
     content_hash: bytes | None = None
+    sender_mxid: UserID | None = None
+    sender: TelegramID | None = None
 
     @classmethod
     def _from_row(cls, row: Record | None) -> Message | None:
@@ -46,7 +49,19 @@ class Message:
             return None
         return cls(**row)
 
-    columns: ClassVar[str] = "mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash"
+    columns: ClassVar[str] = ", ".join(
+        (
+            "mxid",
+            "mx_room",
+            "tgid",
+            "tg_space",
+            "edit_index",
+            "redacted",
+            "content_hash",
+            "sender_mxid",
+            "sender",
+        )
+    )
 
     @classmethod
     async def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> list[Message]:
@@ -108,6 +123,14 @@ class Message:
         )
         return cls._from_row(await cls.db.fetchrow(q, mx_room, tg_space))
 
+    @classmethod
+    async def find_first(cls, mx_room: RoomID, tg_space: TelegramID) -> Message | None:
+        q = (
+            f"SELECT {cls.columns} FROM message WHERE mx_room=$1 AND tg_space=$2 "
+            f"ORDER BY tgid ASC LIMIT 1"
+        )
+        return cls._from_row(await cls.db.fetchrow(q, mx_room, tg_space))
+
     @classmethod
     async def delete_all(cls, mx_room: RoomID) -> None:
         await cls.db.execute("DELETE FROM message WHERE mx_room=$1", mx_room)
@@ -138,6 +161,17 @@ class Message:
             rows = await cls.db.fetch(q, mx_room, tg_space, *mxids)
         return [cls._from_row(row) for row in rows]
 
+    @classmethod
+    async def find_recent(
+        cls, mx_room: RoomID, not_sender: TelegramID, limit: int = 20
+    ) -> list[Message]:
+        q = f"""
+        SELECT {cls.columns} FROM message
+        WHERE mx_room=$1 AND sender<>$2
+        ORDER BY tgid DESC LIMIT $3
+        """
+        return [cls._from_row(row) for row in await cls.db.fetch(q, mx_room, not_sender, limit)]
+
     @classmethod
     async def replace_temp_mxid(cls, temp_mxid: str, mx_room: RoomID, real_mxid: EventID) -> None:
         q = "UPDATE message SET mxid=$1 WHERE mxid=$2 AND mx_room=$3"
@@ -148,6 +182,23 @@ class Message:
         q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2"
         await cls.db.execute(q, temp_mxid, mx_room)
 
+    @classmethod
+    async def bulk_insert(cls, messages: list[Message]) -> None:
+        columns = cls.columns.split(", ")
+        records = [attr.astuple(message) for message in messages]
+        async with cls.db.acquire() as conn, conn.transaction():
+            if cls.db.scheme == Scheme.POSTGRES:
+                await conn.copy_records_to_table("message", records=records, columns=columns)
+            else:
+                await conn.executemany(cls._insert_query, records)
+
+    _insert_query: ClassVar[
+        str
+    ] = """
+        INSERT INTO message (mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash, sender_mxid, sender)
+        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+    """
+
     @property
     def _values(self):
         return (
@@ -158,14 +209,12 @@ class Message:
             self.edit_index,
             self.redacted,
             self.content_hash,
+            self.sender_mxid,
+            self.sender,
         )
 
     async def insert(self) -> None:
-        q = """
-            INSERT INTO message (mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash)
-            VALUES ($1, $2, $3, $4, $5, $6, $7)
-        """
-        await self.db.execute(q, *self._values)
+        await self.db.execute(self._insert_query, *self._values)
 
     async def delete(self) -> None:
         q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2 AND tg_space=$3"

mautrix_telegram/db/portal.py

@@ -22,7 +22,7 @@ from asyncpg import Record
 from attr import dataclass
 import attr
 
-from mautrix.types import ContentURI, EventID, RoomID
+from mautrix.types import BatchID, ContentURI, EventID, RoomID
 from mautrix.util.async_db import Database
 
 from ..types import TelegramID
@@ -44,6 +44,9 @@ class Portal:
     mxid: RoomID | None
     avatar_url: ContentURI | None
     encrypted: bool
+    first_event_id: EventID | None
+    next_batch_id: BatchID | None
+    base_insertion_id: EventID | None
 
     sponsored_event_id: EventID | None
     sponsored_event_ts: int | None
@@ -67,10 +70,29 @@ class Portal:
         data["local_config"] = json.loads(data.pop("config", None) or "{}")
         return cls(**data)
 
-    columns: ClassVar[str] = (
-        "tgid, tg_receiver, peer_type, megagroup, mxid, avatar_url, encrypted, sponsored_event_id,"
-        "sponsored_event_ts, sponsored_msg_random_id, username, title, about, photo_id, "
-        "name_set, avatar_set, config"
+    columns: ClassVar[str] = ", ".join(
+        (
+            "tgid",
+            "tg_receiver",
+            "peer_type",
+            "megagroup",
+            "mxid",
+            "avatar_url",
+            "encrypted",
+            "first_event_id",
+            "next_batch_id",
+            "base_insertion_id",
+            "sponsored_event_id",
+            "sponsored_event_ts",
+            "sponsored_msg_random_id",
+            "username",
+            "title",
+            "about",
+            "photo_id",
+            "name_set",
+            "avatar_set",
+            "config",
+        )
     )
 
     @classmethod
@@ -112,6 +134,9 @@ class Portal:
             self.mxid,
             self.avatar_url,
             self.encrypted,
+            self.first_event_id,
+            self.next_batch_id,
+            self.base_insertion_id,
             self.sponsored_event_id,
             self.sponsored_event_ts,
             self.sponsored_msg_random_id,
@@ -128,9 +153,11 @@ class Portal:
     async def save(self) -> None:
         q = """
         UPDATE portal
-        SET mxid=$4, avatar_url=$5, encrypted=$6, sponsored_event_id=$7, sponsored_event_ts=$8,
-            sponsored_msg_random_id=$9, username=$10, title=$11, about=$12, photo_id=$13,
-            name_set=$14, avatar_set=$15, megagroup=$16, config=$17
+        SET mxid=$4, avatar_url=$5, encrypted=$6,
+            first_event_id=$7, next_batch_id=$8, base_insertion_id=$9,
+            sponsored_event_id=$10, sponsored_event_ts=$11, sponsored_msg_random_id=$12,
+            username=$13, title=$14, about=$15, photo_id=$16, name_set=$17, avatar_set=$18,
+            megagroup=$19, config=$20
         WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)
         """
         await self.db.execute(q, *self._values)
@@ -149,9 +176,11 @@ class Portal:
         q = """
         INSERT INTO portal (
             tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,
+            first_event_id, base_insertion_id, next_batch_id,
             sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,
             username, title, about, photo_id, name_set, avatar_set, megagroup, config
-        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
+                  $19, $20)
         """
         await self.db.execute(q, *self._values)
 

mautrix_telegram/db/puppet.py

@@ -50,6 +50,7 @@ class Puppet:
     avatar_set: bool
     is_bot: bool | None
     is_channel: bool
+    is_premium: bool
 
     custom_mxid: UserID | None
     access_token: str | None
@@ -67,7 +68,8 @@ class Puppet:
     columns: ClassVar[str] = (
         "id, is_registered, displayname, displayname_source, displayname_contact, "
         "displayname_quality, disable_updates, username, phone, photo_id, avatar_url, "
-        "name_set, avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url"
+        "name_set, avatar_set, is_bot, is_channel, is_premium, "
+        "custom_mxid, access_token, next_batch, base_url"
     )
 
     @classmethod
@@ -90,11 +92,6 @@ class Puppet:
         q = f"SELECT {cls.columns} FROM puppet WHERE lower(username)=$1"
         return cls._from_row(await cls.db.fetchrow(q, username.lower()))
 
-    @classmethod
-    async def find_by_displayname(cls, displayname: str) -> Puppet | None:
-        q = f"SELECT {cls.columns} FROM puppet WHERE displayname=$1"
-        return cls._from_row(await cls.db.fetchrow(q, displayname))
-
     @property
     def _values(self):
         return (
@@ -113,6 +110,7 @@ class Puppet:
             self.avatar_set,
             self.is_bot,
             self.is_channel,
+            self.is_premium,
             self.custom_mxid,
             self.access_token,
             self.next_batch,
@@ -125,7 +123,7 @@ class Puppet:
         SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,
             displayname_quality=$6, disable_updates=$7, username=$8, phone=$9, photo_id=$10,
             avatar_url=$11, name_set=$12, avatar_set=$13, is_bot=$14, is_channel=$15,
-            custom_mxid=$16, access_token=$17, next_batch=$18, base_url=$19
+            is_premium=$16, custom_mxid=$17, access_token=$18, next_batch=$19, base_url=$20
         WHERE id=$1
         """
         await self.db.execute(q, *self._values)
@@ -135,8 +133,9 @@ class Puppet:
         INSERT INTO puppet (
             id, is_registered, displayname, displayname_source, displayname_contact,
             displayname_quality, disable_updates, username, phone, photo_id, avatar_url, name_set,
-            avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url
+            avatar_set, is_bot, is_channel, is_premium, custom_mxid, access_token, next_batch,
+            base_url
         ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
-                  $19)
+                  $19, $20)
         """
         await self.db.execute(q, *self._values)

mautrix_telegram/db/reaction.py

@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, ClassVar
 
 from asyncpg import Record
 from attr import dataclass
+from telethon.tl.types import ReactionCustomEmoji, ReactionEmoji, TypeReaction
 
 from mautrix.types import EventID, RoomID
 from mautrix.util.async_db import Database
@@ -58,9 +59,10 @@ class Reaction:
     @classmethod
     async def get_by_sender(
         cls, mxid: EventID, mx_room: RoomID, tg_sender: TelegramID
-    ) -> Reaction | None:
+    ) -> list[Reaction]:
         q = f"SELECT {cls.columns} FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3"
-        return cls._from_row(await cls.db.fetchrow(q, mxid, mx_room, tg_sender))
+        rows = await cls.db.fetch(q, mxid, mx_room, tg_sender)
+        return [cls._from_row(row) for row in rows]
 
     @classmethod
     async def get_all_by_message(cls, mxid: EventID, mx_room: RoomID) -> list[Reaction]:
@@ -68,6 +70,13 @@ class Reaction:
         rows = await cls.db.fetch(q, mxid, mx_room)
         return [cls._from_row(row) for row in rows]
 
+    @property
+    def telegram(self) -> TypeReaction:
+        if self.reaction.isdecimal():
+            return ReactionCustomEmoji(document_id=int(self.reaction))
+        else:
+            return ReactionEmoji(emoticon=self.reaction)
+
     @property
     def _values(self):
         return (
@@ -81,11 +90,11 @@ class Reaction:
     async def save(self) -> None:
         q = """
             INSERT INTO reaction (mxid, mx_room, msg_mxid, tg_sender, reaction)
-            VALUES ($1, $2, $3, $4, $5) ON CONFLICT (msg_mxid, mx_room, tg_sender)
-                DO UPDATE SET mxid=$1, reaction=$5
+            VALUES ($1, $2, $3, $4, $5) ON CONFLICT (msg_mxid, mx_room, tg_sender, reaction)
+                DO UPDATE SET mxid=excluded.mxid
         """
         await self.db.execute(q, *self._values)
 
     async def delete(self) -> None:
-        q = "DELETE FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3"
-        await self.db.execute(q, self.msg_mxid, self.mx_room, self.tg_sender)
+        q = "DELETE FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3 AND reaction=$4"
+        await self.db.execute(q, self.msg_mxid, self.mx_room, self.tg_sender, self.reaction)

mautrix_telegram/db/telegram_file.py

@@ -17,10 +17,11 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, ClassVar
 
+from asyncpg import Record
 from attr import dataclass
 
 from mautrix.types import ContentURI, EncryptedFile
-from mautrix.util.async_db import Database
+from mautrix.util.async_db import Database, Scheme
 
 fake_db = Database.create("") if TYPE_CHECKING else None
 
@@ -40,28 +41,55 @@ class TelegramFile:
     decryption_info: EncryptedFile | None
     thumbnail: TelegramFile | None = None
 
+    columns: ClassVar[str] = (
+        "id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail, "
+        "decryption_info"
+    )
+
     @classmethod
-    async def get(cls, loc_id: str, *, _thumbnail: bool = False) -> TelegramFile | None:
-        q = (
-            "SELECT id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail,"
-            "       decryption_info "
-            "FROM telegram_file WHERE id=$1"
-        )
-        row = await cls.db.fetchrow(q, loc_id)
+    def _from_row(cls, row: Record | None) -> TelegramFile | None:
         if row is None:
             return None
         data = {**row}
-        thumbnail_id = data.pop("thumbnail", None)
-        if _thumbnail:
-            # Don't allow more than one level of recursion
-            thumbnail_id = None
+        data.pop("thumbnail", None)
         decryption_info = data.pop("decryption_info", None)
         return cls(
             **data,
-            thumbnail=(await cls.get(thumbnail_id, _thumbnail=True)) if thumbnail_id else None,
+            thumbnail=None,
             decryption_info=EncryptedFile.parse_json(decryption_info) if decryption_info else None,
         )
 
+    @classmethod
+    async def get_many(cls, loc_ids: list[str]) -> list[TelegramFile]:
+        if cls.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+            q = f"SELECT {cls.columns} FROM telegram_file WHERE id=ANY($1)"
+            rows = await cls.db.fetch(q, loc_ids)
+        else:
+            tgid_placeholders = ("?," * len(loc_ids)).rstrip(",")
+            q = f"SELECT {cls.columns} FROM telegram_file WHERE id IN ({tgid_placeholders})"
+            rows = await cls.db.fetch(q, *loc_ids)
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def get(cls, loc_id: str, *, _thumbnail: bool = False) -> TelegramFile | None:
+        q = f"SELECT {cls.columns} FROM telegram_file WHERE id=$1"
+        row = await cls.db.fetchrow(q, loc_id)
+        file = cls._from_row(row)
+        if file is None:
+            return None
+        try:
+            thumbnail_id = row["thumbnail"]
+        except KeyError:
+            thumbnail_id = None
+        if thumbnail_id and not _thumbnail:
+            file.thumbnail = await cls.get(thumbnail_id, _thumbnail=True)
+        return file
+
+    @classmethod
+    async def find_by_mxc(cls, mxc: ContentURI) -> TelegramFile | None:
+        q = f"SELECT {cls.columns} FROM telegram_file WHERE mxc=$1"
+        return cls._from_row(await cls.db.fetchrow(q, mxc))
+
     async def insert(self) -> None:
         q = (
             "INSERT INTO telegram_file (id, mxc, mime_type, was_converted, size, width, height, "

mautrix_telegram/db/telethon_session.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, Iterable
 import asyncio
 import datetime
 
@@ -124,18 +124,42 @@ class PgSession(MemorySession):
         return updates.State(row["pts"], row["qts"], date, row["seq"], row["unread_count"])
 
     async def set_update_state(self, entity_id: int, row: updates.State) -> None:
-        q = (
-            "INSERT INTO telethon_update_state"
-            "    (session_id, entity_id, pts, qts, date, seq, unread_count) "
-            "VALUES ($1, $2, $3, $4, $5, $6, $7)"
-            "ON CONFLICT (session_id, entity_id) DO UPDATE"
-            "    SET pts=$3, qts=$4, date=$5, seq=$6, unread_count=$7"
-        )
+        q = """
+        INSERT INTO telethon_update_state(session_id, entity_id, pts, qts, date, seq, unread_count)
+        VALUES ($1, $2, $3, $4, $5, $6, $7)
+        ON CONFLICT (session_id, entity_id) DO UPDATE SET
+            pts=excluded.pts, qts=excluded.qts, date=excluded.date, seq=excluded.seq,
+            unread_count=excluded.unread_count
+        """
         ts = row.date.timestamp()
         await self.db.execute(
             q, self.session_id, entity_id, row.pts, row.qts, ts, row.seq, row.unread_count
         )
 
+    async def delete_update_state(self, entity_id: int) -> None:
+        q = "DELETE FROM telethon_update_state WHERE session_id=$1 AND entity_id=$2"
+        await self.db.execute(q, self.session_id, entity_id)
+
+    async def get_update_states(self) -> Iterable[tuple[int, updates.State], ...]:
+        q = (
+            "SELECT entity_id, pts, qts, date, seq, unread_count FROM telethon_update_state "
+            "WHERE session_id=$1"
+        )
+        rows = await self.db.fetch(q, self.session_id)
+        return (
+            (
+                row["entity_id"],
+                updates.State(
+                    row["pts"],
+                    row["qts"],
+                    datetime.datetime.utcfromtimestamp(row["date"]),
+                    row["seq"],
+                    row["unread_count"],
+                ),
+            )
+            for row in rows
+        )
+
     def _entity_values_to_row(
         self, id: int, hash: int, username: str | None, phone: str | int | None, name: str | None
     ) -> tuple[str, int, int, str | None, str | None, str | None]:
@@ -176,25 +200,24 @@ class PgSession(MemorySession):
     async def _select_entity(
         self, constraint: str, *args: str | int | tuple[int, ...]
     ) -> tuple[int, int] | None:
-        row = await self.db.fetchrow(
-            f"SELECT id, hash FROM telethon_entities WHERE {constraint}", *args
-        )
+        q = f"SELECT id, hash FROM telethon_entities WHERE session_id=$1 AND {constraint}"
+        row = await self.db.fetchrow(q, self.session_id, *args)
         if row is None:
             return None
         return row["id"], row["hash"]
 
     async def get_entity_rows_by_phone(self, key: str | int) -> tuple[int, int] | None:
-        return await self._select_entity("phone=$1", str(key))
+        return await self._select_entity("phone=$2", str(key))
 
     async def get_entity_rows_by_username(self, key: str) -> tuple[int, int] | None:
-        return await self._select_entity("username=$1", key)
+        return await self._select_entity("username=$2", key)
 
     async def get_entity_rows_by_name(self, key: str) -> tuple[int, int] | None:
-        return await self._select_entity("name=$1", key)
+        return await self._select_entity("name=$2", key)
 
     async def get_entity_rows_by_id(self, key: int, exact: bool = True) -> tuple[int, int] | None:
         if exact:
-            return await self._select_entity("id=$1", key)
+            return await self._select_entity("id=$2", key)
 
         ids = (
             utils.get_peer_id(PeerUser(key)),
@@ -202,6 +225,6 @@ class PgSession(MemorySession):
             utils.get_peer_id(PeerChannel(key)),
         )
         if self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
-            return await self._select_entity("id=ANY($1)", ids)
+            return await self._select_entity("id=ANY($2)", ids)
         else:
-            return await self._select_entity(f"id IN ($1, $2, $3)", *ids)
+            return await self._select_entity(f"id IN ($2, $3, $4)", *ids)

mautrix_telegram/db/upgrade/__init__.py

@@ -10,4 +10,14 @@ from . import (
     v05_channel_ghosts,
     v06_puppet_avatar_url,
     v07_puppet_phone_number,
+    v08_portal_first_event,
+    v09_puppet_username_index,
+    v10_more_backfill_fields,
+    v11_backfill_queue,
+    v12_message_sender,
+    v13_multiple_reactions,
+    v14_puppet_custom_mxid_index,
+    v15_backfill_anchor_id,
+    v16_backfill_type,
+    v17_message_find_recent,
 )

mautrix_telegram/db/upgrade/v00_latest_revision.py

@@ -13,10 +13,12 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.util.async_db import Connection
+from mautrix.util.async_db import Connection, Scheme
+
+latest_version = 17
 
 
-async def create_v7_tables(conn: Connection) -> int:
+async def create_latest_tables(conn: Connection, scheme: Scheme) -> int:
     await conn.execute(
         """CREATE TABLE "user" (
             mxid TEXT   PRIMARY KEY,
@@ -24,6 +26,7 @@ async def create_v7_tables(conn: Connection) -> int:
             tg_username    TEXT,
             tg_phone       TEXT,
             is_bot         BOOLEAN NOT NULL DEFAULT false,
+            is_premium     BOOLEAN NOT NULL DEFAULT false,
             saved_contacts INTEGER NOT NULL DEFAULT 0
         )"""
     )
@@ -44,6 +47,10 @@ async def create_v7_tables(conn: Connection) -> int:
             megagroup   BOOLEAN,
             config      jsonb,
 
+            first_event_id    TEXT,
+            next_batch_id     TEXT,
+            base_insertion_id TEXT,
+
             sponsored_event_id     TEXT,
             sponsored_event_ts     BIGINT,
             sponsored_msg_random_id bytea,
@@ -60,10 +67,13 @@ async def create_v7_tables(conn: Connection) -> int:
             edit_index   INTEGER,
             redacted     BOOLEAN NOT NULL DEFAULT false,
             content_hash bytea,
+            sender_mxid  TEXT,
+            sender       BIGINT,
             PRIMARY KEY (tgid, tg_space, edit_index),
             UNIQUE (mxid, mx_room, tg_space)
         )"""
     )
+    await conn.execute("CREATE INDEX message_mx_room_and_tgid_idx ON message(mx_room, tgid DESC)")
     await conn.execute(
         """CREATE TABLE reaction (
             mxid      TEXT NOT NULL,
@@ -72,7 +82,7 @@ async def create_v7_tables(conn: Connection) -> int:
             tg_sender BIGINT,
             reaction  TEXT NOT NULL,
 
-            PRIMARY KEY (msg_mxid, mx_room, tg_sender),
+            PRIMARY KEY (msg_mxid, mx_room, tg_sender, reaction),
             UNIQUE (mxid, mx_room)
         )"""
     )
@@ -105,6 +115,7 @@ async def create_v7_tables(conn: Connection) -> int:
             avatar_set          BOOLEAN NOT NULL DEFAULT false,
             is_bot              BOOLEAN,
             is_channel          BOOLEAN NOT NULL DEFAULT false,
+            is_premium          BOOLEAN NOT NULL DEFAULT false,
 
             access_token TEXT,
             custom_mxid  TEXT,
@@ -112,6 +123,8 @@ async def create_v7_tables(conn: Connection) -> int:
             base_url     TEXT
         )"""
     )
+    await conn.execute("CREATE INDEX puppet_username_idx ON puppet(LOWER(username))")
+    await conn.execute("CREATE INDEX puppet_custom_mxid_idx ON puppet(custom_mxid)")
     await conn.execute(
         """CREATE TABLE telegram_file (
             id              TEXT PRIMARY KEY,
@@ -128,6 +141,7 @@ async def create_v7_tables(conn: Connection) -> int:
                 ON UPDATE CASCADE ON DELETE SET NULL
         )"""
     )
+    await conn.execute("CREATE INDEX telegram_file_mxc_idx ON telegram_file(mxc)")
     await conn.execute(
         """CREATE TABLE bot_chat (
             id   BIGINT PRIMARY KEY,
@@ -197,4 +211,31 @@ async def create_v7_tables(conn: Connection) -> int:
             PRIMARY KEY (session_id, entity_id)
         )"""
     )
-    return 7
+    gen = ""
+    if scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+        gen = "GENERATED ALWAYS AS IDENTITY"
+    await conn.execute(
+        f"""
+        CREATE TABLE backfill_queue (
+            queue_id            INTEGER PRIMARY KEY {gen},
+            user_mxid           TEXT,
+            priority            INTEGER NOT NULL,
+            type                TEXT NOT NULL,
+            portal_tgid         BIGINT,
+            portal_tg_receiver  BIGINT,
+            anchor_msg_id       BIGINT,
+            extra_data          jsonb,
+            messages_per_batch  INTEGER NOT NULL,
+            post_batch_delay    INTEGER NOT NULL,
+            max_batches         INTEGER NOT NULL,
+            dispatch_time       TIMESTAMP,
+            completed_at        TIMESTAMP,
+            cooldown_timeout    TIMESTAMP,
+            FOREIGN KEY (user_mxid) REFERENCES "user"(mxid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (portal_tgid, portal_tg_receiver)
+                REFERENCES portal(tgid, tg_receiver) ON DELETE CASCADE
+        )
+        """
+    )
+
+    return latest_version

mautrix_telegram/db/upgrade/v01_initial_revision.py

@@ -18,35 +18,27 @@ from __future__ import annotations
 from mautrix.util.async_db import Connection, Scheme
 
 from . import upgrade_table
-from .v00_latest_revision import create_v7_tables
+from .v00_latest_revision import create_latest_tables, latest_version
 
 legacy_version_query = "SELECT version_num FROM alembic_version"
 last_legacy_version = "bfc0a39bfe02"
 
 
-def table_exists(scheme: str, name: str) -> str:
-    if scheme == Scheme.SQLITE:
-        return f"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='{name}')"
-    elif scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
-        return f"SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name='{name}')"
-    raise RuntimeError("unsupported database scheme")
-
-
-async def first_upgrade_target(conn: Connection, scheme: str) -> int:
-    is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
-    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to v7.
-    # If it's a new db, we'll create the v7 tables directly (see the create_v7_tables call).
-    return 1 if is_legacy else 7
+async def first_upgrade_target(conn: Connection, scheme: Scheme) -> int:
+    is_legacy = await conn.table_exists("alembic_version")
+    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to latest.
+    # If it's a new db, we'll create the latest tables directly (see create_latest_tables call).
+    return 1 if is_legacy else latest_version
 
 
 @upgrade_table.register(description="Initial asyncpg revision", upgrades_to=first_upgrade_target)
-async def upgrade_v1(conn: Connection, scheme: str) -> int:
-    is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
+async def upgrade_v1(conn: Connection, scheme: Scheme) -> int:
+    is_legacy = await conn.table_exists("alembic_version")
     if is_legacy:
         await migrate_legacy_to_v1(conn, scheme)
         return 1
     else:
-        return await create_v7_tables(conn)
+        return await create_latest_tables(conn, scheme)
 
 
 async def drop_constraints(conn: Connection, table: str, contype: str) -> None:
@@ -59,14 +51,14 @@ async def drop_constraints(conn: Connection, table: str, contype: str) -> None:
     await conn.execute(f"ALTER TABLE {table} {drops}")
 
 
-async def migrate_legacy_to_v1(conn: Connection, scheme: str) -> None:
+async def migrate_legacy_to_v1(conn: Connection, scheme: Scheme) -> None:
     legacy_version = await conn.fetchval(legacy_version_query)
     if legacy_version != last_legacy_version:
         raise RuntimeError(
             "Legacy database is not on last version. "
             "Please upgrade the old database with alembic or drop it completely first."
         )
-    if scheme != "sqlite":
+    if scheme != Scheme.SQLITE:
         await drop_constraints(conn, "contact", contype="f")
         await conn.execute(
             """
@@ -131,12 +123,12 @@ async def migrate_legacy_to_v1(conn: Connection, scheme: str) -> None:
     await conn.execute("DROP TABLE alembic_version")
 
 
-async def update_state_store(conn: Connection, scheme: str) -> None:
+async def update_state_store(conn: Connection, scheme: Scheme) -> None:
     # The Matrix state store already has more or less the correct schema, so set the version
     await conn.execute("CREATE TABLE mx_version (version INTEGER PRIMARY KEY)")
     await conn.execute("INSERT INTO mx_version (version) VALUES (2)")
     await conn.execute("UPDATE mx_user_profile SET membership='LEAVE' WHERE membership='LEFT'")
-    if scheme != "sqlite":
+    if scheme != Scheme.SQLITE:
         # Also add the membership type on postgres
         await conn.execute(
             "CREATE TYPE membership AS ENUM ('join', 'leave', 'invite', 'ban', 'knock')"

mautrix_telegram/db/upgrade/v08_portal_first_event.py

@@ -0,0 +1,24 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Track first event ID in portals for infinite backfilling")
+async def upgrade_v8(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN first_event_id TEXT")
+    await conn.execute("ALTER TABLE portal ADD COLUMN next_batch_id TEXT")

mautrix_telegram/db/upgrade/v09_puppet_username_index.py

@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add index to puppet username column")
+async def upgrade_v9(conn: Connection) -> None:
+    await conn.execute("CREATE INDEX puppet_username_idx ON puppet(LOWER(username))")

mautrix_telegram/db/upgrade/v10_more_backfill_fields.py

@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add more portal columns related to infinite backfill")
+async def upgrade_v10(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN base_insertion_id TEXT")

mautrix_telegram/db/upgrade/v11_backfill_queue.py

@@ -0,0 +1,45 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan, Sumner Evans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add the backfill queue table")
+async def upgrade_v11(conn: Connection, scheme: Scheme) -> None:
+    gen = ""
+    if scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+        gen = "GENERATED ALWAYS AS IDENTITY"
+    await conn.execute(
+        f"""
+        CREATE TABLE backfill_queue (
+            queue_id            INTEGER PRIMARY KEY {gen},
+            user_mxid           TEXT,
+            priority            INTEGER NOT NULL,
+            portal_tgid         BIGINT,
+            portal_tg_receiver  BIGINT,
+            messages_per_batch  INTEGER NOT NULL,
+            post_batch_delay    INTEGER NOT NULL,
+            max_batches         INTEGER NOT NULL,
+            dispatch_time       TIMESTAMP,
+            completed_at        TIMESTAMP,
+            cooldown_timeout    TIMESTAMP,
+            FOREIGN KEY (user_mxid) REFERENCES "user"(mxid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (portal_tgid, portal_tg_receiver)
+                REFERENCES portal(tgid, tg_receiver) ON DELETE CASCADE
+        )
+        """
+    )

mautrix_telegram/db/upgrade/v12_message_sender.py

@@ -0,0 +1,24 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store sender in message table")
+async def upgrade_v12(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE message ADD COLUMN sender_mxid TEXT")
+    await conn.execute("ALTER TABLE message ADD COLUMN sender BIGINT")

mautrix_telegram/db/upgrade/v13_multiple_reactions.py

@@ -0,0 +1,54 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Allow multiple reactions from the same user")
+async def upgrade_v13(conn: Connection, scheme: Scheme) -> None:
+    await conn.execute("CREATE INDEX telegram_file_mxc_idx ON telegram_file(mxc)")
+    await conn.execute('ALTER TABLE "user" ADD COLUMN is_premium BOOLEAN NOT NULL DEFAULT false')
+    await conn.execute("ALTER TABLE puppet ADD COLUMN is_premium BOOLEAN NOT NULL DEFAULT false")
+    if scheme == Scheme.POSTGRES:
+        await conn.execute(
+            """
+            ALTER TABLE reaction
+                DROP CONSTRAINT reaction_pkey,
+                ADD CONSTRAINT reaction_pkey PRIMARY KEY (msg_mxid, mx_room, tg_sender, reaction)
+            """
+        )
+    else:
+        await conn.execute(
+            """CREATE TABLE new_reaction (
+                mxid      TEXT NOT NULL,
+                mx_room   TEXT NOT NULL,
+                msg_mxid  TEXT NOT NULL,
+                tg_sender BIGINT,
+                reaction  TEXT NOT NULL,
+
+                PRIMARY KEY (msg_mxid, mx_room, tg_sender, reaction),
+                UNIQUE (mxid, mx_room)
+            )"""
+        )
+        await conn.execute(
+            """
+            INSERT INTO new_reaction (mxid, mx_room, msg_mxid, tg_sender, reaction)
+            SELECT mxid, mx_room, msg_mxid, tg_sender, reaction FROM reaction
+            """
+        )
+        await conn.execute("DROP TABLE reaction")
+        await conn.execute("ALTER TABLE new_reaction RENAME TO reaction")

mautrix_telegram/db/upgrade/v14_puppet_custom_mxid_index.py

@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add index to puppet custom_mxid column")
+async def upgrade_v14(conn: Connection) -> None:
+    await conn.execute("CREATE INDEX IF NOT EXISTS puppet_custom_mxid_idx ON puppet(custom_mxid)")

mautrix_telegram/db/upgrade/v15_backfill_anchor_id.py

@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store lowest message ID in backfill queue")
+async def upgrade_v15(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE backfill_queue ADD COLUMN anchor_msg_id BIGINT")

mautrix_telegram/db/upgrade/v16_backfill_type.py

@@ -0,0 +1,28 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add type for backfill queue items")
+async def upgrade_v16(conn: Connection, scheme: Scheme) -> None:
+    await conn.execute(
+        "ALTER TABLE backfill_queue ADD COLUMN type TEXT NOT NULL DEFAULT 'historical'"
+    )
+    await conn.execute("ALTER TABLE backfill_queue ADD COLUMN extra_data jsonb")
+    if scheme != Scheme.SQLITE:
+        await conn.execute("ALTER TABLE backfill_queue ALTER COLUMN type DROP DEFAULT")

mautrix_telegram/db/upgrade/v17_message_find_recent.py

@@ -0,0 +1,25 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add index for Message.find_recent")
+async def upgrade_v17(conn: Connection) -> None:
+    await conn.execute(
+        "CREATE INDEX IF NOT EXISTS message_mx_room_and_tgid_idx ON message(mx_room, tgid DESC)"
+    )

mautrix_telegram/db/user.py

@@ -37,6 +37,7 @@ class User:
     tg_username: str | None
     tg_phone: str | None
     is_bot: bool
+    is_premium: bool
     saved_contacts: int
 
     @classmethod
@@ -45,7 +46,9 @@ class User:
             return None
         return cls(**row)
 
-    columns: ClassVar[str] = "mxid, tgid, tg_username, tg_phone, is_bot, saved_contacts"
+    columns: ClassVar[str] = ", ".join(
+        ("mxid", "tgid", "tg_username", "tg_phone", "is_bot", "is_premium", "saved_contacts")
+    )
 
     @classmethod
     async def get_by_tgid(cls, tgid: TelegramID) -> User | None:
@@ -78,21 +81,23 @@ class User:
             self.tg_username,
             self.tg_phone,
             self.is_bot,
+            self.is_premium,
             self.saved_contacts,
         )
 
     async def save(self) -> None:
-        q = (
-            'UPDATE "user" SET tgid=$2, tg_username=$3, tg_phone=$4, is_bot=$5, saved_contacts=$6 '
-            "WHERE mxid=$1"
-        )
+        q = """
+        UPDATE "user" SET tgid=$2, tg_username=$3, tg_phone=$4, is_bot=$5, is_premium=$6,
+                          saved_contacts=$7
+        WHERE mxid=$1
+        """
         await self.db.execute(q, *self._values)
 
     async def insert(self) -> None:
-        q = (
-            'INSERT INTO "user" (mxid, tgid, tg_username, tg_phone, is_bot, saved_contacts) '
-            "VALUES ($1, $2, $3, $4, $5, $6)"
-        )
+        q = """
+        INSERT INTO "user" (mxid, tgid, tg_username, tg_phone, is_bot, is_premium, saved_contacts)
+        VALUES ($1, $2, $3, $4, $5, $6, $7)
+        """
         await self.db.execute(q, *self._values)
 
     async def get_contacts(self) -> list[TelegramID]:

mautrix_telegram/example-config.yaml

@@ -7,7 +7,9 @@ homeserver:
     # Whether or not to verify the SSL certificate of the homeserver.
     # Only applies if address starts with https://
     verify_ssl: true
-    asmux: false
+    # What software is the homeserver running?
+    # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here.
+    software: standard
     # Number of retries for all HTTP requests if the homeserver isn't reachable.
     http_retry_count: 4
     # The URL to push real-time bridge status to.
@@ -45,6 +47,7 @@ appservice:
     # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool
     # https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
     # For sqlite, min_size is used as the connection thread pool size and max_size is ignored.
+    # Additionally, SQLite supports init_commands as an array of SQL queries to run on connect (e.g. to set PRAGMAs).
     database_opts:
         min_size: 1
         max_size: 10
@@ -84,7 +87,7 @@ appservice:
     # Whether or not to receive ephemeral events via appservice transactions.
     # Requires MSC2409 support (i.e. Synapse 1.22+).
     # You should disable bridge -> sync_with_custom_puppets when this is enabled.
-    ephemeral_events: false
+    ephemeral_events: true
 
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
@@ -148,21 +151,28 @@ bridge:
     # will not send any more members.
     # -1 means no limit (which means it's limited to 10000 by the server)
     max_initial_member_sync: 100
+    # Maximum number of participants in chats to bridge. Only applies when the portal is being created.
+    # If there are more members when trying to create a room, the room creation will be cancelled.
+    # -1 means no limit (which means all chats can be bridged)
+    max_member_count: -1
     # Whether or not to sync the member list in channels.
     # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
     # list regardless of this setting.
-    sync_channel_members: true
+    sync_channel_members: false
     # Whether or not to skip deleted members when syncing members.
     skip_deleted_members: true
     # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
     # their Telegram account at startup.
-    startup_sync: true
+    startup_sync: false
     # Number of most recently active dialogs to check when syncing chats.
     # Set to 0 to remove limit.
     sync_update_limit: 0
     # Number of most recently active dialogs to create portals for when syncing chats.
     # Set to 0 to remove limit.
-    sync_create_limit: 30
+    sync_create_limit: 15
+    # Should all chats be scheduled to be created later?
+    # This is best used in combination with MSC2716 infinite backfill.
+    sync_deferred_create_all: false
     # Whether or not to sync and create portals for direct chats at startup.
     sync_direct_chats: false
     # The maximum number of simultaneous Telegram deletions to handle.
@@ -174,15 +184,11 @@ bridge:
     # Allow logging in within Matrix. If false, users can only log in using login-qr or the
     # out-of-Matrix login website (see appservice.public config section)
     allow_matrix_login: true
-    # Whether or not to bridge plaintext highlights.
-    # Only enable this if your displayname_template has some static part that the bridge can use to
-    # reliably identify what is a plaintext highlight.
-    plaintext_highlights: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
-    public_portals: true
+    public_portals: false
     # Whether or not to use /sync to get presence, read receipts and typing notifications
     # when double puppeting is enabled
-    sync_with_custom_puppets: true
+    sync_with_custom_puppets: false
     # Whether or not to update the m.direct account data event when double puppeting is enabled.
     # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
     # and is therefore prone to race conditions.
@@ -206,13 +212,13 @@ bridge:
     # Whether or not the !tg join command should do a HTTP request
     # to resolve redirects in invite links.
     invite_link_resolve: false
-    # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Element iOS/Android).
-    inline_images: false
+    # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552.
+    # This is currently not supported in most clients.
+    caption_in_message: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
-    # Maximum number of pixels in an image before sending to Telegram as a document. Defaults to 1280x1280 = 1638400.
-    image_as_file_pixels: 1638400
+    # Maximum number of pixels in an image before sending to Telegram as a document. Defaults to 4096x4096 = 16777216.
+    image_as_file_pixels: 16777216
     # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
     # streaming from/to Matrix and using many connections for Telegram.
     # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
@@ -221,6 +227,9 @@ bridge:
     # Whether or not created rooms should have federation enabled.
     # If false, created portal rooms will never be federated.
     federate_rooms: true
+    # Should the bridge send all unicode reactions as custom emoji reactions to Telegram?
+    # By default, the bridge only uses custom emojis for unicode emojis that aren't allowed in reactions.
+    always_custom_emoji_reaction: false
     # Settings for converting animated stickers.
     animated_sticker:
         # Format to which animated stickers should be converted.
@@ -228,12 +237,24 @@ bridge:
         # png - converts to non-animated png (fastest),
         # gif - converts to animated gif
         # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support
+        # webp - converts to animated webp, requires ffmpeg executable with webp codec/container support
         target: gif
+        # Should video stickers be converted to the specified format as well?
+        convert_from_webm: false
         # Arguments for converter. All converters take width and height.
         args:
             width: 256
             height: 256
-            fps: 25 # only for webm and gif (2, 5, 10, 20 or 25 recommended)
+            fps: 25 # only for webm, webp and gif (2, 5, 10, 20 or 25 recommended)
+    # Settings for converting animated emoji.
+    # Same as animated_sticker, but webm is not supported as the target
+    # (because inline images can only contain images, not videos).
+    animated_emoji:
+        target: webp
+        args:
+            width: 64
+            height: 64
+            fps: 25
     # End-to-bridge encryption support options.
     #
     # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info.
@@ -243,20 +264,48 @@ bridge:
         # Default to encryption, force-enable encryption in all portals the bridge creates
         # This will cause the bridge bot to be in private chats for the encryption to work properly.
         default: false
-        # Database for the encryption data. If set to `default`, will use the appservice database.
-        database: default
-        # Options for automatic key sharing.
-        key_sharing:
-            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
-            # You must use a client that supports requesting keys from other users to use this feature.
-            allow: false
-            # Require the requesting device to have a valid cross-signing signature?
-            # This doesn't require that the bridge has verified the device, only that the user has verified it.
-            # Not yet implemented.
-            require_cross_signing: false
-            # Require devices to be verified by the bridge?
-            # Verification by the bridge is not yet implemented.
-            require_verification: true
+        # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data.
+        appservice: false
+        # Require encryption, drop any unencrypted messages.
+        require: false
+        # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
+        # You must use a client that supports requesting keys from other users to use this feature.
+        allow_key_sharing: false
+        # What level of device verification should be required from users?
+        #
+        # Valid levels:
+        #   unverified - Send keys to all device in the room.
+        #   cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys.
+        #   cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes).
+        #   cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot.
+        #                           Note that creating user signatures from the bridge bot is not currently possible.
+        #   verified - Require manual per-device verification
+        #              (currently only possible by modifying the `trust` column in the `crypto_device` database table).
+        verification_levels:
+            # Minimum level for which the bridge should send keys to when bridging messages from Telegram to Matrix.
+            receive: unverified
+            # Minimum level that the bridge should accept for incoming Matrix messages.
+            send: unverified
+            # Minimum level that the bridge should require for accepting key requests.
+            share: cross-signed-tofu
+        # Options for Megolm room key rotation. These options allow you to
+        # configure the m.room.encryption event content. See:
+        # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for
+        # more information about that event.
+        rotation:
+            # Enable custom Megolm room key rotation settings. Note that these
+            # settings will only apply to rooms created after this option is
+            # set.
+            enable_custom: false
+            # The maximum number of milliseconds a session should be used
+            # before changing it. The Matrix spec recommends 604800000 (a week)
+            # as the default.
+            milliseconds: 604800000
+            # The maximum number of messages that should be sent with a given a
+            # session before changing it. The Matrix spec recommends 100 as the
+            # default.
+            messages: 100
+
     # Whether or not to explicitly set the avatar and room name for private
     # chat portal rooms. This will be implicitly enabled if encryption.default is true.
     private_chat_portal_meta: false
@@ -265,6 +314,8 @@ bridge:
     delivery_receipts: false
     # Whether or not delivery errors should be reported as messages in the Matrix room.
     delivery_error_reports: false
+    # Whether the bridge should send the message status as a custom com.beeper.message_send_status event.
+    message_status_events: false
     # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
     # This field will automatically be changed back to false after it,
     # except if the config file is not writable.
@@ -284,36 +335,65 @@ bridge:
     kick_on_logout: true
     # Should the "* user joined Telegram" notice always be marked as read automatically?
     always_read_joined_telegram_notice: true
+    # Should the bridge auto-create a group chat on Telegram when a ghost is invited to a room?
+    # Requires the user to have sufficient power level and double puppeting enabled.
+    create_group_on_invite: true
     # Settings for backfilling messages from Telegram.
     backfill:
-        # Whether or not the Telegram ghosts of logged in Matrix users should be
-        # invited to private chats when backfilling history from Telegram. This is
-        # usually needed to prevent rate limits and to allow timestamp massaging.
-        invite_own_puppet: true
-        # Maximum number of messages to backfill without using a takeout.
-        # The first time a takeout is used, the user has to manually approve it from a different
-        # device. If initial_limit or missed_limit are higher than this value, the bridge will ask
-        # the user to accept the takeout after logging in before syncing any chats.
-        takeout_limit: 100
-        # Maximum number of messages to backfill initially.
-        # Set to 0 to disable backfilling when creating portal, or -1 to disable the limit.
+        # Allow backfilling at all?
+        enable: true
+        # Use MSC2716 for backfilling?
         #
-        # N.B. Initial backfill will only start after member sync. Make sure your
-        #      max_initial_member_sync is set to a low enough value so it doesn't take forever.
-        initial_limit: 0
-        # Maximum number of messages to backfill if messages were missed while the bridge was
-        # disconnected. Note that this only works for logged in users and only if the chat isn't
-        # older than sync_update_limit
-        # Set to 0 to disable backfilling missed messages.
-        missed_limit: 50
-        # If using double puppeting, should notifications be disabled
-        # while the initial backfill is in progress?
-        disable_notifications: false
+        # This requires a server with MSC2716 support, which is currently an experimental feature in Synapse.
+        # It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml.
+        msc2716: false
+        # Use double puppets for backfilling?
+        #
+        # If using MSC2716, the double puppets must be in the appservice's user ID namespace
+        # (because the bridge can't use the double puppet access token with batch sending).
+        #
+        # Even without MSC2716, bridging old messages with correct timestamps requires the double
+        # puppets to be in an appservice namespace, or the server to be modified to allow
+        # overriding timestamps anyway.
+        double_puppet_backfill: false
         # Whether or not to enable backfilling in normal groups.
         # Normal groups have numerous technical problems in Telegram, and backfilling normal groups
         # will likely cause problems if there are multiple Matrix users in the group.
         normal_groups: false
 
+        # If a backfilled chat is older than this number of hours, mark it as read even if it's unread on Telegram.
+        # Set to -1 to let any chat be unread.
+        unread_hours_threshold: 720
+
+        # Forward backfilling limits. These apply to both MSC2716 and legacy backfill.
+        #
+        # Using a negative initial limit is not recommended, as it would try to backfill everything in a single batch.
+        # MSC2716 and the incremental settings are meant for backfilling everything incrementally rather than at once.
+        forward:
+            # Number of messages to backfill immediately after creating a portal.
+            initial_limit: 10
+            # Number of messages to backfill when syncing chats.
+            sync_limit: 100
+
+        # Settings for incremental backfill of history. These only apply when using MSC2716.
+        incremental:
+            # Maximum number of messages to backfill per batch.
+            messages_per_batch: 100
+            # The number of seconds to wait after backfilling the batch of messages.
+            post_batch_delay: 20
+            # The maximum number of batches to backfill per portal, split by the chat type.
+            # If set to -1, all messages in the chat will eventually be backfilled.
+            max_batches:
+                # Direct chats
+                user: -1
+                # Normal groups. Note that the normal_groups option above must be enabled
+                # for these to be backfilled.
+                normal_group: -1
+                # Supergroups
+                supergroup: 10
+                # Broadcast channels
+                channel: -1
+
     # Overrides for base power levels.
     initial_power_level_overrides:
         user: {}
@@ -455,6 +535,12 @@ telegram:
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
 
+    # Should the bridge request missed updates from Telegram when restarting?
+    catch_up: true
+    # Should incoming updates be handled sequentially to make sure order is preserved on Matrix?
+    sequential_updates: true
+    exit_on_update_error: false
+
     # Telethon connection options.
     connection:
         # The timeout in seconds to be used when connecting.
@@ -480,7 +566,7 @@ telegram:
     # Device info sent to Telegram.
     device_info:
         # "auto" = OS name+version.
-        device_model: auto
+        device_model: mautrix-telegram
         # "auto" = Telethon version.
         system_version: auto
         # "auto" = mautrix-telegram version.

mautrix_telegram/formatter/__init__.py

@@ -1,2 +1,2 @@
 from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram
-from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
+from .from_telegram import telegram_to_matrix

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -18,7 +18,7 @@ from __future__ import annotations
 import re
 
 from telethon import TelegramClient
-from telethon.helpers import add_surrogate, del_surrogate
+from telethon.helpers import add_surrogate, del_surrogate, strip_text
 from telethon.tl.types import MessageEntityItalic, TypeMessageEntity
 
 from mautrix.types import MessageEventContent, RoomID
@@ -59,7 +59,7 @@ async def matrix_to_telegram(
     if html is not None:
         return await _matrix_html_to_telegram(client, html)
     elif text is not None:
-        return _matrix_text_to_telegram(text), []
+        return _matrix_text_to_telegram(text)
     else:
         raise ValueError("text or html must be provided to convert formatting")
 
@@ -73,8 +73,8 @@ async def _matrix_html_to_telegram(
         html = not_command_regex.sub(r"\1", html)
 
         parsed = await MatrixParser(client).parse(add_surrogate(html))
-        text = del_surrogate(parsed.text.strip())
-        text, entities = _cut_long_message(text, parsed.telegram_entities)
+        text, entities = _cut_long_message(parsed.text, parsed.telegram_entities)
+        text = del_surrogate(strip_text(text, entities))
 
         return text, entities
     except Exception as e:
@@ -98,8 +98,13 @@ def _cut_long_message(
     return message, entities
 
 
-def _matrix_text_to_telegram(text: str) -> str:
+def _matrix_text_to_telegram(text: str) -> tuple[str, list[TypeMessageEntity]]:
     text = command_regex.sub(r"/\1", text)
     text = text.replace("\t", " " * 4)
     text = not_command_regex.sub(r"\1", text)
-    return text
+    entities = []
+    surrogated_text = add_surrogate(text)
+    if len(surrogated_text) > MAX_LENGTH:
+        surrogated_text, entities = _cut_long_message(surrogated_text, entities)
+        text = del_surrogate(surrogated_text)
+    return text, entities

mautrix_telegram/formatter/from_telegram.py

@@ -20,14 +20,18 @@ import logging
 import re
 
 from telethon.errors import RPCError
-from telethon.helpers import add_surrogate, del_surrogate, within_surrogate
+from telethon.helpers import add_surrogate, del_surrogate
 from telethon.tl.custom import Message
 from telethon.tl.types import (
+    Channel,
+    InputPeerChannelFromMessage,
+    InputPeerUserFromMessage,
     MessageEntityBlockquote,
     MessageEntityBold,
     MessageEntityBotCommand,
     MessageEntityCashtag,
     MessageEntityCode,
+    MessageEntityCustomEmoji,
     MessageEntityEmail,
     MessageEntityHashtag,
     MessageEntityItalic,
@@ -46,41 +50,47 @@ from telethon.tl.types import (
     PeerUser,
     SponsoredMessage,
     TypeMessageEntity,
+    User,
 )
 
-from mautrix.appservice import IntentAPI
-from mautrix.types import (
-    EventType,
-    Format,
-    InReplyTo,
-    MessageType,
-    RelatesTo,
-    TextMessageEventContent,
-)
+from mautrix.types import Format, MessageType, TextMessageEventContent
 
 from .. import abstract_user as au, portal as po, puppet as pu, user as u
-from ..db import Message as DBMessage
+from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
+from ..tgclient import MautrixTelegramClient
 from ..types import TelegramID
+from ..util.file_transfer import UnicodeCustomEmoji, transfer_custom_emojis_to_matrix
 
 log: logging.Logger = logging.getLogger("mau.fmt.tg")
 
 
-async def telegram_reply_to_matrix(evt: Message, source: au.AbstractUser) -> RelatesTo | None:
-    if evt.reply_to:
-        space = (
-            evt.peer_id.channel_id
-            if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
-            else source.tgid
-        )
-        msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
-        if msg:
-            return RelatesTo(in_reply_to=InReplyTo(event_id=msg.mxid))
-    return None
+async def _get_fwd_entity(client: MautrixTelegramClient, evt: Message) -> Channel | User | None:
+    try:
+        return await client.get_entity(evt.fwd_from.from_id)
+    except (ValueError, RPCError) as e:
+        try:
+            input_peer = await client.get_input_entity(evt.peer_id)
+            if isinstance(evt.fwd_from.from_id, PeerUser):
+                return await client.get_entity(
+                    InputPeerUserFromMessage(
+                        peer=input_peer, msg_id=evt.id, user_id=evt.fwd_from.from_id.user_id
+                    )
+                )
+            elif isinstance(evt.fwd_from.from_id, PeerChannel):
+                return await client.get_entity(
+                    InputPeerChannelFromMessage(
+                        peer=input_peer, msg_id=evt.id, channel_id=evt.fwd_from.from_id.channel_id
+                    )
+                )
+        except (ValueError, RPCError) as e:
+            pass
+        return None
 
 
 async def _add_forward_header(
-    source: au.AbstractUser, content: TextMessageEventContent, fwd_from: MessageFwdHeader
+    client: MautrixTelegramClient, content: TextMessageEventContent, evt: Message
 ) -> None:
+    fwd_from = evt.fwd_from
     fwd_from_html, fwd_from_text = None, None
     if isinstance(fwd_from.from_id, PeerUser):
         user = await u.User.get_by_tgid(TelegramID(fwd_from.from_id.user_id))
@@ -99,12 +109,11 @@ async def _add_forward_header(
                 )
 
         if not fwd_from_text:
-            try:
-                user = await source.client.get_entity(fwd_from.from_id)
-                if user:
-                    fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
-                    fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
-            except (ValueError, RPCError):
+            user = await _get_fwd_entity(client, evt)
+            if user:
+                fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+            else:
                 fwd_from_text = fwd_from_html = "unknown user"
     elif isinstance(fwd_from.from_id, (PeerChannel, PeerChat)):
         from_id = (
@@ -122,12 +131,11 @@ async def _add_forward_header(
             else:
                 fwd_from_html = f"channel <b>{escape(fwd_from_text)}</b>"
         else:
-            try:
-                channel = await source.client.get_entity(fwd_from.from_id)
-                if channel:
-                    fwd_from_text = f"channel {channel.title}"
-                    fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
-            except (ValueError, RPCError):
+            channel = await _get_fwd_entity(client, evt)
+            if channel:
+                fwd_from_text = f"channel {channel.title}"
+                fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
+            else:
                 fwd_from_text = fwd_from_html = "unknown channel"
     elif fwd_from.from_name:
         fwd_from_text = fwd_from.from_name
@@ -145,49 +153,46 @@ async def _add_forward_header(
     )
 
 
-async def _add_reply_header(
-    source: au.AbstractUser, content: TextMessageEventContent, evt: Message, main_intent: IntentAPI
+class ReuploadedCustomEmoji(MessageEntityCustomEmoji):
+    file: DBTelegramFile
+
+    def __init__(self, parent: MessageEntityCustomEmoji, file: DBTelegramFile) -> None:
+        super().__init__(parent.offset, parent.length, parent.document_id)
+        self.file = file
+
+
+async def _convert_custom_emoji(
+    source: au.AbstractUser,
+    entities: list[TypeMessageEntity],
+    client: MautrixTelegramClient | None = None,
 ) -> None:
-    space = (
-        evt.peer_id.channel_id
-        if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
-        else source.tgid
-    )
-
-    msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
-    if not msg:
-        return
-
-    try:
-        event = await main_intent.get_event(msg.mx_room, msg.mxid)
-        if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
-            event = await source.bridge.matrix.e2ee.decrypt(event)
-        if isinstance(event.content, TextMessageEventContent):
-            event.content.trim_reply_fallback()
-        puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
-        content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
-    except Exception:
-        log.exception("Failed to get event to add reply fallback")
-        content.set_reply(msg.mxid)
+    emoji_ids = [
+        entity.document_id for entity in entities if isinstance(entity, MessageEntityCustomEmoji)
+    ]
+    custom_emojis = await transfer_custom_emojis_to_matrix(source, emoji_ids, client=client)
+    if len(custom_emojis) > 0:
+        for i, entity in enumerate(entities):
+            if isinstance(entity, MessageEntityCustomEmoji):
+                entities[i] = ReuploadedCustomEmoji(entity, custom_emojis[entity.document_id])
 
 
 async def telegram_to_matrix(
     evt: Message | SponsoredMessage,
     source: au.AbstractUser,
-    main_intent: IntentAPI | None = None,
-    prefix_text: str | None = None,
-    prefix_html: str | None = None,
+    client: MautrixTelegramClient | None = None,
     override_text: str = None,
     override_entities: list[TypeMessageEntity] = None,
-    no_reply_fallback: bool = False,
     require_html: bool = False,
 ) -> TextMessageEventContent:
+    if not client:
+        client = source.client
     content = TextMessageEventContent(
         msgtype=MessageType.TEXT,
-        body=add_surrogate(override_text or evt.message),
+        body=override_text or evt.message,
     )
     entities = override_entities or evt.entities
     if entities:
+        await _convert_custom_emoji(source, entities, client=client)
         content.format = Format.HTML
         html = await _telegram_entities_to_matrix_catch(add_surrogate(content.body), entities)
         content.formatted_body = del_surrogate(html)
@@ -195,17 +200,8 @@ async def telegram_to_matrix(
     if require_html:
         content.ensure_has_html()
 
-    if prefix_html:
-        content.ensure_has_html()
-        content.formatted_body = prefix_html + content.formatted_body
-    if prefix_text:
-        content.body = prefix_text + content.body
-
     if getattr(evt, "fwd_from", None):
-        await _add_forward_header(source, content, evt.fwd_from)
-
-    if getattr(evt, "reply_to", None) and not no_reply_fallback:
-        await _add_reply_header(source, content, evt, main_intent)
+        await _add_forward_header(client, content, evt)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
         content.ensure_has_html()
@@ -225,9 +221,20 @@ async def _telegram_entities_to_matrix_catch(text: str, entities: list[TypeMessa
     return "[failed conversion in _telegram_entities_to_matrix]"
 
 
+def within_surrogate(text, index):
+    """
+    `True` if ``index`` is within a surrogate (before and after it, not at!).
+    """
+    return (
+        1 < index < len(text)  # in bounds
+        and "\ud800" <= text[index - 1] <= "\udbff"  # current is low surrogate
+        and "\udc00" <= text[index] <= "\udfff"  # previous is high surrogate
+    )
+
+
 async def _telegram_entities_to_matrix(
     text: str,
-    entities: list[TypeMessageEntity],
+    entities: list[TypeMessageEntity | ReuploadedCustomEmoji],
     offset: int = 0,
     length: int = None,
     in_codeblock: bool = False,
@@ -256,9 +263,9 @@ async def _telegram_entities_to_matrix(
         elif relative_offset < last_offset:
             continue
 
-        while within_surrogate(text, relative_offset, length=length):
+        while within_surrogate(text, relative_offset):
             relative_offset += 1
-        while within_surrogate(text, relative_offset + entity.length, length=length):
+        while within_surrogate(text, relative_offset + entity.length):
             entity.length += 1
 
         skip_entity = False
@@ -301,6 +308,17 @@ async def _telegram_entities_to_matrix(
             await _parse_url(
                 html, entity_text, entity.url if entity_type == MessageEntityTextUrl else None
             )
+        elif entity_type == MessageEntityCustomEmoji:
+            html.append(entity_text)
+        elif entity_type == ReuploadedCustomEmoji:
+            if isinstance(entity.file, UnicodeCustomEmoji):
+                html.append(entity.file.emoji)
+            else:
+                html.append(
+                    f"<img data-mx-emoticon data-mau-animated-emoji"
+                    f' src="{escape(entity.file.mxc)}" height="32" width="32"'
+                    f' alt="{entity_text}" title="{entity_text}"/>'
+                )
         elif entity_type in (
             MessageEntityBotCommand,
             MessageEntityHashtag,
@@ -378,7 +396,7 @@ message_link_regex = re.compile(
 )
 
 
-async def _parse_url(html: list[str], entity_text: str, url: str):
+async def _parse_url(html: list[str], entity_text: str, url: str) -> None:
     url = escape(url) if url else entity_text
     if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
         url = "http://" + url

mautrix_telegram/matrix.py

@@ -16,15 +16,14 @@
 from __future__ import annotations
 
 from typing import TYPE_CHECKING
+import sys
 
 from mautrix.bridge import BaseMatrixHandler
-from mautrix.errors import MatrixError
 from mautrix.types import (
     Event,
     EventID,
     EventType,
     MemberStateEventContent,
-    MessageType,
     PresenceEvent,
     PresenceState,
     ReactionEvent,
@@ -36,12 +35,13 @@ from mautrix.types import (
     RoomTopicStateEventContent as TopicContent,
     SingleReceiptEventContent,
     StateEvent,
-    TextMessageEventContent,
     TypingEvent,
     UserID,
 )
 
 from . import commands as com, portal as po, puppet as pu, user as u
+from .commands.portal.util import get_initial_state, user_has_power_level, warn_missing_power
+from .types import TelegramID
 
 if TYPE_CHECKING:
     from .__main__ import TelegramBridge
@@ -61,6 +61,21 @@ class MatrixHandler(BaseMatrixHandler):
 
         self._previously_typing = {}
 
+    async def check_versions(self) -> None:
+        await super().check_versions()
+        if self.config["bridge.backfill.msc2716"] and not (
+            support := self.versions.supports("org.matrix.msc2716")
+        ):
+            self.log.fatal(
+                "Backfilling with MSC2716 is enabled in bridge config, but "
+                + (
+                    "batch sending is not enabled on homeserver"
+                    if support is False
+                    else "homeserver does not support batch sending"
+                )
+            )
+            sys.exit(18)
+
     async def handle_puppet_group_invite(
         self,
         room_id: RoomID,
@@ -69,16 +84,76 @@ class MatrixHandler(BaseMatrixHandler):
         evt: StateEvent,
         members: list[UserID],
     ) -> None:
-        if self.az.bot_mxid not in members:
+        double_puppet = await pu.Puppet.get_by_custom_mxid(invited_by.mxid)
+        if (
+            not double_puppet
+            or self.az.bot_mxid in members
+            or not self.config["bridge.create_group_on_invite"]
+        ):
+            if self.az.bot_mxid not in members:
+                await puppet.default_mxid_intent.leave_room(
+                    room_id,
+                    reason="This ghost does not join multi-user rooms without the bridge bot.",
+                )
+            else:
+                await puppet.default_mxid_intent.send_notice(
+                    room_id,
+                    "This ghost will remain inactive "
+                    "until a Telegram chat is created for this room.",
+                )
+            return
+        elif not await user_has_power_level(
+            evt.room_id, double_puppet.intent, invited_by, "bridge"
+        ):
             await puppet.default_mxid_intent.leave_room(
-                room_id, reason="This ghost does not join multi-user rooms without the bridge bot."
+                room_id, reason="You do not have the permissions to bridge this room."
             )
-        else:
-            await puppet.default_mxid_intent.send_notice(
+            return
+
+        await double_puppet.intent.invite_user(room_id, self.az.bot_mxid)
+
+        title, about, levels, encrypted = await get_initial_state(double_puppet.intent, room_id)
+        if not title:
+            await puppet.default_mxid_intent.leave_room(
+                room_id, reason="Please set a title before inviting Telegram ghosts."
+            )
+            return
+
+        portal = po.Portal(
+            tgid=TelegramID(0),
+            tg_receiver=TelegramID(0),
+            peer_type="channel",
+            mxid=evt.room_id,
+            title=title,
+            about=about,
+            encrypted=encrypted,
+        )
+        await portal.az.intent.ensure_joined(room_id)
+        levels = await portal.az.intent.get_power_levels(room_id)
+        invited_by_level = levels.get_user_level(invited_by.mxid)
+        if invited_by_level > levels.get_user_level(self.az.bot_mxid):
+            levels.users[self.az.bot_mxid] = 100 if invited_by_level >= 100 else invited_by_level
+            await double_puppet.intent.set_power_levels(room_id, levels)
+
+        invites, errors = await portal.get_telegram_users_in_matrix_room(
+            invited_by, pre_create=True
+        )
+        if len(errors) > 0:
+            error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
+            await portal.az.intent.send_notice(
                 room_id,
-                "This ghost will remain inactive until a Telegram chat is created for this room.",
+                f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
+                "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
+                "those users.",
             )
 
+        try:
+            await portal.create_telegram_chat(invited_by, invites=invites, supergroup=True)
+        except ValueError as e:
+            await portal.delete()
+            await portal.az.intent.send_notice(room_id, e.args[0])
+            return
+
     async def handle_invite(
         self, room_id: RoomID, user_id: UserID, inviter: u.User, event_id: EventID
     ) -> None:

mautrix_telegram/portal_util/__init__.py

@@ -1,5 +1,5 @@
 from .deduplication import PortalDedup
-from .media_fallback import make_contact_event_content, make_dice_event_content
+from .message_convert import ConvertedMessage, TelegramMessageConverter
 from .participants import get_users
 from .power_levels import get_base_power_levels, participants_to_power_levels
 from .send_lock import PortalReactionLock, PortalSendLock

mautrix_telegram/portal_util/deduplication.py

@@ -96,13 +96,13 @@ class PortalDedup:
                 )
                 yield media_hash_func(event.media)
 
-    def _hash_event(self, event: TypeMessage) -> bytes:
+    def hash_event(self, event: TypeMessage) -> bytes:
         return hashlib.sha256(
             "-".join(str(a) for a in self._hash_content(event)).encode("utf-8")
         ).digest()
 
     def check_action(self, event: TypeMessage) -> bool:
-        dedup_id = self._hash_event(event) if self._always_force_hash else event.id
+        dedup_id = self.hash_event(event) if self._always_force_hash else event.id
         if dedup_id in self._dedup_action:
             return True
 
@@ -116,7 +116,7 @@ class PortalDedup:
         expected_mxid: DedupMXID | None = None,
         force_hash: bool = False,
     ) -> tuple[bytes, DedupMXID | None]:
-        evt_hash = self._hash_event(event)
+        evt_hash = self.hash_event(event)
         dedup_id = evt_hash if self._always_force_hash or force_hash else event.id
         try:
             found_mxid = self._dedup_mxid[dedup_id]
@@ -133,7 +133,7 @@ class PortalDedup:
     def check(
         self, event: TypeMessage, mxid: DedupMXID = None, force_hash: bool = False
     ) -> tuple[bytes, DedupMXID | None]:
-        evt_hash = self._hash_event(event)
+        evt_hash = self.hash_event(event)
         dedup_id = evt_hash if self._always_force_hash or force_hash else event.id
         if dedup_id in self._dedup:
             return evt_hash, self._dedup_mxid[dedup_id]

mautrix_telegram/portal_util/media_fallback.py

@@ -1,133 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-import html
-
-from telethon.tl.types import MessageMediaContact, MessageMediaDice, PeerUser
-
-from mautrix.types import Format, MessageType, TextMessageEventContent
-
-from .. import abstract_user as au, puppet as pu
-from ..types import TelegramID
-
-try:
-    import phonenumbers
-except ImportError:
-    phonenumbers = None
-
-
-def _format_dice(roll: MessageMediaDice) -> str:
-    if roll.emoticon == "\U0001F3B0":
-        emojis = {
-            0: "\U0001F36B",  # "🍫",
-            1: "\U0001F352",  # "🍒",
-            2: "\U0001F34B",  # "🍋",
-            3: "7\ufe0f\u20e3",  # "7️⃣",
-        }
-        res = roll.value - 1
-        slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
-        return f"{slot1} {slot2} {slot3} ({roll.value})"
-    elif roll.emoticon == "\u26BD":
-        results = {
-            1: "miss",
-            2: "hit the woodwork",
-            3: "goal",  # seems to go in through the center
-            4: "goal",
-            5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
-        }
-    elif roll.emoticon == "\U0001F3B3":
-        results = {
-            1: "miss",
-            2: "1 pin down",
-            3: "3 pins down, split",
-            4: "4 pins down, split",
-            5: "5 pins down",
-            6: "strike 🎉",
-        }
-    # elif roll.emoticon == "\U0001F3C0":
-    #     results = {
-    #         2: "rolled off",
-    #         3: "stuck",
-    #     }
-    # elif roll.emoticon == "\U0001F3AF":
-    #     results = {
-    #         1: "bounced off",
-    #         2: "outer rim",
-    #
-    #         6: "bullseye",
-    #     }
-    else:
-        return str(roll.value)
-    return f"{results[roll.value]} ({roll.value})"
-
-
-def make_dice_event_content(roll: MessageMediaDice) -> TextMessageEventContent:
-    emoji_text = {
-        "\U0001F3AF": " Dart throw",
-        "\U0001F3B2": " Dice roll",
-        "\U0001F3C0": " Basketball throw",
-        "\U0001F3B0": " Slot machine",
-        "\U0001F3B3": " Bowling",
-        "\u26BD": " Football kick",
-    }
-    text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {_format_dice(roll)}"
-    content = TextMessageEventContent(
-        msgtype=MessageType.TEXT, format=Format.HTML, body=text, formatted_body=f"<h4>{text}</h4>"
-    )
-    content["net.maunium.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
-    return content
-
-
-async def make_contact_event_content(
-    source: au.AbstractUser, contact: MessageMediaContact
-) -> TextMessageEventContent:
-    name = " ".join(x for x in [contact.first_name, contact.last_name] if x)
-    formatted_phone = f"+{contact.phone_number}"
-    if phonenumbers is not None:
-        try:
-            parsed = phonenumbers.parse(formatted_phone)
-            fmt = phonenumbers.PhoneNumberFormat.INTERNATIONAL
-            formatted_phone = phonenumbers.format_number(parsed, fmt)
-        except phonenumbers.NumberParseException:
-            pass
-    content = TextMessageEventContent(
-        msgtype=MessageType.TEXT,
-        body=f"Shared contact info for {name}: {formatted_phone}",
-    )
-    content["net.maunium.telegram.contact"] = {
-        "user_id": contact.user_id,
-        "first_name": contact.first_name,
-        "last_name": contact.last_name,
-        "phone_number": contact.phone_number,
-        "vcard": contact.vcard,
-    }
-
-    puppet = await pu.Puppet.get_by_tgid(TelegramID(contact.user_id))
-    if not puppet.displayname:
-        try:
-            entity = await source.client.get_entity(PeerUser(contact.user_id))
-            await puppet.update_info(source, entity)
-        except Exception as e:
-            source.log.warning(f"Failed to sync puppet info of received contact: {e}")
-    else:
-        content.format = Format.HTML
-        content.formatted_body = (
-            f"Shared contact info for "
-            f"<a href='https://matrix.to/#/{puppet.mxid}'>{html.escape(name)}</a>: "
-            f"{html.escape(formatted_phone)}"
-        )
-    return content

mautrix_telegram/portal_util/message_convert.py

@@ -0,0 +1,828 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Any, NamedTuple
+import base64
+import codecs
+import hashlib
+import html
+import mimetypes
+import unicodedata
+
+from attr import dataclass
+from telethon.tl.types import (
+    Document,
+    DocumentAttributeAnimated,
+    DocumentAttributeAudio,
+    DocumentAttributeFilename,
+    DocumentAttributeImageSize,
+    DocumentAttributeSticker,
+    DocumentAttributeVideo,
+    Game,
+    InputPhotoFileLocation,
+    Message,
+    MessageEntityPre,
+    MessageMediaContact,
+    MessageMediaDice,
+    MessageMediaDocument,
+    MessageMediaGame,
+    MessageMediaGeo,
+    MessageMediaGeoLive,
+    MessageMediaPhoto,
+    MessageMediaPoll,
+    MessageMediaUnsupported,
+    MessageMediaVenue,
+    MessageMediaWebPage,
+    PeerChannel,
+    PeerUser,
+    Photo,
+    PhotoCachedSize,
+    PhotoEmpty,
+    PhotoSize,
+    PhotoSizeEmpty,
+    PhotoSizeProgressive,
+    Poll,
+    TypeDocumentAttribute,
+    TypePhotoSize,
+    WebPage,
+)
+from telethon.utils import decode_waveform
+
+from mautrix.appservice import IntentAPI
+from mautrix.types import (
+    EventID,
+    EventType,
+    Format,
+    ImageInfo,
+    LocationMessageEventContent,
+    MediaMessageEventContent,
+    MessageEventContent,
+    MessageType,
+    TextMessageEventContent,
+    ThumbnailInfo,
+)
+from mautrix.util.logging import TraceLogger
+
+from .. import abstract_user as au, formatter, matrix as m, portal as po, puppet as pu, util
+from ..config import Config
+from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
+from ..tgclient import MautrixTelegramClient
+from ..types import TelegramID
+from ..util import sane_mimetypes
+
+try:
+    import phonenumbers
+except ImportError:
+    phonenumbers = None
+
+
+@dataclass
+class ConvertedMessage:
+    content: MessageEventContent
+    caption: MessageEventContent | None = None
+    type: EventType = EventType.ROOM_MESSAGE
+    disappear_seconds: int | None = None
+    disappear_start_immediately: bool = False
+
+
+class DocAttrs(NamedTuple):
+    name: str | None
+    mime_type: str | None
+    is_sticker: bool
+    sticker_alt: str | None
+    width: int
+    height: int
+    is_gif: bool
+    is_audio: bool
+    is_voice: bool
+    duration: int
+    waveform: bytes
+
+
+BEEPER_LINK_PREVIEWS_KEY = "com.beeper.linkpreviews"
+BEEPER_IMAGE_ENCRYPTION_KEY = "beeper:image:encryption"
+
+
+class TelegramMessageConverter:
+    portal: po.Portal
+    matrix: m.MatrixHandler
+    config: Config
+    command_prefix: str
+    log: TraceLogger
+
+    def __init__(self, portal: po.Portal) -> None:
+        self.portal = portal
+        self.matrix = portal.matrix
+        self.config = portal.config
+        self.command_prefix = self.config["bridge.command_prefix"]
+        self.log = portal.log.getChild("msg_conv")
+
+        self._media_converters = {
+            MessageMediaPhoto: self._convert_photo,
+            MessageMediaDocument: self._convert_document,
+            MessageMediaGeo: self._convert_location,
+            MessageMediaGeoLive: self._convert_location,
+            MessageMediaVenue: self._convert_location,
+            MessageMediaPoll: self._convert_poll,
+            MessageMediaDice: self._convert_dice,
+            MessageMediaUnsupported: self._convert_unsupported,
+            MessageMediaGame: self._convert_game,
+            MessageMediaContact: self._convert_contact,
+        }
+        self._allowed_media = tuple(self._media_converters.keys())
+
+    async def convert(
+        self,
+        source: au.AbstractUser,
+        intent: IntentAPI,
+        is_bot: bool,
+        evt: Message,
+        no_reply_fallback: bool = False,
+        deterministic_reply_id: bool = False,
+        client: MautrixTelegramClient | None = None,
+    ) -> ConvertedMessage | None:
+        if not client:
+            client = source.client
+        if hasattr(evt, "media") and isinstance(evt.media, self._allowed_media):
+            convert_media = self._media_converters[type(evt.media)]
+            converted = await convert_media(source=source, intent=intent, evt=evt, client=client)
+        elif evt.message:
+            converted = await self._convert_text(source, intent, is_bot, evt, client)
+        else:
+            self.log.debug("Unhandled Telegram message %d", evt.id)
+            return
+        if converted:
+            if evt.ttl_period and not converted.disappear_seconds:
+                converted.disappear_seconds = evt.ttl_period
+                converted.disappear_start_immediately = True
+            converted.content.external_url = self._get_external_url(evt)
+            converted.content["fi.mau.telegram.source"] = {
+                "space": self.portal.tgid if self.portal.peer_type == "channel" else source.tgid,
+                "chat_id": self.portal.tgid,
+                "peer_type": self.portal.peer_type,
+                "id": evt.id,
+            }
+            if converted.caption:
+                converted.caption["fi.mau.telegram.source"] = converted.content[
+                    "fi.mau.telegram.source"
+                ]
+                converted.caption.external_url = converted.content.external_url
+                if self.portal.get_config("caption_in_message"):
+                    self._caption_to_message(converted)
+            await self._set_reply(
+                source,
+                evt,
+                converted.content,
+                no_fallback=no_reply_fallback,
+                deterministic_id=deterministic_reply_id,
+            )
+        return converted
+
+    @staticmethod
+    def _caption_to_message(converted: ConvertedMessage) -> None:
+        content, caption = converted.content, converted.caption
+        converted.caption = None
+
+        content["filename"] = content.body
+        content["org.matrix.msc1767.caption"] = {
+            "org.matrix.msc1767.text": caption.body,
+        }
+        content.body = caption.body
+        if caption.format == Format.HTML:
+            content["org.matrix.msc1767.caption"][
+                "org.matrix.msc1767.html"
+            ] = caption.formatted_body
+            content["formatted_body"] = caption.formatted_body
+            content["format"] = Format.HTML.value
+
+    def _get_external_url(self, evt: Message) -> str | None:
+        if self.portal.peer_type == "channel" and self.portal.username is not None:
+            return f"https://t.me/{self.portal.username}/{evt.id}"
+        elif self.portal.peer_type != "user":
+            return f"https://t.me/c/{self.portal.tgid}/{evt.id}"
+        return None
+
+    @staticmethod
+    def _int_to_bytes(i: int) -> bytes:
+        return codecs.decode(f"{i:010x}", "hex")
+
+    def _encode_msgid(self, source: au.AbstractUser, evt: Message) -> str:
+        if self.portal.peer_type == "channel":
+            play_id = b"c" + self._int_to_bytes(self.portal.tgid) + self._int_to_bytes(evt.id)
+        elif self.portal.peer_type == "chat":
+            play_id = (
+                b"g"
+                + self._int_to_bytes(self.portal.tgid)
+                + self._int_to_bytes(evt.id)
+                + self._int_to_bytes(source.tgid)
+            )
+        elif self.portal.peer_type == "user":
+            play_id = b"u" + self._int_to_bytes(self.portal.tgid) + self._int_to_bytes(evt.id)
+        else:
+            raise ValueError("Portal has invalid peer type")
+        return base64.b64encode(play_id).decode("utf-8").rstrip("=")
+
+    def deterministic_event_id(self, space: TelegramID, msg_id: TelegramID) -> EventID:
+        hash_content = f"{self.portal.mxid}/telegram/{space}/{msg_id}"
+        hashed = hashlib.sha256(hash_content.encode("utf-8")).digest()
+        b64hash = base64.urlsafe_b64encode(hashed).decode("utf-8").rstrip("=")
+        return EventID(f"${b64hash}:telegram.org")
+
+    async def _set_reply(
+        self,
+        source: au.AbstractUser,
+        evt: Message,
+        content: MessageEventContent,
+        no_fallback: bool = False,
+        deterministic_id: bool = False,
+    ) -> None:
+        if not evt.reply_to:
+            return
+        space = (
+            evt.peer_id.channel_id
+            if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
+            else source.tgid
+        )
+        reply_to_id = TelegramID(evt.reply_to.reply_to_msg_id)
+        msg = await DBMessage.get_one_by_tgid(reply_to_id, space)
+        if not msg or msg.mx_room != self.portal.mxid:
+            if deterministic_id:
+                content.set_reply(self.deterministic_event_id(space, reply_to_id))
+            return
+        elif not isinstance(content, TextMessageEventContent) or no_fallback:
+            # Not a text message, just set the reply metadata and return
+            content.set_reply(msg.mxid)
+            return
+
+        # Text message, try to fetch original message to generate reply fallback.
+        try:
+            event = await self.portal.main_intent.get_event(msg.mx_room, msg.mxid)
+            if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
+                event = await source.bridge.matrix.e2ee.decrypt(event)
+            if isinstance(event.content, TextMessageEventContent):
+                event.content.trim_reply_fallback()
+            puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
+            content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
+        except Exception:
+            self.log.exception("Failed to get event to add reply fallback")
+            content.set_reply(msg.mxid)
+
+    @staticmethod
+    def _photo_size_key(photo: TypePhotoSize) -> int:
+        if isinstance(photo, PhotoSize):
+            return photo.size
+        elif isinstance(photo, PhotoSizeProgressive):
+            return max(photo.sizes)
+        elif isinstance(photo, PhotoSizeEmpty):
+            return 0
+        else:
+            return len(photo.bytes)
+
+    @classmethod
+    def get_largest_photo_size(
+        cls, photo: Photo | Document
+    ) -> tuple[InputPhotoFileLocation | None, TypePhotoSize | None]:
+        if (
+            not photo
+            or isinstance(photo, PhotoEmpty)
+            or (isinstance(photo, Document) and not photo.thumbs)
+        ):
+            return None, None
+
+        largest = max(
+            photo.thumbs if isinstance(photo, Document) else photo.sizes, key=cls._photo_size_key
+        )
+        return (
+            InputPhotoFileLocation(
+                id=photo.id,
+                access_hash=photo.access_hash,
+                file_reference=photo.file_reference,
+                thumb_size=largest.type,
+            ),
+            largest,
+        )
+
+    async def _webpage_to_beeper_link_preview(
+        self, source: au.AbstractUser, intent: IntentAPI, webpage: WebPage
+    ) -> dict[str, Any]:
+        beeper_link_preview: dict[str, Any] = {
+            "matched_url": webpage.url,
+            "og:title": webpage.title,
+            "og:url": webpage.url,
+            "og:description": webpage.description,
+        }
+
+        # Upload an image corresponding to the link preview if it exists.
+        if webpage.photo:
+            loc, largest_size = self.get_largest_photo_size(webpage.photo)
+            if loc is None:
+                return beeper_link_preview
+            beeper_link_preview["og:image:height"] = largest_size.h
+            beeper_link_preview["og:image:width"] = largest_size.w
+            file = await util.transfer_file_to_matrix(
+                source.client,
+                intent,
+                loc,
+                encrypt=self.portal.encrypted,
+                async_upload=self.config["homeserver.async_media"],
+            )
+
+            if file.decryption_info:
+                beeper_link_preview[BEEPER_IMAGE_ENCRYPTION_KEY] = file.decryption_info.serialize()
+            else:
+                beeper_link_preview["og:image"] = file.mxc
+
+        return beeper_link_preview
+
+    async def _convert_text(
+        self,
+        source: au.AbstractUser,
+        intent: IntentAPI,
+        is_bot: bool,
+        evt: Message,
+        client: MautrixTelegramClient,
+    ) -> ConvertedMessage:
+        content = await formatter.telegram_to_matrix(evt, source, client)
+        if is_bot and self.portal.get_config("bot_messages_as_notices"):
+            content.msgtype = MessageType.NOTICE
+
+        if (
+            hasattr(evt, "media")
+            and isinstance(evt.media, MessageMediaWebPage)
+            and isinstance(evt.media.webpage, WebPage)
+        ):
+            content[BEEPER_LINK_PREVIEWS_KEY] = [
+                await self._webpage_to_beeper_link_preview(source, intent, evt.media.webpage)
+            ]
+
+        return ConvertedMessage(content=content)
+
+    async def _convert_photo(
+        self,
+        source: au.AbstractUser,
+        intent: IntentAPI,
+        evt: Message,
+        client: MautrixTelegramClient,
+    ) -> ConvertedMessage | None:
+        media: MessageMediaPhoto = evt.media
+        if media.photo is None and media.ttl_seconds:
+            return ConvertedMessage(
+                content=TextMessageEventContent(
+                    msgtype=MessageType.NOTICE, body="Photo has expired"
+                )
+            )
+        loc, largest_size = self.get_largest_photo_size(media.photo)
+        if loc is None:
+            return ConvertedMessage(
+                content=TextMessageEventContent(
+                    msgtype=MessageType.TEXT,
+                    body="Failed to bridge image",
+                )
+            )
+        file = await util.transfer_file_to_matrix(
+            client,
+            intent,
+            loc,
+            encrypt=self.portal.encrypted,
+            async_upload=self.config["homeserver.async_media"],
+        )
+        if not file:
+            return None
+        info = ImageInfo(
+            height=largest_size.h,
+            width=largest_size.w,
+            orientation=0,
+            mimetype=file.mime_type,
+            size=self._photo_size_key(largest_size),
+        )
+        ext = sane_mimetypes.guess_extension(file.mime_type)
+        name = f"disappearing_image{ext}" if media.ttl_seconds else f"image{ext}"
+        content = MediaMessageEventContent(
+            msgtype=MessageType.IMAGE,
+            info=info,
+            body=name,
+        )
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+        caption_content = (
+            await formatter.telegram_to_matrix(evt, source, client) if evt.message else None
+        )
+        return ConvertedMessage(
+            content=content,
+            caption=caption_content,
+            disappear_seconds=media.ttl_seconds,
+        )
+
+    async def _convert_document(
+        self,
+        source: au.AbstractUser,
+        intent: IntentAPI,
+        evt: Message,
+        client: MautrixTelegramClient,
+    ) -> ConvertedMessage | None:
+        document = evt.media.document
+
+        attrs = _parse_document_attributes(document.attributes)
+
+        if document.size > self.matrix.media_config.upload_size:
+            name = attrs.name or ""
+            caption = f"\n{evt.message}" if evt.message else ""
+            return ConvertedMessage(
+                content=TextMessageEventContent(
+                    msgtype=MessageType.NOTICE, body=f"Too large file {name}{caption}"
+                )
+            )
+
+        thumb_loc, thumb_size = self.get_largest_photo_size(document)
+        if thumb_size and not isinstance(thumb_size, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb_size)}")
+            thumb_loc = None
+            thumb_size = None
+        parallel_id = source.tgid if self.config["bridge.parallel_file_transfer"] else None
+        tgs_convert = self.config["bridge.animated_sticker"]
+        file = await util.transfer_file_to_matrix(
+            client,
+            intent,
+            document,
+            thumb_loc,
+            is_sticker=attrs.is_sticker,
+            tgs_convert=tgs_convert,
+            webm_convert=tgs_convert["target"] if tgs_convert["convert_from_webm"] else None,
+            filename=attrs.name,
+            parallel_id=parallel_id,
+            encrypt=self.portal.encrypted,
+            async_upload=self.config["homeserver.async_media"],
+        )
+        if not file:
+            return None
+
+        info, name = _parse_document_meta(evt, file, attrs, thumb_size)
+
+        event_type = EventType.ROOM_MESSAGE
+        # Elements only support images as stickers, so send animated webm stickers as m.video
+        if attrs.is_sticker and file.mime_type.startswith("image/"):
+            event_type = EventType.STICKER
+            # Tell clients to render the stickers as 256x256 if they're bigger
+            if info.width > 256 or info.height > 256:
+                if info.width > info.height:
+                    info.height = int(info.height / (info.width / 256))
+                    info.width = 256
+                else:
+                    info.width = int(info.width / (info.height / 256))
+                    info.height = 256
+            if info.thumbnail_info:
+                info.thumbnail_info.width = info.width
+                info.thumbnail_info.height = info.height
+        if attrs.is_gif or (attrs.is_sticker and info.mimetype == "video/webm"):
+            if attrs.is_gif:
+                info["fi.mau.telegram.gif"] = True
+            else:
+                info["fi.mau.telegram.animated_sticker"] = True
+            info["fi.mau.loop"] = True
+            info["fi.mau.autoplay"] = True
+            info["fi.mau.hide_controls"] = True
+            info["fi.mau.no_audio"] = True
+        if not name:
+            ext = sane_mimetypes.guess_extension(file.mime_type)
+            name = "unnamed_file" + ext
+
+        content = MediaMessageEventContent(
+            body=name,
+            info=info,
+            msgtype={
+                "video/": MessageType.VIDEO,
+                "audio/": MessageType.AUDIO,
+                "image/": MessageType.IMAGE,
+            }.get(info.mimetype[:6], MessageType.FILE),
+        )
+        if event_type == EventType.STICKER:
+            content.msgtype = None
+        if attrs.is_audio:
+            content["org.matrix.msc1767.audio"] = {"duration": attrs.duration * 1000}
+            if attrs.waveform:
+                content["org.matrix.msc1767.audio"]["waveform"] = [x << 5 for x in attrs.waveform]
+            if attrs.is_voice:
+                content["org.matrix.msc3245.voice"] = {}
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+
+        caption_content = (
+            await formatter.telegram_to_matrix(evt, source, client) if evt.message else None
+        )
+
+        return ConvertedMessage(
+            type=event_type,
+            content=content,
+            caption=caption_content,
+            disappear_seconds=evt.media.ttl_seconds,
+        )
+
+    @staticmethod
+    async def _convert_location(evt: Message, **_) -> ConvertedMessage:
+        long = evt.media.geo.long
+        lat = evt.media.geo.lat
+        long_char = "E" if long > 0 else "W"
+        lat_char = "N" if lat > 0 else "S"
+        geo = f"{round(lat, 6)},{round(long, 6)}"
+
+        body = f"{round(abs(lat), 4)}° {lat_char}, {round(abs(long), 4)}° {long_char}"
+        url = f"https://maps.google.com/?q={geo}"
+
+        if isinstance(evt.media, MessageMediaGeoLive):
+            note = "Live Location (see your Telegram client for live updates)"
+        elif isinstance(evt.media, MessageMediaVenue):
+            note = evt.media.title
+        else:
+            note = "Location"
+
+        content = LocationMessageEventContent(
+            msgtype=MessageType.LOCATION,
+            geo_uri=f"geo:{geo}",
+            body=f"{note}: {body}\n{url}",
+        )
+        content["format"] = str(Format.HTML)
+        content["formatted_body"] = f"{note}: <a href='{url}'>{body}</a>"
+        content["org.matrix.msc3488.location"] = {
+            "uri": content.geo_uri,
+            "description": note,
+        }
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_unsupported(
+        source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
+    ) -> ConvertedMessage:
+        override_text = (
+            "This message is not supported on your version of Mautrix-Telegram. "
+            "Please check https://github.com/mautrix/telegram or ask your "
+            "bridge administrator about possible updates."
+        )
+        content = await formatter.telegram_to_matrix(
+            evt, source, client, override_text=override_text
+        )
+        content.msgtype = MessageType.NOTICE
+        content["fi.mau.telegram.unsupported"] = True
+        return ConvertedMessage(content=content)
+
+    async def _convert_poll(self, source: au.AbstractUser, evt: Message, **_) -> ConvertedMessage:
+        poll: Poll = evt.media.poll
+        poll_id = self._encode_msgid(source, evt)
+
+        _n = 0
+
+        def n() -> int:
+            nonlocal _n
+            _n += 1
+            return _n
+
+        text_answers = "\n".join(f"{n()}. {answer.text}" for answer in poll.answers)
+        html_answers = "\n".join(f"<li>{answer.text}</li>" for answer in poll.answers)
+        vote_command = f"{self.command_prefix} vote {poll_id}"
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT,
+            format=Format.HTML,
+            body=(
+                f"Poll: {poll.question}\n{text_answers}\n"
+                f"Vote with {vote_command} <choice number>"
+            ),
+            formatted_body=(
+                f"<strong>Poll</strong>: {poll.question}<br/>\n"
+                f"<ol>{html_answers}</ol>\n"
+                f"Vote with <code>{vote_command} &lt;choice number&gt;</code>"
+            ),
+        )
+
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_dice(evt: Message, **_) -> ConvertedMessage:
+        roll: MessageMediaDice = evt.media
+        emoji_text = {
+            "\U0001F3AF": " Dart throw",
+            "\U0001F3B2": " Dice roll",
+            "\U0001F3C0": " Basketball throw",
+            "\U0001F3B0": " Slot machine",
+            "\U0001F3B3": " Bowling",
+            "\u26BD": " Football kick",
+        }
+        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {_format_dice(roll)}"
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT,
+            format=Format.HTML,
+            body=text,
+            formatted_body=f"<h4>{text}</h4>",
+        )
+        content["fi.mau.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
+        return ConvertedMessage(content=content)
+
+    async def _convert_game(
+        self, source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
+    ) -> ConvertedMessage:
+        game: Game = evt.media.game
+        play_id = self._encode_msgid(source, evt)
+        command = f"{self.command_prefix} play {play_id}"
+        override_text = f"Run {command} in your bridge management room to play {game.title}"
+        override_entities = [
+            MessageEntityPre(offset=len("Run "), length=len(command), language="")
+        ]
+
+        content = await formatter.telegram_to_matrix(
+            evt, source, client, override_text=override_text, override_entities=override_entities
+        )
+        content.msgtype = MessageType.NOTICE
+        content["fi.mau.telegram.game"] = play_id
+
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_contact(
+        source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
+    ) -> ConvertedMessage:
+        contact: MessageMediaContact = evt.media
+        name = " ".join(x for x in [contact.first_name, contact.last_name] if x)
+        formatted_phone = f"+{contact.phone_number}"
+        if phonenumbers is not None:
+            try:
+                parsed = phonenumbers.parse(formatted_phone)
+                fmt = phonenumbers.PhoneNumberFormat.INTERNATIONAL
+                formatted_phone = phonenumbers.format_number(parsed, fmt)
+            except phonenumbers.NumberParseException:
+                pass
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT,
+            body=f"Shared contact info for {name}: {formatted_phone}",
+        )
+        content["fi.mau.telegram.contact"] = {
+            "user_id": contact.user_id,
+            "first_name": contact.first_name,
+            "last_name": contact.last_name,
+            "phone_number": contact.phone_number,
+            "vcard": contact.vcard,
+        }
+
+        puppet = await pu.Puppet.get_by_tgid(TelegramID(contact.user_id))
+        if not puppet.displayname:
+            try:
+                entity = await client.get_entity(PeerUser(contact.user_id))
+                await puppet.update_info(source, entity, client_override=client)
+            except Exception as e:
+                source.log.warning(f"Failed to sync puppet info of received contact: {e}")
+        else:
+            content.format = Format.HTML
+            content.formatted_body = (
+                f"Shared contact info for "
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>{html.escape(name)}</a>: "
+                f"{html.escape(formatted_phone)}"
+            )
+        return ConvertedMessage(content=content)
+
+
+def _parse_document_attributes(attributes: list[TypeDocumentAttribute]) -> DocAttrs:
+    name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
+    is_gif, is_audio, is_voice, duration, waveform = False, False, False, 0, bytes()
+    for attr in attributes:
+        if isinstance(attr, DocumentAttributeFilename):
+            name = name or attr.file_name
+            mime_type, _ = mimetypes.guess_type(attr.file_name)
+        elif isinstance(attr, DocumentAttributeSticker):
+            is_sticker = True
+            sticker_alt = attr.alt
+        elif isinstance(attr, DocumentAttributeAnimated):
+            is_gif = True
+        elif isinstance(attr, DocumentAttributeVideo):
+            width, height = attr.w, attr.h
+        elif isinstance(attr, DocumentAttributeImageSize):
+            width, height = attr.w, attr.h
+        elif isinstance(attr, DocumentAttributeAudio):
+            is_audio = True
+            is_voice = attr.voice or False
+            duration = attr.duration
+            waveform = decode_waveform(attr.waveform) if attr.waveform else b""
+
+    return DocAttrs(
+        name,
+        mime_type,
+        is_sticker,
+        sticker_alt,
+        width,
+        height,
+        is_gif,
+        is_audio,
+        is_voice,
+        duration,
+        waveform,
+    )
+
+
+def _parse_document_meta(
+    evt: Message, file: DBTelegramFile, attrs: DocAttrs, thumb_size: TypePhotoSize
+) -> tuple[ImageInfo, str]:
+    document = evt.media.document
+    name = attrs.name
+    if attrs.is_sticker:
+        alt = attrs.sticker_alt
+        if len(alt) > 0:
+            try:
+                name = f"{alt} ({unicodedata.name(alt[0]).lower()})"
+            except ValueError:
+                name = alt
+
+    generic_types = ("text/plain", "application/octet-stream")
+    if file.mime_type in generic_types and document.mime_type not in generic_types:
+        mime_type = document.mime_type or file.mime_type
+    elif file.mime_type == "application/ogg":
+        mime_type = "audio/ogg"
+    else:
+        mime_type = file.mime_type or document.mime_type
+    info = ImageInfo(size=file.size, mimetype=mime_type)
+
+    if attrs.mime_type and not file.was_converted:
+        file.mime_type = attrs.mime_type or file.mime_type
+    if file.width and file.height:
+        info.width, info.height = file.width, file.height
+    elif attrs.width and attrs.height:
+        info.width, info.height = attrs.width, attrs.height
+
+    if file.thumbnail:
+        if file.thumbnail.decryption_info:
+            info.thumbnail_file = file.thumbnail.decryption_info
+        else:
+            info.thumbnail_url = file.thumbnail.mxc
+        info.thumbnail_info = ThumbnailInfo(
+            mimetype=file.thumbnail.mime_type,
+            height=file.thumbnail.height or thumb_size.h,
+            width=file.thumbnail.width or thumb_size.w,
+            size=file.thumbnail.size,
+        )
+    elif attrs.is_sticker:
+        # This is a hack for bad clients like Element iOS that require a thumbnail
+        info.thumbnail_info = ImageInfo.deserialize(info.serialize())
+        if file.decryption_info:
+            info.thumbnail_file = file.decryption_info
+        else:
+            info.thumbnail_url = file.mxc
+
+    return info, name
+
+
+def _format_dice(roll: MessageMediaDice) -> str:
+    if roll.emoticon == "\U0001F3B0":
+        emojis = {
+            0: "\U0001F36B",  # "🍫",
+            1: "\U0001F352",  # "🍒",
+            2: "\U0001F34B",  # "🍋",
+            3: "7\ufe0f\u20e3",  # "7️⃣",
+        }
+        res = roll.value - 1
+        slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
+        return f"{slot1} {slot2} {slot3} ({roll.value})"
+    elif roll.emoticon == "\u26BD":
+        results = {
+            1: "miss",
+            2: "hit the woodwork",
+            3: "goal",  # seems to go in through the center
+            4: "goal",
+            5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
+        }
+    elif roll.emoticon == "\U0001F3B3":
+        results = {
+            1: "miss",
+            2: "1 pin down",
+            3: "3 pins down, split",
+            4: "4 pins down, split",
+            5: "5 pins down",
+            6: "strike 🎉",
+        }
+    # elif roll.emoticon == "\U0001F3C0":
+    #     results = {
+    #         2: "rolled off",
+    #         3: "stuck",
+    #     }
+    # elif roll.emoticon == "\U0001F3AF":
+    #     results = {
+    #         1: "bounced off",
+    #         2: "outer rim",
+    #
+    #         6: "bullseye",
+    #     }
+    else:
+        return str(roll.value)
+    return f"{results[roll.value]} ({roll.value})"

mautrix_telegram/portal_util/participants.py

@@ -24,6 +24,7 @@ from telethon.tl.types import (
     ChannelParticipantBanned,
     ChannelParticipantsRecent,
     ChannelParticipantsSearch,
+    ChatParticipantsForbidden,
     InputChannel,
     InputUser,
     TypeChannelParticipant,
@@ -93,6 +94,8 @@ async def get_users(
 ) -> list[TypeUser]:
     if peer_type == "chat":
         chat = await client(GetFullChatRequest(chat_id=tgid))
+        if isinstance(chat.full_chat.participants, ChatParticipantsForbidden):
+            return []
         users = list(_filter_participants(chat.users, chat.full_chat.participants.participants))
         return users[:limit] if limit > 0 else users
     elif peer_type == "channel":

mautrix_telegram/portal_util/power_levels.py

@@ -34,8 +34,12 @@ from ..types import TelegramID
 
 
 def get_base_power_levels(
-    portal: po.Portal, levels: PowerLevelContent = None, entity: TypeChat = None
+    portal: po.Portal,
+    levels: PowerLevelContent = None,
+    entity: TypeChat | None = None,
+    dbr: ChatBannedRights | None = None,
 ) -> PowerLevelContent:
+    is_initial = not levels
     levels = levels or PowerLevelContent()
     if portal.peer_type == "user":
         overrides = portal.config["bridge.initial_power_level_overrides.user"]
@@ -51,7 +55,7 @@ def get_base_power_levels(
         levels.events_default = overrides.get("events_default", 0)
     else:
         overrides = portal.config["bridge.initial_power_level_overrides.group"]
-        dbr = entity.default_banned_rights
+        dbr = dbr or entity.default_banned_rights
         if not dbr:
             portal.log.debug(f"default_banned_rights is None in {entity}")
             dbr = ChatBannedRights(
@@ -80,16 +84,14 @@ def get_base_power_levels(
         levels.events_default = overrides.get(
             "events_default",
             50
-            if (
-                portal.peer_type == "channel"
-                and not entity.megagroup
-                or entity.default_banned_rights.send_messages
-            )
+            if portal.peer_type == "channel" and not portal.megagroup or dbr.send_messages
             else 0,
         )
     for evt_type, value in overrides.get("events", {}).items():
         levels.events[EventType.find(evt_type)] = value
-    levels.users = overrides.get("users", {})
+    userlevel_overrides = overrides.get("users", {})
+    if is_initial:
+        levels.users.update(userlevel_overrides)
     if portal.main_intent.mxid not in levels.users:
         levels.users[portal.main_intent.mxid] = 100
     return levels

mautrix_telegram/portal_util/sponsored_message.py

@@ -20,6 +20,7 @@ import html
 
 from telethon.tl.functions.channels import GetSponsoredMessagesRequest
 from telethon.tl.types import Channel, InputChannel, PeerChannel, PeerUser, SponsoredMessage, User
+from telethon.tl.types.messages import SponsoredMessages, SponsoredMessagesEmpty
 
 from mautrix.types import MessageType, TextMessageEventContent
 
@@ -32,8 +33,9 @@ async def get_sponsored_message(
     entity: InputChannel,
 ) -> tuple[SponsoredMessage | None, int | None, Channel | User | None]:
     resp = await user.client(GetSponsoredMessagesRequest(entity))
-    if len(resp.messages) == 0:
+    if isinstance(resp, SponsoredMessagesEmpty):
         return None, None, None
+    assert isinstance(resp, SponsoredMessages)
     msg = resp.messages[0]
     if isinstance(msg.from_id, PeerUser):
         entities = resp.users
@@ -83,7 +85,7 @@ async def make_sponsored_message_content(
     else:
         sponsor_name = sponsor_name_html = "unknown entity"
 
-    content["net.maunium.telegram.sponsored"] = sponsored_meta
+    content["fi.mau.telegram.sponsored"] = sponsored_meta
     content.formatted_body += (
         f"<br/><br/>Sponsored message from {sponsor_name_html} "
         f"- <a href='{content.external_url}'>{action}</a>"

mautrix_telegram/puppet.py

@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, AsyncGenerator, AsyncIterable, Awaitable, cast
 from difflib import SequenceMatcher
 import unicodedata
 
+from telethon import utils
 from telethon.tl.types import (
     Channel,
     ChatPhoto,
@@ -29,8 +30,6 @@ from telethon.tl.types import (
     PeerChat,
     PeerUser,
     TypeChatPhoto,
-    TypeInputPeer,
-    TypeInputUser,
     TypePeer,
     TypeUserProfilePhoto,
     UpdateUserName,
@@ -48,6 +47,7 @@ from mautrix.util.simple_template import SimpleTemplate
 from . import abstract_user as au, portal as p, util
 from .config import Config
 from .db import Puppet as DBPuppet
+from .tgclient import MautrixTelegramClient
 from .types import TelegramID
 
 if TYPE_CHECKING:
@@ -80,6 +80,7 @@ class Puppet(DBPuppet, BasePuppet):
         avatar_set: bool = False,
         is_bot: bool = False,
         is_channel: bool = False,
+        is_premium: bool = False,
         custom_mxid: UserID | None = None,
         access_token: str | None = None,
         next_batch: SyncToken | None = None,
@@ -101,6 +102,7 @@ class Puppet(DBPuppet, BasePuppet):
             avatar_set=avatar_set,
             is_bot=is_bot,
             is_channel=is_channel,
+            is_premium=is_premium,
             custom_mxid=custom_mxid,
             access_token=access_token,
             next_batch=next_batch,
@@ -145,9 +147,6 @@ class Puppet(DBPuppet, BasePuppet):
     def plain_displayname(self) -> str:
         return self.displayname_template.parse(self.displayname) or self.displayname
 
-    def get_input_entity(self, user: au.AbstractUser) -> Awaitable[TypeInputPeer | TypeInputUser]:
-        return user.client.get_input_entity(self.peer)
-
     def intent_for(self, portal: p.Portal) -> IntentAPI:
         if portal.tgid == self.tgid:
             return self.default_mxid_intent
@@ -253,13 +252,22 @@ class Puppet(DBPuppet, BasePuppet):
         except Exception:
             source.log.exception(f"Failed to update info of {self.tgid}")
 
-    async def update_info(self, source: au.AbstractUser, info: User | Channel) -> None:
+    async def update_info(
+        self,
+        source: au.AbstractUser,
+        info: User | Channel,
+        client_override: MautrixTelegramClient | None = None,
+    ) -> None:
         is_bot = False if isinstance(info, Channel) else info.bot
+        is_premium = False if isinstance(info, Channel) else info.premium
         is_channel = isinstance(info, Channel)
-        changed = is_bot != self.is_bot or is_channel != self.is_channel
+        changed = (
+            is_bot != self.is_bot or is_channel != self.is_channel or is_premium != self.is_premium
+        )
 
         self.is_bot = is_bot
         self.is_channel = is_channel
+        self.is_premium = is_premium
 
         if self.username != info.username:
             self.username = info.username
@@ -271,8 +279,16 @@ class Puppet(DBPuppet, BasePuppet):
 
         if not self.disable_updates:
             try:
-                changed = await self.update_displayname(source, info) or changed
-                changed = await self.update_avatar(source, info.photo) or changed
+                changed = (
+                    await self.update_displayname(source, info, client_override=client_override)
+                    or changed
+                )
+                changed = (
+                    await self.update_avatar(
+                        source, info.photo, entity=info, client_override=client_override
+                    )
+                    or changed
+                )
             except Exception:
                 self.log.exception(f"Failed to update info from source {source.tgid}")
 
@@ -287,7 +303,10 @@ class Puppet(DBPuppet, BasePuppet):
             await portal.update_info_from_puppet(self)
 
     async def update_displayname(
-        self, source: au.AbstractUser, info: User | Channel | UpdateUserName
+        self,
+        source: au.AbstractUser,
+        info: User | Channel | UpdateUserName,
+        client_override: MautrixTelegramClient | None = None,
     ) -> bool:
         if self.disable_updates:
             return False
@@ -314,7 +333,7 @@ class Puppet(DBPuppet, BasePuppet):
             return False
 
         if isinstance(info, UpdateUserName):
-            info = await source.client.get_entity(self.peer)
+            info = await (client_override or source.client).get_entity(self.peer)
         if isinstance(info, Channel) or not info.contact:
             self.displayname_contact = False
         elif not self.displayname_contact:
@@ -351,7 +370,11 @@ class Puppet(DBPuppet, BasePuppet):
         return False
 
     async def update_avatar(
-        self, source: au.AbstractUser, photo: TypeUserProfilePhoto | TypeChatPhoto
+        self,
+        source: au.AbstractUser,
+        photo: TypeUserProfilePhoto | TypeChatPhoto,
+        entity: User | None = None,
+        client_override: MautrixTelegramClient | None = None,
     ) -> bool:
         if self.disable_updates:
             return False
@@ -370,11 +393,22 @@ class Puppet(DBPuppet, BasePuppet):
                 self.photo_id = ""
                 self.avatar_url = None
             elif self.photo_id != photo_id or not self.avatar_url:
+                client = client_override or source.client
+                try:
+                    peer = await client.get_input_entity(entity or self.peer)
+                except ValueError:
+                    if entity:
+                        peer = utils.get_input_peer(entity, check_hash=False)
+                    else:
+                        self.log.warning(f"Couldn't get input entity to update avatar")
+                        return False
                 file = await util.transfer_file_to_matrix(
-                    client=source.client,
+                    client=client,
                     intent=self.default_mxid_intent,
                     location=InputPeerPhotoFileLocation(
-                        peer=await self.get_input_entity(source), photo_id=photo.photo_id, big=True
+                        peer=peer,
+                        photo_id=photo.photo_id,
+                        big=True,
                     ),
                     async_upload=self.config["homeserver.async_media"],
                 )
@@ -393,7 +427,7 @@ class Puppet(DBPuppet, BasePuppet):
 
     async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
         portal: p.Portal = await p.Portal.get_by_mxid(room_id)
-        return portal and not portal.backfill_lock.locked and portal.peer_type != "user"
+        return portal and portal.peer_type != "user"
 
     # endregion
     # region Getters
@@ -406,7 +440,7 @@ class Puppet(DBPuppet, BasePuppet):
     @classmethod
     @async_getter_lock
     async def get_by_tgid(
-        cls, tgid: TelegramID, *, create: bool = True, is_channel: bool = False
+        cls, tgid: TelegramID, /, *, create: bool = True, is_channel: bool = False
     ) -> Puppet | None:
         if tgid is None:
             return None
@@ -459,7 +493,7 @@ class Puppet(DBPuppet, BasePuppet):
 
     @classmethod
     @async_getter_lock
-    async def get_by_custom_mxid(cls, mxid: UserID) -> Puppet | None:
+    async def get_by_custom_mxid(cls, mxid: UserID, /) -> Puppet | None:
         try:
             return cls.by_custom_mxid[mxid]
         except KeyError:
@@ -512,23 +546,4 @@ class Puppet(DBPuppet, BasePuppet):
 
         return None
 
-    @classmethod
-    async def find_by_displayname(cls, displayname: str) -> Puppet | None:
-        if not displayname:
-            return None
-
-        for _, puppet in cls.by_tgid.items():
-            if puppet.displayname and puppet.displayname == displayname:
-                return puppet
-
-        puppet = cast(cls, await super().find_by_displayname(displayname))
-        if puppet:
-            try:
-                return cls.by_tgid[puppet.tgid]
-            except KeyError:
-                puppet._add_to_cache()
-                return puppet
-
-        return None
-
     # endregion

mautrix_telegram/scripts/unicodemojipack/__main__.py

@@ -0,0 +1,397 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Any, Literal, TypedDict
+from pathlib import Path
+import argparse
+import asyncio
+import io
+import json
+import logging
+import math
+import mimetypes
+import pickle
+import random
+import string
+
+from lottie.exporters import export_tgs
+from lottie.exporters.cairo import export_png
+from lottie.exporters.tgs_validator import Severity, TgsValidator
+from lottie.importers.svg import import_svg
+from lottie.objects import Animation
+from lottie.utils.stripper import float_strip
+from PIL import Image
+from telethon import TelegramClient
+from telethon.custom import Conversation, Message
+from telethon.tl.functions.messages import GetStickerSetRequest
+from telethon.tl.types import (
+    Document,
+    DocumentAttributeCustomEmoji,
+    DocumentAttributeFilename,
+    DocumentAttributeImageSize,
+    InputMediaUploadedDocument,
+    InputStickerSetShortName,
+)
+import aiohttp
+
+mimetypes.add_type("image/webp", ".webp")
+
+parser = argparse.ArgumentParser(description="mautrix-telegram unicode emoji packer")
+parser.add_argument(
+    "-i", "--api-id", type=int, required=True, metavar="<api id>", help="Telegram API ID"
+)
+parser.add_argument(
+    "-a", "--api-hash", type=str, required=True, metavar="<api hash>", help="Telegram API hash"
+)
+parser.add_argument(
+    "-s",
+    "--session",
+    type=str,
+    default="unicodemojipacker.session",
+    metavar="<file name>",
+    help="Telethon session name",
+)
+parser.add_argument(
+    "-o",
+    "--output",
+    type=str,
+    default="mautrix_telegram/unicodemojipack.json",
+    metavar="<file name>",
+    help="Path to save created emoji pack document IDs",
+)
+parser.add_argument(
+    "-f",
+    "--font-directory",
+    type=Path,
+    required=True,
+    metavar="<directory path>",
+    help="Path to the Noto color emoji files",
+)
+parser.add_argument(
+    "-m",
+    "--media-directory",
+    type=Path,
+    required=True,
+    metavar="<directory path>",
+    help="Path to save converted tgs and webp emoji files",
+)
+args = parser.parse_args()
+font_dir: Path = args.font_directory
+media_dir: Path = args.media_directory
+
+EMOJI_DATA_URL = "https://raw.githubusercontent.com/iamcal/emoji-data/master/emoji.json"
+
+
+def unified_to_unicode(unified: str) -> str:
+    return (
+        "".join(rf"\U{chunk:0>8}" for chunk in unified.split("-"))
+        .encode("ascii")
+        .decode("unicode_escape")
+    )
+
+
+def tag_to_str(unified: str) -> str:
+    return "".join(chr(int(x.removeprefix("E00"), 16)) for x in unified.split("-"))
+
+
+EmojiType = Literal["webp", "tgs"]
+PackType = Literal["Animated emoji", "Static emoji"]
+
+
+class Emoji(TypedDict):
+    hex: str
+    emoji: str
+    type: EmojiType
+    filename: str
+
+
+class EmojiData(TypedDict):
+    tgs: list[Emoji]
+    webp: list[Emoji]
+
+
+def parse_emoji_data(tone: dict[str, Any], emoji: dict[str, Any]) -> Emoji:
+    hex = (tone["non_qualified"] or tone["unified"]).replace("-FE0F", "")
+    filename_hex = hex.replace("-", "_").lower()
+    filename = f"svg/emoji_u{filename_hex}.svg"
+    if emoji["category"] == "Flags" and emoji["subcategory"] in (
+        "country-flag",
+        "subdivision-flag",
+    ):
+        filename = f"third_party/region-flags/waved-svg/emoji_u{filename_hex}.svg"
+
+    with (font_dir / filename).open() as f:
+        lot: Animation = import_svg(f)
+    float_strip(lot)
+    lot.tgs_sanitize()
+
+    output = io.BytesIO()
+    export_tgs(lot, output)
+
+    validator = TgsValidator()
+    validator(lot)
+    validator.check_size(len(output.getvalue()))
+    errors = [err for err in validator.errors if err.severity != Severity.Note]
+    if errors or ("region-flags" in filename and len(output.getvalue()) > 32768):
+        lot.scale(100, 100)
+
+        png_out = io.BytesIO()
+        export_png(lot, png_out)
+        img = Image.open(png_out)
+        output = io.BytesIO()
+        output.name = "image.webp"
+        img.save(output, "webp")
+
+        media_type: EmojiType = "webp"
+    else:
+        media_type: EmojiType = "tgs"
+    path = media_dir / f"{filename_hex}.{media_type}"
+    with path.open("wb") as f:
+        f.write(output.getvalue())
+    print(
+        "Converted", filename, "->", path.name, "//" if errors else "", "\n".join(map(str, errors))
+    )
+
+    return {
+        "hex": hex,
+        "emoji": unified_to_unicode(tone["unified"]),
+        "type": media_type,
+        "filename": path.name,
+    }
+
+
+async def load_emoji_data() -> EmojiData:
+    cache_path = media_dir / "conversion-cache.json"
+    try:
+        with cache_path.open() as f:
+            return json.load(f)
+    except FileNotFoundError:
+        pass
+    async with aiohttp.ClientSession() as sess, sess.get(EMOJI_DATA_URL) as resp:
+        raw_emoji_data = sorted(
+            await resp.json(content_type=None),
+            key=lambda dat: dat["sort_order"],
+        )
+    tgs_emoji = []
+    webp_emoji = []
+    for emoji in raw_emoji_data:
+        for tone in (emoji, *emoji.get("skin_variations", {}).values()):
+            parsed_emoji = parse_emoji_data(tone, emoji)
+            if parsed_emoji["type"] == "tgs":
+                tgs_emoji.append(parsed_emoji)
+            else:
+                webp_emoji.append(parsed_emoji)
+    full_data = {"tgs": tgs_emoji, "webp": webp_emoji}
+    with cache_path.open("w") as f:
+        json.dump(full_data, f, ensure_ascii=False)
+    return full_data
+
+
+async def create_pack(conv: Conversation, name: str, pack_type: str) -> None:
+    await conv.send_message("/newemojipack")
+    resp: Message = await conv.get_response()
+    assert "A new set of custom emoji" in resp.raw_text
+    assert "Please choose the type" in resp.raw_text
+    await conv.send_message(pack_type)
+    resp = await conv.get_response()
+    if pack_type == "Animated emoji":
+        assert "When ready to upload, tell me the name of your set." in resp.raw_text
+    else:
+        assert "Now choose a name for your set." in resp.raw_text
+    await conv.send_message(name)
+    resp = await conv.get_response()
+    if pack_type == "Animated emoji":
+        assert "Now send me the first animated emoji" in resp.raw_text
+    else:
+        assert "Now send me the custom emoji" in resp.raw_text
+
+
+async def publish_pack(conv: Conversation, shortname: str) -> None:
+    await conv.send_message("/publish")
+
+    resp: Message = await conv.get_response()
+    assert "You can send me a custom emoji from your emoji set" in resp.raw_text
+    await conv.send_message("/skip")
+
+    resp = await conv.get_response()
+    assert "Please provide a short name for your emoji set" in resp.raw_text
+    await conv.send_message(shortname)
+
+    resp = await conv.get_response()
+    assert "I've just published your emoji set" in resp.raw_text
+
+
+async def send_emoji(
+    conv: Conversation, file: bytes | Path | InputMediaUploadedDocument, emoji: str
+) -> None:
+    await conv.send_file(file)
+    resp: Message = await conv.get_response()
+    assert "Send me a replacement emoji that corresponds to your custom emoji" in resp.raw_text
+    await conv.send_message(emoji)
+    resp = await conv.get_response()
+    if "Sorry, too many attempts" in resp.raw_text:
+        print(resp.raw_text)
+        input("Press enter to continue")
+        await conv.send_message(emoji)
+        resp = await conv.get_response()
+    while "Please send an emoji that best describes your custom emoji." in resp.raw_text:
+        emoji = input(f"{emoji} was rejected, provide replacement: ")
+        await conv.send_message(emoji)
+        resp = await conv.get_response()
+    assert "Congratulations" in resp.raw_text
+
+
+class CachedPack(TypedDict):
+    name: str
+    short_name: str
+    part: int
+    type: PackType
+    published: bool
+    collected: bool
+    emojis: list[Emoji]
+
+
+class CachedData(TypedDict):
+    packs: list[CachedPack]
+
+
+def _split_packs_int(
+    emoji_list: list[Emoji], pack_type: PackType, current_part: int, total_parts: int
+) -> tuple[list[CachedPack], int]:
+    packs = []
+    current_pack: CachedPack | None = None
+    for i, emoji in enumerate(emoji_list):
+        if i % 200 == 0:
+            current_part += 1
+            random_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=8))
+            short_name = f"mxtg_unicodemoji_{random_id}"
+            name = f"mautrix-telegram unicodemoji ({current_part}/{total_parts})"
+            current_pack = {
+                "type": pack_type,
+                "short_name": short_name,
+                "part": current_part,
+                "name": name,
+                "published": False,
+                "collected": False,
+                "emojis": [],
+            }
+            packs.append(current_pack)
+        current_pack["emojis"].append(emoji)
+    return packs, current_part
+
+
+def split_packs(emoji_data: EmojiData) -> list[CachedPack]:
+    total_parts = math.ceil(len(emoji_data["tgs"]) / 200) + math.ceil(
+        len(emoji_data["webp"]) / 200
+    )
+    current_part = 0
+    animated_packs, current_part = _split_packs_int(
+        emoji_data["tgs"], "Animated emoji", current_part, total_parts
+    )
+    static_packs, current_part = _split_packs_int(
+        emoji_data["webp"], "Static emoji", current_part, total_parts
+    )
+    return animated_packs + static_packs
+
+
+async def create_and_fill_pack(
+    client: TelegramClient, conv: Conversation, pack: CachedPack
+) -> None:
+    if pack["short_name"] == "mxtg_unicodemoji_xvzs6743":
+        print("Continuing pack", pack["name"])
+    else:
+        print("Creating pack", pack["name"])
+        await create_pack(conv, pack["name"], pack["type"])
+    total = len(pack["emojis"])
+    for i, emoji in enumerate(pack["emojis"]):
+        if pack["short_name"] == "mxtg_unicodemoji_xvzs6743" and i < 87:
+            continue
+        print(f"Adding emoji {i+1}/{total}", emoji["hex"], emoji["emoji"])
+        emoji_file = media_dir / emoji["filename"]
+        if emoji["type"] == "webp":
+            attrs = [
+                DocumentAttributeImageSize(w=100, h=100),
+                DocumentAttributeFilename(file_name="image.webp"),
+            ]
+            with emoji_file.open("rb") as f:
+                file_handle = await client.upload_file(f, file_name="emoji.webp")
+            emoji_file = InputMediaUploadedDocument(
+                file_handle, mime_type="image/webp", attributes=attrs
+            )
+        await send_emoji(conv, emoji_file, emoji["emoji"])
+        await asyncio.sleep(2)
+    print("Publishing pack", pack["short_name"])
+    await publish_pack(conv, pack["short_name"])
+
+
+async def main():
+    logging.basicConfig(level=logging.INFO)
+
+    emoji_data = await load_emoji_data()
+
+    split_cache = media_dir / "split-cache.json"
+    try:
+        with split_cache.open() as f:
+            packs: list[CachedPack] = json.load(f)
+    except FileNotFoundError:
+        packs = split_packs(emoji_data)
+        with split_cache.open("w") as f:
+            json.dump(packs, f)
+
+    doc_id_file = Path(args.output)
+    try:
+        with doc_id_file.open() as f:
+            doc_ids = json.load(f)
+    except FileNotFoundError:
+        doc_ids = {}
+
+    client = TelegramClient(args.session, args.api_id, args.api_hash, flood_sleep_threshold=3600)
+    await client.start()
+    async with client.conversation("Stickers", max_messages=20000) as conv:
+        for pack in packs:
+            if not pack["published"]:
+                await create_and_fill_pack(client, conv, pack)
+                pack["published"] = True
+                with split_cache.open("w") as f:
+                    json.dump(packs, f, ensure_ascii=False)
+            if not pack["collected"] or True:
+                print("Collecting document IDs from pack", pack["short_name"])
+                stickers = await client(
+                    GetStickerSetRequest(InputStickerSetShortName(pack["short_name"]), 0)
+                )
+                doc: Document
+                for i, doc in enumerate(stickers.documents):
+                    attr = next(
+                        attr
+                        for attr in doc.attributes
+                        if isinstance(attr, DocumentAttributeCustomEmoji)
+                    )
+                    base_emoji = attr.alt.replace("\ufe0f", "")
+                    emoji = pack["emojis"][i]["emoji"].replace("\ufe0f", "")
+                    doc_ids[emoji] = doc.id
+                    print(f"Mapped {emoji} (fallback: {base_emoji}) -> {doc_ids[emoji]}")
+                pack["collected"] = True
+                with split_cache.open("w") as f:
+                    json.dump(packs, f, ensure_ascii=False)
+                with doc_id_file.open("w") as f:
+                    json.dump(doc_ids, f, ensure_ascii=False)
+                print("Pack completed")
+                await asyncio.sleep(5)
+    with open(args.output.replace(".json", ".pickle"), "wb") as f:
+        pickle.dump(doc_ids, f)
+    print("Wrote pickle")
+
+
+asyncio.run(main())

mautrix_telegram/user.py

@@ -15,21 +15,31 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, AsyncGenerator, AsyncIterable, Awaitable, NamedTuple, cast
-from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, Awaitable, NamedTuple, cast
+from datetime import datetime
 import asyncio
+import time
 
-from telethon.errors import AuthKeyDuplicatedError, RPCError, UnauthorizedError
+from telethon.errors import (
+    AuthKeyDuplicatedError,
+    RPCError,
+    TakeoutInitDelayError,
+    UnauthorizedError,
+)
 from telethon.tl.custom import Dialog
 from telethon.tl.functions.account import UpdateStatusRequest
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
+from telethon.tl.functions.help import GetAppConfigRequest
+from telethon.tl.functions.messages import GetAvailableReactionsRequest
 from telethon.tl.functions.updates import GetStateRequest
 from telethon.tl.functions.users import GetUsersRequest
 from telethon.tl.types import (
     Chat,
     ChatForbidden,
     InputUserSelf,
+    Message,
     NotifyPeer,
+    PeerUser,
     TypeUpdate,
     UpdateFolderPeers,
     UpdateNewChannelMessage,
@@ -41,6 +51,7 @@ from telethon.tl.types import (
     User as TLUser,
 )
 from telethon.tl.types.contacts import ContactsNotModified
+from telethon.tl.types.messages import AvailableReactions
 
 from mautrix.appservice import DOUBLE_PUPPET_SOURCE_KEY
 from mautrix.bridge import BaseUser, async_getter_lock
@@ -50,9 +61,10 @@ from mautrix.types import PushActionType, PushRuleKind, PushRuleScope, RoomID, R
 from mautrix.util.bridge_state import BridgeState, BridgeStateEvent
 from mautrix.util.opt_prometheus import Gauge
 
-from . import portal as po, puppet as pu
+from . import portal as po, puppet as pu, util
 from .abstract_user import AbstractUser
-from .db import Message as DBMessage, PgSession, User as DBUser
+from .db import Backfill, BackfillType, Message as DBMessage, PgSession, User as DBUser
+from .tgclient import MautrixTelegramClient
 from .types import TelegramID
 
 if TYPE_CHECKING:
@@ -81,7 +93,17 @@ class User(DBUser, AbstractUser, BaseUser):
 
     _ensure_started_lock: asyncio.Lock
     _track_connection_task: asyncio.Task | None
+    _backfill_task: asyncio.Task | None
+    wakeup_backfill_task: asyncio.Event
     _is_backfilling: bool
+    takeout_retry_immediate: asyncio.Event
+    takeout_requested: bool
+
+    _available_emoji_reactions: set[str] | None
+    _available_emoji_reactions_hash: int | None
+    _available_emoji_reactions_fetched: float
+    _available_emoji_reactions_lock: asyncio.Lock
+    _app_config: dict[str, Any] | None
 
     def __init__(
         self,
@@ -90,6 +112,7 @@ class User(DBUser, AbstractUser, BaseUser):
         tg_username: str | None = None,
         tg_phone: str | None = None,
         is_bot: bool = False,
+        is_premium: bool = False,
         saved_contacts: int = 0,
     ) -> None:
         super().__init__(
@@ -98,6 +121,7 @@ class User(DBUser, AbstractUser, BaseUser):
             tg_username=tg_username,
             tg_phone=tg_phone,
             is_bot=is_bot,
+            is_premium=is_premium,
             saved_contacts=saved_contacts,
         )
         AbstractUser.__init__(self)
@@ -107,6 +131,17 @@ class User(DBUser, AbstractUser, BaseUser):
         self._is_backfilling = False
         self._portals_cache = None
 
+        self._backfill_task = None
+        self.wakeup_backfill_task = asyncio.Event()
+        self.takeout_retry_immediate = asyncio.Event()
+        self.takeout_requested = False
+
+        self._available_emoji_reactions = None
+        self._available_emoji_reactions_hash = None
+        self._available_emoji_reactions_fetched = 0
+        self._available_emoji_reactions_lock = asyncio.Lock()
+        self._app_config = None
+
         (
             self.relaybot_whitelisted,
             self.whitelisted,
@@ -129,6 +164,10 @@ class User(DBUser, AbstractUser, BaseUser):
     def human_tg_id(self) -> str:
         return f"@{self.tg_username}" if self.tg_username else f"+{self.tg_phone}" or None
 
+    @property
+    def peer(self) -> PeerUser | None:
+        return PeerUser(user_id=self.tgid) if self.tgid else None
+
     # TODO replace with proper displayname getting everywhere
     @property
     def displayname(self) -> str:
@@ -226,6 +265,14 @@ class User(DBUser, AbstractUser, BaseUser):
             self.client and self.client._sender and self.client._sender._transport_connected()
         )
 
+    @property
+    def _bridge_state_info(self) -> dict[str, Any]:
+        if self.takeout_requested:
+            return {
+                "takeout_requested": True,
+            }
+        return {}
+
     async def _track_connection(self) -> None:
         self.log.debug("Starting loop to track connection state")
         while True:
@@ -238,6 +285,7 @@ class User(DBUser, AbstractUser, BaseUser):
                     if self._is_backfilling
                     else BridgeStateEvent.CONNECTED,
                     ttl=3600,
+                    info=self._bridge_state_info,
                 )
             else:
                 await self.push_bridge_state(
@@ -262,7 +310,7 @@ class User(DBUser, AbstractUser, BaseUser):
         else:
             state_event = BridgeStateEvent.UNKNOWN_ERROR
             ttl = 240
-        return [BridgeState(state_event=state_event, ttl=ttl)]
+        return [BridgeState(state_event=state_event, ttl=ttl, info=self._bridge_state_info)]
 
     async def get_puppet(self) -> pu.Puppet | None:
         if not self.tgid:
@@ -280,11 +328,16 @@ class User(DBUser, AbstractUser, BaseUser):
         if self._track_connection_task:
             self._track_connection_task.cancel()
             self._track_connection_task = None
+        if self._backfill_task:
+            self._backfill_task.cancel()
+            self._backfill_task = None
         await super().stop()
         self._track_metric(METRIC_CONNECTED, False)
 
     async def post_login(self, info: TLUser = None, first_login: bool = False) -> None:
-        if self.config["metrics.enabled"] and not self._track_connection_task:
+        if (
+            self.config["metrics.enabled"] or self.config["homeserver.status_endpoint"]
+        ) and not self._track_connection_task:
             self._track_connection_task = asyncio.create_task(self._track_connection())
 
         try:
@@ -294,6 +347,8 @@ class User(DBUser, AbstractUser, BaseUser):
             return
 
         self._track_metric(METRIC_LOGGED_IN, True)
+        if not self._backfill_task or self._backfill_task.done():
+            self._backfill_task = asyncio.create_task(self._try_handle_backfill_requests_loop())
 
         try:
             puppet = await pu.Puppet.get_by_tgid(self.tgid)
@@ -303,7 +358,7 @@ class User(DBUser, AbstractUser, BaseUser):
         except Exception:
             self.log.exception("Failed to automatically enable custom puppet")
 
-        if not self.is_bot and self.config["bridge.startup_sync"]:
+        if not self.is_bot and (self.config["bridge.startup_sync"] or first_login):
             try:
                 self._is_backfilling = True
                 await self.sync_dialogs()
@@ -313,6 +368,123 @@ class User(DBUser, AbstractUser, BaseUser):
             finally:
                 self._is_backfilling = False
 
+    @property
+    def _takeout_options(self) -> dict[str, bool | int]:
+        return {
+            "users": True,
+            "chats": self.config["bridge.backfill.normal_groups"],
+            "megagroups": True,
+            "channels": True,
+            "files": True,
+            "max_file_size": min(self.bridge.matrix.media_config.upload_size, 2000 * 1024 * 1024),
+        }
+
+    async def _try_handle_backfill_requests_loop(self) -> None:
+        if not self.config["bridge.backfill.enable"]:
+            return
+        try:
+            await self._handle_backfill_requests_loop()
+        except Exception:
+            self.log.exception("Fatal error in backfill request loop")
+
+    async def _handle_backfill_requests_loop(self) -> None:
+        while True:
+            req = await Backfill.get_next(self.mxid)
+            if not req:
+                try:
+                    await asyncio.wait_for(self.wakeup_backfill_task.wait(), timeout=300)
+                except asyncio.TimeoutError:
+                    pass
+                self.wakeup_backfill_task.clear()
+            else:
+                try:
+                    await self._takeout_and_backfill(req)
+                except Exception:
+                    self.log.exception("Error in takeout backfill loop, retrying in an hour")
+                    await asyncio.sleep(3600)
+
+    async def _check_server_notice_edit(self, message: Message) -> None:
+        if "Data export request" in message.message and "Accepted" in message.message:
+            self.log.debug(
+                f"Received an edit to message {message.id} that looks like the data export"
+                " was accepted, marking takeout as retriable"
+            )
+            self.takeout_retry_immediate.set()
+
+    async def _takeout_and_backfill(self, first_req: Backfill, first_attempt: bool = True) -> None:
+        self.takeout_retry_immediate.clear()
+        self.takeout_requested = True
+        try:
+            async with self.client.takeout(**self._takeout_options) as takeout_client:
+                self.takeout_requested = False
+                self.log.info("Acquired takeout client successfully")
+                await self._backfill_loop_with_client(takeout_client, first_req)
+                self.log.info("Backfills finished, exiting takeout")
+        except TakeoutInitDelayError as e:
+            if first_attempt:
+                self.log.info(
+                    f"Takeout requested, will wait for retry request or {e.seconds} seconds"
+                )
+            else:
+                self.log.warning(
+                    f"Got takeout init delay again after retry, waiting for {e.seconds} seconds"
+                )
+            try:
+                await asyncio.wait_for(self.takeout_retry_immediate.wait(), timeout=e.seconds)
+                self.log.info("Retrying takeout")
+            except asyncio.TimeoutError:
+                self.log.info("Takeout timeout expired")
+            await self._takeout_and_backfill(first_req, first_attempt=False)
+
+    async def _backfill_loop_with_client(
+        self, client: MautrixTelegramClient, first_req: Backfill
+    ) -> None:
+        missed_reqs = 0
+        while missed_reqs < 10:
+            req = first_req or await Backfill.get_next(self.mxid)
+            first_req = None
+            if not req:
+                missed_reqs += 1
+                try:
+                    await asyncio.wait_for(self.wakeup_backfill_task.wait(), timeout=30)
+                except asyncio.TimeoutError:
+                    pass
+                self.wakeup_backfill_task.clear()
+                continue
+            missed_reqs = 0
+            self.log.info("Backfill request %s", req)
+            try:
+                portal = await po.Portal.get_by_tgid(
+                    TelegramID(req.portal_tgid), tg_receiver=TelegramID(req.portal_tg_receiver)
+                )
+                await req.mark_dispatched()
+                if req.type == BackfillType.HISTORICAL:
+                    await portal.backfill(self, client, req=req)
+                elif req.type == BackfillType.SYNC_DIALOG:
+                    await self._backfill_sync_dialog(portal, client, req.extra_data)
+                await req.mark_done()
+                await asyncio.sleep(req.post_batch_delay)
+            except Exception:
+                self.log.exception("Error handling backfill request for %s", req.portal_tgid)
+                await req.set_cooldown_timeout(1800)
+
+    async def _backfill_sync_dialog(
+        self, portal: po.Portal, client: MautrixTelegramClient, post_sync_args: dict[str, Any]
+    ) -> None:
+        if portal.mxid:
+            self.log.debug("Portal already exists, skipping dialog sync backfill queue item")
+            return
+        self.log.info(f"Creating portal for {portal.tgid_log} as part of backfill loop")
+        try:
+            await portal.create_matrix_room(
+                self, client=client, update_if_exists=False, invites=[self.mxid]
+            )
+        except Exception:
+            self.log.exception(f"Error while creating {portal.tgid_log}")
+        else:
+            puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+            await self._post_sync_dialog(portal, puppet, was_created=True, **post_sync_args)
+
     async def update(self, update: TypeUpdate) -> bool:
         if not self.is_bot:
             return False
@@ -363,6 +535,9 @@ class User(DBUser, AbstractUser, BaseUser):
         if self.is_bot != info.bot:
             self.is_bot = info.bot
             changed = True
+        if self.is_premium != info.premium:
+            self.is_premium = info.premium
+            changed = True
         if self.tg_username != info.username:
             self.tg_username = info.username
             changed = True
@@ -408,10 +583,11 @@ class User(DBUser, AbstractUser, BaseUser):
                 pass
             self.tgid = None
         ok = await self.client.log_out()
-        await self.client.session.delete()
+        sess = self.client.session
+        await self.stop()
+        await sess.delete()
         await self.delete()
         self.by_mxid.pop(self.mxid, None)
-        await self.stop()
         self._track_metric(METRIC_LOGGED_IN, False)
         return ok
 
@@ -469,19 +645,18 @@ class User(DBUser, AbstractUser, BaseUser):
         if active and tag_info is None:
             tag_info = RoomTagInfo(order=0.5)
             tag_info[DOUBLE_PUPPET_SOURCE_KEY] = self.bridge.name
-            self.log.debug("Adding tag {tag} to {portal.mxid}/{portal.tgid}")
+            self.log.debug(f"Adding tag {tag} to {portal.mxid}/{portal.tgid}")
             await puppet.intent.set_room_tag(portal.mxid, tag, tag_info)
         elif (
             not active and tag_info and tag_info.get(DOUBLE_PUPPET_SOURCE_KEY) == self.bridge.name
         ):
-            self.log.debug("Removing tag {tag} from {portal.mxid}/{portal.tgid}")
+            self.log.debug(f"Removing tag {tag} from {portal.mxid}/{portal.tgid}")
             await puppet.intent.remove_room_tag(portal.mxid, tag)
 
-    async def _mute_room(self, puppet: pu.Puppet, portal: po.Portal, mute_until: datetime) -> None:
+    async def _mute_room(self, puppet: pu.Puppet, portal: po.Portal, mute_until: float) -> None:
         if not self.config["bridge.mute_bridging"] or not portal or not portal.mxid:
             return
-        now = datetime.utcnow().replace(tzinfo=timezone.utc)
-        if mute_until is not None and mute_until > now:
+        if mute_until is not None and mute_until > time.time():
             self.log.debug(
                 f"Muting {portal.mxid}/{portal.tgid} (muted until {mute_until} on Telegram)"
             )
@@ -537,15 +712,28 @@ class User(DBUser, AbstractUser, BaseUser):
         portal = await po.Portal.get_by_entity(
             update.peer.peer, tg_receiver=self.tgid, create=False
         )
-        await self._mute_room(puppet, portal, update.notify_settings.mute_until)
+        await self._mute_room(puppet, portal, update.notify_settings.mute_until.timestamp())
 
     async def _sync_dialog(
         self, portal: po.Portal, dialog: Dialog, should_create: bool, puppet: pu.Puppet | None
     ) -> None:
         was_created = False
+        post_sync_args = {
+            "last_message_ts": cast(datetime, dialog.date).timestamp(),
+            "unread_count": dialog.unread_count,
+            "max_read_id": dialog.dialog.read_inbox_max_id,
+            "mute_until": (
+                dialog.dialog.notify_settings.mute_until.timestamp()
+                if dialog.dialog.notify_settings.mute_until
+                else None
+            ),
+            "pinned": dialog.pinned,
+            "archived": dialog.archived,
+        }
         if portal.mxid:
+            self.log.debug(f"Backfilling and updating {portal.tgid_log} (dialog sync)")
             try:
-                await portal.backfill(self, last_id=dialog.message.id)
+                await portal.forward_backfill(self, initial=False, last_tgid=dialog.message.id)
             except Exception:
                 self.log.exception(f"Error while backfilling {portal.tgid_log}")
             try:
@@ -553,31 +741,73 @@ class User(DBUser, AbstractUser, BaseUser):
             except Exception:
                 self.log.exception(f"Error while updating {portal.tgid_log}")
         elif should_create:
+            self.log.debug(f"Creating portal for {portal.tgid_log} immediately (dialog sync)")
             try:
                 await portal.create_matrix_room(self, dialog.entity, invites=[self.mxid])
                 was_created = True
             except Exception:
                 self.log.exception(f"Error while creating {portal.tgid_log}")
+        elif self.config["bridge.sync_deferred_create_all"]:
+            self.log.debug(f"Enqueuing deferred dialog sync for {portal.tgid_log}")
+            await portal.enqueue_backfill(
+                self,
+                priority=40,
+                type=BackfillType.SYNC_DIALOG,
+                extra_data=post_sync_args,
+            )
         if portal.mxid and puppet and puppet.is_real_user:
-            tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
-            if dialog.unread_count == 0:
-                # This is usually more reliable than finding a specific message
-                # e.g. if the last read message is a service message that isn't in the message db
-                last_read = await DBMessage.find_last(portal.mxid, tg_space)
-            else:
-                last_read = await DBMessage.get_one_by_tgid(
-                    portal.tgid, tg_space, dialog.dialog.read_inbox_max_id
+            await self._post_sync_dialog(
+                portal=portal,
+                puppet=puppet,
+                was_created=was_created,
+                **post_sync_args,
+            )
+        self.log.debug(f"_sync_dialog finished for {portal.tgid_log}")
+
+    async def _post_sync_dialog(
+        self,
+        portal: po.Portal,
+        puppet: pu.Puppet,
+        was_created: bool,
+        max_read_id: int,
+        last_message_ts: float,
+        unread_count: int,
+        mute_until: float,
+        pinned: bool,
+        archived: bool,
+    ) -> None:
+        self.log.debug(
+            f"Running dialog post-sync for {portal.tgid_log} with args "
+            f"{was_created=}, {max_read_id=}, {last_message_ts=}, {unread_count=}, "
+            f"{mute_until=}, {pinned=}, {archived=}"
+        )
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        unread_threshold_hours = self.config["bridge.backfill.unread_hours_threshold"]
+        force_read = (
+            was_created
+            and unread_threshold_hours >= 0
+            and last_message_ts + (unread_threshold_hours * 60 * 60) < time.time()
+        )
+        if unread_count == 0 or force_read:
+            # This is usually more reliable than finding a specific message
+            # e.g. if the last read message is a service message that isn't in the message db
+            last_read = await DBMessage.find_last(portal.mxid, tg_space)
+            if force_read:
+                self.log.debug(
+                    f"Marking {portal.tgid_log} as read because the last message is from "
+                    f"{last_message_ts} (unread threshold is {unread_threshold_hours} hours)"
                 )
+        else:
+            last_read = await DBMessage.get_one_by_tgid(portal.tgid, tg_space, max_read_id)
+        try:
             if last_read:
                 await puppet.intent.mark_read(last_read.mx_room, last_read.mxid)
             if was_created or not self.config["bridge.tag_only_on_create"]:
-                await self._mute_room(puppet, portal, dialog.dialog.notify_settings.mute_until)
-                await self._tag_room(
-                    puppet, portal, self.config["bridge.pinned_tag"], dialog.pinned
-                )
-                await self._tag_room(
-                    puppet, portal, self.config["bridge.archive_tag"], dialog.archived
-                )
+                await self._mute_room(puppet, portal, mute_until)
+                await self._tag_room(puppet, portal, self.config["bridge.pinned_tag"], pinned)
+                await self._tag_room(puppet, portal, self.config["bridge.archive_tag"], archived)
+        except Exception:
+            self.log.exception(f"Error updating read status and tags for {portal.tgid_log}")
 
     async def get_cached_portals(self) -> dict[tuple[TelegramID, TelegramID], po.Portal]:
         if self._portals_cache is None:
@@ -594,9 +824,7 @@ class User(DBUser, AbstractUser, BaseUser):
         update_limit = self.config["bridge.sync_update_limit"] or None
         create_limit = self.config["bridge.sync_create_limit"]
         index = 0
-        self.log.debug(
-            f"Syncing dialogs (update_limit={update_limit}, create_limit={create_limit})"
-        )
+        self.log.debug(f"Syncing dialogs ({update_limit=}, {create_limit=})")
         await self.push_bridge_state(BridgeStateEvent.BACKFILLING)
         puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
         dialog: Dialog
@@ -617,11 +845,12 @@ class User(DBUser, AbstractUser, BaseUser):
                 continue
             portal = await po.Portal.get_by_entity(entity, tg_receiver=self.tgid)
             new_portal_cache[portal.tgid_full] = portal
+            should_create = not create_limit or index < create_limit
             coro = self._sync_dialog(
                 portal=portal,
                 dialog=dialog,
                 puppet=puppet,
-                should_create=not create_limit or index < create_limit,
+                should_create=should_create,
             )
             creators.append(asyncio.create_task(coro))
             index += 1
@@ -680,8 +909,51 @@ class User(DBUser, AbstractUser, BaseUser):
             await puppet.update_info(self, user)
             contacts[user.id] = puppet.contact_info
         await self.set_contacts(contacts.keys())
+        self.log.debug("Contact syncing complete")
         return contacts
 
+    async def get_available_reactions(self) -> set[str]:
+        if self._available_emoji_reactions_fetched + 12 * 60 * 60 > time.monotonic():
+            return self._available_emoji_reactions
+        async with self._available_emoji_reactions_lock:
+            if self._available_emoji_reactions_fetched + 12 * 60 * 60 > time.monotonic():
+                return self._available_emoji_reactions
+            self.log.debug("Fetching available emoji reactions")
+            available_reactions = await self.client(
+                GetAvailableReactionsRequest(hash=self._available_emoji_reactions_hash or 0)
+            )
+            if isinstance(available_reactions, AvailableReactions):
+                self._available_emoji_reactions = {
+                    react.reaction
+                    for react in available_reactions.reactions
+                    if self.is_premium or not react.premium
+                }
+                self._available_emoji_reactions_hash = available_reactions.hash
+                self._available_emoji_reactions_fetched = time.monotonic()
+                self.log.debug(
+                    "Got available emoji reactions: %s", self._available_emoji_reactions
+                )
+            return self._available_emoji_reactions
+
+    def tl_to_json(self) -> Any:
+        pass
+
+    async def get_app_config(self) -> dict[str, Any]:
+        if not self._app_config:
+            cfg = await self.client(GetAppConfigRequest())
+            self._app_config = util.parse_tl_json(cfg)
+        return self._app_config
+
+    async def get_max_reactions(self, is_premium: bool | None = None) -> int:
+        if is_premium is None:
+            is_premium = self.is_premium
+        cfg = await self.get_app_config()
+        return (
+            cfg.get("reactions_user_max_premium", 3)
+            if is_premium
+            else cfg.get("reactions_user_max_default", 1)
+        )
+
     # endregion
     # region Class instance lookup
 
@@ -710,7 +982,7 @@ class User(DBUser, AbstractUser, BaseUser):
     @classmethod
     @async_getter_lock
     async def get_by_mxid(
-        cls, mxid: UserID, *, check_db: bool = True, create: bool = True
+        cls, mxid: UserID, /, *, check_db: bool = True, create: bool = True
     ) -> User | None:
         if not mxid or pu.Puppet.get_id_from_mxid(mxid) or mxid == cls.az.bot_mxid:
             return None
@@ -738,7 +1010,7 @@ class User(DBUser, AbstractUser, BaseUser):
 
     @classmethod
     @async_getter_lock
-    async def get_by_tgid(cls, tgid: TelegramID) -> User | None:
+    async def get_by_tgid(cls, tgid: TelegramID, /) -> User | None:
         try:
             return cls.by_tgid[tgid]
         except KeyError:

mautrix_telegram/util/__init__.py

@@ -1,4 +1,11 @@
 from .color_log import ColorFormatter
-from .file_transfer import convert_image, transfer_file_to_matrix
+from .file_transfer import (
+    UnicodeCustomEmoji,
+    convert_image,
+    transfer_custom_emojis_to_matrix,
+    transfer_file_to_matrix,
+    unicode_custom_emoji_map,
+)
 from .parallel_file_transfer import parallel_transfer_to_telegram
 from .recursive_dict import recursive_del, recursive_get, recursive_set
+from .tl_json import parse_tl_json

mautrix_telegram/util/file_transfer.py

@@ -15,11 +15,13 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import Optional, Union
+from typing import NamedTuple, Optional, Union
 from io import BytesIO
 from sqlite3 import IntegrityError
 import asyncio
 import logging
+import pickle
+import pkgutil
 import tempfile
 import time
 
@@ -31,6 +33,7 @@ from telethon.errors import (
     LocationInvalidError,
     SecurityError,
 )
+from telethon.tl.functions.messages import GetCustomEmojiDocumentsRequest
 from telethon.tl.types import (
     Document,
     InputDocumentFileLocation,
@@ -41,15 +44,17 @@ from telethon.tl.types import (
     PhotoSize,
     TypePhotoSize,
 )
-import magic
 
 from mautrix.appservice import IntentAPI
+from mautrix.util import magic, variation_selector
 
+from .. import abstract_user as au
 from ..db import TelegramFile as DBTelegramFile
 from ..tgclient import MautrixTelegramClient
 from ..util import sane_mimetypes
 from .parallel_file_transfer import parallel_transfer_to_matrix
 from .tgs_converter import convert_tgs_to
+from .webm_converter import convert_webm_to
 
 try:
     from PIL import Image
@@ -125,9 +130,9 @@ def _read_video_thumbnail(
 
 def _location_to_id(location: TypeLocation) -> str:
     if isinstance(location, Document):
-        return f"{location.id}-{location.access_hash}"
+        return str(location.id)
     elif isinstance(location, (InputDocumentFileLocation, InputPhotoFileLocation)):
-        return f"{location.id}-{location.access_hash}-{location.thumb_size}"
+        return f"{location.id}-{location.thumb_size}"
     elif isinstance(location, InputFileLocation):
         return f"{location.volume_id}-{location.local_id}"
     elif isinstance(location, InputPeerPhotoFileLocation):
@@ -155,6 +160,8 @@ async def transfer_thumbnail_to_matrix(
 
     if custom_data:
         loc_id += "-mau_custom_thumbnail"
+    if encrypt:
+        loc_id += "-encrypted"
 
     db_file = await DBTelegramFile.get(loc_id)
     if db_file:
@@ -172,7 +179,7 @@ async def transfer_thumbnail_to_matrix(
     else:
         file = await client.download_file(thumbnail_loc)
         width, height = None, None
-        mime_type = magic.from_buffer(file, mime=True)
+        mime_type = magic.mimetype(file)
 
     decryption_info = None
     upload_mime_type = mime_type
@@ -207,9 +214,71 @@ async def transfer_thumbnail_to_matrix(
 
 transfer_locks: dict[str, asyncio.Lock] = {}
 
+unicode_custom_emoji_map = pickle.loads(
+    pkgutil.get_data("mautrix_telegram", "unicodemojipack.pickle")
+)
+reverse_unicode_custom_emoji_map = {
+    doc_id: emoji for emoji, doc_id in unicode_custom_emoji_map.items()
+}
+
 TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
 
 
+class UnicodeCustomEmoji(NamedTuple):
+    emoji: str
+
+
+async def transfer_custom_emojis_to_matrix(
+    source: au.AbstractUser, emoji_ids: list[int], client: MautrixTelegramClient | None = None
+) -> dict[int, DBTelegramFile | UnicodeCustomEmoji]:
+    if not client:
+        client = source.client
+    emoji_ids = set(emoji_ids)
+    existing_unicode = {}
+    for emoji_id in emoji_ids:
+        try:
+            existing_unicode[emoji_id] = UnicodeCustomEmoji(
+                variation_selector.add(reverse_unicode_custom_emoji_map[emoji_id])
+            )
+        except KeyError:
+            pass
+    emoji_ids -= existing_unicode.keys()
+    if not emoji_ids:
+        return existing_unicode
+    existing = await DBTelegramFile.get_many([str(id) for id in emoji_ids])
+    file_map = {int(file.id): file for file in existing} | existing_unicode
+    not_existing_ids = list(emoji_ids - file_map.keys())
+    if not_existing_ids:
+        log.debug(f"Transferring custom emojis through {source.mxid}: {not_existing_ids}")
+
+        documents: list[Document] = await client(
+            GetCustomEmojiDocumentsRequest(document_id=not_existing_ids)
+        )
+
+        tgs_args = source.config["bridge.animated_emoji"]
+        webm_convert = tgs_args["target"]
+
+        transfer_sema = asyncio.Semaphore(5)
+
+        async def transfer(document: Document) -> None:
+            async with transfer_sema:
+                file_map[document.id] = await transfer_file_to_matrix(
+                    client,
+                    source.bridge.az.intent,
+                    document,
+                    is_sticker=True,
+                    tgs_convert=tgs_args,
+                    webm_convert=webm_convert,
+                    filename=f"emoji-{document.id}",
+                    # Emojis are used as inline images and can't be encrypted
+                    encrypt=False,
+                    async_upload=source.config["homeserver.async_media"],
+                )
+
+        await asyncio.gather(*[transfer(doc) for doc in documents])
+    return file_map
+
+
 async def transfer_file_to_matrix(
     client: MautrixTelegramClient,
     intent: IntentAPI,
@@ -218,6 +287,7 @@ async def transfer_file_to_matrix(
     *,
     is_sticker: bool = False,
     tgs_convert: dict | None = None,
+    webm_convert: str | None = None,
     filename: str | None = None,
     encrypt: bool = False,
     parallel_id: int | None = None,
@@ -226,6 +296,8 @@ async def transfer_file_to_matrix(
     location_id = _location_to_id(location)
     if not location_id:
         return None
+    if encrypt:
+        location_id += "-encrypted"
 
     db_file = await DBTelegramFile.get(location_id)
     if db_file:
@@ -245,6 +317,7 @@ async def transfer_file_to_matrix(
             thumbnail,
             is_sticker,
             tgs_convert,
+            webm_convert,
             filename,
             encrypt,
             parallel_id,
@@ -260,6 +333,7 @@ async def _unlocked_transfer_file_to_matrix(
     thumbnail: TypeThumbnail,
     is_sticker: bool,
     tgs_convert: dict | None,
+    webm_convert: str | None,
     filename: str | None,
     encrypt: bool,
     parallel_id: int | None,
@@ -287,13 +361,10 @@ async def _unlocked_transfer_file_to_matrix(
             return None
 
         width, height = None, None
-        mime_type = magic.from_buffer(file, mime=True)
+        mime_type = magic.mimetype(file)
 
         image_converted = False
-        # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
-        is_tgs = mime_type == "application/gzip" or (
-            mime_type == "application/octet-stream" and magic.from_buffer(file).startswith("gzip")
-        )
+        is_tgs = mime_type == "application/gzip"
         if is_sticker and tgs_convert and is_tgs:
             converted_anim = await convert_tgs_to(
                 file, tgs_convert["target"], **tgs_convert["args"]
@@ -303,6 +374,12 @@ async def _unlocked_transfer_file_to_matrix(
             width, height = converted_anim.width, converted_anim.height
             image_converted = mime_type != "application/gzip"
             thumbnail = None
+        elif is_sticker and webm_convert and webm_convert != "webm" and mime_type == "video/webm":
+            converted_anim = await convert_webm_to(file, webm_convert)
+            mime_type = converted_anim.mime
+            file = converted_anim.data
+            image_converted = mime_type != "video/webm"
+            thumbnail = None
 
         decryption_info = None
         upload_mime_type = mime_type

mautrix_telegram/util/tgs_converter.py

@@ -99,7 +99,7 @@ if lottieconverter:
     converters["png"] = tgs_to_png
     converters["gif"] = tgs_to_gif
 
-if lottieconverter and ffmpeg:
+if lottieconverter and ffmpeg.ffmpeg_path:
 
     async def tgs_to_webm(
         file: bytes, width: int, height: int, fps: int = 30, **_: Any
@@ -126,7 +126,33 @@ if lottieconverter and ffmpeg:
                 log.error(str(e))
         return ConvertedSticker("application/gzip", file)
 
+    async def tgs_to_webp(
+        file: bytes, width: int, height: int, fps: int = 30, **_: Any
+    ) -> ConvertedSticker:
+        with tempfile.TemporaryDirectory(prefix="tgs_") as tmpdir:
+            file_template = tmpdir + "/out_"
+            try:
+                await _run_lottieconverter(
+                    args=("-", file_template, "pngs", f"{width}x{height}", str(fps)),
+                    input_data=file,
+                )
+                first_frame_name = min(os.listdir(tmpdir))
+                with open(f"{tmpdir}/{first_frame_name}", "rb") as first_frame_file:
+                    first_frame_data = first_frame_file.read()
+                webp_data = await ffmpeg.convert_path(
+                    input_args=("-framerate", str(fps), "-pattern_type", "glob"),
+                    input_file=f"{file_template}*.png",
+                    output_args=("-c:v", "libwebp_anim", "-pix_fmt", "yuva420p", "-f", "webp"),
+                    output_path_override="-",
+                    output_extension=None,
+                )
+                return ConvertedSticker("image/webp", webp_data, "image/png", first_frame_data)
+            except ffmpeg.ConverterError as e:
+                log.error(str(e))
+        return ConvertedSticker("application/gzip", file)
+
     converters["webm"] = tgs_to_webm
+    converters["webp"] = tgs_to_webp
 
 
 async def convert_tgs_to(

mautrix_telegram/util/tl_json.py

@@ -0,0 +1,39 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from telethon.tl.types import (
+    JsonArray,
+    JsonBool,
+    JsonNull,
+    JsonNumber,
+    JsonObject,
+    JsonObjectValue,
+    JsonString,
+    TypeJSONValue,
+)
+
+from mautrix.types import JSON
+
+
+def parse_tl_json(val: TypeJSONValue) -> JSON:
+    if isinstance(val, JsonObject):
+        return {entry.key: parse_tl_json(entry.value) for entry in val.value}
+    elif isinstance(val, JsonArray):
+        return [parse_tl_json(item) for item in val.value]
+    elif isinstance(val, (JsonBool, JsonNumber, JsonString)):
+        return val.value
+    elif isinstance(val, JsonNull):
+        return None
+    raise ValueError(f"Unsupported type {type(val)} in TL JSON object")

mautrix_telegram/util/webm_converter.py

@@ -0,0 +1,52 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import logging
+
+from mautrix.util import ffmpeg
+
+from .tgs_converter import ConvertedSticker
+
+log: logging.Logger = logging.getLogger("mau.util.webm")
+
+
+converter_args = {
+    "gif": {
+        "output_args": ("-vf", "split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse"),
+    },
+    "png": {
+        "input_args": ("-ss", "0"),
+        "output_args": ("-frames:v", "1"),
+    },
+    "webp": {},
+}
+
+
+async def convert_webm_to(file: bytes, convert_to: str) -> ConvertedSticker:
+    if convert_to in ("png", "gif", "webp"):
+        try:
+            converted_data = await ffmpeg.convert_bytes(
+                data=file,
+                output_extension=f".{convert_to}",
+                **converter_args[convert_to],
+            )
+            return ConvertedSticker(f"image/{convert_to}", converted_data)
+        except ffmpeg.ConverterError as e:
+            log.error(str(e))
+    elif convert_to != "disable":
+        log.warning(f"Unable to convert webm animated sticker, type {convert_to} not supported")
+    return ConvertedSticker("video/webm", file)

mautrix_telegram/web/common/auth_api.py

@@ -128,7 +128,7 @@ class AuthAPI(abc.ABC):
                 status=200,
                 message=(
                     "Code requested successfully. Check your SMS "
-                    "or Telegram client and enter the code below."
+                    "or Telegram app and enter the code below."
                 ),
             )
         except PhoneNumberInvalidError:
@@ -279,6 +279,14 @@ class AuthAPI(abc.ABC):
                 errcode="phone_code_expired",
                 error="Phone code expired.",
             )
+        except PhoneNumberUnoccupiedError:
+            return self.get_login_response(
+                mxid=user.mxid,
+                state="code",
+                status=403,
+                errcode="phone_number_unoccupied",
+                error="That phone number has not been registered.",
+            )
         except SessionPasswordNeededError:
             if not password_in_data:
                 if user.command_status and user.command_status["action"] == "Login":

mautrix_telegram/web/provisioning/__init__.py

@@ -25,6 +25,7 @@ from telethon.tl.types import ChannelForbidden, ChatForbidden, TypeChat, User as
 from telethon.utils import get_peer_id, resolve_id
 
 from mautrix.appservice import AppService
+from mautrix.client import Client
 from mautrix.errors import IntentError, MatrixRequestError
 from mautrix.types import UserID
 
@@ -53,7 +54,7 @@ class ProvisioningAPI(AuthAPI):
 
         self.app = web.Application(loop=bridge.loop, middlewares=[self.error_middleware])
 
-        portal_prefix = "/v1/portal/{mxid:![^/]+}"
+        portal_prefix = "/v1/portal/{mxid}"
         self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal_by_mxid)
         self.app.router.add_route("GET", "/v1/portal/{tgid:-[0-9]+}", self.get_portal_by_tgid)
         self.app.router.add_route(
@@ -62,19 +63,24 @@ class ProvisioningAPI(AuthAPI):
         self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat)
         self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat)
 
-        user_prefix = "/v1/user/{mxid:@[^:]*:[^/]+}"
+        user_prefix = "/v1/user/{mxid}"
         self.app.router.add_route("GET", f"{user_prefix}", self.get_user_info)
         self.app.router.add_route("GET", f"{user_prefix}/chats", self.get_chats)
         self.app.router.add_route("GET", f"{user_prefix}/contacts", self.get_contacts)
+        self.app.router.add_route(
+            "GET", f"{user_prefix}/resolve_identifier/{{identifier}}", self.resolve_identifier
+        )
         self.app.router.add_route("POST", f"{user_prefix}/pm/{{identifier}}", self.start_dm)
 
+        self.app.router.add_route("POST", f"{user_prefix}/retry_takeout", self.retry_takeout)
+
         self.app.router.add_route("POST", f"{user_prefix}/logout", self.logout)
         self.app.router.add_route("POST", f"{user_prefix}/login/bot_token", self.send_bot_token)
         self.app.router.add_route("POST", f"{user_prefix}/login/request_code", self.request_code)
         self.app.router.add_route("POST", f"{user_prefix}/login/send_code", self.send_code)
         self.app.router.add_route("POST", f"{user_prefix}/login/send_password", self.send_password)
 
-        self.app.router.add_route("GET", "/bridge", self.bridge_info)
+        self.app.router.add_route("GET", "/v1/bridge", self.bridge_info)
 
     async def get_portal_by_mxid(self, request: web.Request) -> web.Response:
         err = self.check_authorization(request)
@@ -401,40 +407,80 @@ class ProvisioningAPI(AuthAPI):
             return err
         return web.json_response(data=await user.sync_contacts())
 
-    async def start_dm(self, request: web.Request) -> web.Response:
+    async def _resolve_id(
+        self, request: web.Request
+    ) -> tuple[Portal | None, User | None, TLUser | None, web.Response | None]:
         data, user, err = await self.get_user_request_info(request, expect_logged_in=True)
         if err is not None:
-            return err
+            return None, user, None, err
         try:
             identifier: str | int = request.match_info["identifier"]
             if isinstance(identifier, str) and identifier.isdecimal():
                 identifier = int(identifier)
             target = await user.client.get_entity(identifier)
         except ValueError:
-            return web.json_response(
-                {
-                    "error": "Invalid user identifier or user not found.",
-                    "errcode": "M_NOT_FOUND",
-                },
-                status=404,
+            return (
+                None,
+                user,
+                None,
+                web.json_response(
+                    {
+                        "error": "Invalid user identifier or user not found.",
+                        "errcode": "M_NOT_FOUND",
+                    },
+                    status=404,
+                ),
             )
 
         if not target:
-            return web.json_response(
-                {
-                    "error": "User not found.",
-                    "errcode": "M_NOT_FOUND",
-                },
-                status=404,
+            return (
+                None,
+                user,
+                None,
+                web.json_response(
+                    {
+                        "error": "User not found.",
+                        "errcode": "M_NOT_FOUND",
+                    },
+                    status=404,
+                ),
             )
         elif not isinstance(target, TLUser):
-            return web.json_response(
-                {
-                    "error": "Identifier is not a user.",
-                },
-                status=400,
+            return (
+                None,
+                user,
+                None,
+                web.json_response(
+                    {
+                        "error": "Identifier is not a user.",
+                        "errcode": "FI.MAU.TELEGRAM_ID_NOT_USER",
+                    },
+                    status=400,
+                ),
             )
         portal = await Portal.get_by_entity(target, tg_receiver=user.tgid)
+        return portal, user, target, None
+
+    async def resolve_identifier(self, request: web.Request) -> web.Response:
+        portal, user, target, err = await self._resolve_id(request)
+        if err is not None:
+            return err
+        puppet = await portal.get_dm_puppet()
+        await puppet.update_info(user, target)
+        return web.json_response(
+            {
+                "room_id": portal.mxid,
+                "just_created": False,
+                "id": portal.tgid,
+                "contact_info": puppet.contact_info,
+            },
+            status=200,
+        )
+
+    async def start_dm(self, request: web.Request) -> web.Response:
+        portal, user, target, err = await self._resolve_id(request)
+        if err is not None:
+            return err
         puppet = await portal.get_dm_puppet()
         if portal.mxid:
             just_created = False
@@ -451,6 +497,22 @@ class ProvisioningAPI(AuthAPI):
             status=201 if just_created else 200,
         )
 
+    async def retry_takeout(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(
+            request, expect_logged_in=True, want_data=False
+        )
+        if err is not None:
+            return err
+        if not user.takeout_requested:
+            return web.json_response(
+                {
+                    "error": "There was no takeout requested",
+                },
+                status=400,
+            )
+        user.takeout_retry_immediate.set()
+        return web.json_response({}, status=200)
+
     async def send_bot_token(self, request: web.Request) -> web.Response:
         data, user, err = await self.get_user_request_info(request)
         if err is not None:
@@ -596,6 +658,12 @@ class ProvisioningAPI(AuthAPI):
             return None, self.get_login_response(
                 error="User ID not given.", errcode="mxid_empty", status=400
             )
+        try:
+            Client.parse_user_id(mxid)
+        except ValueError:
+            return None, self.get_login_response(
+                error="Invalid user ID", errcode="mxid_invalid", status=400
+            )
 
         user = await User.get_and_start_by_mxid(mxid, even_if_no_session=True)
         if require_puppeting and not user.puppet_whitelisted:

optional-requirements.txt

@@ -2,8 +2,7 @@
 # Uncommented lines after the group definition insert things into that group.
 
 #/speedups
-cryptg>=0.1,<0.3
-cchardet
+cryptg>=0.1,<0.4
 aiodns
 brotli
 
@@ -18,7 +17,7 @@ moviepy>=1,<2
 phonenumbers>=8,<9
 
 #/metrics
-prometheus_client>=0.6,<0.15
+prometheus_client>=0.6,<0.16
 
 #/e2be
 python-olm>=3,<4

requirements.txt

@@ -3,10 +3,9 @@ python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.16.0,<0.17
-#telethon>=1.24,<1.25
-# Fork to make session storage async and update to layer 138
-tulir-telethon==1.25.0a7
-asyncpg>=0.20,<0.26
+mautrix>=0.18.8,<0.19
+#telethon>=1.25.4,<1.27
+tulir-telethon==1.27.0a1
+asyncpg>=0.20,<0.28
 mako>=1,<2
 setuptools

setup.py

@@ -66,7 +66,7 @@ setuptools.setup(
     ],
     package_data={"mautrix_telegram": [
         "web/public/*.mako", "web/public/*.png", "web/public/*.css",
-        "example-config.yaml",
+        "example-config.yaml", "unicodemojipack.pickle",
     ]},
     data_files=[
         (".", ["mautrix_telegram/example-config.yaml"]),