Bump version to 0.13.0

Update changelog
Catch errors updating initial profile. Fixes #860
2023-02-26 17:24:11 +02:00 · 2023-02-26 15:05:49 +02:00 · 2023-02-22 01:31:32 +02:00 · 2023-02-22 01:02:47 +02:00 · 2023-02-21 22:34:21 +02:00 · 2023-02-15 23:01:09 +02:00
76 changed files with 5137 additions and 1926 deletions
@@ -0,0 +1,7 @@
+---
+name: Bug report
+about: If something is definitely wrong in the bridge (rather than just a setup issue),
+  file a bug report. Remember to include relevant logs.
+labels: bug
+
+---
@@ -0,0 +1,7 @@
+contact_links:
+  - name: Troubleshooting docs & FAQ
+    url: https://docs.mau.fi/bridges/general/troubleshooting.html
+    about: Check this first if you're having problems setting up the bridge.
+  - name: Support room
+    url: https://matrix.to/#/#telegram:maunium.net
+    about: For setup issues not answered by the troubleshooting docs, ask in the Matrix room.
@@ -0,0 +1,6 @@
+---
+name: Enhancement request
+about: Submit a feature request or other suggestion
+labels: enhancement
+
+---
@@ -9,14 +9,14 @@ jobs:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@v2
      with:
-        python-version: "3.10"
+        python-version: "3.11"
    - uses: isort/isort-action@master
      with:
        sortPaths: "./mautrix_telegram"
    - uses: psf/black@stable
      with:
        src: "./mautrix_telegram"
-        version: "22.3.0"
+        version: "23.1.0"
    - name: pre-commit
      run: |
        pip install pre-commit
@@ -14,5 +14,8 @@ __pycache__
 /registration.yaml
 *.log*
 *.db
-*.pickle
+/*.pickle
 *.bak
+/*.session
+/*.session-journal
+/*.json
@@ -1,66 +1,3 @@
-image: docker:stable
-
-stages:
- build
- manifest
-
-default:
-  before_script:
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-
-build amd64:
-  stage: build
-  tags:
-  - amd64
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  after_script:
-  - |
-    if [[ "$CI_COMMIT_BRANCH" == "master" && "$CI_JOB_STATUS" == "success" ]]; then
-      apk add --update curl jq
-      rm -rf /var/cache/apk/*
-
-      jq -n '
-         {
-           password:  env.BEEPER_DEV_ADMIN_NIGHTLY_PASS,
-           bridge:    env.BEEPER_BRIDGE_TYPE,
-           image:     "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
-           channel:   "STABLE"
-         }
-        ' | curl "$BEEPER_DEV_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
-
-      jq -n '
-         {
-           password:    env.BEEPER_PROD_ADMIN_NIGHTLY_PASS,
-           bridge:      env.BEEPER_BRIDGE_TYPE,
-           image:       "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
-           channel:     "INTERNAL",
-           deployNext:  true
-         }
-         ' | curl "$BEEPER_PROD_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
-    fi
-
-build arm64:
-  stage: build
-  tags:
-  - arm64
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-
-manifest:
-  stage: manifest
-  before_script:
-  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
-  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+include:
+- project: 'mautrix/ci'
+  file: '/python.yml'
@@ -1,6 +1,6 @@
 repos:
  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.4.0
    hooks:
      - id: trailing-whitespace
        exclude_types: [markdown]
@@ -8,13 +8,13 @@ repos:
      - id: check-yaml
      - id: check-added-large-files
  - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.1.0
    hooks:
      - id: black
        language_version: python3
        files: ^mautrix_telegram/.*\.pyi?$
  - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.12.0
    hooks:
      - id: isort
        files: ^mautrix_telegram/.*\.pyi?$
@@ -1,3 +1,147 @@
+# v0.13.0 (2023-02-26)
+
+### Added
+* Added `allow_contact_info` config option to specify whether personal names
+  and avatars for other users should be bridged.
+  * The option is only safe to enable on single-user instances, using it
+    anywhere else will cause ghost user profiles to flip back and forth between
+    personal and default ones.
+* Added config option to notify Matrix room if bridging an incoming message
+  fails.
+
+### Improved
+* Updated Docker image to Alpine 3.17.
+* Updated to Telegram API layer 152.
+* Improved handling users getting logged out.
+* Removed support for creating accounts, as Telegram only allows requesting SMS
+  login codes on the official mobile clients now.
+* Replaced moviepy with calling ffmpeg directly for generating video thumbnails.
+
+### Fixed
+* Fixed handling Telegram chat upgrades when backfilling is enabled.
+* Fixed file transfers failing if transfering the thumbnail fails.
+* Fixed bridging unnamed files with unrecognized mime types.
+* Fixed enqueueing more backfill.
+* Fixed timestamps not being saved in `telegram_file` table.
+* Fixed issues with old events being replayed if the bridge was shut down
+  uncleanly.
+
+# v0.12.2 (2022-11-26)
+
+### Added
+* Added built-in custom emoji packs to allow reacting with any standard unicode
+  emoji from Matrix (note that only premium users can use custom emojis).
+* Added infinite backfill using [MSC2716].
+  * The new system includes a backwards compatibility mechanism which uses the
+    old method of just sending events to the room. By default, MSC2716 is not
+    enabled and the legacy method will be used.
+
+### Improved
+* Redacting reactions on Matrix no longer removes the user's other reactions to
+  the same message (premium users can have up to 3 reactions per message).
+* Changes to default user permissions on Telegram are now bridged.
+* Added database index to make reaction polling more efficient
+  (thanks to [@AndrewFerr] in [#862]).
+
+### Fixed
+* Fixed provisioning API not working with URL-encoded parameters.
+
+[MSC2716]: https://github.com/matrix-org/matrix-spec-proposals/pull/2716
+[@AndrewFerr]: https://github.com/AndrewFerr
+[#862]: https://github.com/mautrix/telegram/pull/862
+
+# v0.12.1 (2022-09-26)
+
+### Added
+* Support for custom emojis in reactions.
+  * Like other bridges with custom emoji reactions, they're bridged as `mxc://`
+    URIs, so client support is required to render them properly.
+
+### Improved
+* The bridge will now poll for reactions to 20 most recent messages when
+  receiving a read receipt. This works around Telegram's bad protocol that
+  doesn't notify clients on reactions to other users' messages.
+* The docker image now has an option to bypass the startup script by setting
+  the `MAUTRIX_DIRECT_STARTUP` environment variable. Additionally, it will
+  refuse to run as a non-root user if that variable is not set (and print an
+  error message suggesting to either set the variable or use a custom command).
+* Moved environment variable overrides for config fields to mautrix-python.
+  The new system also allows loading JSON values to enable overriding maps like
+  `login_shared_secret_map`.
+
+### Fixed
+* `ChatParticipantsForbidden` is handled properly when syncing non-supergroup
+  info.
+* Fixed some bugs with file transfers when using SQLite.
+* Fixed error when attempting to log in again after logging out.
+* Fixed QR login not working.
+* Fixed error syncing chats if bridging a message had previously been
+  interrupted.
+
+# v0.12.0 (2022-08-26)
+
+**N.B.** This release requires a homeserver with Matrix v1.1 support, which
+bumps up the minimum homeserver versions to Synapse 1.54 and Dendrite 0.8.7.
+Minimum Conduit version remains at 0.4.0.
+
+### Added
+* Added provisioning API for resolving Telegram identifiers (like usernames).
+* Added support for bridging Telegram custom emojis to Matrix.
+* Added option to not bridge chats with lots of members.
+* Added option to include captions in the same message as the media to
+  implement [MSC2530]. Sending captions the same way is also supported and
+  enabled by default.
+* Added commands to kick or ban relaybot users from Telegram.
+* Added support for Telegram's disappearing messages.
+* Added support for bridging forwarded messages as forwards on Telegram.
+  * Forwarding is not allowed in relay mode as the bot wouldn't be able to
+    specify who sent the message.
+  * Matrix doesn't have real forwarding (there's no forwarding metadata), so
+    only messages bridged from Telegram can be forwarded.
+  * Double puppeted messages from Telegram currently can't be forwarded without
+    removing the `fi.mau.double_puppet_source` key from the content.
+  * If forwarding fails (e.g. due to it being blocked in the source chat), the
+    bridge will automatically fall back to sending it as a normal new message.
+* Added options to make encryption more secure.
+  * The `encryption` -> `verification_levels` config options can be used to
+    make the bridge require encrypted messages to come from cross-signed
+    devices, with trust-on-first-use validation of the cross-signing master
+    key.
+  * The `encryption` -> `require` option can be used to make the bridge ignore
+    any unencrypted messages.
+  * Key rotation settings can be configured with the `encryption` -> `rotation`
+    config.
+
+### Improved
+* Improved handling the bridge user leaving chats on Telegram, and new users
+  being added on Telegram.
+* Improved animated sticker conversion options: added support for animated webp
+  and added option to convert video stickers (webm) to the specified image
+  format.
+* Audio and video metadata is now bridged properly to Telegram.
+* Added database index on Telegram usernames (used when bridging username
+  @-mentions in messages).
+* Changed `/login/send_code` provisioning API to return a proper error when the
+  phone number is not registered on Telegram.
+  * The same login code can be used for registering an account, but registering
+    is not currently supported in the provisioning API.
+* Removed `plaintext_highlights` config option (the code using it was already
+  removed in v0.11.0).
+* Enabled appservice ephemeral events by default for new installations.
+  * Existing bridges can turn it on by enabling `ephemeral_events` and disabling
+    `sync_with_custom_puppets` in the config, then regenerating the registration
+    file.
+* Updated to API layer 144 so that Telegram would send new message types like
+  premium stickers to the bridge.
+* Updated Docker image to Alpine 3.16 and made it smaller.
+
+### Fixed
+* Fixed command prefix in game and poll messages (thanks to [@cynhr] in [#804]).
+
+[MSC2530]: https://github.com/matrix-org/matrix-spec-proposals/pull/2530
+[@cynhr]: https://github.com/cynhr
+[#804]: https://github.com/mautrix/telegram/pull/804
+
 # v0.11.3 (2022-04-17)

 **N.B.** This release drops support for old homeservers which don't support the
@@ -1,31 +1,25 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.15
-
-ARG TARGETARCH=amd64
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.17

 RUN apk add --no-cache \
      python3 py3-pip py3-setuptools py3-wheel \
-      py3-virtualenv \
      py3-pillow \
      py3-aiohttp \
      py3-magic \
      py3-ruamel.yaml \
      py3-commonmark \
-      py3-prometheus-client \
+      py3-phonenumbers \
+      py3-mako \
+      #py3-prometheus-client \ (pulls in twisted unnecessarily)
      # Indirect dependencies
      py3-idna \
-      #moviepy
-        py3-decorator \
-        py3-tqdm \
-        py3-requests \
-        #imageio
-          py3-numpy \
+      py3-rsa \
      #py3-telethon \ (outdated)
        # Optional for socks proxies
        py3-pysocks \
        py3-pyaes \
        # cryptg
          py3-cffi \
-	  py3-qrcode \
+          py3-qrcode \
      py3-brotli \
      # Other dependencies
      ffmpeg \
@@ -46,13 +40,15 @@ COPY requirements.txt /opt/mautrix-telegram/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
 WORKDIR /opt/mautrix-telegram
 RUN apk add --virtual .build-deps python3-dev libffi-dev build-base \
- && pip3 install -r requirements.txt -r optional-requirements.txt \
- && apk del .build-deps
+ && pip3 install /cryptg-*.whl \
+ && pip3 install --no-cache-dir -r requirements.txt -r optional-requirements.txt \
+ && apk del .build-deps \
+ && rm -f /cryptg-*.whl

 COPY . /opt/mautrix-telegram
-RUN apk add git && pip3 install .[all] && apk del git \
+RUN apk add git && pip3 install --no-cache-dir .[all] && apk del git \
  # This doesn't make the image smaller, but it's needed so that the `version` command works properly
-  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
+  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram .git build

 VOLUME /data
 ENV UID=1337 GID=1337 \
@@ -11,13 +11,13 @@ A Matrix-Telegram hybrid puppeting/relaybot bridge.
 ## Sponsors
 * [Joel Lehtonen / Zouppen](https://github.com/zouppen)

-### Documentation
+## Documentation
 All setup and usage instructions are located on
 [docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
 Some quick links:

-* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
-  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
+* [Bridge setup](https://docs.mau.fi/bridges/python/setup.html?bridge=telegram)
+  (or [with Docker](https://docs.mau.fi/bridges/general/docker-setup.html?bridge=telegram))
 * Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
  [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
  [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)
@@ -24,6 +24,7 @@
 * Telegram → Matrix
  * [x] Message content (text, formatting, files, etc..)
  * [ ] Advanced message content/media
+    * [x] Custom emojis
    * [x] Polls
 	* [x] Games
 	* [ ] Buttons
@@ -54,7 +55,7 @@
  * [x] Automatic portal creation
    * [x] At startup
    * [x] When receiving invite or message
-  * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
+  * [x] Portal creation by inviting Matrix puppet of Telegram user to new room
  * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
  * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
  * [ ] ‡ Calls (hard, not yet supported by Telethon)
@@ -1,3 +1,3 @@
 pre-commit>=2.10.1,<3
 isort>=5.10.1,<6
-black>=22.3,<23
+black>=23,<24
@@ -1,4 +1,18 @@
 #!/bin/sh
+if [ ! -z "$MAUTRIX_DIRECT_STARTUP" ]; then
+	if [ $(id -u) == 0 ]; then
+		echo "|------------------------------------------|"
+		echo "| Warning: running bridge unsafely as root |"
+		echo "|------------------------------------------|"
+	fi
+	exec python3 -m mautrix_telegram -c /data/config.yaml
+elif [ $(id -u) != 0 ]; then
+	echo "The startup script must run as root. It will use su-exec to drop permissions before running the bridge."
+	echo "To bypass the startup script, either set the `MAUTRIX_DIRECT_STARTUP` environment variable,"
+	echo "or just use `python3 -m mautrix_telegram -c /data/config.yaml` as the run command."
+	echo "Note that the config and registration will not be auto-generated when bypassing the startup script."
+	exit 1
+fi

 # Define functions.
 function fixperms {
@@ -1,2 +1,2 @@
-__version__ = "0.11.3"
+__version__ = "0.13.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"
@@ -103,7 +103,9 @@ class TelegramBridge(Bridge):
    def prepare_stop(self) -> None:
        for puppet in Puppet.by_custom_mxid.values():
            puppet.stop()
-        self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
+        self.add_shutdown_actions(user.stop() for user in User.by_tgid.values())
+        if self.bot:
+            self.add_shutdown_actions(self.bot.stop())

    async def get_user(self, user_id: UserID, create: bool = True) -> User | None:
        user = await User.get_by_mxid(user_id, create=create)
@@ -22,6 +22,7 @@ import logging
 import platform
 import time

+from telethon.errors import AuthKeyError, UnauthorizedError
 from telethon.network import (
    Connection,
    ConnectionTcpFull,
@@ -38,7 +39,9 @@ from telethon.tl.types import (
    PeerChat,
    PeerUser,
    TypeUpdate,
+    UpdateChannel,
    UpdateChannelUserTyping,
+    UpdateChatDefaultBannedRights,
    UpdateChatParticipantAdmin,
    UpdateChatParticipants,
    UpdateChatUserTyping,
@@ -57,10 +60,11 @@ from telethon.tl.types import (
    UpdateReadChannelInbox,
    UpdateReadHistoryInbox,
    UpdateReadHistoryOutbox,
+    UpdateShort,
    UpdateShortChatMessage,
    UpdateShortMessage,
+    UpdateUser,
    UpdateUserName,
-    UpdateUserPhoto,
    UpdateUserStatus,
    UpdateUserTyping,
    User,
@@ -71,6 +75,7 @@ from telethon.tl.types import (
 from mautrix.appservice import AppService
 from mautrix.errors import MatrixError
 from mautrix.types import PresenceState, UserID
+from mautrix.util import background_task
 from mautrix.util.logging import TraceLogger
 from mautrix.util.opt_prometheus import Counter, Histogram

@@ -223,11 +228,32 @@ class AbstractUser(ABC):
            connection=connection,
            proxy=proxy,
            raise_last_call_error=True,
+            catch_up=self.config["telegram.catch_up"],
+            sequential_updates=self.config["telegram.sequential_updates"],
            loop=self.loop,
            base_logger=base_logger,
+            update_error_callback=self._telethon_update_error_callback,
        )
        self.client.add_event_handler(self._update_catch)

+    @abstractmethod
+    async def on_signed_out(self, err: UnauthorizedError | AuthKeyError) -> None:
+        pass
+
+    async def _telethon_update_error_callback(self, err: Exception) -> None:
+        if isinstance(err, (UnauthorizedError, AuthKeyError)):
+            background_task.create(self.on_signed_out(err))
+            return
+        if self.config["telegram.exit_on_update_error"]:
+            self.log.critical(f"Stopping due to update handling error {type(err).__name__}")
+            self.bridge.manual_stop(50)
+        else:
+            self.log.info("Recreating Telethon connection in 60 seconds")
+            await asyncio.sleep(60)
+            self.log.debug("Now recreating Telethon connection")
+            await self.stop()
+            await self.start()
+
    @abstractmethod
    async def update(self, update: TypeUpdate) -> bool:
        return False
@@ -282,22 +308,25 @@ class AbstractUser(ABC):
    async def ensure_started(self, even_if_no_session=False) -> AbstractUser:
        if self.connected:
            return self
-        if even_if_no_session or await PgSession.has(self.mxid):
+        session_exists = await PgSession.has(self.mxid)
+        if even_if_no_session or session_exists:
            self.log.debug(
-                "Starting client due to ensure_started"
-                f"(even_if_no_session={even_if_no_session})"
+                f"Starting client due to ensure_started({even_if_no_session=}, {session_exists=})"
            )
            await self.start(delete_unless_authenticated=not even_if_no_session)
        return self

    async def stop(self) -> None:
-        await self.client.disconnect()
-        self.client = None
+        if self.client:
+            await self.client.disconnect()
+            self.client = None

    # region Telegram update handling

    async def _update(self, update: TypeUpdate) -> None:
-        asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
+        if isinstance(update, UpdateShort):
+            update = update.update
+        background_task.create(self._handle_entity_updates(getattr(update, "_entities", {})))
        if isinstance(
            update,
            (
@@ -324,9 +353,11 @@ class AbstractUser(ABC):
            await self.update_admin(update)
        elif isinstance(update, UpdateChatParticipants):
            await self.update_participants(update)
+        elif isinstance(update, UpdateChatDefaultBannedRights):
+            await self.update_default_banned_rights(update)
        elif isinstance(update, (UpdatePinnedMessages, UpdatePinnedChannelMessages)):
            await self.update_pinned_messages(update)
-        elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
+        elif isinstance(update, (UpdateUserName, UpdateUser)):
            await self.update_others_info(update)
        elif isinstance(update, UpdateReadHistoryOutbox):
            await self.update_read_receipt(update)
@@ -338,6 +369,8 @@ class AbstractUser(ABC):
            await self.update_pinned_dialogs(update)
        elif isinstance(update, UpdateNotifySettings):
            await self.update_notify_settings(update)
+        elif isinstance(update, UpdateChannel):
+            await self.update_channel(update)
        else:
            self.log.trace("Unhandled update: %s", update)

@@ -368,6 +401,12 @@ class AbstractUser(ABC):
        if portal and portal.mxid:
            await portal.update_power_levels(update.participants.participants)

+    @staticmethod
+    async def update_default_banned_rights(update: UpdateChatDefaultBannedRights) -> None:
+        portal = await po.Portal.get_by_entity(update.peer)
+        if portal and portal.mxid:
+            await portal.update_default_banned_rights(update.default_banned_rights)
+
    async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
        if not isinstance(update.peer, PeerUser):
            self.log.debug("Unexpected read receipt peer: %s", update.peer)
@@ -396,6 +435,7 @@ class AbstractUser(ABC):
        if not puppet.is_real_user:
            return

+        self.log.debug("Handling own read receipt: %s", update)
        if isinstance(update, UpdateReadChannelInbox):
            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
        elif isinstance(update.peer, PeerChat):
@@ -409,6 +449,7 @@ class AbstractUser(ABC):
            return

        if not portal or not portal.mxid:
+            self.log.debug(f"Dropping own read receipt in unknown chat ({update.peer})")
            return

        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
@@ -416,6 +457,9 @@ class AbstractUser(ABC):
            TelegramID(update.max_id), tg_space, edit_index=-1
        )
        if not message:
+            self.log.debug(
+                f"Dropping own read receipt: unknown message {update.max_id}@{tg_space}"
+            )
            return

        await puppet.intent.mark_read(portal.mxid, message.mxid)
@@ -462,18 +506,23 @@ class AbstractUser(ABC):
        except Exception:
            self.log.exception("Failed to handle entity updates")

-    async def update_others_info(self, update: UpdateUserName | UpdateUserPhoto) -> None:
+    async def update_others_info(self, update: UpdateUserName | UpdateUser) -> None:
        # TODO duplication not checked
        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
        if isinstance(update, UpdateUserName):
-            puppet.username = update.username
+            if len(update.usernames) > 1:
+                self.log.warning(
+                    "Got update with multiple usernames (%s) for %s, only saving first one",
+                    update.usernames,
+                    update.user_id,
+                )
+            puppet.username = update.usernames[0].username if update.usernames else None
            if await puppet.update_displayname(self, update):
                await puppet.save()
                await puppet.update_portals_meta()
-        elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo):
-                await puppet.save()
-                await puppet.update_portals_meta()
+        elif isinstance(update, UpdateUser):
+            info = await self.client.get_entity(puppet.peer)
+            await puppet.update_info(self, info)
        else:
            self.log.warning(f"Unexpected other user info update: {type(update)}")

@@ -491,9 +540,7 @@ class AbstractUser(ABC):
        self, update: UpdateMessage
    ) -> tuple[UpdateMessageContent, pu.Puppet | None, po.Portal | None]:
        if isinstance(update, UpdateShortChatMessage):
-            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
-            if not portal:
-                self.log.warning(f"Received message in chat with unknown type {update.chat_id}")
+            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
            sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id))
        elif isinstance(update, UpdateShortMessage):
            portal = await po.Portal.get_by_tgid(
@@ -517,6 +564,8 @@ class AbstractUser(ABC):
                sender = await pu.Puppet.get_by_tgid(self.tgid)
            elif isinstance(update.from_id, (PeerUser, PeerChannel)):
                sender = await pu.Puppet.get_by_peer(update.from_id)
+            elif isinstance(update.peer_id, PeerUser):
+                sender = await pu.Puppet.get_by_peer(update.peer_id)
            else:
                sender = None
        else:
@@ -568,6 +617,43 @@ class AbstractUser(ABC):
            return
        await portal.handle_telegram_reactions(self, TelegramID(update.msg_id), update.reactions)

+    async def update_channel(self, update: UpdateChannel) -> None:
+        portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        if not portal:
+            return
+        if getattr(update, "mau_telethon_is_leave", False):
+            self.log.debug("UpdateChannel has mau_telethon_is_leave, leaving portal")
+            await portal.delete_telegram_user(self.tgid, sender=None)
+        elif chan := getattr(update, "mau_channel", None):
+            if not portal.mxid:
+                background_task.create(self._delayed_create_channel(chan))
+            else:
+                self.log.debug("Updating channel info with data fetched by Telethon")
+                await portal.update_info(self, chan)
+                await portal.invite_to_matrix(self.mxid)
+
+    async def _delayed_create_channel(self, chan: Channel) -> None:
+        self.log.debug(
+            f"Waiting 5 seconds before handling UpdateChannel for non-existent portal {chan.id}"
+        )
+        await asyncio.sleep(5)
+        portal = await po.Portal.get_by_tgid(TelegramID(chan.id))
+        if portal.mxid:
+            self.log.debug(
+                "Portal started existing after waiting 5 seconds, "
+                f"dropping UpdateChannel for {portal.tgid}"
+            )
+            return
+        else:
+            self.log.info(
+                f"Creating Matrix room for {portal.tgid}"
+                " with data fetched by Telethon due to UpdateChannel"
+            )
+            await portal.create_matrix_room(self, chan, invites=[self.mxid])
+
+    async def _check_server_notice_edit(self, message: Message) -> None:
+        pass
+
    async def update_message(self, original_update: UpdateMessage) -> None:
        update, sender, portal = await self.get_message_details(original_update)
        if not portal:
@@ -601,7 +687,7 @@ class AbstractUser(ABC):

        if isinstance(update, MessageService):
            if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.trace(
+                self.log.debug(
                    "Received %s in %s by %d, unregistering portal...",
                    update.action,
                    portal.tgid_log,
@@ -610,7 +696,7 @@ class AbstractUser(ABC):
                await self.unregister_portal(update.action.chat_id, update.action.chat_id)
                await self.register_portal(portal)
                return
-            self.log.trace(
+            self.log.debug(
                "Handling action %s to %s by %d",
                update.action,
                portal.tgid_log,
@@ -619,6 +705,8 @@ class AbstractUser(ABC):
            return await portal.handle_telegram_action(self, sender, update)

        if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
+            if sender and sender.tgid == 777000:
+                await self._check_server_notice_edit(update)
            return await portal.handle_telegram_edit(self, sender, update)
        return await portal.handle_telegram_message(self, sender, update)

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,10 +13,18 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Tuple
-import logging
+from __future__ import annotations

-from telethon.errors import ChannelInvalidError, ChannelPrivateError
+from typing import TYPE_CHECKING, Awaitable, Callable, Literal
+import logging
+import time
+
+from telethon.errors import (
+    AuthKeyError,
+    ChannelInvalidError,
+    ChannelPrivateError,
+    UnauthorizedError,
+)
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.patched import Message, MessageService
@@ -26,6 +34,7 @@ from telethon.tl.types import (
    ChatForbidden,
    ChatParticipantAdmin,
    ChatParticipantCreator,
+    ChatParticipantsForbidden,
    InputChannel,
    InputUser,
    MessageActionChatAddUser,
@@ -35,31 +44,64 @@ from telethon.tl.types import (
    PeerChannel,
    PeerChat,
    PeerUser,
+    TypeChannelParticipant,
+    TypeChatParticipant,
+    TypeInputPeer,
    TypePeer,
    UpdateNewChannelMessage,
    UpdateNewMessage,
    User,
 )
+from telethon.utils import add_surrogate, del_surrogate

-from mautrix.types import UserID
+from mautrix.errors import MBadState, MForbidden
+from mautrix.types import RoomID, UserID

 from . import portal as po, puppet as pu, user as u
 from .abstract_user import AbstractUser
-from .db import BotChat
+from .db import BotChat, Message as DBMessage
 from .types import TelegramID

+if TYPE_CHECKING:
+    from asyncio import Future
+
 ReplyFunc = Callable[[str], Awaitable[Message]]
+BanFunc = Callable[[RoomID, UserID, str], Awaitable[None]]
+TelegramAdminPermission = Literal[
+    "change_info",
+    "post_messages",
+    "edit_messages",
+    "delete_messages",
+    "ban_users",
+    "invite_users",
+    "pin_messages",
+    "add_admins",
+    "anonymous",
+    "manage_call",
+    "other",
+]


 class Bot(AbstractUser):
    log: logging.Logger = logging.getLogger("mau.user.bot")

    token: str
-    chats: Dict[int, str]
-    tg_whitelist: List[int]
+    chats: dict[int, str]
+    tg_whitelist: list[int]
    whitelist_group_admins: bool
-    _me_info: Optional[User]
-    _me_mxid: Optional[UserID]
+    _me_info: User | None
+    _me_mxid: UserID | None
+    _admin_cache: dict[
+        tuple[int, int],
+        tuple[ChatParticipantAdmin | ChatParticipantCreator | None, float],
+    ]
+    _login_wait_fut: Future | None
+    required_permissions: dict[str, TelegramAdminPermission] = {
+        "portal": None,
+        "invite": "invite_users",
+        "mxban": "ban_users",
+        "mxkick": "ban_users",
+    }

    def __init__(self, token: str) -> None:
        super().__init__()
@@ -73,14 +115,16 @@ class Bot(AbstractUser):
        self.is_relaybot = True
        self.is_bot = True
        self.chats = {}
+        self._admin_cache = {}
        self.tg_whitelist = []
        self.whitelist_group_admins = (
            self.config["bridge.relaybot.whitelist_group_admins"] or False
        )
        self._me_info = None
        self._me_mxid = None
+        self._login_wait_fut = self.loop.create_future()

-    async def get_me(self, use_cache: bool = True) -> Tuple[User, UserID]:
+    async def get_me(self, use_cache: bool = True) -> tuple[User, UserID]:
        if not use_cache or not self._me_mxid:
            self._me_info = await self.client.get_me()
            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
@@ -98,7 +142,7 @@ class Bot(AbstractUser):
            if isinstance(user_id, int):
                self.tg_whitelist.append(user_id)

-    async def start(self, delete_unless_authenticated: bool = False) -> "Bot":
+    async def start(self, delete_unless_authenticated: bool = False) -> Bot:
        self.chats = {chat.id: chat.type for chat in await BotChat.all()}
        await super().start(delete_unless_authenticated)
        if not await self.is_logged_in():
@@ -106,12 +150,19 @@ class Bot(AbstractUser):
        await self.post_login()
        return self

+    async def on_signed_out(self, err: UnauthorizedError | AuthKeyError) -> None:
+        self.log.fatal("Relay bot got signed out, crashing bridge", exc_info=err)
+        self.bridge.manual_stop(51)
+
    async def post_login(self) -> None:
        await self.init_permissions()
        info = await self.client.get_me()
        self.tgid = TelegramID(info.id)
        self.tg_username = info.username
        self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
+        if self._login_wait_fut:
+            self._login_wait_fut.set_result(None)
+            self._login_wait_fut = None

        chat_ids = [chat_id for chat_id, chat_type in self.chats.items() if chat_type == "chat"]
        response = await self.client(GetChatsRequest(chat_ids))
@@ -148,7 +199,46 @@ class Bot(AbstractUser):
            pass
        await BotChat.delete_by_id(chat_id)

-    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
+    async def _get_admin_participant(
+        self, chat: TypePeer | TypeInputPeer, tgid: TelegramID
+    ) -> TypeChatParticipant | TypeChannelParticipant | None:
+        chan_id = chat.channel_id if isinstance(chat, PeerChannel) else chat.chat_id
+        try:
+            cached, created = self._admin_cache[chan_id, tgid]
+            if created + 60 < time.time():
+                return cached
+        except KeyError:
+            pass
+        if isinstance(chat, PeerChannel):
+            p = await self.client(GetParticipantRequest(chat, tgid))
+            pcp = p.participant
+            self._admin_cache[chat.channel_id, tgid] = (pcp, time.time())
+            return pcp
+        elif isinstance(chat, PeerChat):
+            chat = await self.client(GetFullChatRequest(chat.chat_id))
+            if isinstance(chat.full_chat.participants, ChatParticipantsForbidden):
+                return None
+            participants = chat.full_chat.participants.participants
+            for p in participants:
+                self._admin_cache[chat.channel_id, tgid] = (p, time.time())
+                if p.user_id == tgid:
+                    return p
+        return None
+
+    @staticmethod
+    def _has_participant_permission(
+        pcp: TypeChatParticipant | TypeChannelParticipant | None,
+        permission: TelegramAdminPermission | None,
+    ) -> bool:
+        if isinstance(pcp, (ChannelParticipantCreator, ChannelParticipantAdmin)):
+            return permission is None or getattr(pcp.admin_rights, permission, False)
+        elif isinstance(pcp, (ChatParticipantCreator, ChatParticipantAdmin)):
+            return True
+        return False
+
+    async def _can_use_commands(
+        self, chat: TypePeer, tgid: TelegramID, permission: TelegramAdminPermission | None = None
+    ) -> bool:
        if tgid in self.tg_whitelist:
            return True

@@ -158,22 +248,20 @@ class Bot(AbstractUser):
            return True

        if self.whitelist_group_admins:
-            if isinstance(chat, PeerChannel):
-                p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(
-                    p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin)
-                )
-            elif isinstance(chat, PeerChat):
-                chat = await self.client(GetFullChatRequest(chat.chat_id))
-                participants = chat.full_chat.participants.participants
-                for p in participants:
-                    if p.user_id == tgid:
-                        return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+            pcp = await self._get_admin_participant(chat, tgid)
+            return self._has_participant_permission(pcp, permission)
        return False

-    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
-        # FIXME event.from_id is not int
-        if not await self._can_use_commands(event.to_id, TelegramID(event.from_id)):
+    async def check_can_use_command(self, event: Message, reply: ReplyFunc, command: str) -> bool:
+        if command not in self.required_permissions:
+            # Unknown command
+            return False
+        elif not isinstance(event.from_id, PeerUser):
+            await reply("Channels can't use commands")
+            return False
+        elif not await self._can_use_commands(
+            event.to_id, TelegramID(event.from_id.user_id), self.required_permissions[command]
+        ):
            await reply("You do not have the permission to use that command.")
            return False
        return True
@@ -193,6 +281,8 @@ class Bot(AbstractUser):
                )
            else:
                return await reply("Portal is not public. Use `/invite <mxid>` to get an invite.")
+        else:
+            return await reply("Couldn't create portal room")

    async def handle_command_invite(
        self, portal: po.Portal, reply: ReplyFunc, mxid_input: UserID
@@ -213,9 +303,59 @@ class Bot(AbstractUser):
                f"Just invite [{displayname}](tg://user?id={user.tgid})"
            )
        else:
-            await portal.invite_to_matrix(user.mxid)
+            try:
+                await portal.invite_to_matrix(user.mxid)
+            except MBadState:
+                try:
+                    await portal.main_intent.unban_user(
+                        portal.mxid, user.mxid, reason="Invited from Telegram"
+                    )
+                except Exception:
+                    return await reply(f"Failed to unban `{user.mxid}` from the portal.")
+                await portal.invite_to_matrix(user.mxid)
+                return await reply(f"Unbanned and invited `{user.mxid}` to the portal.")
            return await reply(f"Invited `{user.mxid}` to the portal.")

+    async def handle_command_ban(
+        self,
+        message: Message,
+        portal: po.Portal,
+        reply: ReplyFunc,
+        reason: str,
+        action: Literal["kick", "ban"] = "ban",
+    ) -> Message:
+        if not message.reply_to:
+            return await reply("You must reply to a relaybot message when using that command")
+        reply_to_id = TelegramID(message.reply_to.reply_to_msg_id)
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        msg = await DBMessage.get_one_by_tgid(reply_to_id, tg_space)
+        if not msg or msg.sender != self.tgid or not msg.sender_mxid:
+            return await reply("Target message is not a relayed message")
+        puppet = await pu.Puppet.get_by_peer(message.from_id)
+        actioned = "Banned" if action == "ban" else "Kicked"
+        try:
+            intent = puppet.intent_for(portal)
+            func: BanFunc = intent.ban_user if action == "ban" else intent.kick_user
+            await func(portal.mxid, msg.sender_mxid, reason)
+        except MForbidden as e:
+            self.log.warning(
+                f"Failed to {action} {msg.sender_mxid} from {portal.mxid} as {puppet.mxid}: {e}, "
+                f"falling back to bridge bot"
+            )
+            reason_prefix = f"{actioned} by {puppet.displayname or puppet.tgid}"
+            reason = f"{reason_prefix}: {reason}" if reason else reason_prefix
+            try:
+                func: BanFunc = (
+                    self.az.intent.ban_user if action == "ban" else self.az.intent.kick_user
+                )
+                await func(portal.mxid, msg.sender_mxid, reason)
+            except MForbidden as e:
+                self.log.warning(
+                    f"Failed to {action} {msg.sender_mxid} from {portal.mxid} as bridge bot: {e}"
+                )
+                return await reply(f"Failed to {action} `{msg.sender_mxid}`")
+        return await reply(f"Successfully {actioned.lower()} `{msg.sender_mxid}`")
+
    @staticmethod
    def handle_command_id(message: Message, reply: ReplyFunc) -> Awaitable[Message]:
        # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
@@ -233,53 +373,46 @@ class Bot(AbstractUser):
        else:
            return reply("Failed to find chat ID.")

-    def match_command(self, text: str, command: str) -> bool:
-        text = text.lower()
-        command = f"/{command.lower()}"
-        command_targeted = f"{command}@{self.tg_username.lower()}"
+    def parse_command(self, message: Message) -> tuple[str | None, str | None]:
+        if not message.entities or len(message.entities) < 1 or not message.message:
+            return None, None
+        cmd_entity = message.entities[0]
+        if not isinstance(cmd_entity, MessageEntityBotCommand) or cmd_entity.offset != 0:
+            return None, None
+        surrogated_text = add_surrogate(message.message)
+        command: str = del_surrogate(surrogated_text[: cmd_entity.length]).lower()
+        rest_of_message: str = ""
+        if len(surrogated_text) > cmd_entity.length + 1:
+            rest_of_message: str = del_surrogate(surrogated_text[cmd_entity.length + 1 :])
+        command, *target = command.split("@", 1)
+        if not command.startswith("/"):
+            return None, None
+        elif target and target[0] != self.tg_username.lower():
+            return None, None
+        return command[1:], rest_of_message

-        is_plain_command = text == command or text == command_targeted
-        if is_plain_command:
-            return True
-
-        is_arg_command = text.startswith(command + " ") or text.startswith(command_targeted + " ")
-        if is_arg_command:
-            return True
-
-        return False
-
-    async def handle_command(self, message: Message) -> None:
+    async def handle_command(self, message: Message, command: str, args: str) -> None:
        def reply(reply_text: str) -> Awaitable[Message]:
            return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)

-        text = message.message
-
-        if self.match_command(text, "start"):
+        if command == "start":
            pcm = self.config["bridge.relaybot.private_chat.message"]
            if pcm:
                await reply(pcm)
-            return
-        elif self.match_command(text, "id"):
+        elif command == "id":
            await self.handle_command_id(message, reply)
-            return
-        elif message.is_private:
-            return
-
-        portal = await po.Portal.get_by_entity(message.to_id)
-
-        is_portal_cmd = self.match_command(text, "portal")
-        is_invite_cmd = self.match_command(text, "invite")
-        if is_portal_cmd or is_invite_cmd:
-            if not await self.check_can_use_commands(message, reply):
+        elif not message.is_private:
+            if not await self.check_can_use_command(message, reply, command):
                return
-            if is_portal_cmd:
+            portal = await po.Portal.get_by_entity(message.to_id)
+            if command == "portal":
                await self.handle_command_portal(portal, reply)
-            elif is_invite_cmd:
-                try:
-                    mxid = text[text.index(" ") + 1 :]
-                except ValueError:
-                    mxid = ""
-                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
+            elif command == "invite":
+                await self.handle_command_invite(portal, reply, mxid_input=UserID(args))
+            elif command == "mxban":
+                await self.handle_command_ban(message, portal, reply, reason=args)
+            elif command == "mxkick":
+                await self.handle_command_ban(message, portal, reply, reason=args, action="kick")

    async def handle_service_message(self, message: MessageService) -> None:
        to_peer = message.to_id
@@ -302,21 +435,18 @@ class Bot(AbstractUser):
            await self.add_chat(TelegramID(action.channel_id), "channel")

    async def update(self, update) -> bool:
+        if self._login_wait_fut:
+            await self._login_wait_fut
        if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
            return False
        if isinstance(update.message, MessageService):
            await self.handle_service_message(update.message)
            return False

-        is_command = (
-            isinstance(update.message, Message)
-            and update.message.entities
-            and len(update.message.entities) > 0
-            and isinstance(update.message.entities[0], MessageEntityBotCommand)
-            and update.message.entities[0].offset == 0
-        )
-        if is_command:
-            await self.handle_command(update.message)
+        if isinstance(update.message, Message):
+            command, args = self.parse_command(update.message)
+            if command:
+                await self.handle_command(update.message, command, args)
        return False

    def is_in_chat(self, peer_id) -> bool:
@@ -21,6 +21,7 @@ import asyncio
 from telethon.tl.types import ChannelForbidden, ChatForbidden

 from mautrix.types import EventID, RoomID
+from mautrix.util import background_task

 from ... import portal as po
 from ...types import TelegramID
@@ -184,7 +185,7 @@ async def confirm_bridge(evt: CommandEvent) -> EventID | None:
        if not ok:
            return None
        elif coro:
-            asyncio.create_task(coro)
+            background_task.create(coro)
            await evt.reply("Cleaning up previous portal room...")
    elif portal.mxid:
        evt.sender.command_status = None
@@ -251,7 +252,7 @@ async def _locked_confirm_bridge(
    await portal.save()
    await portal.update_bridge_info()

-    asyncio.create_task(portal.update_matrix_room(user, entity, levels=levels))
+    background_task.create(portal.update_matrix_room(user, entity, levels=levels))

    await warn_missing_power(levels, evt)

@@ -29,6 +29,7 @@ from .. import SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler

@command_handler(
    needs_auth=False,
+    needs_puppeting=False,
    help_section=SECTION_PORTAL_MANAGEMENT,
    help_text="View or change per-portal settings.",
    help_args="<`help`|_subcommand_> [...]",
@@ -98,7 +99,7 @@ def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
                "exceptions": evt.config["bridge.bridge_notices.exceptions"],
            },
            "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
-            "inline_images": evt.config["bridge.inline_images"],
+            "caption_in_message": evt.config["bridge.caption_in_message"],
            "message_formats": evt.config["bridge.message_formats"],
            "emote_format": evt.config["bridge.emote_format"],
            "state_event_formats": evt.config["bridge.state_event_formats"],
@@ -81,4 +81,3 @@ async def create(evt: CommandEvent) -> EventID:
    except ValueError as e:
        await portal.delete()
        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
@@ -68,5 +68,5 @@ async def user_has_power_level(
        await intent.get_power_levels(room_id)
    except MatrixRequestError:
        return False
-    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
+    event_type = EventType.find(f"fi.mau.telegram.{event}", t_class=EventType.Class.STATE)
    return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)
@@ -22,7 +22,6 @@ import io
 from telethon.errors import (
    AccessTokenExpiredError,
    AccessTokenInvalidError,
-    FirstNameInvalidError,
    FloodWaitError,
    PasswordHashInvalidError,
    PhoneCodeExpiredError,
@@ -31,7 +30,6 @@ from telethon.errors import (
    PhoneNumberBannedError,
    PhoneNumberFloodError,
    PhoneNumberInvalidError,
-    PhoneNumberOccupiedError,
    PhoneNumberUnoccupiedError,
    SessionPasswordNeededError,
 )
@@ -46,6 +44,7 @@ from mautrix.types import (
    TextMessageEventContent,
    UserID,
 )
+from mautrix.util import background_task
 from mautrix.util.format_duration import format_duration as fmt_duration

 from ... import user as u
@@ -93,70 +92,6 @@ async def ping_bot(evt: CommandEvent) -> EventID:
    )


-@command_handler(
-    needs_auth=False,
-    management_only=True,
-    help_section=SECTION_AUTH,
-    help_args="<_phone_> <_full name_>",
-    help_text="Register to Telegram",
-)
-async def register(evt: CommandEvent) -> EventID:
-    if await evt.sender.is_logged_in():
-        return await evt.reply("You are already logged in.")
-    elif len(evt.args) < 1:
-        return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
-
-    phone_number = evt.args[0]
-    if len(evt.args) == 2:
-        full_name = evt.args[1], ""
-    else:
-        full_name = " ".join(evt.args[1:-1]), evt.args[-1]
-
-    await _request_code(
-        evt,
-        phone_number,
-        {
-            "next": enter_code_register,
-            "action": "Register",
-            "full_name": full_name,
-        },
-    )
-    return await evt.reply(
-        "By signing up for Telegram, you agree to "
-        "the terms of service: https://telegram.org/tos"
-    )
-
-
-async def enter_code_register(evt: CommandEvent) -> EventID:
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
-    try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        first_name, last_name = evt.sender.command_status["full_name"]
-        user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
-        asyncio.create_task(evt.sender.post_login(user, first_login=True))
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully registered to Telegram.")
-    except PhoneNumberOccupiedError:
-        return await evt.reply(
-            "That phone number has already been registered. "
-            "You can log in with `$cmdprefix+sp login`."
-        )
-    except FirstNameInvalidError:
-        return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
-    except PhoneCodeExpiredError:
-        return await evt.reply(
-            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`."
-        )
-    except PhoneCodeInvalidError:
-        return await evt.reply("Invalid phone code.")
-    except Exception:
-        evt.log.exception("Error sending phone code")
-        return await evt.reply(
-            "Unhandled exception while sending code. Check console for more details."
-        )
-
-
@command_handler(
    needs_auth=False,
    management_only=True,
@@ -215,6 +150,10 @@ async def login_qr(evt: CommandEvent) -> EventID:
            return await evt.reply(
                "Your account has two-factor authentication. Please send your password here."
            )
+        try:
+            await evt.main_intent.redact(evt.room_id, qr_event_id, reason="QR code scanned")
+        except Exception:
+            pass
    else:
        timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
        timeout.set_edit(qr_event_id)
@@ -312,7 +251,7 @@ async def _request_code(
    except PhoneNumberUnoccupiedError:
        return await evt.reply(
            "That phone number has not been registered. "
-            "Please register with `$cmdprefix+sp register <phone>`."
+            "Please sign up to Telegram using an official mobile client first."
        )
    except PhoneNumberInvalidError:
        return await evt.reply("That phone number is not valid.")
@@ -377,6 +316,7 @@ async def enter_password(evt: CommandEvent) -> EventID | None:
            "This bridge instance does not allow in-Matrix login. "
            "Please use `$cmdprefix+sp login` to get login instructions"
        )
+    await evt.redact()
    try:
        await _sign_in(
            evt,
@@ -426,7 +366,7 @@ async def _finish_sign_in(evt: CommandEvent, user: User, login_as: u.User = None
            f"[{existing_user.displayname}] (https://matrix.to/#/{existing_user.mxid})"
            " was logged out from the account."
        )
-    asyncio.create_task(login_as.post_login(user, first_login=True))
+    background_task.create(login_as.post_login(user, first_login=True))
    evt.sender.command_status = None
    name = f"@{user.username}" if user.username else f"+{user.phone}"
    if login_as != evt.sender:
@@ -18,6 +18,7 @@ from __future__ import annotations
 from typing import cast
 import base64
 import codecs
+import math
 import re

 from aiohttp import ClientSession, InvalidURL
@@ -66,6 +67,7 @@ from ...types import TelegramID

@command_handler(
    needs_auth=False,
+    needs_puppeting=False,
    help_section=SECTION_MISC,
    help_args="<_caption_>",
    help_text="Set a caption for the next image you send",
@@ -132,15 +134,16 @@ async def search(evt: CommandEvent) -> EventID:

@command_handler(
    help_section=SECTION_CREATING_PORTALS,
-    help_args="<_identifier_>",
-    help_text="Open a private chat with the given Telegram user. The identifier is "
-    "either the internal user ID, the username or the phone number. "
-    "**N.B.** The phone numbers you start chats with must already be in "
-    "your contacts.",
+    help_args="<_username_>",
+    help_text=(
+        "Open a private chat with the given Telegram user. You can also use a "
+        "phone number instead of username, but you must have the number in "
+        "your Telegram contacts for that to work."
+    ),
 )
 async def pm(evt: CommandEvent) -> EventID:
    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
+        return await evt.reply("**Usage:** `$cmdprefix+sp pm <username>`")

    try:
        id = "".join(evt.args).translate({ord(c): None for c in "+()- "})
@@ -233,7 +236,10 @@ async def join(evt: CommandEvent) -> EventID | None:
                    updates.stringify(),
                )
                raise e
-            return await evt.reply(f"Created room for {portal.title}")
+            if portal.mxid:
+                return await evt.reply(f"Created room for {portal.title}")
+            else:
+                return await evt.reply(f"Couldn't create room for {portal.title}")
    return None


@@ -423,6 +429,9 @@ async def backfill(evt: CommandEvent) -> None:
    if not evt.is_portal:
        await evt.reply("You can only use backfill in portal rooms")
        return
+    elif not evt.config["bridge.backfill.enable"]:
+        await evt.reply("Backfilling is disabled in the bridge config")
+        return
    try:
        limit = int(evt.args[0])
    except (ValueError, IndexError):
@@ -431,16 +440,14 @@ async def backfill(evt: CommandEvent) -> None:
    if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
        await evt.reply("Backfilling normal groups is disabled in the bridge config")
        return
-    try:
-        await portal.backfill(evt.sender, limit=limit)
-    except TakeoutInitDelayError:
-        msg = (
-            "Please accept the data export request from a mobile device, "
-            "then re-run the backfill command."
-        )
-        if portal.peer_type == "user":
-            from mautrix.appservice import IntentAPI
-
-            await portal.main_intent.send_notice(evt.room_id, msg)
-        else:
-            await evt.reply(msg)
+    if portal.backfill_msc2716:
+        messages_per_batch = evt.config["bridge.backfill.incremental.messages_per_batch"]
+        batches = math.ceil(limit / messages_per_batch)
+        rounded = ""
+        if batches * messages_per_batch != limit:
+            rounded = f" (rounded message limit to {batches}*{messages_per_batch})"
+        await portal.enqueue_backfill(evt.sender, priority=0, max_batches=batches)
+        await evt.reply(f"Backfill queued{rounded}")
+    else:
+        output = await portal.forward_backfill(evt.sender, initial=False, override_limit=limit)
+        await evt.reply(output)
@@ -35,12 +35,6 @@ Permissions = NamedTuple(


 class Config(BaseBridgeConfig):
-    def __getitem__(self, key: str) -> Any:
-        try:
-            return os.environ[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
-        except KeyError:
-            return super().__getitem__(key)
-
    @property
    def forbidden_defaults(self) -> List[ForbiddenDefault]:
        return [
@@ -63,8 +57,6 @@ class Config(BaseBridgeConfig):
        super().do_update(helper)
        copy, copy_dict, base = helper

-        copy("homeserver.asmux")
-
        if "appservice.protocol" in self and "appservice.address" not in self:
            protocol, hostname, port = (
                self["appservice.protocol"],
@@ -109,8 +101,10 @@ class Config(BaseBridgeConfig):
        copy("bridge.displayname_preference")
        copy("bridge.displayname_max_length")
        copy("bridge.allow_avatar_remove")
+        copy("bridge.allow_contact_info")

        copy("bridge.max_initial_member_sync")
+        copy("bridge.max_member_count")
        copy("bridge.sync_channel_members")
        copy("bridge.skip_deleted_members")
        copy("bridge.startup_sync")
@@ -120,16 +114,17 @@ class Config(BaseBridgeConfig):
        else:
            copy("bridge.sync_update_limit")
            copy("bridge.sync_create_limit")
+        copy("bridge.sync_deferred_create_all")
        copy("bridge.sync_direct_chats")
        copy("bridge.max_telegram_delete")
        copy("bridge.sync_matrix_state")
        copy("bridge.allow_matrix_login")
-        copy("bridge.plaintext_highlights")
        copy("bridge.public_portals")
        copy("bridge.sync_with_custom_puppets")
        copy("bridge.sync_direct_chat_list")
        copy("bridge.double_puppet_server_map")
        copy("bridge.double_puppet_allow_discovery")
+        copy("bridge.create_group_on_invite")
        if "bridge.login_shared_secret" in self:
            base["bridge.login_shared_secret_map"] = {
                base["homeserver.domain"]: self["bridge.login_shared_secret"]
@@ -138,24 +133,26 @@ class Config(BaseBridgeConfig):
            copy("bridge.login_shared_secret_map")
        copy("bridge.telegram_link_preview")
        copy("bridge.invite_link_resolve")
-        copy("bridge.inline_images")
+        copy("bridge.caption_in_message")
        copy("bridge.image_as_file_size")
        copy("bridge.image_as_file_pixels")
        copy("bridge.parallel_file_transfer")
        copy("bridge.federate_rooms")
+        copy("bridge.always_custom_emoji_reaction")
        copy("bridge.animated_sticker.target")
+        copy("bridge.animated_sticker.convert_from_webm")
        copy("bridge.animated_sticker.args.width")
        copy("bridge.animated_sticker.args.height")
        copy("bridge.animated_sticker.args.fps")
-        copy("bridge.encryption.allow")
-        copy("bridge.encryption.default")
-        copy("bridge.encryption.database")
-        copy("bridge.encryption.key_sharing.allow")
-        copy("bridge.encryption.key_sharing.require_cross_signing")
-        copy("bridge.encryption.key_sharing.require_verification")
+        copy("bridge.animated_emoji.target")
+        copy("bridge.animated_emoji.args.width")
+        copy("bridge.animated_emoji.args.height")
+        copy("bridge.animated_emoji.args.fps")
        copy("bridge.private_chat_portal_meta")
        copy("bridge.delivery_receipts")
        copy("bridge.delivery_error_reports")
+        copy("bridge.incoming_bridge_error_reports")
+        copy("bridge.message_status_events")
        copy("bridge.resend_bridge_info")
        copy("bridge.mute_bridging")
        copy("bridge.pinned_tag")
@@ -164,12 +161,19 @@ class Config(BaseBridgeConfig):
        copy("bridge.bridge_matrix_leave")
        copy("bridge.kick_on_logout")
        copy("bridge.always_read_joined_telegram_notice")
-        copy("bridge.backfill.invite_own_puppet")
-        copy("bridge.backfill.takeout_limit")
-        copy("bridge.backfill.initial_limit")
-        copy("bridge.backfill.missed_limit")
-        copy("bridge.backfill.disable_notifications")
+        copy("bridge.backfill.enable")
+        copy("bridge.backfill.msc2716")
+        copy("bridge.backfill.double_puppet_backfill")
        copy("bridge.backfill.normal_groups")
+        copy("bridge.backfill.unread_hours_threshold")
+        copy("bridge.backfill.forward.initial_limit")
+        copy("bridge.backfill.forward.sync_limit")
+        copy("bridge.backfill.incremental.messages_per_batch")
+        copy("bridge.backfill.incremental.post_batch_delay")
+        copy("bridge.backfill.incremental.max_batches.user")
+        copy("bridge.backfill.incremental.max_batches.normal_group")
+        copy("bridge.backfill.incremental.max_batches.supergroup")
+        copy("bridge.backfill.incremental.max_batches.channel")

        copy("bridge.initial_power_level_overrides.group")
        copy("bridge.initial_power_level_overrides.user")
@@ -228,6 +232,10 @@ class Config(BaseBridgeConfig):
        copy("telegram.api_hash")
        copy("telegram.bot_token")

+        copy("telegram.catch_up")
+        copy("telegram.sequential_updates")
+        copy("telegram.exit_on_update_error")
+
        copy("telegram.connection.timeout")
        copy("telegram.connection.retries")
        copy("telegram.connection.retry_delay")
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from mautrix.util.async_db import Database

+from .backfill_queue import Backfill, BackfillType
 from .bot_chat import BotChat
 from .disappearing_message import DisappearingMessage
 from .message import Message
@@ -38,6 +39,7 @@ def init(db: Database) -> None:
        BotChat,
        PgSession,
        DisappearingMessage,
+        Backfill,
    ):
        table.db = db

@@ -54,4 +56,5 @@ __all__ = [
    "BotChat",
    "PgSession",
    "DisappearingMessage",
+    "Backfill",
 ]
@@ -0,0 +1,235 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan, Sumner Evans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar
+from datetime import datetime, timedelta
+from enum import Enum
+import json
+
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.types import UserID
+from mautrix.util.async_db import Connection, Database
+
+from ..types import TelegramID
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+class BackfillType(Enum):
+    HISTORICAL = "historical"
+    SYNC_DIALOG = "sync_dialog"
+
+
+@dataclass
+class Backfill:
+    db: ClassVar[Database] = fake_db
+
+    queue_id: int | None
+    user_mxid: UserID
+    priority: int
+    type: BackfillType
+    portal_tgid: TelegramID
+    portal_tg_receiver: TelegramID
+    anchor_msg_id: TelegramID | None
+    extra_data: dict[str, Any]
+    messages_per_batch: int
+    post_batch_delay: int
+    max_batches: int
+    dispatch_time: datetime | None
+    completed_at: datetime | None
+    cooldown_timeout: datetime | None
+
+    @staticmethod
+    def new(
+        user_mxid: UserID,
+        priority: int,
+        type: BackfillType,
+        portal_tgid: TelegramID,
+        portal_tg_receiver: TelegramID,
+        messages_per_batch: int,
+        anchor_msg_id: TelegramID | None = None,
+        extra_data: dict[str, Any] | None = None,
+        post_batch_delay: int = 0,
+        max_batches: int = -1,
+    ) -> "Backfill":
+        return Backfill(
+            queue_id=None,
+            user_mxid=user_mxid,
+            priority=priority,
+            type=type,
+            portal_tgid=portal_tgid,
+            portal_tg_receiver=portal_tg_receiver,
+            anchor_msg_id=anchor_msg_id,
+            extra_data=extra_data or {},
+            messages_per_batch=messages_per_batch,
+            post_batch_delay=post_batch_delay,
+            max_batches=max_batches,
+            dispatch_time=None,
+            completed_at=None,
+            cooldown_timeout=None,
+        )
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> Backfill | None:
+        if row is None:
+            return None
+        data = {**row}
+        type = BackfillType(data.pop("type"))
+        extra_data = json.loads(data.pop("extra_data", None) or "{}")
+        return cls(**data, type=type, extra_data=extra_data)
+
+    columns = [
+        "user_mxid",
+        "priority",
+        "type",
+        "portal_tgid",
+        "portal_tg_receiver",
+        "anchor_msg_id",
+        "extra_data",
+        "messages_per_batch",
+        "post_batch_delay",
+        "max_batches",
+        "dispatch_time",
+        "completed_at",
+        "cooldown_timeout",
+    ]
+    columns_str = ",".join(columns)
+
+    @classmethod
+    async def get_next(cls, user_mxid: UserID) -> Backfill | None:
+        q = f"""
+        SELECT queue_id, {cls.columns_str}
+        FROM backfill_queue
+        WHERE user_mxid=$1
+            AND (
+                dispatch_time IS NULL
+                OR (
+                    dispatch_time < $2
+                    AND completed_at IS NULL
+                )
+            )
+            AND (
+                cooldown_timeout IS NULL
+                OR cooldown_timeout < current_timestamp
+            )
+        ORDER BY priority, queue_id
+        LIMIT 1
+        """
+        return cls._from_row(
+            await cls.db.fetchrow(q, user_mxid, datetime.now() - timedelta(minutes=15))
+        )
+
+    @classmethod
+    async def delete_existing(
+        cls,
+        user_mxid: UserID,
+        portal_tgid: int,
+        portal_tg_receiver: int,
+        type: BackfillType,
+    ) -> Backfill | None:
+        q = f"""
+        WITH deleted_entries AS (
+            DELETE FROM backfill_queue
+            WHERE user_mxid=$1
+              AND portal_tgid=$2
+              AND portal_tg_receiver=$3
+              AND type=$4
+              AND dispatch_time IS NULL
+              AND completed_at IS NULL
+            RETURNING 1
+        )
+        WITH dispatched_entries AS (
+            SELECT 1 FROM backfill_queue
+            WHERE user_mxid=$1
+              AND portal_tgid=$2
+              AND portal_tg_receiver=$3
+              AND type=$4
+              AND dispatch_time IS NOT NULL
+              AND completed_at IS NULL
+        )
+        """
+        return cls._from_row(
+            await cls.db.fetchrow(q, user_mxid, portal_tgid, portal_tg_receiver, type.value)
+        )
+
+    @classmethod
+    async def delete_all(cls, user_mxid: UserID, conn: Connection | None = None) -> None:
+        await (conn or cls.db).execute("DELETE FROM backfill_queue WHERE user_mxid=$1", user_mxid)
+
+    @classmethod
+    async def delete_for_portal(cls, tgid: int, tg_receiver: int) -> None:
+        q = "DELETE FROM backfill_queue WHERE portal_tgid=$1 AND portal_tg_receiver=$2"
+        await cls.db.execute(q, tgid, tg_receiver)
+
+    async def insert(self) -> list[Backfill]:
+        delete_q = f"""
+        DELETE FROM backfill_queue
+        WHERE user_mxid=$1
+          AND portal_tgid=$2
+          AND portal_tg_receiver=$3
+          AND type=$4
+          AND dispatch_time IS NULL
+          AND completed_at IS NULL
+        RETURNING queue_id, {self.columns_str}
+        """
+        q = f"""
+        INSERT INTO backfill_queue ({self.columns_str})
+        VALUES ({','.join(f'${i+1}' for i in range(len(self.columns)))})
+        RETURNING queue_id
+        """
+        async with self.db.acquire() as conn, conn.transaction():
+            deleted_rows = await conn.fetch(
+                delete_q,
+                self.user_mxid,
+                self.portal_tgid,
+                self.portal_tg_receiver,
+                self.type.value,
+            )
+            self.queue_id = await conn.fetchval(
+                q,
+                self.user_mxid,
+                self.priority,
+                self.type.value,
+                self.portal_tgid,
+                self.portal_tg_receiver,
+                self.anchor_msg_id,
+                json.dumps(self.extra_data) if self.extra_data else None,
+                self.messages_per_batch,
+                self.post_batch_delay,
+                self.max_batches,
+                self.dispatch_time,
+                self.completed_at,
+                self.cooldown_timeout,
+            )
+        return [self._from_row(row) for row in deleted_rows]
+
+    async def mark_dispatched(self) -> None:
+        q = "UPDATE backfill_queue SET dispatch_time=$1 WHERE queue_id=$2"
+        await self.db.execute(q, datetime.now(), self.queue_id)
+
+    async def mark_done(self) -> None:
+        q = "UPDATE backfill_queue SET completed_at=$1 WHERE queue_id=$2"
+        await self.db.execute(q, datetime.now(), self.queue_id)
+
+    async def set_cooldown_timeout(self, timeout: int) -> None:
+        """
+        Set the backfill request to cooldown for ``timeout`` seconds.
+        """
+        q = "UPDATE backfill_queue SET cooldown_timeout=$1 WHERE queue_id=$2"
+        await self.db.execute(q, datetime.now() + timedelta(seconds=timeout), self.queue_id)
@@ -19,8 +19,9 @@ from typing import TYPE_CHECKING, ClassVar

 from asyncpg import Record
 from attr import dataclass
+import attr

-from mautrix.types import EventID, RoomID
+from mautrix.types import EventID, RoomID, UserID
 from mautrix.util.async_db import Database, Scheme

 from ..types import TelegramID
@@ -39,6 +40,8 @@ class Message:
    edit_index: int
    redacted: bool = False
    content_hash: bytes | None = None
+    sender_mxid: UserID | None = None
+    sender: TelegramID | None = None

    @classmethod
    def _from_row(cls, row: Record | None) -> Message | None:
@@ -46,7 +49,19 @@ class Message:
            return None
        return cls(**row)

-    columns: ClassVar[str] = "mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash"
+    columns: ClassVar[str] = ", ".join(
+        (
+            "mxid",
+            "mx_room",
+            "tgid",
+            "tg_space",
+            "edit_index",
+            "redacted",
+            "content_hash",
+            "sender_mxid",
+            "sender",
+        )
+    )

    @classmethod
    async def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> list[Message]:
@@ -108,6 +123,14 @@ class Message:
        )
        return cls._from_row(await cls.db.fetchrow(q, mx_room, tg_space))

+    @classmethod
+    async def find_first(cls, mx_room: RoomID, tg_space: TelegramID) -> Message | None:
+        q = (
+            f"SELECT {cls.columns} FROM message WHERE mx_room=$1 AND tg_space=$2 "
+            f"ORDER BY tgid ASC LIMIT 1"
+        )
+        return cls._from_row(await cls.db.fetchrow(q, mx_room, tg_space))
+
    @classmethod
    async def delete_all(cls, mx_room: RoomID) -> None:
        await cls.db.execute("DELETE FROM message WHERE mx_room=$1", mx_room)
@@ -138,6 +161,17 @@ class Message:
            rows = await cls.db.fetch(q, mx_room, tg_space, *mxids)
        return [cls._from_row(row) for row in rows]

+    @classmethod
+    async def find_recent(
+        cls, mx_room: RoomID, not_sender: TelegramID, limit: int = 20
+    ) -> list[Message]:
+        q = f"""
+        SELECT {cls.columns} FROM message
+        WHERE mx_room=$1 AND sender<>$2
+        ORDER BY tgid DESC LIMIT $3
+        """
+        return [cls._from_row(row) for row in await cls.db.fetch(q, mx_room, not_sender, limit)]
+
    @classmethod
    async def replace_temp_mxid(cls, temp_mxid: str, mx_room: RoomID, real_mxid: EventID) -> None:
        q = "UPDATE message SET mxid=$1 WHERE mxid=$2 AND mx_room=$3"
@@ -148,6 +182,23 @@ class Message:
        q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2"
        await cls.db.execute(q, temp_mxid, mx_room)

+    @classmethod
+    async def bulk_insert(cls, messages: list[Message]) -> None:
+        columns = cls.columns.split(", ")
+        records = [attr.astuple(message) for message in messages]
+        async with cls.db.acquire() as conn, conn.transaction():
+            if cls.db.scheme == Scheme.POSTGRES:
+                await conn.copy_records_to_table("message", records=records, columns=columns)
+            else:
+                await conn.executemany(cls._insert_query, records)
+
+    _insert_query: ClassVar[
+        str
+    ] = """
+        INSERT INTO message (mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash, sender_mxid, sender)
+        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+    """
+
    @property
    def _values(self):
        return (
@@ -158,14 +209,12 @@ class Message:
            self.edit_index,
            self.redacted,
            self.content_hash,
+            self.sender_mxid,
+            self.sender,
        )

    async def insert(self) -> None:
-        q = """
-            INSERT INTO message (mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash)
-            VALUES ($1, $2, $3, $4, $5, $6, $7)
-        """
-        await self.db.execute(q, *self._values)
+        await self.db.execute(self._insert_query, *self._values)

    async def delete(self) -> None:
        q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2 AND tg_space=$3"
@@ -22,7 +22,7 @@ from asyncpg import Record
 from attr import dataclass
 import attr

-from mautrix.types import ContentURI, EventID, RoomID
+from mautrix.types import BatchID, ContentURI, EventID, RoomID
 from mautrix.util.async_db import Database

 from ..types import TelegramID
@@ -44,6 +44,9 @@ class Portal:
    mxid: RoomID | None
    avatar_url: ContentURI | None
    encrypted: bool
+    first_event_id: EventID | None
+    next_batch_id: BatchID | None
+    base_insertion_id: EventID | None

    sponsored_event_id: EventID | None
    sponsored_event_ts: int | None
@@ -67,10 +70,29 @@ class Portal:
        data["local_config"] = json.loads(data.pop("config", None) or "{}")
        return cls(**data)

-    columns: ClassVar[str] = (
-        "tgid, tg_receiver, peer_type, megagroup, mxid, avatar_url, encrypted, sponsored_event_id,"
-        "sponsored_event_ts, sponsored_msg_random_id, username, title, about, photo_id, "
-        "name_set, avatar_set, config"
+    columns: ClassVar[str] = ", ".join(
+        (
+            "tgid",
+            "tg_receiver",
+            "peer_type",
+            "megagroup",
+            "mxid",
+            "avatar_url",
+            "encrypted",
+            "first_event_id",
+            "next_batch_id",
+            "base_insertion_id",
+            "sponsored_event_id",
+            "sponsored_event_ts",
+            "sponsored_msg_random_id",
+            "username",
+            "title",
+            "about",
+            "photo_id",
+            "name_set",
+            "avatar_set",
+            "config",
+        )
    )

    @classmethod
@@ -112,6 +134,9 @@ class Portal:
            self.mxid,
            self.avatar_url,
            self.encrypted,
+            self.first_event_id,
+            self.next_batch_id,
+            self.base_insertion_id,
            self.sponsored_event_id,
            self.sponsored_event_ts,
            self.sponsored_msg_random_id,
@@ -128,9 +153,11 @@ class Portal:
    async def save(self) -> None:
        q = """
        UPDATE portal
-        SET mxid=$4, avatar_url=$5, encrypted=$6, sponsored_event_id=$7, sponsored_event_ts=$8,
-            sponsored_msg_random_id=$9, username=$10, title=$11, about=$12, photo_id=$13,
-            name_set=$14, avatar_set=$15, megagroup=$16, config=$17
+        SET mxid=$4, avatar_url=$5, encrypted=$6,
+            first_event_id=$7, next_batch_id=$8, base_insertion_id=$9,
+            sponsored_event_id=$10, sponsored_event_ts=$11, sponsored_msg_random_id=$12,
+            username=$13, title=$14, about=$15, photo_id=$16, name_set=$17, avatar_set=$18,
+            megagroup=$19, config=$20
        WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)
        """
        await self.db.execute(q, *self._values)
@@ -140,7 +167,10 @@ class Portal:
            "UPDATE portal SET tgid=$1, tg_receiver=$1, peer_type=$2 "
            "WHERE tgid=$3 AND tg_receiver=$3"
        )
-        await self.db.execute(q, id, peer_type, self.tgid)
+        clear_queue = "DELETE FROM backfill_queue WHERE portal_tgid=$1 AND portal_tg_receiver=$2"
+        async with self.db.acquire() as conn, conn.transaction():
+            await conn.execute(clear_queue, self.tgid, self.tg_receiver)
+            await conn.execute(q, id, peer_type, self.tgid)
        self.tgid = id
        self.tg_receiver = id
        self.peer_type = peer_type
@@ -149,9 +179,11 @@ class Portal:
        q = """
        INSERT INTO portal (
            tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,
+            first_event_id, base_insertion_id, next_batch_id,
            sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,
            username, title, about, photo_id, name_set, avatar_set, megagroup, config
-        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
+                  $19, $20)
        """
        await self.db.execute(q, *self._values)

@@ -50,6 +50,7 @@ class Puppet:
    avatar_set: bool
    is_bot: bool | None
    is_channel: bool
+    is_premium: bool

    custom_mxid: UserID | None
    access_token: str | None
@@ -67,7 +68,8 @@ class Puppet:
    columns: ClassVar[str] = (
        "id, is_registered, displayname, displayname_source, displayname_contact, "
        "displayname_quality, disable_updates, username, phone, photo_id, avatar_url, "
-        "name_set, avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url"
+        "name_set, avatar_set, is_bot, is_channel, is_premium, "
+        "custom_mxid, access_token, next_batch, base_url"
    )

    @classmethod
@@ -90,11 +92,6 @@ class Puppet:
        q = f"SELECT {cls.columns} FROM puppet WHERE lower(username)=$1"
        return cls._from_row(await cls.db.fetchrow(q, username.lower()))

-    @classmethod
-    async def find_by_displayname(cls, displayname: str) -> Puppet | None:
-        q = f"SELECT {cls.columns} FROM puppet WHERE displayname=$1"
-        return cls._from_row(await cls.db.fetchrow(q, displayname))
-
    @property
    def _values(self):
        return (
@@ -113,6 +110,7 @@ class Puppet:
            self.avatar_set,
            self.is_bot,
            self.is_channel,
+            self.is_premium,
            self.custom_mxid,
            self.access_token,
            self.next_batch,
@@ -125,7 +123,7 @@ class Puppet:
        SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,
            displayname_quality=$6, disable_updates=$7, username=$8, phone=$9, photo_id=$10,
            avatar_url=$11, name_set=$12, avatar_set=$13, is_bot=$14, is_channel=$15,
-            custom_mxid=$16, access_token=$17, next_batch=$18, base_url=$19
+            is_premium=$16, custom_mxid=$17, access_token=$18, next_batch=$19, base_url=$20
        WHERE id=$1
        """
        await self.db.execute(q, *self._values)
@@ -135,8 +133,9 @@ class Puppet:
        INSERT INTO puppet (
            id, is_registered, displayname, displayname_source, displayname_contact,
            displayname_quality, disable_updates, username, phone, photo_id, avatar_url, name_set,
-            avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url
+            avatar_set, is_bot, is_channel, is_premium, custom_mxid, access_token, next_batch,
+            base_url
        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
-                  $19)
+                  $19, $20)
        """
        await self.db.execute(q, *self._values)
@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, ClassVar

 from asyncpg import Record
 from attr import dataclass
+from telethon.tl.types import ReactionCustomEmoji, ReactionEmoji, TypeReaction

 from mautrix.types import EventID, RoomID
 from mautrix.util.async_db import Database
@@ -58,9 +59,10 @@ class Reaction:
    @classmethod
    async def get_by_sender(
        cls, mxid: EventID, mx_room: RoomID, tg_sender: TelegramID
-    ) -> Reaction | None:
+    ) -> list[Reaction]:
        q = f"SELECT {cls.columns} FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3"
-        return cls._from_row(await cls.db.fetchrow(q, mxid, mx_room, tg_sender))
+        rows = await cls.db.fetch(q, mxid, mx_room, tg_sender)
+        return [cls._from_row(row) for row in rows]

    @classmethod
    async def get_all_by_message(cls, mxid: EventID, mx_room: RoomID) -> list[Reaction]:
@@ -68,6 +70,13 @@ class Reaction:
        rows = await cls.db.fetch(q, mxid, mx_room)
        return [cls._from_row(row) for row in rows]

+    @property
+    def telegram(self) -> TypeReaction:
+        if self.reaction.isdecimal():
+            return ReactionCustomEmoji(document_id=int(self.reaction))
+        else:
+            return ReactionEmoji(emoticon=self.reaction)
+
    @property
    def _values(self):
        return (
@@ -81,11 +90,11 @@ class Reaction:
    async def save(self) -> None:
        q = """
            INSERT INTO reaction (mxid, mx_room, msg_mxid, tg_sender, reaction)
-            VALUES ($1, $2, $3, $4, $5) ON CONFLICT (msg_mxid, mx_room, tg_sender)
-                DO UPDATE SET mxid=$1, reaction=$5
+            VALUES ($1, $2, $3, $4, $5) ON CONFLICT (msg_mxid, mx_room, tg_sender, reaction)
+                DO UPDATE SET mxid=excluded.mxid
        """
        await self.db.execute(q, *self._values)

    async def delete(self) -> None:
-        q = "DELETE FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3"
-        await self.db.execute(q, self.msg_mxid, self.mx_room, self.tg_sender)
+        q = "DELETE FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3 AND reaction=$4"
+        await self.db.execute(q, self.msg_mxid, self.mx_room, self.tg_sender, self.reaction)
@@ -17,10 +17,11 @@ from __future__ import annotations

 from typing import TYPE_CHECKING, ClassVar

+from asyncpg import Record
 from attr import dataclass

 from mautrix.types import ContentURI, EncryptedFile
-from mautrix.util.async_db import Database
+from mautrix.util.async_db import Database, Scheme

 fake_db = Database.create("") if TYPE_CHECKING else None

@@ -40,33 +41,60 @@ class TelegramFile:
    decryption_info: EncryptedFile | None
    thumbnail: TelegramFile | None = None

+    columns: ClassVar[str] = (
+        "id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail, "
+        "decryption_info"
+    )
+
    @classmethod
-    async def get(cls, loc_id: str, *, _thumbnail: bool = False) -> TelegramFile | None:
-        q = (
-            "SELECT id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail,"
-            "       decryption_info "
-            "FROM telegram_file WHERE id=$1"
-        )
-        row = await cls.db.fetchrow(q, loc_id)
+    def _from_row(cls, row: Record | None) -> TelegramFile | None:
        if row is None:
            return None
        data = {**row}
-        thumbnail_id = data.pop("thumbnail", None)
-        if _thumbnail:
-            # Don't allow more than one level of recursion
-            thumbnail_id = None
+        data.pop("thumbnail", None)
        decryption_info = data.pop("decryption_info", None)
        return cls(
            **data,
-            thumbnail=(await cls.get(thumbnail_id, _thumbnail=True)) if thumbnail_id else None,
+            thumbnail=None,
            decryption_info=EncryptedFile.parse_json(decryption_info) if decryption_info else None,
        )

+    @classmethod
+    async def get_many(cls, loc_ids: list[str]) -> list[TelegramFile]:
+        if cls.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+            q = f"SELECT {cls.columns} FROM telegram_file WHERE id=ANY($1)"
+            rows = await cls.db.fetch(q, loc_ids)
+        else:
+            tgid_placeholders = ("?," * len(loc_ids)).rstrip(",")
+            q = f"SELECT {cls.columns} FROM telegram_file WHERE id IN ({tgid_placeholders})"
+            rows = await cls.db.fetch(q, *loc_ids)
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def get(cls, loc_id: str, *, _thumbnail: bool = False) -> TelegramFile | None:
+        q = f"SELECT {cls.columns} FROM telegram_file WHERE id=$1"
+        row = await cls.db.fetchrow(q, loc_id)
+        file = cls._from_row(row)
+        if file is None:
+            return None
+        try:
+            thumbnail_id = row["thumbnail"]
+        except KeyError:
+            thumbnail_id = None
+        if thumbnail_id and not _thumbnail:
+            file.thumbnail = await cls.get(thumbnail_id, _thumbnail=True)
+        return file
+
+    @classmethod
+    async def find_by_mxc(cls, mxc: ContentURI) -> TelegramFile | None:
+        q = f"SELECT {cls.columns} FROM telegram_file WHERE mxc=$1"
+        return cls._from_row(await cls.db.fetchrow(q, mxc))
+
    async def insert(self) -> None:
        q = (
-            "INSERT INTO telegram_file (id, mxc, mime_type, was_converted, size, width, height, "
-            "                           thumbnail, decryption_info) "
-            "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"
+            "INSERT INTO telegram_file (id, mxc, mime_type, was_converted, timestamp,"
+            "                           size, width, height, thumbnail, decryption_info) "
+            "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)"
        )
        await self.db.execute(
            q,
@@ -74,6 +102,7 @@ class TelegramFile:
            self.mxc,
            self.mime_type,
            self.was_converted,
+            self.timestamp,
            self.size,
            self.width,
            self.height,
@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations

-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, Iterable
 import asyncio
 import datetime

@@ -123,19 +123,79 @@ class PgSession(MemorySession):
        date = datetime.datetime.utcfromtimestamp(row["date"])
        return updates.State(row["pts"], row["qts"], date, row["seq"], row["unread_count"])

+    _set_update_state_q = """
+    INSERT INTO telethon_update_state (session_id, entity_id, pts, qts, date, seq, unread_count)
+    VALUES ($1, $2, $3, $4, $5, $6, $7)
+    ON CONFLICT (session_id, entity_id) DO UPDATE SET
+        pts=excluded.pts, qts=excluded.qts, date=excluded.date, seq=excluded.seq,
+        unread_count=excluded.unread_count
+    """
+
    async def set_update_state(self, entity_id: int, row: updates.State) -> None:
-        q = (
-            "INSERT INTO telethon_update_state"
-            "    (session_id, entity_id, pts, qts, date, seq, unread_count) "
-            "VALUES ($1, $2, $3, $4, $5, $6, $7)"
-            "ON CONFLICT (session_id, entity_id) DO UPDATE"
-            "    SET pts=$3, qts=$4, date=$5, seq=$6, unread_count=$7"
-        )
+        q = self._set_update_state_q
        ts = row.date.timestamp()
        await self.db.execute(
            q, self.session_id, entity_id, row.pts, row.qts, ts, row.seq, row.unread_count
        )

+    async def set_update_states(self, rows: list[tuple[int, updates.State]]) -> None:
+        rows = [
+            (
+                self.session_id,
+                entity_id,
+                row.pts,
+                row.qts,
+                row.date.timestamp(),
+                row.seq,
+                row.unread_count,
+            )
+            for entity_id, row in rows
+        ]
+        if self.db.scheme == Scheme.POSTGRES:
+            q = """
+            INSERT INTO telethon_update_state (
+                session_id, entity_id, pts, qts, date, seq, unread_count
+            )
+            VALUES (
+                $1,
+                unnest($2::bigint[]), unnest($3::bigint[]), unnest($4::bigint[]),
+                unnest($5::bigint[]), unnest($6::bigint[]), unnest($7::integer[])
+            )
+            ON CONFLICT (session_id, entity_id) DO UPDATE SET
+                pts=excluded.pts, qts=excluded.qts, date=excluded.date, seq=excluded.seq,
+                unread_count=excluded.unread_count
+            """
+            _, entity_ids, ptses, qtses, timestamps, seqs, unread_counts = zip(*rows)
+            await self.db.execute(
+                q, self.session_id, entity_ids, ptses, qtses, timestamps, seqs, unread_counts
+            )
+        else:
+            await self.db.executemany(self._set_update_state_q, rows)
+
+    async def delete_update_state(self, entity_id: int) -> None:
+        q = "DELETE FROM telethon_update_state WHERE session_id=$1 AND entity_id=$2"
+        await self.db.execute(q, self.session_id, entity_id)
+
+    async def get_update_states(self) -> Iterable[tuple[int, updates.State], ...]:
+        q = (
+            "SELECT entity_id, pts, qts, date, seq, unread_count FROM telethon_update_state "
+            "WHERE session_id=$1"
+        )
+        rows = await self.db.fetch(q, self.session_id)
+        return (
+            (
+                row["entity_id"],
+                updates.State(
+                    row["pts"],
+                    row["qts"],
+                    datetime.datetime.utcfromtimestamp(row["date"]),
+                    row["seq"],
+                    row["unread_count"],
+                ),
+            )
+            for row in rows
+        )
+
    def _entity_values_to_row(
        self, id: int, hash: int, username: str | None, phone: str | int | None, name: str | None
    ) -> tuple[str, int, int, str | None, str | None, str | None]:
@@ -176,25 +236,24 @@ class PgSession(MemorySession):
    async def _select_entity(
        self, constraint: str, *args: str | int | tuple[int, ...]
    ) -> tuple[int, int] | None:
-        row = await self.db.fetchrow(
-            f"SELECT id, hash FROM telethon_entities WHERE {constraint}", *args
-        )
+        q = f"SELECT id, hash FROM telethon_entities WHERE session_id=$1 AND {constraint}"
+        row = await self.db.fetchrow(q, self.session_id, *args)
        if row is None:
            return None
        return row["id"], row["hash"]

    async def get_entity_rows_by_phone(self, key: str | int) -> tuple[int, int] | None:
-        return await self._select_entity("phone=$1", str(key))
+        return await self._select_entity("phone=$2", str(key))

    async def get_entity_rows_by_username(self, key: str) -> tuple[int, int] | None:
-        return await self._select_entity("username=$1", key)
+        return await self._select_entity("username=$2", key)

    async def get_entity_rows_by_name(self, key: str) -> tuple[int, int] | None:
-        return await self._select_entity("name=$1", key)
+        return await self._select_entity("name=$2", key)

    async def get_entity_rows_by_id(self, key: int, exact: bool = True) -> tuple[int, int] | None:
        if exact:
-            return await self._select_entity("id=$1", key)
+            return await self._select_entity("id=$2", key)

        ids = (
            utils.get_peer_id(PeerUser(key)),
@@ -202,6 +261,6 @@ class PgSession(MemorySession):
            utils.get_peer_id(PeerChannel(key)),
        )
        if self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
-            return await self._select_entity("id=ANY($1)", ids)
+            return await self._select_entity("id=ANY($2)", ids)
        else:
-            return await self._select_entity(f"id IN ($1, $2, $3)", *ids)
+            return await self._select_entity(f"id IN ($2, $3, $4)", *ids)
@@ -10,4 +10,14 @@ from . import (
    v05_channel_ghosts,
    v06_puppet_avatar_url,
    v07_puppet_phone_number,
+    v08_portal_first_event,
+    v09_puppet_username_index,
+    v10_more_backfill_fields,
+    v11_backfill_queue,
+    v12_message_sender,
+    v13_multiple_reactions,
+    v14_puppet_custom_mxid_index,
+    v15_backfill_anchor_id,
+    v16_backfill_type,
+    v17_message_find_recent,
 )
@@ -13,10 +13,12 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.util.async_db import Connection
+from mautrix.util.async_db import Connection, Scheme
+
+latest_version = 17


-async def create_v7_tables(conn: Connection) -> int:
+async def create_latest_tables(conn: Connection, scheme: Scheme) -> int:
    await conn.execute(
        """CREATE TABLE "user" (
            mxid TEXT   PRIMARY KEY,
@@ -24,6 +26,7 @@ async def create_v7_tables(conn: Connection) -> int:
            tg_username    TEXT,
            tg_phone       TEXT,
            is_bot         BOOLEAN NOT NULL DEFAULT false,
+            is_premium     BOOLEAN NOT NULL DEFAULT false,
            saved_contacts INTEGER NOT NULL DEFAULT 0
        )"""
    )
@@ -44,6 +47,10 @@ async def create_v7_tables(conn: Connection) -> int:
            megagroup   BOOLEAN,
            config      jsonb,

+            first_event_id    TEXT,
+            next_batch_id     TEXT,
+            base_insertion_id TEXT,
+
            sponsored_event_id     TEXT,
            sponsored_event_ts     BIGINT,
            sponsored_msg_random_id bytea,
@@ -60,10 +67,13 @@ async def create_v7_tables(conn: Connection) -> int:
            edit_index   INTEGER,
            redacted     BOOLEAN NOT NULL DEFAULT false,
            content_hash bytea,
+            sender_mxid  TEXT,
+            sender       BIGINT,
            PRIMARY KEY (tgid, tg_space, edit_index),
            UNIQUE (mxid, mx_room, tg_space)
        )"""
    )
+    await conn.execute("CREATE INDEX message_mx_room_and_tgid_idx ON message(mx_room, tgid DESC)")
    await conn.execute(
        """CREATE TABLE reaction (
            mxid      TEXT NOT NULL,
@@ -72,7 +82,7 @@ async def create_v7_tables(conn: Connection) -> int:
            tg_sender BIGINT,
            reaction  TEXT NOT NULL,

-            PRIMARY KEY (msg_mxid, mx_room, tg_sender),
+            PRIMARY KEY (msg_mxid, mx_room, tg_sender, reaction),
            UNIQUE (mxid, mx_room)
        )"""
    )
@@ -105,6 +115,7 @@ async def create_v7_tables(conn: Connection) -> int:
            avatar_set          BOOLEAN NOT NULL DEFAULT false,
            is_bot              BOOLEAN,
            is_channel          BOOLEAN NOT NULL DEFAULT false,
+            is_premium          BOOLEAN NOT NULL DEFAULT false,

            access_token TEXT,
            custom_mxid  TEXT,
@@ -112,6 +123,8 @@ async def create_v7_tables(conn: Connection) -> int:
            base_url     TEXT
        )"""
    )
+    await conn.execute("CREATE INDEX puppet_username_idx ON puppet(LOWER(username))")
+    await conn.execute("CREATE INDEX puppet_custom_mxid_idx ON puppet(custom_mxid)")
    await conn.execute(
        """CREATE TABLE telegram_file (
            id              TEXT PRIMARY KEY,
@@ -128,6 +141,7 @@ async def create_v7_tables(conn: Connection) -> int:
                ON UPDATE CASCADE ON DELETE SET NULL
        )"""
    )
+    await conn.execute("CREATE INDEX telegram_file_mxc_idx ON telegram_file(mxc)")
    await conn.execute(
        """CREATE TABLE bot_chat (
            id   BIGINT PRIMARY KEY,
@@ -197,4 +211,31 @@ async def create_v7_tables(conn: Connection) -> int:
            PRIMARY KEY (session_id, entity_id)
        )"""
    )
-    return 7
+    gen = ""
+    if scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+        gen = "GENERATED ALWAYS AS IDENTITY"
+    await conn.execute(
+        f"""
+        CREATE TABLE backfill_queue (
+            queue_id            INTEGER PRIMARY KEY {gen},
+            user_mxid           TEXT,
+            priority            INTEGER NOT NULL,
+            type                TEXT NOT NULL,
+            portal_tgid         BIGINT,
+            portal_tg_receiver  BIGINT,
+            anchor_msg_id       BIGINT,
+            extra_data          jsonb,
+            messages_per_batch  INTEGER NOT NULL,
+            post_batch_delay    INTEGER NOT NULL,
+            max_batches         INTEGER NOT NULL,
+            dispatch_time       TIMESTAMP,
+            completed_at        TIMESTAMP,
+            cooldown_timeout    TIMESTAMP,
+            FOREIGN KEY (user_mxid) REFERENCES "user"(mxid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (portal_tgid, portal_tg_receiver)
+                REFERENCES portal(tgid, tg_receiver) ON DELETE CASCADE
+        )
+        """
+    )
+
+    return latest_version
@@ -18,35 +18,27 @@ from __future__ import annotations
 from mautrix.util.async_db import Connection, Scheme

 from . import upgrade_table
-from .v00_latest_revision import create_v7_tables
+from .v00_latest_revision import create_latest_tables, latest_version

 legacy_version_query = "SELECT version_num FROM alembic_version"
 last_legacy_version = "bfc0a39bfe02"


-def table_exists(scheme: str, name: str) -> str:
-    if scheme == Scheme.SQLITE:
-        return f"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='{name}')"
-    elif scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
-        return f"SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name='{name}')"
-    raise RuntimeError("unsupported database scheme")
-
-
-async def first_upgrade_target(conn: Connection, scheme: str) -> int:
-    is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
-    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to v7.
-    # If it's a new db, we'll create the v7 tables directly (see the create_v7_tables call).
-    return 1 if is_legacy else 7
+async def first_upgrade_target(conn: Connection, scheme: Scheme) -> int:
+    is_legacy = await conn.table_exists("alembic_version")
+    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to latest.
+    # If it's a new db, we'll create the latest tables directly (see create_latest_tables call).
+    return 1 if is_legacy else latest_version


@upgrade_table.register(description="Initial asyncpg revision", upgrades_to=first_upgrade_target)
-async def upgrade_v1(conn: Connection, scheme: str) -> int:
-    is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
+async def upgrade_v1(conn: Connection, scheme: Scheme) -> int:
+    is_legacy = await conn.table_exists("alembic_version")
    if is_legacy:
        await migrate_legacy_to_v1(conn, scheme)
        return 1
    else:
-        return await create_v7_tables(conn)
+        return await create_latest_tables(conn, scheme)


 async def drop_constraints(conn: Connection, table: str, contype: str) -> None:
@@ -59,14 +51,14 @@ async def drop_constraints(conn: Connection, table: str, contype: str) -> None:
    await conn.execute(f"ALTER TABLE {table} {drops}")


-async def migrate_legacy_to_v1(conn: Connection, scheme: str) -> None:
+async def migrate_legacy_to_v1(conn: Connection, scheme: Scheme) -> None:
    legacy_version = await conn.fetchval(legacy_version_query)
    if legacy_version != last_legacy_version:
        raise RuntimeError(
            "Legacy database is not on last version. "
            "Please upgrade the old database with alembic or drop it completely first."
        )
-    if scheme != "sqlite":
+    if scheme != Scheme.SQLITE:
        await drop_constraints(conn, "contact", contype="f")
        await conn.execute(
            """
@@ -131,12 +123,12 @@ async def migrate_legacy_to_v1(conn: Connection, scheme: str) -> None:
    await conn.execute("DROP TABLE alembic_version")


-async def update_state_store(conn: Connection, scheme: str) -> None:
+async def update_state_store(conn: Connection, scheme: Scheme) -> None:
    # The Matrix state store already has more or less the correct schema, so set the version
    await conn.execute("CREATE TABLE mx_version (version INTEGER PRIMARY KEY)")
    await conn.execute("INSERT INTO mx_version (version) VALUES (2)")
    await conn.execute("UPDATE mx_user_profile SET membership='LEAVE' WHERE membership='LEFT'")
-    if scheme != "sqlite":
+    if scheme != Scheme.SQLITE:
        # Also add the membership type on postgres
        await conn.execute(
            "CREATE TYPE membership AS ENUM ('join', 'leave', 'invite', 'ban', 'knock')"
@@ -0,0 +1,24 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Track first event ID in portals for infinite backfilling")
+async def upgrade_v8(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN first_event_id TEXT")
+    await conn.execute("ALTER TABLE portal ADD COLUMN next_batch_id TEXT")
@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add index to puppet username column")
+async def upgrade_v9(conn: Connection) -> None:
+    await conn.execute("CREATE INDEX puppet_username_idx ON puppet(LOWER(username))")
@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add more portal columns related to infinite backfill")
+async def upgrade_v10(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN base_insertion_id TEXT")
@@ -0,0 +1,45 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan, Sumner Evans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add the backfill queue table")
+async def upgrade_v11(conn: Connection, scheme: Scheme) -> None:
+    gen = ""
+    if scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+        gen = "GENERATED ALWAYS AS IDENTITY"
+    await conn.execute(
+        f"""
+        CREATE TABLE backfill_queue (
+            queue_id            INTEGER PRIMARY KEY {gen},
+            user_mxid           TEXT,
+            priority            INTEGER NOT NULL,
+            portal_tgid         BIGINT,
+            portal_tg_receiver  BIGINT,
+            messages_per_batch  INTEGER NOT NULL,
+            post_batch_delay    INTEGER NOT NULL,
+            max_batches         INTEGER NOT NULL,
+            dispatch_time       TIMESTAMP,
+            completed_at        TIMESTAMP,
+            cooldown_timeout    TIMESTAMP,
+            FOREIGN KEY (user_mxid) REFERENCES "user"(mxid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (portal_tgid, portal_tg_receiver)
+                REFERENCES portal(tgid, tg_receiver) ON DELETE CASCADE
+        )
+        """
+    )
@@ -0,0 +1,24 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store sender in message table")
+async def upgrade_v12(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE message ADD COLUMN sender_mxid TEXT")
+    await conn.execute("ALTER TABLE message ADD COLUMN sender BIGINT")
@@ -0,0 +1,54 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Allow multiple reactions from the same user")
+async def upgrade_v13(conn: Connection, scheme: Scheme) -> None:
+    await conn.execute("CREATE INDEX telegram_file_mxc_idx ON telegram_file(mxc)")
+    await conn.execute('ALTER TABLE "user" ADD COLUMN is_premium BOOLEAN NOT NULL DEFAULT false')
+    await conn.execute("ALTER TABLE puppet ADD COLUMN is_premium BOOLEAN NOT NULL DEFAULT false")
+    if scheme == Scheme.POSTGRES:
+        await conn.execute(
+            """
+            ALTER TABLE reaction
+                DROP CONSTRAINT reaction_pkey,
+                ADD CONSTRAINT reaction_pkey PRIMARY KEY (msg_mxid, mx_room, tg_sender, reaction)
+            """
+        )
+    else:
+        await conn.execute(
+            """CREATE TABLE new_reaction (
+                mxid      TEXT NOT NULL,
+                mx_room   TEXT NOT NULL,
+                msg_mxid  TEXT NOT NULL,
+                tg_sender BIGINT,
+                reaction  TEXT NOT NULL,
+
+                PRIMARY KEY (msg_mxid, mx_room, tg_sender, reaction),
+                UNIQUE (mxid, mx_room)
+            )"""
+        )
+        await conn.execute(
+            """
+            INSERT INTO new_reaction (mxid, mx_room, msg_mxid, tg_sender, reaction)
+            SELECT mxid, mx_room, msg_mxid, tg_sender, reaction FROM reaction
+            """
+        )
+        await conn.execute("DROP TABLE reaction")
+        await conn.execute("ALTER TABLE new_reaction RENAME TO reaction")
@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add index to puppet custom_mxid column")
+async def upgrade_v14(conn: Connection) -> None:
+    await conn.execute("CREATE INDEX IF NOT EXISTS puppet_custom_mxid_idx ON puppet(custom_mxid)")
@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store lowest message ID in backfill queue")
+async def upgrade_v15(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE backfill_queue ADD COLUMN anchor_msg_id BIGINT")
@@ -0,0 +1,28 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add type for backfill queue items")
+async def upgrade_v16(conn: Connection, scheme: Scheme) -> None:
+    await conn.execute(
+        "ALTER TABLE backfill_queue ADD COLUMN type TEXT NOT NULL DEFAULT 'historical'"
+    )
+    await conn.execute("ALTER TABLE backfill_queue ADD COLUMN extra_data jsonb")
+    if scheme != Scheme.SQLITE:
+        await conn.execute("ALTER TABLE backfill_queue ALTER COLUMN type DROP DEFAULT")
@@ -0,0 +1,25 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add index for Message.find_recent")
+async def upgrade_v17(conn: Connection) -> None:
+    await conn.execute(
+        "CREATE INDEX IF NOT EXISTS message_mx_room_and_tgid_idx ON message(mx_room, tgid DESC)"
+    )
@@ -21,9 +21,10 @@ from asyncpg import Record
 from attr import dataclass

 from mautrix.types import UserID
-from mautrix.util.async_db import Database, Scheme
+from mautrix.util.async_db import Connection, Database, Scheme

 from ..types import TelegramID
+from .backfill_queue import Backfill

 fake_db = Database.create("") if TYPE_CHECKING else None

@@ -37,6 +38,7 @@ class User:
    tg_username: str | None
    tg_phone: str | None
    is_bot: bool
+    is_premium: bool
    saved_contacts: int

    @classmethod
@@ -45,7 +47,9 @@ class User:
            return None
        return cls(**row)

-    columns: ClassVar[str] = "mxid, tgid, tg_username, tg_phone, is_bot, saved_contacts"
+    columns: ClassVar[str] = ", ".join(
+        ("mxid", "tgid", "tg_username", "tg_phone", "is_bot", "is_premium", "saved_contacts")
+    )

    @classmethod
    async def get_by_tgid(cls, tgid: TelegramID) -> User | None:
@@ -70,6 +74,20 @@ class User:
    async def delete(self) -> None:
        await self.db.execute('DELETE FROM "user" WHERE mxid=$1', self.mxid)

+    async def remove_tgid(self) -> None:
+        async with self.db.acquire() as conn, conn.transaction():
+            if self.tgid:
+                await conn.execute('DELETE FROM contact WHERE "user"=$1', self.tgid)
+                await conn.execute('DELETE FROM user_portal WHERE "user"=$1', self.tgid)
+            await Backfill.delete_all(self.mxid, conn=conn)
+            self.tgid = None
+            self.tg_username = None
+            self.tg_phone = None
+            self.is_bot = False
+            self.is_premium = False
+            self.saved_contacts = 0
+            await self.save(conn=conn)
+
    @property
    def _values(self):
        return (
@@ -78,21 +96,23 @@ class User:
            self.tg_username,
            self.tg_phone,
            self.is_bot,
+            self.is_premium,
            self.saved_contacts,
        )

-    async def save(self) -> None:
-        q = (
-            'UPDATE "user" SET tgid=$2, tg_username=$3, tg_phone=$4, is_bot=$5, saved_contacts=$6 '
-            "WHERE mxid=$1"
-        )
-        await self.db.execute(q, *self._values)
+    async def save(self, conn: Connection | None = None) -> None:
+        q = """
+        UPDATE "user" SET tgid=$2, tg_username=$3, tg_phone=$4, is_bot=$5, is_premium=$6,
+                          saved_contacts=$7
+        WHERE mxid=$1
+        """
+        await (conn or self.db).execute(q, *self._values)

    async def insert(self) -> None:
-        q = (
-            'INSERT INTO "user" (mxid, tgid, tg_username, tg_phone, is_bot, saved_contacts) '
-            "VALUES ($1, $2, $3, $4, $5, $6)"
-        )
+        q = """
+        INSERT INTO "user" (mxid, tgid, tg_username, tg_phone, is_bot, is_premium, saved_contacts)
+        VALUES ($1, $2, $3, $4, $5, $6, $7)
+        """
        await self.db.execute(q, *self._values)

    async def get_contacts(self) -> list[TelegramID]:
@@ -7,7 +7,9 @@ homeserver:
    # Whether or not to verify the SSL certificate of the homeserver.
    # Only applies if address starts with https://
    verify_ssl: true
-    asmux: false
+    # What software is the homeserver running?
+    # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here.
+    software: standard
    # Number of retries for all HTTP requests if the homeserver isn't reachable.
    http_retry_count: 4
    # The URL to push real-time bridge status to.
@@ -45,6 +47,7 @@ appservice:
    # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool
    # https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
    # For sqlite, min_size is used as the connection thread pool size and max_size is ignored.
+    # Additionally, SQLite supports init_commands as an array of SQL queries to run on connect (e.g. to set PRAGMAs).
    database_opts:
        min_size: 1
        max_size: 10
@@ -84,7 +87,7 @@ appservice:
    # Whether or not to receive ephemeral events via appservice transactions.
    # Requires MSC2409 support (i.e. Synapse 1.22+).
    # You should disable bridge -> sync_with_custom_puppets when this is enabled.
-    ephemeral_events: false
+    ephemeral_events: true

    # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
    as_token: "This value is generated when generating the registration"
@@ -142,27 +145,37 @@ bridge:
    # as there's no way to determine whether an avatar is removed or just hidden from some users. If
    # you're on a single-user instance, this should be safe to enable.
    allow_avatar_remove: false
+    # Should contact names and profile pictures be allowed?
+    # This is only safe to enable on single-user instances.
+    allow_contact_info: false

    # Maximum number of members to sync per portal when starting up. Other members will be
    # synced when they send messages. The maximum is 10000, after which the Telegram server
    # will not send any more members.
    # -1 means no limit (which means it's limited to 10000 by the server)
    max_initial_member_sync: 100
+    # Maximum number of participants in chats to bridge. Only applies when the portal is being created.
+    # If there are more members when trying to create a room, the room creation will be cancelled.
+    # -1 means no limit (which means all chats can be bridged)
+    max_member_count: -1
    # Whether or not to sync the member list in channels.
    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
    # list regardless of this setting.
-    sync_channel_members: true
+    sync_channel_members: false
    # Whether or not to skip deleted members when syncing members.
    skip_deleted_members: true
    # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
    # their Telegram account at startup.
-    startup_sync: true
+    startup_sync: false
    # Number of most recently active dialogs to check when syncing chats.
    # Set to 0 to remove limit.
    sync_update_limit: 0
    # Number of most recently active dialogs to create portals for when syncing chats.
    # Set to 0 to remove limit.
-    sync_create_limit: 30
+    sync_create_limit: 15
+    # Should all chats be scheduled to be created later?
+    # This is best used in combination with MSC2716 infinite backfill.
+    sync_deferred_create_all: false
    # Whether or not to sync and create portals for direct chats at startup.
    sync_direct_chats: false
    # The maximum number of simultaneous Telegram deletions to handle.
@@ -174,15 +187,11 @@ bridge:
    # Allow logging in within Matrix. If false, users can only log in using login-qr or the
    # out-of-Matrix login website (see appservice.public config section)
    allow_matrix_login: true
-    # Whether or not to bridge plaintext highlights.
-    # Only enable this if your displayname_template has some static part that the bridge can use to
-    # reliably identify what is a plaintext highlight.
-    plaintext_highlights: false
    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
-    public_portals: true
+    public_portals: false
    # Whether or not to use /sync to get presence, read receipts and typing notifications
    # when double puppeting is enabled
-    sync_with_custom_puppets: true
+    sync_with_custom_puppets: false
    # Whether or not to update the m.direct account data event when double puppeting is enabled.
    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
    # and is therefore prone to race conditions.
@@ -206,13 +215,13 @@ bridge:
    # Whether or not the !tg join command should do a HTTP request
    # to resolve redirects in invite links.
    invite_link_resolve: false
-    # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Element iOS/Android).
-    inline_images: false
+    # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552.
+    # This is currently not supported in most clients.
+    caption_in_message: false
    # Maximum size of image in megabytes before sending to Telegram as a document.
    image_as_file_size: 10
-    # Maximum number of pixels in an image before sending to Telegram as a document. Defaults to 1280x1280 = 1638400.
-    image_as_file_pixels: 1638400
+    # Maximum number of pixels in an image before sending to Telegram as a document. Defaults to 4096x4096 = 16777216.
+    image_as_file_pixels: 16777216
    # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
    # streaming from/to Matrix and using many connections for Telegram.
    # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
@@ -221,6 +230,9 @@ bridge:
    # Whether or not created rooms should have federation enabled.
    # If false, created portal rooms will never be federated.
    federate_rooms: true
+    # Should the bridge send all unicode reactions as custom emoji reactions to Telegram?
+    # By default, the bridge only uses custom emojis for unicode emojis that aren't allowed in reactions.
+    always_custom_emoji_reaction: false
    # Settings for converting animated stickers.
    animated_sticker:
        # Format to which animated stickers should be converted.
@@ -228,12 +240,24 @@ bridge:
        # png - converts to non-animated png (fastest),
        # gif - converts to animated gif
        # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support
+        # webp - converts to animated webp, requires ffmpeg executable with webp codec/container support
        target: gif
+        # Should video stickers be converted to the specified format as well?
+        convert_from_webm: false
        # Arguments for converter. All converters take width and height.
        args:
            width: 256
            height: 256
-            fps: 25 # only for webm and gif (2, 5, 10, 20 or 25 recommended)
+            fps: 25 # only for webm, webp and gif (2, 5, 10, 20 or 25 recommended)
+    # Settings for converting animated emoji.
+    # Same as animated_sticker, but webm is not supported as the target
+    # (because inline images can only contain images, not videos).
+    animated_emoji:
+        target: webp
+        args:
+            width: 64
+            height: 64
+            fps: 25
    # End-to-bridge encryption support options.
    #
    # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info.
@@ -243,20 +267,48 @@ bridge:
        # Default to encryption, force-enable encryption in all portals the bridge creates
        # This will cause the bridge bot to be in private chats for the encryption to work properly.
        default: false
-        # Database for the encryption data. If set to `default`, will use the appservice database.
-        database: default
-        # Options for automatic key sharing.
-        key_sharing:
-            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
-            # You must use a client that supports requesting keys from other users to use this feature.
-            allow: false
-            # Require the requesting device to have a valid cross-signing signature?
-            # This doesn't require that the bridge has verified the device, only that the user has verified it.
-            # Not yet implemented.
-            require_cross_signing: false
-            # Require devices to be verified by the bridge?
-            # Verification by the bridge is not yet implemented.
-            require_verification: true
+        # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data.
+        appservice: false
+        # Require encryption, drop any unencrypted messages.
+        require: false
+        # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
+        # You must use a client that supports requesting keys from other users to use this feature.
+        allow_key_sharing: false
+        # What level of device verification should be required from users?
+        #
+        # Valid levels:
+        #   unverified - Send keys to all device in the room.
+        #   cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys.
+        #   cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes).
+        #   cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot.
+        #                           Note that creating user signatures from the bridge bot is not currently possible.
+        #   verified - Require manual per-device verification
+        #              (currently only possible by modifying the `trust` column in the `crypto_device` database table).
+        verification_levels:
+            # Minimum level for which the bridge should send keys to when bridging messages from Telegram to Matrix.
+            receive: unverified
+            # Minimum level that the bridge should accept for incoming Matrix messages.
+            send: unverified
+            # Minimum level that the bridge should require for accepting key requests.
+            share: cross-signed-tofu
+        # Options for Megolm room key rotation. These options allow you to
+        # configure the m.room.encryption event content. See:
+        # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for
+        # more information about that event.
+        rotation:
+            # Enable custom Megolm room key rotation settings. Note that these
+            # settings will only apply to rooms created after this option is
+            # set.
+            enable_custom: false
+            # The maximum number of milliseconds a session should be used
+            # before changing it. The Matrix spec recommends 604800000 (a week)
+            # as the default.
+            milliseconds: 604800000
+            # The maximum number of messages that should be sent with a given a
+            # session before changing it. The Matrix spec recommends 100 as the
+            # default.
+            messages: 100
+
    # Whether or not to explicitly set the avatar and room name for private
    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
    private_chat_portal_meta: false
@@ -265,6 +317,10 @@ bridge:
    delivery_receipts: false
    # Whether or not delivery errors should be reported as messages in the Matrix room.
    delivery_error_reports: false
+    # Should errors in incoming message handling send a message to the Matrix room?
+    incoming_bridge_error_reports: false
+    # Whether the bridge should send the message status as a custom com.beeper.message_send_status event.
+    message_status_events: false
    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
    # This field will automatically be changed back to false after it,
    # except if the config file is not writable.
@@ -284,36 +340,65 @@ bridge:
    kick_on_logout: true
    # Should the "* user joined Telegram" notice always be marked as read automatically?
    always_read_joined_telegram_notice: true
+    # Should the bridge auto-create a group chat on Telegram when a ghost is invited to a room?
+    # Requires the user to have sufficient power level and double puppeting enabled.
+    create_group_on_invite: true
    # Settings for backfilling messages from Telegram.
    backfill:
-        # Whether or not the Telegram ghosts of logged in Matrix users should be
-        # invited to private chats when backfilling history from Telegram. This is
-        # usually needed to prevent rate limits and to allow timestamp massaging.
-        invite_own_puppet: true
-        # Maximum number of messages to backfill without using a takeout.
-        # The first time a takeout is used, the user has to manually approve it from a different
-        # device. If initial_limit or missed_limit are higher than this value, the bridge will ask
-        # the user to accept the takeout after logging in before syncing any chats.
-        takeout_limit: 100
-        # Maximum number of messages to backfill initially.
-        # Set to 0 to disable backfilling when creating portal, or -1 to disable the limit.
+        # Allow backfilling at all?
+        enable: true
+        # Use MSC2716 for backfilling?
        #
-        # N.B. Initial backfill will only start after member sync. Make sure your
-        #      max_initial_member_sync is set to a low enough value so it doesn't take forever.
-        initial_limit: 0
-        # Maximum number of messages to backfill if messages were missed while the bridge was
-        # disconnected. Note that this only works for logged in users and only if the chat isn't
-        # older than sync_update_limit
-        # Set to 0 to disable backfilling missed messages.
-        missed_limit: 50
-        # If using double puppeting, should notifications be disabled
-        # while the initial backfill is in progress?
-        disable_notifications: false
+        # This requires a server with MSC2716 support, which is currently an experimental feature in Synapse.
+        # It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml.
+        msc2716: false
+        # Use double puppets for backfilling?
+        #
+        # If using MSC2716, the double puppets must be in the appservice's user ID namespace
+        # (because the bridge can't use the double puppet access token with batch sending).
+        #
+        # Even without MSC2716, bridging old messages with correct timestamps requires the double
+        # puppets to be in an appservice namespace, or the server to be modified to allow
+        # overriding timestamps anyway.
+        double_puppet_backfill: false
        # Whether or not to enable backfilling in normal groups.
        # Normal groups have numerous technical problems in Telegram, and backfilling normal groups
        # will likely cause problems if there are multiple Matrix users in the group.
        normal_groups: false

+        # If a backfilled chat is older than this number of hours, mark it as read even if it's unread on Telegram.
+        # Set to -1 to let any chat be unread.
+        unread_hours_threshold: 720
+
+        # Forward backfilling limits. These apply to both MSC2716 and legacy backfill.
+        #
+        # Using a negative initial limit is not recommended, as it would try to backfill everything in a single batch.
+        # MSC2716 and the incremental settings are meant for backfilling everything incrementally rather than at once.
+        forward:
+            # Number of messages to backfill immediately after creating a portal.
+            initial_limit: 10
+            # Number of messages to backfill when syncing chats.
+            sync_limit: 100
+
+        # Settings for incremental backfill of history. These only apply when using MSC2716.
+        incremental:
+            # Maximum number of messages to backfill per batch.
+            messages_per_batch: 100
+            # The number of seconds to wait after backfilling the batch of messages.
+            post_batch_delay: 20
+            # The maximum number of batches to backfill per portal, split by the chat type.
+            # If set to -1, all messages in the chat will eventually be backfilled.
+            max_batches:
+                # Direct chats
+                user: -1
+                # Normal groups. Note that the normal_groups option above must be enabled
+                # for these to be backfilled.
+                normal_group: -1
+                # Supergroups
+                supergroup: 10
+                # Broadcast channels
+                channel: -1
+
    # Overrides for base power levels.
    initial_power_level_overrides:
        user: {}
@@ -455,6 +540,12 @@ telegram:
    # (Optional) Create your own bot at https://t.me/BotFather
    bot_token: disabled

+    # Should the bridge request missed updates from Telegram when restarting?
+    catch_up: true
+    # Should incoming updates be handled sequentially to make sure order is preserved on Matrix?
+    sequential_updates: true
+    exit_on_update_error: false
+
    # Telethon connection options.
    connection:
        # The timeout in seconds to be used when connecting.
@@ -480,7 +571,7 @@ telegram:
    # Device info sent to Telegram.
    device_info:
        # "auto" = OS name+version.
-        device_model: auto
+        device_model: mautrix-telegram
        # "auto" = Telethon version.
        system_version: auto
        # "auto" = mautrix-telegram version.
@@ -1,2 +1,2 @@
 from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram
-from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
+from .from_telegram import telegram_to_matrix
@@ -18,7 +18,7 @@ from __future__ import annotations
 import re

 from telethon import TelegramClient
-from telethon.helpers import add_surrogate, del_surrogate
+from telethon.helpers import add_surrogate, del_surrogate, strip_text
 from telethon.tl.types import MessageEntityItalic, TypeMessageEntity

 from mautrix.types import MessageEventContent, RoomID
@@ -59,7 +59,7 @@ async def matrix_to_telegram(
    if html is not None:
        return await _matrix_html_to_telegram(client, html)
    elif text is not None:
-        return _matrix_text_to_telegram(text), []
+        return _matrix_text_to_telegram(text)
    else:
        raise ValueError("text or html must be provided to convert formatting")

@@ -73,8 +73,8 @@ async def _matrix_html_to_telegram(
        html = not_command_regex.sub(r"\1", html)

        parsed = await MatrixParser(client).parse(add_surrogate(html))
-        text = del_surrogate(parsed.text.strip())
-        text, entities = _cut_long_message(text, parsed.telegram_entities)
+        text, entities = _cut_long_message(parsed.text, parsed.telegram_entities)
+        text = del_surrogate(strip_text(text, entities))

        return text, entities
    except Exception as e:
@@ -98,8 +98,13 @@ def _cut_long_message(
    return message, entities


-def _matrix_text_to_telegram(text: str) -> str:
+def _matrix_text_to_telegram(text: str) -> tuple[str, list[TypeMessageEntity]]:
    text = command_regex.sub(r"/\1", text)
    text = text.replace("\t", " " * 4)
    text = not_command_regex.sub(r"\1", text)
-    return text
+    entities = []
+    surrogated_text = add_surrogate(text)
+    if len(surrogated_text) > MAX_LENGTH:
+        surrogated_text, entities = _cut_long_message(surrogated_text, entities)
+        text = del_surrogate(surrogated_text)
+    return text, entities
@@ -20,14 +20,18 @@ import logging
 import re

 from telethon.errors import RPCError
-from telethon.helpers import add_surrogate, del_surrogate, within_surrogate
+from telethon.helpers import add_surrogate, del_surrogate
 from telethon.tl.custom import Message
 from telethon.tl.types import (
+    Channel,
+    InputPeerChannelFromMessage,
+    InputPeerUserFromMessage,
    MessageEntityBlockquote,
    MessageEntityBold,
    MessageEntityBotCommand,
    MessageEntityCashtag,
    MessageEntityCode,
+    MessageEntityCustomEmoji,
    MessageEntityEmail,
    MessageEntityHashtag,
    MessageEntityItalic,
@@ -46,41 +50,47 @@ from telethon.tl.types import (
    PeerUser,
    SponsoredMessage,
    TypeMessageEntity,
+    User,
 )

-from mautrix.appservice import IntentAPI
-from mautrix.types import (
-    EventType,
-    Format,
-    InReplyTo,
-    MessageType,
-    RelatesTo,
-    TextMessageEventContent,
-)
+from mautrix.types import Format, MessageType, TextMessageEventContent

 from .. import abstract_user as au, portal as po, puppet as pu, user as u
-from ..db import Message as DBMessage
+from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
+from ..tgclient import MautrixTelegramClient
 from ..types import TelegramID
+from ..util.file_transfer import UnicodeCustomEmoji, transfer_custom_emojis_to_matrix

 log: logging.Logger = logging.getLogger("mau.fmt.tg")


-async def telegram_reply_to_matrix(evt: Message, source: au.AbstractUser) -> RelatesTo | None:
-    if evt.reply_to:
-        space = (
-            evt.peer_id.channel_id
-            if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
-            else source.tgid
-        )
-        msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
-        if msg:
-            return RelatesTo(in_reply_to=InReplyTo(event_id=msg.mxid))
-    return None
+async def _get_fwd_entity(client: MautrixTelegramClient, evt: Message) -> Channel | User | None:
+    try:
+        return await client.get_entity(evt.fwd_from.from_id)
+    except (ValueError, RPCError) as e:
+        try:
+            input_peer = await client.get_input_entity(evt.peer_id)
+            if isinstance(evt.fwd_from.from_id, PeerUser):
+                return await client.get_entity(
+                    InputPeerUserFromMessage(
+                        peer=input_peer, msg_id=evt.id, user_id=evt.fwd_from.from_id.user_id
+                    )
+                )
+            elif isinstance(evt.fwd_from.from_id, PeerChannel):
+                return await client.get_entity(
+                    InputPeerChannelFromMessage(
+                        peer=input_peer, msg_id=evt.id, channel_id=evt.fwd_from.from_id.channel_id
+                    )
+                )
+        except (ValueError, RPCError) as e:
+            pass
+        return None


 async def _add_forward_header(
-    source: au.AbstractUser, content: TextMessageEventContent, fwd_from: MessageFwdHeader
+    client: MautrixTelegramClient, content: TextMessageEventContent, evt: Message
 ) -> None:
+    fwd_from = evt.fwd_from
    fwd_from_html, fwd_from_text = None, None
    if isinstance(fwd_from.from_id, PeerUser):
        user = await u.User.get_by_tgid(TelegramID(fwd_from.from_id.user_id))
@@ -99,12 +109,11 @@ async def _add_forward_header(
                )

        if not fwd_from_text:
-            try:
-                user = await source.client.get_entity(fwd_from.from_id)
-                if user:
-                    fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
-                    fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
-            except (ValueError, RPCError):
+            user = await _get_fwd_entity(client, evt)
+            if user:
+                fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+            else:
                fwd_from_text = fwd_from_html = "unknown user"
    elif isinstance(fwd_from.from_id, (PeerChannel, PeerChat)):
        from_id = (
@@ -122,12 +131,11 @@ async def _add_forward_header(
            else:
                fwd_from_html = f"channel <b>{escape(fwd_from_text)}</b>"
        else:
-            try:
-                channel = await source.client.get_entity(fwd_from.from_id)
-                if channel:
-                    fwd_from_text = f"channel {channel.title}"
-                    fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
-            except (ValueError, RPCError):
+            channel = await _get_fwd_entity(client, evt)
+            if channel:
+                fwd_from_text = f"channel {channel.title}"
+                fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
+            else:
                fwd_from_text = fwd_from_html = "unknown channel"
    elif fwd_from.from_name:
        fwd_from_text = fwd_from.from_name
@@ -145,49 +153,46 @@ async def _add_forward_header(
    )


-async def _add_reply_header(
-    source: au.AbstractUser, content: TextMessageEventContent, evt: Message, main_intent: IntentAPI
+class ReuploadedCustomEmoji(MessageEntityCustomEmoji):
+    file: DBTelegramFile
+
+    def __init__(self, parent: MessageEntityCustomEmoji, file: DBTelegramFile) -> None:
+        super().__init__(parent.offset, parent.length, parent.document_id)
+        self.file = file
+
+
+async def _convert_custom_emoji(
+    source: au.AbstractUser,
+    entities: list[TypeMessageEntity],
+    client: MautrixTelegramClient | None = None,
 ) -> None:
-    space = (
-        evt.peer_id.channel_id
-        if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
-        else source.tgid
-    )
-
-    msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
-    if not msg:
-        return
-
-    try:
-        event = await main_intent.get_event(msg.mx_room, msg.mxid)
-        if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
-            event = await source.bridge.matrix.e2ee.decrypt(event)
-        if isinstance(event.content, TextMessageEventContent):
-            event.content.trim_reply_fallback()
-        puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
-        content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
-    except Exception:
-        log.exception("Failed to get event to add reply fallback")
-        content.set_reply(msg.mxid)
+    emoji_ids = [
+        entity.document_id for entity in entities if isinstance(entity, MessageEntityCustomEmoji)
+    ]
+    custom_emojis = await transfer_custom_emojis_to_matrix(source, emoji_ids, client=client)
+    if len(custom_emojis) > 0:
+        for i, entity in enumerate(entities):
+            if isinstance(entity, MessageEntityCustomEmoji):
+                entities[i] = ReuploadedCustomEmoji(entity, custom_emojis[entity.document_id])


 async def telegram_to_matrix(
    evt: Message | SponsoredMessage,
    source: au.AbstractUser,
-    main_intent: IntentAPI | None = None,
-    prefix_text: str | None = None,
-    prefix_html: str | None = None,
+    client: MautrixTelegramClient | None = None,
    override_text: str = None,
    override_entities: list[TypeMessageEntity] = None,
-    no_reply_fallback: bool = False,
    require_html: bool = False,
 ) -> TextMessageEventContent:
+    if not client:
+        client = source.client
    content = TextMessageEventContent(
        msgtype=MessageType.TEXT,
-        body=add_surrogate(override_text or evt.message),
+        body=override_text or evt.message,
    )
    entities = override_entities or evt.entities
    if entities:
+        await _convert_custom_emoji(source, entities, client=client)
        content.format = Format.HTML
        html = await _telegram_entities_to_matrix_catch(add_surrogate(content.body), entities)
        content.formatted_body = del_surrogate(html)
@@ -195,17 +200,8 @@ async def telegram_to_matrix(
    if require_html:
        content.ensure_has_html()

-    if prefix_html:
-        content.ensure_has_html()
-        content.formatted_body = prefix_html + content.formatted_body
-    if prefix_text:
-        content.body = prefix_text + content.body
-
    if getattr(evt, "fwd_from", None):
-        await _add_forward_header(source, content, evt.fwd_from)
-
-    if getattr(evt, "reply_to", None) and not no_reply_fallback:
-        await _add_reply_header(source, content, evt, main_intent)
+        await _add_forward_header(client, content, evt)

    if isinstance(evt, Message) and evt.post and evt.post_author:
        content.ensure_has_html()
@@ -225,9 +221,20 @@ async def _telegram_entities_to_matrix_catch(text: str, entities: list[TypeMessa
    return "[failed conversion in _telegram_entities_to_matrix]"


+def within_surrogate(text, index):
+    """
+    `True` if ``index`` is within a surrogate (before and after it, not at!).
+    """
+    return (
+        1 < index < len(text)  # in bounds
+        and "\ud800" <= text[index - 1] <= "\udbff"  # current is low surrogate
+        and "\udc00" <= text[index] <= "\udfff"  # previous is high surrogate
+    )
+
+
 async def _telegram_entities_to_matrix(
    text: str,
-    entities: list[TypeMessageEntity],
+    entities: list[TypeMessageEntity | ReuploadedCustomEmoji],
    offset: int = 0,
    length: int = None,
    in_codeblock: bool = False,
@@ -256,9 +263,9 @@ async def _telegram_entities_to_matrix(
        elif relative_offset < last_offset:
            continue

-        while within_surrogate(text, relative_offset, length=length):
+        while within_surrogate(text, relative_offset):
            relative_offset += 1
-        while within_surrogate(text, relative_offset + entity.length, length=length):
+        while within_surrogate(text, relative_offset + entity.length):
            entity.length += 1

        skip_entity = False
@@ -301,6 +308,17 @@ async def _telegram_entities_to_matrix(
            await _parse_url(
                html, entity_text, entity.url if entity_type == MessageEntityTextUrl else None
            )
+        elif entity_type == MessageEntityCustomEmoji:
+            html.append(entity_text)
+        elif entity_type == ReuploadedCustomEmoji:
+            if isinstance(entity.file, UnicodeCustomEmoji):
+                html.append(entity.file.emoji)
+            else:
+                html.append(
+                    f"<img data-mx-emoticon data-mau-animated-emoji"
+                    f' src="{escape(entity.file.mxc)}" height="32" width="32"'
+                    f' alt="{entity_text}" title="{entity_text}"/>'
+                )
        elif entity_type in (
            MessageEntityBotCommand,
            MessageEntityHashtag,
@@ -378,7 +396,7 @@ message_link_regex = re.compile(
 )


-async def _parse_url(html: list[str], entity_text: str, url: str):
+async def _parse_url(html: list[str], entity_text: str, url: str) -> None:
    url = escape(url) if url else entity_text
    if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
        url = "http://" + url
@@ -16,15 +16,14 @@
 from __future__ import annotations

 from typing import TYPE_CHECKING
+import sys

 from mautrix.bridge import BaseMatrixHandler
-from mautrix.errors import MatrixError
 from mautrix.types import (
    Event,
    EventID,
    EventType,
    MemberStateEventContent,
-    MessageType,
    PresenceEvent,
    PresenceState,
    ReactionEvent,
@@ -36,12 +35,13 @@ from mautrix.types import (
    RoomTopicStateEventContent as TopicContent,
    SingleReceiptEventContent,
    StateEvent,
-    TextMessageEventContent,
    TypingEvent,
    UserID,
 )

 from . import commands as com, portal as po, puppet as pu, user as u
+from .commands.portal.util import get_initial_state, user_has_power_level, warn_missing_power
+from .types import TelegramID

 if TYPE_CHECKING:
    from .__main__ import TelegramBridge
@@ -61,6 +61,21 @@ class MatrixHandler(BaseMatrixHandler):

        self._previously_typing = {}

+    async def check_versions(self) -> None:
+        await super().check_versions()
+        if self.config["bridge.backfill.msc2716"] and not (
+            support := self.versions.supports("org.matrix.msc2716")
+        ):
+            self.log.fatal(
+                "Backfilling with MSC2716 is enabled in bridge config, but "
+                + (
+                    "batch sending is not enabled on homeserver"
+                    if support is False
+                    else "homeserver does not support batch sending"
+                )
+            )
+            sys.exit(18)
+
    async def handle_puppet_group_invite(
        self,
        room_id: RoomID,
@@ -69,16 +84,76 @@ class MatrixHandler(BaseMatrixHandler):
        evt: StateEvent,
        members: list[UserID],
    ) -> None:
-        if self.az.bot_mxid not in members:
+        double_puppet = await pu.Puppet.get_by_custom_mxid(invited_by.mxid)
+        if (
+            not double_puppet
+            or self.az.bot_mxid in members
+            or not self.config["bridge.create_group_on_invite"]
+        ):
+            if self.az.bot_mxid not in members:
+                await puppet.default_mxid_intent.leave_room(
+                    room_id,
+                    reason="This ghost does not join multi-user rooms without the bridge bot.",
+                )
+            else:
+                await puppet.default_mxid_intent.send_notice(
+                    room_id,
+                    "This ghost will remain inactive "
+                    "until a Telegram chat is created for this room.",
+                )
+            return
+        elif not await user_has_power_level(
+            evt.room_id, double_puppet.intent, invited_by, "bridge"
+        ):
            await puppet.default_mxid_intent.leave_room(
-                room_id, reason="This ghost does not join multi-user rooms without the bridge bot."
+                room_id, reason="You do not have the permissions to bridge this room."
            )
-        else:
-            await puppet.default_mxid_intent.send_notice(
+            return
+
+        await double_puppet.intent.invite_user(room_id, self.az.bot_mxid)
+
+        title, about, levels, encrypted = await get_initial_state(double_puppet.intent, room_id)
+        if not title:
+            await puppet.default_mxid_intent.leave_room(
+                room_id, reason="Please set a title before inviting Telegram ghosts."
+            )
+            return
+
+        portal = po.Portal(
+            tgid=TelegramID(0),
+            tg_receiver=TelegramID(0),
+            peer_type="channel",
+            mxid=evt.room_id,
+            title=title,
+            about=about,
+            encrypted=encrypted,
+        )
+        await portal.az.intent.ensure_joined(room_id)
+        levels = await portal.az.intent.get_power_levels(room_id)
+        invited_by_level = levels.get_user_level(invited_by.mxid)
+        if invited_by_level > levels.get_user_level(self.az.bot_mxid):
+            levels.users[self.az.bot_mxid] = 100 if invited_by_level >= 100 else invited_by_level
+            await double_puppet.intent.set_power_levels(room_id, levels)
+
+        invites, errors = await portal.get_telegram_users_in_matrix_room(
+            invited_by, pre_create=True
+        )
+        if len(errors) > 0:
+            error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
+            await portal.az.intent.send_notice(
                room_id,
-                "This ghost will remain inactive until a Telegram chat is created for this room.",
+                f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
+                "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
+                "those users.",
            )

+        try:
+            await portal.create_telegram_chat(invited_by, invites=invites, supergroup=True)
+        except ValueError as e:
+            await portal.delete()
+            await portal.az.intent.send_notice(room_id, e.args[0])
+            return
+
    async def handle_invite(
        self, room_id: RoomID, user_id: UserID, inviter: u.User, event_id: EventID
    ) -> None:
@@ -1,5 +1,5 @@
 from .deduplication import PortalDedup
-from .media_fallback import make_contact_event_content, make_dice_event_content
+from .message_convert import ConvertedMessage, TelegramMessageConverter
 from .participants import get_users
 from .power_levels import get_base_power_levels, participants_to_power_levels
 from .send_lock import PortalReactionLock, PortalSendLock
@@ -96,13 +96,13 @@ class PortalDedup:
                )
                yield media_hash_func(event.media)

-    def _hash_event(self, event: TypeMessage) -> bytes:
+    def hash_event(self, event: TypeMessage) -> bytes:
        return hashlib.sha256(
            "-".join(str(a) for a in self._hash_content(event)).encode("utf-8")
        ).digest()

    def check_action(self, event: TypeMessage) -> bool:
-        dedup_id = self._hash_event(event) if self._always_force_hash else event.id
+        dedup_id = self.hash_event(event) if self._always_force_hash else event.id
        if dedup_id in self._dedup_action:
            return True

@@ -116,7 +116,7 @@ class PortalDedup:
        expected_mxid: DedupMXID | None = None,
        force_hash: bool = False,
    ) -> tuple[bytes, DedupMXID | None]:
-        evt_hash = self._hash_event(event)
+        evt_hash = self.hash_event(event)
        dedup_id = evt_hash if self._always_force_hash or force_hash else event.id
        try:
            found_mxid = self._dedup_mxid[dedup_id]
@@ -133,7 +133,7 @@ class PortalDedup:
    def check(
        self, event: TypeMessage, mxid: DedupMXID = None, force_hash: bool = False
    ) -> tuple[bytes, DedupMXID | None]:
-        evt_hash = self._hash_event(event)
+        evt_hash = self.hash_event(event)
        dedup_id = evt_hash if self._always_force_hash or force_hash else event.id
        if dedup_id in self._dedup:
            return evt_hash, self._dedup_mxid[dedup_id]
@@ -1,133 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-import html
-
-from telethon.tl.types import MessageMediaContact, MessageMediaDice, PeerUser
-
-from mautrix.types import Format, MessageType, TextMessageEventContent
-
-from .. import abstract_user as au, puppet as pu
-from ..types import TelegramID
-
-try:
-    import phonenumbers
-except ImportError:
-    phonenumbers = None
-
-
-def _format_dice(roll: MessageMediaDice) -> str:
-    if roll.emoticon == "\U0001F3B0":
-        emojis = {
-            0: "\U0001F36B",  # "🍫",
-            1: "\U0001F352",  # "🍒",
-            2: "\U0001F34B",  # "🍋",
-            3: "7\ufe0f\u20e3",  # "7️⃣",
-        }
-        res = roll.value - 1
-        slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
-        return f"{slot1} {slot2} {slot3} ({roll.value})"
-    elif roll.emoticon == "\u26BD":
-        results = {
-            1: "miss",
-            2: "hit the woodwork",
-            3: "goal",  # seems to go in through the center
-            4: "goal",
-            5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
-        }
-    elif roll.emoticon == "\U0001F3B3":
-        results = {
-            1: "miss",
-            2: "1 pin down",
-            3: "3 pins down, split",
-            4: "4 pins down, split",
-            5: "5 pins down",
-            6: "strike 🎉",
-        }
-    # elif roll.emoticon == "\U0001F3C0":
-    #     results = {
-    #         2: "rolled off",
-    #         3: "stuck",
-    #     }
-    # elif roll.emoticon == "\U0001F3AF":
-    #     results = {
-    #         1: "bounced off",
-    #         2: "outer rim",
-    #
-    #         6: "bullseye",
-    #     }
-    else:
-        return str(roll.value)
-    return f"{results[roll.value]} ({roll.value})"
-
-
-def make_dice_event_content(roll: MessageMediaDice) -> TextMessageEventContent:
-    emoji_text = {
-        "\U0001F3AF": " Dart throw",
-        "\U0001F3B2": " Dice roll",
-        "\U0001F3C0": " Basketball throw",
-        "\U0001F3B0": " Slot machine",
-        "\U0001F3B3": " Bowling",
-        "\u26BD": " Football kick",
-    }
-    text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {_format_dice(roll)}"
-    content = TextMessageEventContent(
-        msgtype=MessageType.TEXT, format=Format.HTML, body=text, formatted_body=f"<h4>{text}</h4>"
-    )
-    content["net.maunium.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
-    return content
-
-
-async def make_contact_event_content(
-    source: au.AbstractUser, contact: MessageMediaContact
-) -> TextMessageEventContent:
-    name = " ".join(x for x in [contact.first_name, contact.last_name] if x)
-    formatted_phone = f"+{contact.phone_number}"
-    if phonenumbers is not None:
-        try:
-            parsed = phonenumbers.parse(formatted_phone)
-            fmt = phonenumbers.PhoneNumberFormat.INTERNATIONAL
-            formatted_phone = phonenumbers.format_number(parsed, fmt)
-        except phonenumbers.NumberParseException:
-            pass
-    content = TextMessageEventContent(
-        msgtype=MessageType.TEXT,
-        body=f"Shared contact info for {name}: {formatted_phone}",
-    )
-    content["net.maunium.telegram.contact"] = {
-        "user_id": contact.user_id,
-        "first_name": contact.first_name,
-        "last_name": contact.last_name,
-        "phone_number": contact.phone_number,
-        "vcard": contact.vcard,
-    }
-
-    puppet = await pu.Puppet.get_by_tgid(TelegramID(contact.user_id))
-    if not puppet.displayname:
-        try:
-            entity = await source.client.get_entity(PeerUser(contact.user_id))
-            await puppet.update_info(source, entity)
-        except Exception as e:
-            source.log.warning(f"Failed to sync puppet info of received contact: {e}")
-    else:
-        content.format = Format.HTML
-        content.formatted_body = (
-            f"Shared contact info for "
-            f"<a href='https://matrix.to/#/{puppet.mxid}'>{html.escape(name)}</a>: "
-            f"{html.escape(formatted_phone)}"
-        )
-    return content
@@ -0,0 +1,832 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Any, NamedTuple
+import base64
+import codecs
+import hashlib
+import html
+import mimetypes
+import unicodedata
+
+from attr import dataclass
+from telethon.tl.types import (
+    Document,
+    DocumentAttributeAnimated,
+    DocumentAttributeAudio,
+    DocumentAttributeFilename,
+    DocumentAttributeImageSize,
+    DocumentAttributeSticker,
+    DocumentAttributeVideo,
+    Game,
+    InputPhotoFileLocation,
+    Message,
+    MessageEntityPre,
+    MessageMediaContact,
+    MessageMediaDice,
+    MessageMediaDocument,
+    MessageMediaGame,
+    MessageMediaGeo,
+    MessageMediaGeoLive,
+    MessageMediaPhoto,
+    MessageMediaPoll,
+    MessageMediaUnsupported,
+    MessageMediaVenue,
+    MessageMediaWebPage,
+    PeerChannel,
+    PeerUser,
+    Photo,
+    PhotoCachedSize,
+    PhotoEmpty,
+    PhotoSize,
+    PhotoSizeEmpty,
+    PhotoSizeProgressive,
+    Poll,
+    TypeDocumentAttribute,
+    TypePhotoSize,
+    WebPage,
+)
+from telethon.utils import decode_waveform
+
+from mautrix.appservice import IntentAPI
+from mautrix.types import (
+    EventID,
+    EventType,
+    Format,
+    ImageInfo,
+    LocationMessageEventContent,
+    MediaMessageEventContent,
+    MessageEventContent,
+    MessageType,
+    TextMessageEventContent,
+    ThumbnailInfo,
+)
+from mautrix.util.logging import TraceLogger
+
+from .. import abstract_user as au, formatter, matrix as m, portal as po, puppet as pu, util
+from ..config import Config
+from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
+from ..tgclient import MautrixTelegramClient
+from ..types import TelegramID
+from ..util import sane_mimetypes
+
+try:
+    import phonenumbers
+except ImportError:
+    phonenumbers = None
+
+
+@dataclass
+class ConvertedMessage:
+    content: MessageEventContent
+    caption: MessageEventContent | None = None
+    type: EventType = EventType.ROOM_MESSAGE
+    disappear_seconds: int | None = None
+    disappear_start_immediately: bool = False
+
+
+class DocAttrs(NamedTuple):
+    name: str | None
+    mime_type: str | None
+    is_sticker: bool
+    sticker_alt: str | None
+    width: int
+    height: int
+    is_gif: bool
+    is_audio: bool
+    is_voice: bool
+    duration: int
+    waveform: bytes
+
+
+BEEPER_LINK_PREVIEWS_KEY = "com.beeper.linkpreviews"
+BEEPER_IMAGE_ENCRYPTION_KEY = "beeper:image:encryption"
+
+
+class TelegramMessageConverter:
+    portal: po.Portal
+    matrix: m.MatrixHandler
+    config: Config
+    command_prefix: str
+    log: TraceLogger
+
+    def __init__(self, portal: po.Portal) -> None:
+        self.portal = portal
+        self.matrix = portal.matrix
+        self.config = portal.config
+        self.command_prefix = self.config["bridge.command_prefix"]
+        self.log = portal.log.getChild("msg_conv")
+
+        self._media_converters = {
+            MessageMediaPhoto: self._convert_photo,
+            MessageMediaDocument: self._convert_document,
+            MessageMediaGeo: self._convert_location,
+            MessageMediaGeoLive: self._convert_location,
+            MessageMediaVenue: self._convert_location,
+            MessageMediaPoll: self._convert_poll,
+            MessageMediaDice: self._convert_dice,
+            MessageMediaUnsupported: self._convert_unsupported,
+            MessageMediaGame: self._convert_game,
+            MessageMediaContact: self._convert_contact,
+        }
+        self._allowed_media = tuple(self._media_converters.keys())
+
+    async def convert(
+        self,
+        source: au.AbstractUser,
+        intent: IntentAPI,
+        is_bot: bool,
+        evt: Message,
+        no_reply_fallback: bool = False,
+        deterministic_reply_id: bool = False,
+        client: MautrixTelegramClient | None = None,
+    ) -> ConvertedMessage | None:
+        if not client:
+            client = source.client
+        if hasattr(evt, "media") and isinstance(evt.media, self._allowed_media):
+            convert_media = self._media_converters[type(evt.media)]
+            converted = await convert_media(source=source, intent=intent, evt=evt, client=client)
+        elif evt.message:
+            converted = await self._convert_text(source, intent, is_bot, evt, client)
+        else:
+            self.log.debug("Unhandled Telegram message %d", evt.id)
+            return
+        if converted:
+            if evt.ttl_period and not converted.disappear_seconds:
+                converted.disappear_seconds = evt.ttl_period
+                converted.disappear_start_immediately = True
+            converted.content.external_url = self._get_external_url(evt)
+            converted.content["fi.mau.telegram.source"] = {
+                "space": self.portal.tgid if self.portal.peer_type == "channel" else source.tgid,
+                "chat_id": self.portal.tgid,
+                "peer_type": self.portal.peer_type,
+                "id": evt.id,
+            }
+            if converted.caption:
+                converted.caption["fi.mau.telegram.source"] = converted.content[
+                    "fi.mau.telegram.source"
+                ]
+                converted.caption.external_url = converted.content.external_url
+                if self.portal.get_config("caption_in_message"):
+                    self._caption_to_message(converted)
+            await self._set_reply(
+                source,
+                evt,
+                converted.content,
+                no_fallback=no_reply_fallback,
+                deterministic_id=deterministic_reply_id,
+            )
+        return converted
+
+    @staticmethod
+    def _caption_to_message(converted: ConvertedMessage) -> None:
+        content, caption = converted.content, converted.caption
+        converted.caption = None
+
+        content["filename"] = content.body
+        content["org.matrix.msc1767.caption"] = {
+            "org.matrix.msc1767.text": caption.body,
+        }
+        content.body = caption.body
+        if caption.format == Format.HTML:
+            content["org.matrix.msc1767.caption"][
+                "org.matrix.msc1767.html"
+            ] = caption.formatted_body
+            content["formatted_body"] = caption.formatted_body
+            content["format"] = Format.HTML.value
+
+    def _get_external_url(self, evt: Message) -> str | None:
+        if self.portal.peer_type == "channel" and self.portal.username is not None:
+            return f"https://t.me/{self.portal.username}/{evt.id}"
+        elif self.portal.peer_type != "user":
+            return f"https://t.me/c/{self.portal.tgid}/{evt.id}"
+        return None
+
+    @staticmethod
+    def _int_to_bytes(i: int) -> bytes:
+        return codecs.decode(f"{i:010x}", "hex")
+
+    def _encode_msgid(self, source: au.AbstractUser, evt: Message) -> str:
+        if self.portal.peer_type == "channel":
+            play_id = b"c" + self._int_to_bytes(self.portal.tgid) + self._int_to_bytes(evt.id)
+        elif self.portal.peer_type == "chat":
+            play_id = (
+                b"g"
+                + self._int_to_bytes(self.portal.tgid)
+                + self._int_to_bytes(evt.id)
+                + self._int_to_bytes(source.tgid)
+            )
+        elif self.portal.peer_type == "user":
+            play_id = b"u" + self._int_to_bytes(self.portal.tgid) + self._int_to_bytes(evt.id)
+        else:
+            raise ValueError("Portal has invalid peer type")
+        return base64.b64encode(play_id).decode("utf-8").rstrip("=")
+
+    def deterministic_event_id(self, space: TelegramID, msg_id: TelegramID) -> EventID:
+        hash_content = f"{self.portal.mxid}/telegram/{space}/{msg_id}"
+        hashed = hashlib.sha256(hash_content.encode("utf-8")).digest()
+        b64hash = base64.urlsafe_b64encode(hashed).decode("utf-8").rstrip("=")
+        return EventID(f"${b64hash}:telegram.org")
+
+    async def _set_reply(
+        self,
+        source: au.AbstractUser,
+        evt: Message,
+        content: MessageEventContent,
+        no_fallback: bool = False,
+        deterministic_id: bool = False,
+    ) -> None:
+        if not evt.reply_to:
+            return
+        space = (
+            evt.peer_id.channel_id
+            if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
+            else source.tgid
+        )
+        reply_to_id = TelegramID(evt.reply_to.reply_to_msg_id)
+        msg = await DBMessage.get_one_by_tgid(reply_to_id, space)
+        if not msg or msg.mx_room != self.portal.mxid:
+            if deterministic_id:
+                content.set_reply(self.deterministic_event_id(space, reply_to_id))
+            return
+        elif not isinstance(content, TextMessageEventContent) or no_fallback:
+            # Not a text message, just set the reply metadata and return
+            content.set_reply(msg.mxid)
+            return
+
+        # Text message, try to fetch original message to generate reply fallback.
+        try:
+            event = await self.portal.main_intent.get_event(msg.mx_room, msg.mxid)
+            if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
+                event = await source.bridge.matrix.e2ee.decrypt(event)
+            if isinstance(event.content, TextMessageEventContent):
+                event.content.trim_reply_fallback()
+            puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
+            content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
+        except Exception:
+            self.log.exception("Failed to get event to add reply fallback")
+            content.set_reply(msg.mxid)
+
+    @staticmethod
+    def _photo_size_key(photo: TypePhotoSize) -> int:
+        if isinstance(photo, PhotoSize):
+            return photo.size
+        elif isinstance(photo, PhotoSizeProgressive):
+            return max(photo.sizes)
+        elif isinstance(photo, PhotoSizeEmpty):
+            return 0
+        else:
+            return len(photo.bytes)
+
+    @classmethod
+    def get_largest_photo_size(
+        cls, photo: Photo | Document
+    ) -> tuple[InputPhotoFileLocation | None, TypePhotoSize | None]:
+        if (
+            not photo
+            or isinstance(photo, PhotoEmpty)
+            or (isinstance(photo, Document) and not photo.thumbs)
+        ):
+            return None, None
+
+        largest = max(
+            photo.thumbs if isinstance(photo, Document) else photo.sizes, key=cls._photo_size_key
+        )
+        return (
+            InputPhotoFileLocation(
+                id=photo.id,
+                access_hash=photo.access_hash,
+                file_reference=photo.file_reference,
+                thumb_size=largest.type,
+            ),
+            largest,
+        )
+
+    async def _webpage_to_beeper_link_preview(
+        self, source: au.AbstractUser, intent: IntentAPI, webpage: WebPage
+    ) -> dict[str, Any]:
+        beeper_link_preview: dict[str, Any] = {
+            "matched_url": webpage.url,
+            "og:title": webpage.title,
+            "og:url": webpage.url,
+            "og:description": webpage.description,
+        }
+
+        # Upload an image corresponding to the link preview if it exists.
+        if webpage.photo:
+            loc, largest_size = self.get_largest_photo_size(webpage.photo)
+            if loc is None:
+                return beeper_link_preview
+            beeper_link_preview["og:image:height"] = largest_size.h
+            beeper_link_preview["og:image:width"] = largest_size.w
+            file = await util.transfer_file_to_matrix(
+                source.client,
+                intent,
+                loc,
+                encrypt=self.portal.encrypted,
+                async_upload=self.config["homeserver.async_media"],
+            )
+
+            if file.decryption_info:
+                beeper_link_preview[BEEPER_IMAGE_ENCRYPTION_KEY] = file.decryption_info.serialize()
+            else:
+                beeper_link_preview["og:image"] = file.mxc
+
+        return beeper_link_preview
+
+    async def _convert_text(
+        self,
+        source: au.AbstractUser,
+        intent: IntentAPI,
+        is_bot: bool,
+        evt: Message,
+        client: MautrixTelegramClient,
+    ) -> ConvertedMessage:
+        content = await formatter.telegram_to_matrix(evt, source, client)
+        if is_bot and self.portal.get_config("bot_messages_as_notices"):
+            content.msgtype = MessageType.NOTICE
+
+        if (
+            hasattr(evt, "media")
+            and isinstance(evt.media, MessageMediaWebPage)
+            and isinstance(evt.media.webpage, WebPage)
+        ):
+            content[BEEPER_LINK_PREVIEWS_KEY] = [
+                await self._webpage_to_beeper_link_preview(source, intent, evt.media.webpage)
+            ]
+
+        return ConvertedMessage(content=content)
+
+    async def _convert_photo(
+        self,
+        source: au.AbstractUser,
+        intent: IntentAPI,
+        evt: Message,
+        client: MautrixTelegramClient,
+    ) -> ConvertedMessage | None:
+        media: MessageMediaPhoto = evt.media
+        if media.photo is None and media.ttl_seconds:
+            return ConvertedMessage(
+                content=TextMessageEventContent(
+                    msgtype=MessageType.NOTICE, body="Photo has expired"
+                )
+            )
+        loc, largest_size = self.get_largest_photo_size(media.photo)
+        if loc is None:
+            return ConvertedMessage(
+                content=TextMessageEventContent(
+                    msgtype=MessageType.TEXT,
+                    body="Failed to bridge image",
+                )
+            )
+        file = await util.transfer_file_to_matrix(
+            client,
+            intent,
+            loc,
+            encrypt=self.portal.encrypted,
+            async_upload=self.config["homeserver.async_media"],
+        )
+        if not file:
+            return None
+        info = ImageInfo(
+            height=largest_size.h,
+            width=largest_size.w,
+            orientation=0,
+            mimetype=file.mime_type,
+            size=self._photo_size_key(largest_size),
+        )
+        if media.spoiler:
+            info["fi.mau.telegram.spoiler"] = True
+        ext = sane_mimetypes.guess_extension(file.mime_type)
+        name = f"disappearing_image{ext}" if media.ttl_seconds else f"image{ext}"
+        content = MediaMessageEventContent(
+            msgtype=MessageType.IMAGE,
+            info=info,
+            body=name,
+        )
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+        caption_content = (
+            await formatter.telegram_to_matrix(evt, source, client) if evt.message else None
+        )
+        return ConvertedMessage(
+            content=content,
+            caption=caption_content,
+            disappear_seconds=media.ttl_seconds,
+        )
+
+    async def _convert_document(
+        self,
+        source: au.AbstractUser,
+        intent: IntentAPI,
+        evt: Message,
+        client: MautrixTelegramClient,
+    ) -> ConvertedMessage | None:
+        document = evt.media.document
+
+        attrs = _parse_document_attributes(document.attributes)
+
+        if document.size > self.matrix.media_config.upload_size:
+            name = attrs.name or ""
+            caption = f"\n{evt.message}" if evt.message else ""
+            return ConvertedMessage(
+                content=TextMessageEventContent(
+                    msgtype=MessageType.NOTICE, body=f"Too large file {name}{caption}"
+                )
+            )
+
+        thumb_loc, thumb_size = self.get_largest_photo_size(document)
+        if thumb_size and not isinstance(thumb_size, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb_size)}")
+            thumb_loc = None
+            thumb_size = None
+        parallel_id = source.tgid if self.config["bridge.parallel_file_transfer"] else None
+        tgs_convert = self.config["bridge.animated_sticker"]
+        file = await util.transfer_file_to_matrix(
+            client,
+            intent,
+            document,
+            thumb_loc,
+            is_sticker=attrs.is_sticker,
+            tgs_convert=tgs_convert,
+            webm_convert=tgs_convert["target"] if tgs_convert["convert_from_webm"] else None,
+            filename=attrs.name,
+            parallel_id=parallel_id,
+            encrypt=self.portal.encrypted,
+            async_upload=self.config["homeserver.async_media"],
+        )
+        if not file:
+            return None
+
+        info, name = _parse_document_meta(evt, file, attrs, thumb_size)
+
+        event_type = EventType.ROOM_MESSAGE
+        # Elements only support images as stickers, so send animated webm stickers as m.video
+        if attrs.is_sticker and file.mime_type.startswith("image/"):
+            event_type = EventType.STICKER
+            # Tell clients to render the stickers as 256x256 if they're bigger
+            if info.width > 256 or info.height > 256:
+                if info.width > info.height:
+                    info.height = int(info.height / (info.width / 256))
+                    info.width = 256
+                else:
+                    info.width = int(info.width / (info.height / 256))
+                    info.height = 256
+            if info.thumbnail_info:
+                info.thumbnail_info.width = info.width
+                info.thumbnail_info.height = info.height
+        if attrs.is_gif or (attrs.is_sticker and info.mimetype == "video/webm"):
+            if attrs.is_gif:
+                info["fi.mau.telegram.gif"] = True
+            else:
+                info["fi.mau.telegram.animated_sticker"] = True
+            info["fi.mau.loop"] = True
+            info["fi.mau.autoplay"] = True
+            info["fi.mau.hide_controls"] = True
+            info["fi.mau.no_audio"] = True
+        if evt.media.spoiler:
+            info["fi.mau.telegram.spoiler"] = True
+        if not name:
+            ext = sane_mimetypes.guess_extension(file.mime_type) or ""
+            name = "unnamed_file" + ext
+
+        content = MediaMessageEventContent(
+            body=name,
+            info=info,
+            msgtype={
+                "video/": MessageType.VIDEO,
+                "audio/": MessageType.AUDIO,
+                "image/": MessageType.IMAGE,
+            }.get(info.mimetype[:6], MessageType.FILE),
+        )
+        if event_type == EventType.STICKER:
+            content.msgtype = None
+        if attrs.is_audio:
+            content["org.matrix.msc1767.audio"] = {"duration": attrs.duration * 1000}
+            if attrs.waveform:
+                content["org.matrix.msc1767.audio"]["waveform"] = [x << 5 for x in attrs.waveform]
+            if attrs.is_voice:
+                content["org.matrix.msc3245.voice"] = {}
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+
+        caption_content = (
+            await formatter.telegram_to_matrix(evt, source, client) if evt.message else None
+        )
+
+        return ConvertedMessage(
+            type=event_type,
+            content=content,
+            caption=caption_content,
+            disappear_seconds=evt.media.ttl_seconds,
+        )
+
+    @staticmethod
+    async def _convert_location(evt: Message, **_) -> ConvertedMessage:
+        long = evt.media.geo.long
+        lat = evt.media.geo.lat
+        long_char = "E" if long > 0 else "W"
+        lat_char = "N" if lat > 0 else "S"
+        geo = f"{round(lat, 6)},{round(long, 6)}"
+
+        body = f"{round(abs(lat), 4)}° {lat_char}, {round(abs(long), 4)}° {long_char}"
+        url = f"https://maps.google.com/?q={geo}"
+
+        if isinstance(evt.media, MessageMediaGeoLive):
+            note = "Live Location (see your Telegram client for live updates)"
+        elif isinstance(evt.media, MessageMediaVenue):
+            note = evt.media.title
+        else:
+            note = "Location"
+
+        content = LocationMessageEventContent(
+            msgtype=MessageType.LOCATION,
+            geo_uri=f"geo:{geo}",
+            body=f"{note}: {body}\n{url}",
+        )
+        content["format"] = str(Format.HTML)
+        content["formatted_body"] = f"{note}: <a href='{url}'>{body}</a>"
+        content["org.matrix.msc3488.location"] = {
+            "uri": content.geo_uri,
+            "description": note,
+        }
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_unsupported(
+        source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
+    ) -> ConvertedMessage:
+        override_text = (
+            "This message is not supported on your version of Mautrix-Telegram. "
+            "Please check https://github.com/mautrix/telegram or ask your "
+            "bridge administrator about possible updates."
+        )
+        content = await formatter.telegram_to_matrix(
+            evt, source, client, override_text=override_text
+        )
+        content.msgtype = MessageType.NOTICE
+        content["fi.mau.telegram.unsupported"] = True
+        return ConvertedMessage(content=content)
+
+    async def _convert_poll(self, source: au.AbstractUser, evt: Message, **_) -> ConvertedMessage:
+        poll: Poll = evt.media.poll
+        poll_id = self._encode_msgid(source, evt)
+
+        _n = 0
+
+        def n() -> int:
+            nonlocal _n
+            _n += 1
+            return _n
+
+        text_answers = "\n".join(f"{n()}. {answer.text}" for answer in poll.answers)
+        html_answers = "\n".join(f"<li>{answer.text}</li>" for answer in poll.answers)
+        vote_command = f"{self.command_prefix} vote {poll_id}"
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT,
+            format=Format.HTML,
+            body=(
+                f"Poll: {poll.question}\n{text_answers}\n"
+                f"Vote with {vote_command} <choice number>"
+            ),
+            formatted_body=(
+                f"<strong>Poll</strong>: {poll.question}<br/>\n"
+                f"<ol>{html_answers}</ol>\n"
+                f"Vote with <code>{vote_command} &lt;choice number&gt;</code>"
+            ),
+        )
+
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_dice(evt: Message, **_) -> ConvertedMessage:
+        roll: MessageMediaDice = evt.media
+        emoji_text = {
+            "\U0001F3AF": " Dart throw",
+            "\U0001F3B2": " Dice roll",
+            "\U0001F3C0": " Basketball throw",
+            "\U0001F3B0": " Slot machine",
+            "\U0001F3B3": " Bowling",
+            "\u26BD": " Football kick",
+        }
+        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {_format_dice(roll)}"
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT,
+            format=Format.HTML,
+            body=text,
+            formatted_body=f"<h4>{text}</h4>",
+        )
+        content["fi.mau.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
+        return ConvertedMessage(content=content)
+
+    async def _convert_game(
+        self, source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
+    ) -> ConvertedMessage:
+        game: Game = evt.media.game
+        play_id = self._encode_msgid(source, evt)
+        command = f"{self.command_prefix} play {play_id}"
+        override_text = f"Run {command} in your bridge management room to play {game.title}"
+        override_entities = [
+            MessageEntityPre(offset=len("Run "), length=len(command), language="")
+        ]
+
+        content = await formatter.telegram_to_matrix(
+            evt, source, client, override_text=override_text, override_entities=override_entities
+        )
+        content.msgtype = MessageType.NOTICE
+        content["fi.mau.telegram.game"] = play_id
+
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_contact(
+        source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
+    ) -> ConvertedMessage:
+        contact: MessageMediaContact = evt.media
+        name = " ".join(x for x in [contact.first_name, contact.last_name] if x)
+        formatted_phone = f"+{contact.phone_number}"
+        if phonenumbers is not None:
+            try:
+                parsed = phonenumbers.parse(formatted_phone)
+                fmt = phonenumbers.PhoneNumberFormat.INTERNATIONAL
+                formatted_phone = phonenumbers.format_number(parsed, fmt)
+            except phonenumbers.NumberParseException:
+                pass
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT,
+            body=f"Shared contact info for {name}: {formatted_phone}",
+        )
+        content["fi.mau.telegram.contact"] = {
+            "user_id": contact.user_id,
+            "first_name": contact.first_name,
+            "last_name": contact.last_name,
+            "phone_number": contact.phone_number,
+            "vcard": contact.vcard,
+        }
+
+        puppet = await pu.Puppet.get_by_tgid(TelegramID(contact.user_id))
+        if not puppet.displayname:
+            try:
+                entity = await client.get_entity(PeerUser(contact.user_id))
+                await puppet.update_info(source, entity, client_override=client)
+            except Exception as e:
+                source.log.warning(f"Failed to sync puppet info of received contact: {e}")
+        else:
+            content.format = Format.HTML
+            content.formatted_body = (
+                f"Shared contact info for "
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>{html.escape(name)}</a>: "
+                f"{html.escape(formatted_phone)}"
+            )
+        return ConvertedMessage(content=content)
+
+
+def _parse_document_attributes(attributes: list[TypeDocumentAttribute]) -> DocAttrs:
+    name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
+    is_gif, is_audio, is_voice, duration, waveform = False, False, False, 0, bytes()
+    for attr in attributes:
+        if isinstance(attr, DocumentAttributeFilename):
+            name = name or attr.file_name
+            mime_type, _ = mimetypes.guess_type(attr.file_name)
+        elif isinstance(attr, DocumentAttributeSticker):
+            is_sticker = True
+            sticker_alt = attr.alt
+        elif isinstance(attr, DocumentAttributeAnimated):
+            is_gif = True
+        elif isinstance(attr, DocumentAttributeVideo):
+            width, height = attr.w, attr.h
+        elif isinstance(attr, DocumentAttributeImageSize):
+            width, height = attr.w, attr.h
+        elif isinstance(attr, DocumentAttributeAudio):
+            is_audio = True
+            is_voice = attr.voice or False
+            duration = attr.duration
+            waveform = decode_waveform(attr.waveform) if attr.waveform else b""
+
+    return DocAttrs(
+        name,
+        mime_type,
+        is_sticker,
+        sticker_alt,
+        width,
+        height,
+        is_gif,
+        is_audio,
+        is_voice,
+        duration,
+        waveform,
+    )
+
+
+def _parse_document_meta(
+    evt: Message, file: DBTelegramFile, attrs: DocAttrs, thumb_size: TypePhotoSize
+) -> tuple[ImageInfo, str]:
+    document = evt.media.document
+    name = attrs.name
+    if attrs.is_sticker:
+        alt = attrs.sticker_alt
+        if len(alt) > 0:
+            try:
+                name = f"{alt} ({unicodedata.name(alt[0]).lower()})"
+            except ValueError:
+                name = alt
+
+    generic_types = ("text/plain", "application/octet-stream")
+    if file.mime_type in generic_types and document.mime_type not in generic_types:
+        mime_type = document.mime_type or file.mime_type
+    elif file.mime_type == "application/ogg":
+        mime_type = "audio/ogg"
+    else:
+        mime_type = file.mime_type or document.mime_type
+    info = ImageInfo(size=file.size, mimetype=mime_type)
+
+    if attrs.mime_type and not file.was_converted:
+        file.mime_type = attrs.mime_type or file.mime_type
+    if file.width and file.height:
+        info.width, info.height = file.width, file.height
+    elif attrs.width and attrs.height:
+        info.width, info.height = attrs.width, attrs.height
+
+    if file.thumbnail:
+        if file.thumbnail.decryption_info:
+            info.thumbnail_file = file.thumbnail.decryption_info
+        else:
+            info.thumbnail_url = file.thumbnail.mxc
+        info.thumbnail_info = ThumbnailInfo(
+            mimetype=file.thumbnail.mime_type,
+            height=file.thumbnail.height or thumb_size.h,
+            width=file.thumbnail.width or thumb_size.w,
+            size=file.thumbnail.size,
+        )
+    elif attrs.is_sticker:
+        # This is a hack for bad clients like Element iOS that require a thumbnail
+        info.thumbnail_info = ImageInfo.deserialize(info.serialize())
+        if file.decryption_info:
+            info.thumbnail_file = file.decryption_info
+        else:
+            info.thumbnail_url = file.mxc
+
+    return info, name
+
+
+def _format_dice(roll: MessageMediaDice) -> str:
+    if roll.emoticon == "\U0001F3B0":
+        emojis = {
+            0: "\U0001F36B",  # "🍫",
+            1: "\U0001F352",  # "🍒",
+            2: "\U0001F34B",  # "🍋",
+            3: "7\ufe0f\u20e3",  # "7️⃣",
+        }
+        res = roll.value - 1
+        slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
+        return f"{slot1} {slot2} {slot3} ({roll.value})"
+    elif roll.emoticon == "\u26BD":
+        results = {
+            1: "miss",
+            2: "hit the woodwork",
+            3: "goal",  # seems to go in through the center
+            4: "goal",
+            5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
+        }
+    elif roll.emoticon == "\U0001F3B3":
+        results = {
+            1: "miss",
+            2: "1 pin down",
+            3: "3 pins down, split",
+            4: "4 pins down, split",
+            5: "5 pins down",
+            6: "strike 🎉",
+        }
+    # elif roll.emoticon == "\U0001F3C0":
+    #     results = {
+    #         2: "rolled off",
+    #         3: "stuck",
+    #     }
+    # elif roll.emoticon == "\U0001F3AF":
+    #     results = {
+    #         1: "bounced off",
+    #         2: "outer rim",
+    #
+    #         6: "bullseye",
+    #     }
+    else:
+        return str(roll.value)
+    return f"{results[roll.value]} ({roll.value})"
@@ -24,6 +24,7 @@ from telethon.tl.types import (
    ChannelParticipantBanned,
    ChannelParticipantsRecent,
    ChannelParticipantsSearch,
+    ChatParticipantsForbidden,
    InputChannel,
    InputUser,
    TypeChannelParticipant,
@@ -93,6 +94,8 @@ async def get_users(
 ) -> list[TypeUser]:
    if peer_type == "chat":
        chat = await client(GetFullChatRequest(chat_id=tgid))
+        if isinstance(chat.full_chat.participants, ChatParticipantsForbidden):
+            return []
        users = list(_filter_participants(chat.users, chat.full_chat.participants.participants))
        return users[:limit] if limit > 0 else users
    elif peer_type == "channel":
@@ -34,8 +34,12 @@ from ..types import TelegramID


 def get_base_power_levels(
-    portal: po.Portal, levels: PowerLevelContent = None, entity: TypeChat = None
+    portal: po.Portal,
+    levels: PowerLevelContent = None,
+    entity: TypeChat | None = None,
+    dbr: ChatBannedRights | None = None,
 ) -> PowerLevelContent:
+    is_initial = not levels
    levels = levels or PowerLevelContent()
    if portal.peer_type == "user":
        overrides = portal.config["bridge.initial_power_level_overrides.user"]
@@ -51,7 +55,7 @@ def get_base_power_levels(
        levels.events_default = overrides.get("events_default", 0)
    else:
        overrides = portal.config["bridge.initial_power_level_overrides.group"]
-        dbr = entity.default_banned_rights
+        dbr = dbr or entity.default_banned_rights
        if not dbr:
            portal.log.debug(f"default_banned_rights is None in {entity}")
            dbr = ChatBannedRights(
@@ -80,16 +84,14 @@ def get_base_power_levels(
        levels.events_default = overrides.get(
            "events_default",
            50
-            if (
-                portal.peer_type == "channel"
-                and not entity.megagroup
-                or entity.default_banned_rights.send_messages
-            )
+            if portal.peer_type == "channel" and not portal.megagroup or dbr.send_messages
            else 0,
        )
    for evt_type, value in overrides.get("events", {}).items():
        levels.events[EventType.find(evt_type)] = value
-    levels.users = overrides.get("users", {})
+    userlevel_overrides = overrides.get("users", {})
+    if is_initial:
+        levels.users.update(userlevel_overrides)
    if portal.main_intent.mxid not in levels.users:
        levels.users[portal.main_intent.mxid] = 100
    return levels
@@ -20,6 +20,7 @@ import html

 from telethon.tl.functions.channels import GetSponsoredMessagesRequest
 from telethon.tl.types import Channel, InputChannel, PeerChannel, PeerUser, SponsoredMessage, User
+from telethon.tl.types.messages import SponsoredMessages, SponsoredMessagesEmpty

 from mautrix.types import MessageType, TextMessageEventContent

@@ -32,8 +33,9 @@ async def get_sponsored_message(
    entity: InputChannel,
 ) -> tuple[SponsoredMessage | None, int | None, Channel | User | None]:
    resp = await user.client(GetSponsoredMessagesRequest(entity))
-    if len(resp.messages) == 0:
+    if isinstance(resp, SponsoredMessagesEmpty):
        return None, None, None
+    assert isinstance(resp, SponsoredMessages)
    msg = resp.messages[0]
    if isinstance(msg.from_id, PeerUser):
        entities = resp.users
@@ -83,7 +85,7 @@ async def make_sponsored_message_content(
    else:
        sponsor_name = sponsor_name_html = "unknown entity"

-    content["net.maunium.telegram.sponsored"] = sponsored_meta
+    content["fi.mau.telegram.sponsored"] = sponsored_meta
    content.formatted_body += (
        f"<br/><br/>Sponsored message from {sponsor_name_html} "
        f"- <a href='{content.external_url}'>{action}</a>"
@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, AsyncGenerator, AsyncIterable, Awaitable, cast
 from difflib import SequenceMatcher
 import unicodedata

+from telethon import utils
 from telethon.tl.types import (
    Channel,
    ChatPhoto,
@@ -29,8 +30,6 @@ from telethon.tl.types import (
    PeerChat,
    PeerUser,
    TypeChatPhoto,
-    TypeInputPeer,
-    TypeInputUser,
    TypePeer,
    TypeUserProfilePhoto,
    UpdateUserName,
@@ -48,6 +47,7 @@ from mautrix.util.simple_template import SimpleTemplate
 from . import abstract_user as au, portal as p, util
 from .config import Config
 from .db import Puppet as DBPuppet
+from .tgclient import MautrixTelegramClient
 from .types import TelegramID

 if TYPE_CHECKING:
@@ -80,6 +80,7 @@ class Puppet(DBPuppet, BasePuppet):
        avatar_set: bool = False,
        is_bot: bool = False,
        is_channel: bool = False,
+        is_premium: bool = False,
        custom_mxid: UserID | None = None,
        access_token: str | None = None,
        next_batch: SyncToken | None = None,
@@ -101,6 +102,7 @@ class Puppet(DBPuppet, BasePuppet):
            avatar_set=avatar_set,
            is_bot=is_bot,
            is_channel=is_channel,
+            is_premium=is_premium,
            custom_mxid=custom_mxid,
            access_token=access_token,
            next_batch=next_batch,
@@ -145,9 +147,6 @@ class Puppet(DBPuppet, BasePuppet):
    def plain_displayname(self) -> str:
        return self.displayname_template.parse(self.displayname) or self.displayname

-    def get_input_entity(self, user: au.AbstractUser) -> Awaitable[TypeInputPeer | TypeInputUser]:
-        return user.client.get_input_entity(self.peer)
-
    def intent_for(self, portal: p.Portal) -> IntentAPI:
        if portal.tgid == self.tgid:
            return self.default_mxid_intent
@@ -253,13 +252,22 @@ class Puppet(DBPuppet, BasePuppet):
        except Exception:
            source.log.exception(f"Failed to update info of {self.tgid}")

-    async def update_info(self, source: au.AbstractUser, info: User | Channel) -> None:
+    async def update_info(
+        self,
+        source: au.AbstractUser,
+        info: User | Channel,
+        client_override: MautrixTelegramClient | None = None,
+    ) -> None:
        is_bot = False if isinstance(info, Channel) else info.bot
+        is_premium = False if isinstance(info, Channel) else info.premium
        is_channel = isinstance(info, Channel)
-        changed = is_bot != self.is_bot or is_channel != self.is_channel
+        changed = (
+            is_bot != self.is_bot or is_channel != self.is_channel or is_premium != self.is_premium
+        )

        self.is_bot = is_bot
        self.is_channel = is_channel
+        self.is_premium = is_premium

        if self.username != info.username:
            self.username = info.username
@@ -271,8 +279,16 @@ class Puppet(DBPuppet, BasePuppet):

        if not self.disable_updates:
            try:
-                changed = await self.update_displayname(source, info) or changed
-                changed = await self.update_avatar(source, info.photo) or changed
+                changed = (
+                    await self.update_displayname(source, info, client_override=client_override)
+                    or changed
+                )
+                changed = (
+                    await self.update_avatar(
+                        source, info.photo, entity=info, client_override=client_override
+                    )
+                    or changed
+                )
            except Exception:
                self.log.exception(f"Failed to update info from source {source.tgid}")

@@ -287,7 +303,10 @@ class Puppet(DBPuppet, BasePuppet):
            await portal.update_info_from_puppet(self)

    async def update_displayname(
-        self, source: au.AbstractUser, info: User | Channel | UpdateUserName
+        self,
+        source: au.AbstractUser,
+        info: User | Channel | UpdateUserName,
+        client_override: MautrixTelegramClient | None = None,
    ) -> bool:
        if self.disable_updates:
            return False
@@ -314,14 +333,16 @@ class Puppet(DBPuppet, BasePuppet):
            return False

        if isinstance(info, UpdateUserName):
-            info = await source.client.get_entity(self.peer)
-        if isinstance(info, Channel) or not info.contact:
-            self.displayname_contact = False
-        elif not self.displayname_contact:
-            if not self.displayname:
-                self.displayname_contact = True
-            else:
-                return False
+            info = await (client_override or source.client).get_entity(self.peer)
+        is_contact_name = not isinstance(info, Channel) and info.contact
+        # Reject name change if the contact status is moving in an unwanted direction,
+        # and we already have a name for the ghost.
+        if (
+            is_contact_name != self.displayname_contact
+            and is_contact_name != self.config["bridge.allow_contact_info"]
+            and self.displayname
+        ):
+            return False

        displayname, quality = self.get_displayname(info)
        needs_reset = displayname != self.displayname or not self.name_set
@@ -329,12 +350,14 @@ class Puppet(DBPuppet, BasePuppet):
        if needs_reset and is_high_quality:
            allow_because = f"{allow_because} and quality {quality} >= {self.displayname_quality}"
            self.log.debug(
-                f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
-                f"because {allow_because}) from {self.displayname} to {displayname}"
+                f"Updating displayname of {self.id} (src: {source.tgid}, "
+                f"contact: {is_contact_name}, allowed because {allow_because}) "
+                f"from {self.displayname} to {displayname}"
            )
            self.log.trace("Displayname source data: %s", info)
            self.displayname = displayname
            self.displayname_source = source.tgid
+            self.displayname_contact = is_contact_name
            self.displayname_quality = quality
            try:
                await self.default_mxid_intent.set_displayname(
@@ -351,10 +374,24 @@ class Puppet(DBPuppet, BasePuppet):
        return False

    async def update_avatar(
-        self, source: au.AbstractUser, photo: TypeUserProfilePhoto | TypeChatPhoto
+        self,
+        source: au.AbstractUser,
+        photo: TypeUserProfilePhoto | TypeChatPhoto,
+        entity: User | None = None,
+        client_override: MautrixTelegramClient | None = None,
    ) -> bool:
        if self.disable_updates:
            return False
+        if (
+            isinstance(photo, UserProfilePhoto)
+            and photo.personal
+            and not self.config["bridge.allow_contact_info"]
+        ):
+            self.log.trace(
+                "Dropping user avatar as it's personal "
+                "and contact info is disabled in bridge config"
+            )
+            return False

        if photo is None or isinstance(photo, (UserProfilePhotoEmpty, ChatPhotoEmpty)):
            photo_id = ""
@@ -370,11 +407,22 @@ class Puppet(DBPuppet, BasePuppet):
                self.photo_id = ""
                self.avatar_url = None
            elif self.photo_id != photo_id or not self.avatar_url:
+                client = client_override or source.client
+                try:
+                    peer = await client.get_input_entity(entity or self.peer)
+                except ValueError:
+                    if entity:
+                        peer = utils.get_input_peer(entity, check_hash=False)
+                    else:
+                        self.log.warning(f"Couldn't get input entity to update avatar")
+                        return False
                file = await util.transfer_file_to_matrix(
-                    client=source.client,
+                    client=client,
                    intent=self.default_mxid_intent,
                    location=InputPeerPhotoFileLocation(
-                        peer=await self.get_input_entity(source), photo_id=photo.photo_id, big=True
+                        peer=peer,
+                        photo_id=photo.photo_id,
+                        big=True,
                    ),
                    async_upload=self.config["homeserver.async_media"],
                )
@@ -393,7 +441,7 @@ class Puppet(DBPuppet, BasePuppet):

    async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
        portal: p.Portal = await p.Portal.get_by_mxid(room_id)
-        return portal and not portal.backfill_lock.locked and portal.peer_type != "user"
+        return portal and portal.peer_type != "user"

    # endregion
    # region Getters
@@ -406,7 +454,7 @@ class Puppet(DBPuppet, BasePuppet):
    @classmethod
    @async_getter_lock
    async def get_by_tgid(
-        cls, tgid: TelegramID, *, create: bool = True, is_channel: bool = False
+        cls, tgid: TelegramID, /, *, create: bool = True, is_channel: bool = False
    ) -> Puppet | None:
        if tgid is None:
            return None
@@ -459,7 +507,7 @@ class Puppet(DBPuppet, BasePuppet):

    @classmethod
    @async_getter_lock
-    async def get_by_custom_mxid(cls, mxid: UserID) -> Puppet | None:
+    async def get_by_custom_mxid(cls, mxid: UserID, /) -> Puppet | None:
        try:
            return cls.by_custom_mxid[mxid]
        except KeyError:
@@ -512,23 +560,4 @@ class Puppet(DBPuppet, BasePuppet):

        return None

-    @classmethod
-    async def find_by_displayname(cls, displayname: str) -> Puppet | None:
-        if not displayname:
-            return None
-
-        for _, puppet in cls.by_tgid.items():
-            if puppet.displayname and puppet.displayname == displayname:
-                return puppet
-
-        puppet = cast(cls, await super().find_by_displayname(displayname))
-        if puppet:
-            try:
-                return cls.by_tgid[puppet.tgid]
-            except KeyError:
-                puppet._add_to_cache()
-                return puppet
-
-        return None
-
    # endregion
@@ -0,0 +1,397 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Any, Literal, TypedDict
+from pathlib import Path
+import argparse
+import asyncio
+import io
+import json
+import logging
+import math
+import mimetypes
+import pickle
+import random
+import string
+
+from lottie.exporters import export_tgs
+from lottie.exporters.cairo import export_png
+from lottie.exporters.tgs_validator import Severity, TgsValidator
+from lottie.importers.svg import import_svg
+from lottie.objects import Animation
+from lottie.utils.stripper import float_strip
+from PIL import Image
+from telethon import TelegramClient
+from telethon.custom import Conversation, Message
+from telethon.tl.functions.messages import GetStickerSetRequest
+from telethon.tl.types import (
+    Document,
+    DocumentAttributeCustomEmoji,
+    DocumentAttributeFilename,
+    DocumentAttributeImageSize,
+    InputMediaUploadedDocument,
+    InputStickerSetShortName,
+)
+import aiohttp
+
+mimetypes.add_type("image/webp", ".webp")
+
+parser = argparse.ArgumentParser(description="mautrix-telegram unicode emoji packer")
+parser.add_argument(
+    "-i", "--api-id", type=int, required=True, metavar="<api id>", help="Telegram API ID"
+)
+parser.add_argument(
+    "-a", "--api-hash", type=str, required=True, metavar="<api hash>", help="Telegram API hash"
+)
+parser.add_argument(
+    "-s",
+    "--session",
+    type=str,
+    default="unicodemojipacker.session",
+    metavar="<file name>",
+    help="Telethon session name",
+)
+parser.add_argument(
+    "-o",
+    "--output",
+    type=str,
+    default="mautrix_telegram/unicodemojipack.json",
+    metavar="<file name>",
+    help="Path to save created emoji pack document IDs",
+)
+parser.add_argument(
+    "-f",
+    "--font-directory",
+    type=Path,
+    required=True,
+    metavar="<directory path>",
+    help="Path to the Noto color emoji files",
+)
+parser.add_argument(
+    "-m",
+    "--media-directory",
+    type=Path,
+    required=True,
+    metavar="<directory path>",
+    help="Path to save converted tgs and webp emoji files",
+)
+args = parser.parse_args()
+font_dir: Path = args.font_directory
+media_dir: Path = args.media_directory
+
+EMOJI_DATA_URL = "https://raw.githubusercontent.com/iamcal/emoji-data/master/emoji.json"
+
+
+def unified_to_unicode(unified: str) -> str:
+    return (
+        "".join(rf"\U{chunk:0>8}" for chunk in unified.split("-"))
+        .encode("ascii")
+        .decode("unicode_escape")
+    )
+
+
+def tag_to_str(unified: str) -> str:
+    return "".join(chr(int(x.removeprefix("E00"), 16)) for x in unified.split("-"))
+
+
+EmojiType = Literal["webp", "tgs"]
+PackType = Literal["Animated emoji", "Static emoji"]
+
+
+class Emoji(TypedDict):
+    hex: str
+    emoji: str
+    type: EmojiType
+    filename: str
+
+
+class EmojiData(TypedDict):
+    tgs: list[Emoji]
+    webp: list[Emoji]
+
+
+def parse_emoji_data(tone: dict[str, Any], emoji: dict[str, Any]) -> Emoji:
+    hex = (tone["non_qualified"] or tone["unified"]).replace("-FE0F", "")
+    filename_hex = hex.replace("-", "_").lower()
+    filename = f"svg/emoji_u{filename_hex}.svg"
+    if emoji["category"] == "Flags" and emoji["subcategory"] in (
+        "country-flag",
+        "subdivision-flag",
+    ):
+        filename = f"third_party/region-flags/waved-svg/emoji_u{filename_hex}.svg"
+
+    with (font_dir / filename).open() as f:
+        lot: Animation = import_svg(f)
+    float_strip(lot)
+    lot.tgs_sanitize()
+
+    output = io.BytesIO()
+    export_tgs(lot, output)
+
+    validator = TgsValidator()
+    validator(lot)
+    validator.check_size(len(output.getvalue()))
+    errors = [err for err in validator.errors if err.severity != Severity.Note]
+    if errors or ("region-flags" in filename and len(output.getvalue()) > 32768):
+        lot.scale(100, 100)
+
+        png_out = io.BytesIO()
+        export_png(lot, png_out)
+        img = Image.open(png_out)
+        output = io.BytesIO()
+        output.name = "image.webp"
+        img.save(output, "webp")
+
+        media_type: EmojiType = "webp"
+    else:
+        media_type: EmojiType = "tgs"
+    path = media_dir / f"{filename_hex}.{media_type}"
+    with path.open("wb") as f:
+        f.write(output.getvalue())
+    print(
+        "Converted", filename, "->", path.name, "//" if errors else "", "\n".join(map(str, errors))
+    )
+
+    return {
+        "hex": hex,
+        "emoji": unified_to_unicode(tone["unified"]),
+        "type": media_type,
+        "filename": path.name,
+    }
+
+
+async def load_emoji_data() -> EmojiData:
+    cache_path = media_dir / "conversion-cache.json"
+    try:
+        with cache_path.open() as f:
+            return json.load(f)
+    except FileNotFoundError:
+        pass
+    async with aiohttp.ClientSession() as sess, sess.get(EMOJI_DATA_URL) as resp:
+        raw_emoji_data = sorted(
+            await resp.json(content_type=None),
+            key=lambda dat: dat["sort_order"],
+        )
+    tgs_emoji = []
+    webp_emoji = []
+    for emoji in raw_emoji_data:
+        for tone in (emoji, *emoji.get("skin_variations", {}).values()):
+            parsed_emoji = parse_emoji_data(tone, emoji)
+            if parsed_emoji["type"] == "tgs":
+                tgs_emoji.append(parsed_emoji)
+            else:
+                webp_emoji.append(parsed_emoji)
+    full_data = {"tgs": tgs_emoji, "webp": webp_emoji}
+    with cache_path.open("w") as f:
+        json.dump(full_data, f, ensure_ascii=False)
+    return full_data
+
+
+async def create_pack(conv: Conversation, name: str, pack_type: str) -> None:
+    await conv.send_message("/newemojipack")
+    resp: Message = await conv.get_response()
+    assert "A new set of custom emoji" in resp.raw_text
+    assert "Please choose the type" in resp.raw_text
+    await conv.send_message(pack_type)
+    resp = await conv.get_response()
+    if pack_type == "Animated emoji":
+        assert "When ready to upload, tell me the name of your set." in resp.raw_text
+    else:
+        assert "Now choose a name for your set." in resp.raw_text
+    await conv.send_message(name)
+    resp = await conv.get_response()
+    if pack_type == "Animated emoji":
+        assert "Now send me the first animated emoji" in resp.raw_text
+    else:
+        assert "Now send me the custom emoji" in resp.raw_text
+
+
+async def publish_pack(conv: Conversation, shortname: str) -> None:
+    await conv.send_message("/publish")
+
+    resp: Message = await conv.get_response()
+    assert "You can send me a custom emoji from your emoji set" in resp.raw_text
+    await conv.send_message("/skip")
+
+    resp = await conv.get_response()
+    assert "Please provide a short name for your emoji set" in resp.raw_text
+    await conv.send_message(shortname)
+
+    resp = await conv.get_response()
+    assert "I've just published your emoji set" in resp.raw_text
+
+
+async def send_emoji(
+    conv: Conversation, file: bytes | Path | InputMediaUploadedDocument, emoji: str
+) -> None:
+    await conv.send_file(file)
+    resp: Message = await conv.get_response()
+    assert "Send me a replacement emoji that corresponds to your custom emoji" in resp.raw_text
+    await conv.send_message(emoji)
+    resp = await conv.get_response()
+    if "Sorry, too many attempts" in resp.raw_text:
+        print(resp.raw_text)
+        input("Press enter to continue")
+        await conv.send_message(emoji)
+        resp = await conv.get_response()
+    while "Please send an emoji that best describes your custom emoji." in resp.raw_text:
+        emoji = input(f"{emoji} was rejected, provide replacement: ")
+        await conv.send_message(emoji)
+        resp = await conv.get_response()
+    assert "Congratulations" in resp.raw_text
+
+
+class CachedPack(TypedDict):
+    name: str
+    short_name: str
+    part: int
+    type: PackType
+    published: bool
+    collected: bool
+    emojis: list[Emoji]
+
+
+class CachedData(TypedDict):
+    packs: list[CachedPack]
+
+
+def _split_packs_int(
+    emoji_list: list[Emoji], pack_type: PackType, current_part: int, total_parts: int
+) -> tuple[list[CachedPack], int]:
+    packs = []
+    current_pack: CachedPack | None = None
+    for i, emoji in enumerate(emoji_list):
+        if i % 200 == 0:
+            current_part += 1
+            random_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=8))
+            short_name = f"mxtg_unicodemoji_{random_id}"
+            name = f"mautrix-telegram unicodemoji ({current_part}/{total_parts})"
+            current_pack = {
+                "type": pack_type,
+                "short_name": short_name,
+                "part": current_part,
+                "name": name,
+                "published": False,
+                "collected": False,
+                "emojis": [],
+            }
+            packs.append(current_pack)
+        current_pack["emojis"].append(emoji)
+    return packs, current_part
+
+
+def split_packs(emoji_data: EmojiData) -> list[CachedPack]:
+    total_parts = math.ceil(len(emoji_data["tgs"]) / 200) + math.ceil(
+        len(emoji_data["webp"]) / 200
+    )
+    current_part = 0
+    animated_packs, current_part = _split_packs_int(
+        emoji_data["tgs"], "Animated emoji", current_part, total_parts
+    )
+    static_packs, current_part = _split_packs_int(
+        emoji_data["webp"], "Static emoji", current_part, total_parts
+    )
+    return animated_packs + static_packs
+
+
+async def create_and_fill_pack(
+    client: TelegramClient, conv: Conversation, pack: CachedPack
+) -> None:
+    if pack["short_name"] == "mxtg_unicodemoji_xvzs6743":
+        print("Continuing pack", pack["name"])
+    else:
+        print("Creating pack", pack["name"])
+        await create_pack(conv, pack["name"], pack["type"])
+    total = len(pack["emojis"])
+    for i, emoji in enumerate(pack["emojis"]):
+        if pack["short_name"] == "mxtg_unicodemoji_xvzs6743" and i < 87:
+            continue
+        print(f"Adding emoji {i+1}/{total}", emoji["hex"], emoji["emoji"])
+        emoji_file = media_dir / emoji["filename"]
+        if emoji["type"] == "webp":
+            attrs = [
+                DocumentAttributeImageSize(w=100, h=100),
+                DocumentAttributeFilename(file_name="image.webp"),
+            ]
+            with emoji_file.open("rb") as f:
+                file_handle = await client.upload_file(f, file_name="emoji.webp")
+            emoji_file = InputMediaUploadedDocument(
+                file_handle, mime_type="image/webp", attributes=attrs
+            )
+        await send_emoji(conv, emoji_file, emoji["emoji"])
+        await asyncio.sleep(2)
+    print("Publishing pack", pack["short_name"])
+    await publish_pack(conv, pack["short_name"])
+
+
+async def main():
+    logging.basicConfig(level=logging.INFO)
+
+    emoji_data = await load_emoji_data()
+
+    split_cache = media_dir / "split-cache.json"
+    try:
+        with split_cache.open() as f:
+            packs: list[CachedPack] = json.load(f)
+    except FileNotFoundError:
+        packs = split_packs(emoji_data)
+        with split_cache.open("w") as f:
+            json.dump(packs, f)
+
+    doc_id_file = Path(args.output)
+    try:
+        with doc_id_file.open() as f:
+            doc_ids = json.load(f)
+    except FileNotFoundError:
+        doc_ids = {}
+
+    client = TelegramClient(args.session, args.api_id, args.api_hash, flood_sleep_threshold=3600)
+    await client.start()
+    async with client.conversation("Stickers", max_messages=20000) as conv:
+        for pack in packs:
+            if not pack["published"]:
+                await create_and_fill_pack(client, conv, pack)
+                pack["published"] = True
+                with split_cache.open("w") as f:
+                    json.dump(packs, f, ensure_ascii=False)
+            if not pack["collected"] or True:
+                print("Collecting document IDs from pack", pack["short_name"])
+                stickers = await client(
+                    GetStickerSetRequest(InputStickerSetShortName(pack["short_name"]), 0)
+                )
+                doc: Document
+                for i, doc in enumerate(stickers.documents):
+                    attr = next(
+                        attr
+                        for attr in doc.attributes
+                        if isinstance(attr, DocumentAttributeCustomEmoji)
+                    )
+                    base_emoji = attr.alt.replace("\ufe0f", "")
+                    emoji = pack["emojis"][i]["emoji"].replace("\ufe0f", "")
+                    doc_ids[emoji] = doc.id
+                    print(f"Mapped {emoji} (fallback: {base_emoji}) -> {doc_ids[emoji]}")
+                pack["collected"] = True
+                with split_cache.open("w") as f:
+                    json.dump(packs, f, ensure_ascii=False)
+                with doc_id_file.open("w") as f:
+                    json.dump(doc_ids, f, ensure_ascii=False)
+                print("Pack completed")
+                await asyncio.sleep(5)
+    with open(args.output.replace(".json", ".pickle"), "wb") as f:
+        pickle.dump(doc_ids, f)
+    print("Wrote pickle")
+
+
+asyncio.run(main())
@@ -15,21 +15,32 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations

-from typing import TYPE_CHECKING, AsyncGenerator, AsyncIterable, Awaitable, NamedTuple, cast
-from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, Awaitable, NamedTuple, cast
+from datetime import datetime
 import asyncio
+import time

-from telethon.errors import AuthKeyDuplicatedError, RPCError, UnauthorizedError
+from telethon.errors import (
+    AuthKeyDuplicatedError,
+    AuthKeyError,
+    RPCError,
+    TakeoutInitDelayError,
+    UnauthorizedError,
+)
 from telethon.tl.custom import Dialog
 from telethon.tl.functions.account import UpdateStatusRequest
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
+from telethon.tl.functions.help import GetAppConfigRequest
+from telethon.tl.functions.messages import GetAvailableReactionsRequest
 from telethon.tl.functions.updates import GetStateRequest
 from telethon.tl.functions.users import GetUsersRequest
 from telethon.tl.types import (
    Chat,
    ChatForbidden,
    InputUserSelf,
+    Message,
    NotifyPeer,
+    PeerUser,
    TypeUpdate,
    UpdateFolderPeers,
    UpdateNewChannelMessage,
@@ -41,18 +52,21 @@ from telethon.tl.types import (
    User as TLUser,
 )
 from telethon.tl.types.contacts import ContactsNotModified
+from telethon.tl.types.messages import AvailableReactions

 from mautrix.appservice import DOUBLE_PUPPET_SOURCE_KEY
 from mautrix.bridge import BaseUser, async_getter_lock
 from mautrix.client import Client
 from mautrix.errors import MatrixRequestError, MNotFound
 from mautrix.types import PushActionType, PushRuleKind, PushRuleScope, RoomID, RoomTagInfo, UserID
+from mautrix.util import background_task
 from mautrix.util.bridge_state import BridgeState, BridgeStateEvent
 from mautrix.util.opt_prometheus import Gauge

-from . import portal as po, puppet as pu
+from . import portal as po, puppet as pu, util
 from .abstract_user import AbstractUser
-from .db import Message as DBMessage, PgSession, User as DBUser
+from .db import Backfill, BackfillType, Message as DBMessage, PgSession, User as DBUser
+from .tgclient import MautrixTelegramClient
 from .types import TelegramID

 if TYPE_CHECKING:
@@ -81,7 +95,17 @@ class User(DBUser, AbstractUser, BaseUser):

    _ensure_started_lock: asyncio.Lock
    _track_connection_task: asyncio.Task | None
+    _backfill_task: asyncio.Task | None
+    wakeup_backfill_task: asyncio.Event
    _is_backfilling: bool
+    takeout_retry_immediate: asyncio.Event
+    takeout_requested: bool
+
+    _available_emoji_reactions: set[str] | None
+    _available_emoji_reactions_hash: int | None
+    _available_emoji_reactions_fetched: float
+    _available_emoji_reactions_lock: asyncio.Lock
+    _app_config: dict[str, Any] | None

    def __init__(
        self,
@@ -90,6 +114,7 @@ class User(DBUser, AbstractUser, BaseUser):
        tg_username: str | None = None,
        tg_phone: str | None = None,
        is_bot: bool = False,
+        is_premium: bool = False,
        saved_contacts: int = 0,
    ) -> None:
        super().__init__(
@@ -98,6 +123,7 @@ class User(DBUser, AbstractUser, BaseUser):
            tg_username=tg_username,
            tg_phone=tg_phone,
            is_bot=is_bot,
+            is_premium=is_premium,
            saved_contacts=saved_contacts,
        )
        AbstractUser.__init__(self)
@@ -107,6 +133,17 @@ class User(DBUser, AbstractUser, BaseUser):
        self._is_backfilling = False
        self._portals_cache = None

+        self._backfill_task = None
+        self.wakeup_backfill_task = asyncio.Event()
+        self.takeout_retry_immediate = asyncio.Event()
+        self.takeout_requested = False
+
+        self._available_emoji_reactions = None
+        self._available_emoji_reactions_hash = None
+        self._available_emoji_reactions_fetched = 0
+        self._available_emoji_reactions_lock = asyncio.Lock()
+        self._app_config = None
+
        (
            self.relaybot_whitelisted,
            self.whitelisted,
@@ -129,6 +166,10 @@ class User(DBUser, AbstractUser, BaseUser):
    def human_tg_id(self) -> str:
        return f"@{self.tg_username}" if self.tg_username else f"+{self.tg_phone}" or None

+    @property
+    def peer(self) -> PeerUser | None:
+        return PeerUser(user_id=self.tgid) if self.tgid else None
+
    # TODO replace with proper displayname getting everywhere
    @property
    def displayname(self) -> str:
@@ -168,17 +209,30 @@ class User(DBUser, AbstractUser, BaseUser):
        async with self._ensure_started_lock:
            return cast(User, await super().ensure_started(even_if_no_session))

+    async def on_signed_out(self, err: UnauthorizedError | AuthKeyError) -> None:
+        error_code = "tg-auth-error"
+        if isinstance(err, AuthKeyDuplicatedError):
+            error_code = "tg-auth-key-duplicated"
+            message = None
+        else:
+            message = str(err)
+        self.log.warning(f"User got signed out with {err}, deleting data...")
+        try:
+            await self.log_out(
+                state=BridgeStateEvent.BAD_CREDENTIALS,
+                error=error_code,
+                message=message,
+                delete=False,
+            )
+        except Exception:
+            self.log.exception("Error handling external logout")
+
    async def start(self, delete_unless_authenticated: bool = False) -> User:
        try:
            await super().start()
-        except AuthKeyDuplicatedError:
+        except AuthKeyDuplicatedError as e:
            self.log.warning("Got AuthKeyDuplicatedError in start()")
-            await self.push_bridge_state(
-                BridgeStateEvent.BAD_CREDENTIALS, error="tg-auth-key-duplicated"
-            )
-            await self.client.disconnect()
-            await self.client.session.delete()
-            self.client = None
+            await self.on_signed_out(e)
            if not delete_unless_authenticated:
                # The caller wants the client to be connected, so restart the connection.
                await super().start()
@@ -198,12 +252,7 @@ class User(DBUser, AbstractUser, BaseUser):
            if delete_unless_authenticated or self.tgid:
                self.log.error(f"Authorization error in start(): {type(e)}: {e}")
            if self.tgid:
-                await self.push_bridge_state(
-                    BridgeStateEvent.BAD_CREDENTIALS,
-                    error="tg-auth-error",
-                    message=str(e),
-                    ttl=3600,
-                )
+                await self.on_signed_out(e)
        except RPCError as e:
            self.log.error(f"Unknown RPC error in start(): {type(e)}: {e}")
            if self.tgid:
@@ -211,10 +260,10 @@ class User(DBUser, AbstractUser, BaseUser):
        else:
            # Authenticated, run post login
            self.log.debug(f"Ensuring post_login() for {self.name}")
-            asyncio.create_task(self.post_login())
+            background_task.create(self.post_login())
            return self
        # Not authenticated, delete data if necessary
-        if delete_unless_authenticated:
+        if delete_unless_authenticated and self.client is not None:
            self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
            await self.client.disconnect()
            await self.client.session.delete()
@@ -226,6 +275,14 @@ class User(DBUser, AbstractUser, BaseUser):
            self.client and self.client._sender and self.client._sender._transport_connected()
        )

+    @property
+    def _bridge_state_info(self) -> dict[str, Any]:
+        if self.takeout_requested:
+            return {
+                "takeout_requested": True,
+            }
+        return {}
+
    async def _track_connection(self) -> None:
        self.log.debug("Starting loop to track connection state")
        while True:
@@ -237,17 +294,18 @@ class User(DBUser, AbstractUser, BaseUser):
                    BridgeStateEvent.BACKFILLING
                    if self._is_backfilling
                    else BridgeStateEvent.CONNECTED,
-                    ttl=3600,
+                    info=self._bridge_state_info,
                )
            else:
                await self.push_bridge_state(
-                    BridgeStateEvent.TRANSIENT_DISCONNECT, ttl=240, error="tg-not-connected"
+                    BridgeStateEvent.TRANSIENT_DISCONNECT, error="tg-not-connected"
                )

    async def fill_bridge_state(self, state: BridgeState) -> None:
        await super().fill_bridge_state(state)
-        state.remote_id = str(self.tgid)
-        state.remote_name = self.human_tg_id
+        if self.tgid:
+            state.remote_id = str(self.tgid)
+            state.remote_name = self.human_tg_id

    async def get_bridge_states(self) -> list[BridgeState]:
        if not self.tgid:
@@ -262,7 +320,7 @@ class User(DBUser, AbstractUser, BaseUser):
        else:
            state_event = BridgeStateEvent.UNKNOWN_ERROR
            ttl = 240
-        return [BridgeState(state_event=state_event, ttl=ttl)]
+        return [BridgeState(state_event=state_event, ttl=ttl, info=self._bridge_state_info)]

    async def get_puppet(self) -> pu.Puppet | None:
        if not self.tgid:
@@ -280,11 +338,16 @@ class User(DBUser, AbstractUser, BaseUser):
        if self._track_connection_task:
            self._track_connection_task.cancel()
            self._track_connection_task = None
+        if self._backfill_task:
+            self._backfill_task.cancel()
+            self._backfill_task = None
        await super().stop()
        self._track_metric(METRIC_CONNECTED, False)

    async def post_login(self, info: TLUser = None, first_login: bool = False) -> None:
-        if self.config["metrics.enabled"] and not self._track_connection_task:
+        if (
+            self.config["metrics.enabled"] or self.config["homeserver.status_endpoint"]
+        ) and not self._track_connection_task:
            self._track_connection_task = asyncio.create_task(self._track_connection())

        try:
@@ -294,6 +357,8 @@ class User(DBUser, AbstractUser, BaseUser):
            return

        self._track_metric(METRIC_LOGGED_IN, True)
+        if not self._backfill_task or self._backfill_task.done():
+            self._backfill_task = asyncio.create_task(self._try_handle_backfill_requests_loop())

        try:
            puppet = await pu.Puppet.get_by_tgid(self.tgid)
@@ -303,7 +368,7 @@ class User(DBUser, AbstractUser, BaseUser):
        except Exception:
            self.log.exception("Failed to automatically enable custom puppet")

-        if not self.is_bot and self.config["bridge.startup_sync"]:
+        if not self.is_bot and (self.config["bridge.startup_sync"] or first_login):
            try:
                self._is_backfilling = True
                await self.sync_dialogs()
@@ -313,6 +378,123 @@ class User(DBUser, AbstractUser, BaseUser):
            finally:
                self._is_backfilling = False

+    @property
+    def _takeout_options(self) -> dict[str, bool | int]:
+        return {
+            "users": True,
+            "chats": self.config["bridge.backfill.normal_groups"],
+            "megagroups": True,
+            "channels": True,
+            "files": True,
+            "max_file_size": min(self.bridge.matrix.media_config.upload_size, 2000 * 1024 * 1024),
+        }
+
+    async def _try_handle_backfill_requests_loop(self) -> None:
+        if not self.config["bridge.backfill.enable"]:
+            return
+        try:
+            await self._handle_backfill_requests_loop()
+        except Exception:
+            self.log.exception("Fatal error in backfill request loop")
+
+    async def _handle_backfill_requests_loop(self) -> None:
+        while True:
+            req = await Backfill.get_next(self.mxid)
+            if not req:
+                try:
+                    await asyncio.wait_for(self.wakeup_backfill_task.wait(), timeout=300)
+                except asyncio.TimeoutError:
+                    pass
+                self.wakeup_backfill_task.clear()
+            else:
+                try:
+                    await self._takeout_and_backfill(req)
+                except Exception:
+                    self.log.exception("Error in takeout backfill loop, retrying in an hour")
+                    await asyncio.sleep(3600)
+
+    async def _check_server_notice_edit(self, message: Message) -> None:
+        if "Data export request" in message.message and "Accepted" in message.message:
+            self.log.debug(
+                f"Received an edit to message {message.id} that looks like the data export"
+                " was accepted, marking takeout as retriable"
+            )
+            self.takeout_retry_immediate.set()
+
+    async def _takeout_and_backfill(self, first_req: Backfill, first_attempt: bool = True) -> None:
+        self.takeout_retry_immediate.clear()
+        self.takeout_requested = True
+        try:
+            async with self.client.takeout(**self._takeout_options) as takeout_client:
+                self.takeout_requested = False
+                self.log.info("Acquired takeout client successfully")
+                await self._backfill_loop_with_client(takeout_client, first_req)
+                self.log.info("Backfills finished, exiting takeout")
+        except TakeoutInitDelayError as e:
+            if first_attempt:
+                self.log.info(
+                    f"Takeout requested, will wait for retry request or {e.seconds} seconds"
+                )
+            else:
+                self.log.warning(
+                    f"Got takeout init delay again after retry, waiting for {e.seconds} seconds"
+                )
+            try:
+                await asyncio.wait_for(self.takeout_retry_immediate.wait(), timeout=e.seconds)
+                self.log.info("Retrying takeout")
+            except asyncio.TimeoutError:
+                self.log.info("Takeout timeout expired")
+            await self._takeout_and_backfill(first_req, first_attempt=False)
+
+    async def _backfill_loop_with_client(
+        self, client: MautrixTelegramClient, first_req: Backfill
+    ) -> None:
+        missed_reqs = 0
+        while missed_reqs < 10:
+            req = first_req or await Backfill.get_next(self.mxid)
+            first_req = None
+            if not req:
+                missed_reqs += 1
+                try:
+                    await asyncio.wait_for(self.wakeup_backfill_task.wait(), timeout=30)
+                except asyncio.TimeoutError:
+                    pass
+                self.wakeup_backfill_task.clear()
+                continue
+            missed_reqs = 0
+            self.log.info("Backfill request %s", req)
+            try:
+                portal = await po.Portal.get_by_tgid(
+                    TelegramID(req.portal_tgid), tg_receiver=TelegramID(req.portal_tg_receiver)
+                )
+                await req.mark_dispatched()
+                if req.type == BackfillType.HISTORICAL:
+                    await portal.backfill(self, client, req=req)
+                elif req.type == BackfillType.SYNC_DIALOG:
+                    await self._backfill_sync_dialog(portal, client, req.extra_data)
+                await req.mark_done()
+                await asyncio.sleep(req.post_batch_delay)
+            except Exception:
+                self.log.exception("Error handling backfill request for %s", req.portal_tgid)
+                await req.set_cooldown_timeout(1800)
+
+    async def _backfill_sync_dialog(
+        self, portal: po.Portal, client: MautrixTelegramClient, post_sync_args: dict[str, Any]
+    ) -> None:
+        if portal.mxid:
+            self.log.debug("Portal already exists, skipping dialog sync backfill queue item")
+            return
+        self.log.info(f"Creating portal for {portal.tgid_log} as part of backfill loop")
+        try:
+            await portal.create_matrix_room(
+                self, client=client, update_if_exists=False, invites=[self.mxid]
+            )
+        except Exception:
+            self.log.exception(f"Error while creating {portal.tgid_log}")
+        else:
+            puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+            await self._post_sync_dialog(portal, puppet, was_created=True, **post_sync_args)
+
    async def update(self, update: TypeUpdate) -> bool:
        if not self.is_bot:
            return False
@@ -363,6 +545,9 @@ class User(DBUser, AbstractUser, BaseUser):
        if self.is_bot != info.bot:
            self.is_bot = info.bot
            changed = True
+        if self.is_premium != info.premium:
+            self.is_premium = info.premium
+            changed = True
        if self.tg_username != info.username:
            self.tg_username = info.username
            changed = True
@@ -392,26 +577,46 @@ class User(DBUser, AbstractUser, BaseUser):
                except MatrixRequestError:
                    pass

-    async def log_out(self) -> bool:
+    async def log_out(
+        self,
+        delete: bool = True,
+        do_logout: bool = True,
+        state: BridgeStateEvent = BridgeStateEvent.LOGGED_OUT,
+        error: str | None = None,
+        message: str | None = None,
+    ) -> bool:
        puppet = await pu.Puppet.get_by_tgid(self.tgid)
-        if puppet.is_real_user:
+        if puppet is not None and puppet.is_real_user:
            await puppet.switch_mxid(None, None)
        try:
            await self.kick_from_portals()
        except Exception:
            self.log.exception("Failed to kick user from portals on logout")
-        await self.push_bridge_state(BridgeStateEvent.LOGGED_OUT)
        if self.tgid:
            try:
                del self.by_tgid[self.tgid]
            except KeyError:
                pass
-            self.tgid = None
-        ok = await self.client.log_out()
-        await self.client.session.delete()
-        await self.delete()
-        self.by_mxid.pop(self.mxid, None)
-        await self.stop()
+        ok = False
+        if self.client is not None:
+            sess = self.client.session
+            # Try to send a logout request. If it succeeds, this also disconnects the client and
+            # deletes the session, but we do those again later just to be safe.
+            if do_logout:
+                ok = await self.client.log_out()
+            # Force-disconnect the client and set it to None
+            await self.stop()
+            await sess.delete()
+
+        # TODO send a management room notice for non-manual logouts?
+        await self.push_bridge_state(state, error=error, message=message)
+        if delete:
+            await self.delete()
+            self.by_mxid.pop(self.mxid, None)
+            self.log.info("User deleted")
+        else:
+            await self.remove_tgid()
+            self.log.info("User telegram ID cleared")
        self._track_metric(METRIC_LOGGED_IN, False)
        return ok

@@ -469,19 +674,18 @@ class User(DBUser, AbstractUser, BaseUser):
        if active and tag_info is None:
            tag_info = RoomTagInfo(order=0.5)
            tag_info[DOUBLE_PUPPET_SOURCE_KEY] = self.bridge.name
-            self.log.debug("Adding tag {tag} to {portal.mxid}/{portal.tgid}")
+            self.log.debug(f"Adding tag {tag} to {portal.mxid}/{portal.tgid}")
            await puppet.intent.set_room_tag(portal.mxid, tag, tag_info)
        elif (
            not active and tag_info and tag_info.get(DOUBLE_PUPPET_SOURCE_KEY) == self.bridge.name
        ):
-            self.log.debug("Removing tag {tag} from {portal.mxid}/{portal.tgid}")
+            self.log.debug(f"Removing tag {tag} from {portal.mxid}/{portal.tgid}")
            await puppet.intent.remove_room_tag(portal.mxid, tag)

-    async def _mute_room(self, puppet: pu.Puppet, portal: po.Portal, mute_until: datetime) -> None:
+    async def _mute_room(self, puppet: pu.Puppet, portal: po.Portal, mute_until: float) -> None:
        if not self.config["bridge.mute_bridging"] or not portal or not portal.mxid:
            return
-        now = datetime.utcnow().replace(tzinfo=timezone.utc)
-        if mute_until is not None and mute_until > now:
+        if mute_until is not None and mute_until > time.time():
            self.log.debug(
                f"Muting {portal.mxid}/{portal.tgid} (muted until {mute_until} on Telegram)"
            )
@@ -537,15 +741,28 @@ class User(DBUser, AbstractUser, BaseUser):
        portal = await po.Portal.get_by_entity(
            update.peer.peer, tg_receiver=self.tgid, create=False
        )
-        await self._mute_room(puppet, portal, update.notify_settings.mute_until)
+        await self._mute_room(puppet, portal, update.notify_settings.mute_until.timestamp())

    async def _sync_dialog(
        self, portal: po.Portal, dialog: Dialog, should_create: bool, puppet: pu.Puppet | None
    ) -> None:
        was_created = False
+        post_sync_args = {
+            "last_message_ts": cast(datetime, dialog.date).timestamp(),
+            "unread_count": dialog.unread_count,
+            "max_read_id": dialog.dialog.read_inbox_max_id,
+            "mute_until": (
+                dialog.dialog.notify_settings.mute_until.timestamp()
+                if dialog.dialog.notify_settings.mute_until
+                else None
+            ),
+            "pinned": dialog.pinned,
+            "archived": dialog.archived,
+        }
        if portal.mxid:
+            self.log.debug(f"Backfilling and updating {portal.tgid_log} (dialog sync)")
            try:
-                await portal.backfill(self, last_id=dialog.message.id)
+                await portal.forward_backfill(self, initial=False, last_tgid=dialog.message.id)
            except Exception:
                self.log.exception(f"Error while backfilling {portal.tgid_log}")
            try:
@@ -553,31 +770,73 @@ class User(DBUser, AbstractUser, BaseUser):
            except Exception:
                self.log.exception(f"Error while updating {portal.tgid_log}")
        elif should_create:
+            self.log.debug(f"Creating portal for {portal.tgid_log} immediately (dialog sync)")
            try:
                await portal.create_matrix_room(self, dialog.entity, invites=[self.mxid])
                was_created = True
            except Exception:
                self.log.exception(f"Error while creating {portal.tgid_log}")
+        elif self.config["bridge.sync_deferred_create_all"]:
+            self.log.debug(f"Enqueuing deferred dialog sync for {portal.tgid_log}")
+            await portal.enqueue_backfill(
+                self,
+                priority=40,
+                type=BackfillType.SYNC_DIALOG,
+                extra_data=post_sync_args,
+            )
        if portal.mxid and puppet and puppet.is_real_user:
-            tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
-            if dialog.unread_count == 0:
-                # This is usually more reliable than finding a specific message
-                # e.g. if the last read message is a service message that isn't in the message db
-                last_read = await DBMessage.find_last(portal.mxid, tg_space)
-            else:
-                last_read = await DBMessage.get_one_by_tgid(
-                    portal.tgid, tg_space, dialog.dialog.read_inbox_max_id
+            await self._post_sync_dialog(
+                portal=portal,
+                puppet=puppet,
+                was_created=was_created,
+                **post_sync_args,
+            )
+        self.log.debug(f"_sync_dialog finished for {portal.tgid_log}")
+
+    async def _post_sync_dialog(
+        self,
+        portal: po.Portal,
+        puppet: pu.Puppet,
+        was_created: bool,
+        max_read_id: int,
+        last_message_ts: float,
+        unread_count: int,
+        mute_until: float,
+        pinned: bool,
+        archived: bool,
+    ) -> None:
+        self.log.debug(
+            f"Running dialog post-sync for {portal.tgid_log} with args "
+            f"{was_created=}, {max_read_id=}, {last_message_ts=}, {unread_count=}, "
+            f"{mute_until=}, {pinned=}, {archived=}"
+        )
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        unread_threshold_hours = self.config["bridge.backfill.unread_hours_threshold"]
+        force_read = (
+            was_created
+            and unread_threshold_hours >= 0
+            and last_message_ts + (unread_threshold_hours * 60 * 60) < time.time()
+        )
+        if unread_count == 0 or force_read:
+            # This is usually more reliable than finding a specific message
+            # e.g. if the last read message is a service message that isn't in the message db
+            last_read = await DBMessage.find_last(portal.mxid, tg_space)
+            if force_read:
+                self.log.debug(
+                    f"Marking {portal.tgid_log} as read because the last message is from "
+                    f"{last_message_ts} (unread threshold is {unread_threshold_hours} hours)"
                )
+        else:
+            last_read = await DBMessage.get_one_by_tgid(portal.tgid, tg_space, max_read_id)
+        try:
            if last_read:
                await puppet.intent.mark_read(last_read.mx_room, last_read.mxid)
            if was_created or not self.config["bridge.tag_only_on_create"]:
-                await self._mute_room(puppet, portal, dialog.dialog.notify_settings.mute_until)
-                await self._tag_room(
-                    puppet, portal, self.config["bridge.pinned_tag"], dialog.pinned
-                )
-                await self._tag_room(
-                    puppet, portal, self.config["bridge.archive_tag"], dialog.archived
-                )
+                await self._mute_room(puppet, portal, mute_until)
+                await self._tag_room(puppet, portal, self.config["bridge.pinned_tag"], pinned)
+                await self._tag_room(puppet, portal, self.config["bridge.archive_tag"], archived)
+        except Exception:
+            self.log.exception(f"Error updating read status and tags for {portal.tgid_log}")

    async def get_cached_portals(self) -> dict[tuple[TelegramID, TelegramID], po.Portal]:
        if self._portals_cache is None:
@@ -594,9 +853,7 @@ class User(DBUser, AbstractUser, BaseUser):
        update_limit = self.config["bridge.sync_update_limit"] or None
        create_limit = self.config["bridge.sync_create_limit"]
        index = 0
-        self.log.debug(
-            f"Syncing dialogs (update_limit={update_limit}, create_limit={create_limit})"
-        )
+        self.log.debug(f"Syncing dialogs ({update_limit=}, {create_limit=})")
        await self.push_bridge_state(BridgeStateEvent.BACKFILLING)
        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
        dialog: Dialog
@@ -617,11 +874,12 @@ class User(DBUser, AbstractUser, BaseUser):
                continue
            portal = await po.Portal.get_by_entity(entity, tg_receiver=self.tgid)
            new_portal_cache[portal.tgid_full] = portal
+            should_create = not create_limit or index < create_limit
            coro = self._sync_dialog(
                portal=portal,
                dialog=dialog,
                puppet=puppet,
-                should_create=not create_limit or index < create_limit,
+                should_create=should_create,
            )
            creators.append(asyncio.create_task(coro))
            index += 1
@@ -680,8 +938,51 @@ class User(DBUser, AbstractUser, BaseUser):
            await puppet.update_info(self, user)
            contacts[user.id] = puppet.contact_info
        await self.set_contacts(contacts.keys())
+        self.log.debug("Contact syncing complete")
        return contacts

+    async def get_available_reactions(self) -> set[str]:
+        if self._available_emoji_reactions_fetched + 12 * 60 * 60 > time.monotonic():
+            return self._available_emoji_reactions
+        async with self._available_emoji_reactions_lock:
+            if self._available_emoji_reactions_fetched + 12 * 60 * 60 > time.monotonic():
+                return self._available_emoji_reactions
+            self.log.debug("Fetching available emoji reactions")
+            available_reactions = await self.client(
+                GetAvailableReactionsRequest(hash=self._available_emoji_reactions_hash or 0)
+            )
+            if isinstance(available_reactions, AvailableReactions):
+                self._available_emoji_reactions = {
+                    react.reaction
+                    for react in available_reactions.reactions
+                    if self.is_premium or not react.premium
+                }
+                self._available_emoji_reactions_hash = available_reactions.hash
+                self._available_emoji_reactions_fetched = time.monotonic()
+                self.log.debug(
+                    "Got available emoji reactions: %s", self._available_emoji_reactions
+                )
+            return self._available_emoji_reactions
+
+    def tl_to_json(self) -> Any:
+        pass
+
+    async def get_app_config(self) -> dict[str, Any]:
+        if not self._app_config:
+            cfg = await self.client(GetAppConfigRequest())
+            self._app_config = util.parse_tl_json(cfg)
+        return self._app_config
+
+    async def get_max_reactions(self, is_premium: bool | None = None) -> int:
+        if is_premium is None:
+            is_premium = self.is_premium
+        cfg = await self.get_app_config()
+        return (
+            cfg.get("reactions_user_max_premium", 3)
+            if is_premium
+            else cfg.get("reactions_user_max_default", 1)
+        )
+
    # endregion
    # region Class instance lookup

@@ -710,7 +1011,7 @@ class User(DBUser, AbstractUser, BaseUser):
    @classmethod
    @async_getter_lock
    async def get_by_mxid(
-        cls, mxid: UserID, *, check_db: bool = True, create: bool = True
+        cls, mxid: UserID, /, *, check_db: bool = True, create: bool = True
    ) -> User | None:
        if not mxid or pu.Puppet.get_id_from_mxid(mxid) or mxid == cls.az.bot_mxid:
            return None
@@ -738,7 +1039,7 @@ class User(DBUser, AbstractUser, BaseUser):

    @classmethod
    @async_getter_lock
-    async def get_by_tgid(cls, tgid: TelegramID) -> User | None:
+    async def get_by_tgid(cls, tgid: TelegramID, /) -> User | None:
        try:
            return cls.by_tgid[tgid]
        except KeyError:
@@ -1,4 +1,11 @@
 from .color_log import ColorFormatter
-from .file_transfer import convert_image, transfer_file_to_matrix
+from .file_transfer import (
+    UnicodeCustomEmoji,
+    convert_image,
+    transfer_custom_emojis_to_matrix,
+    transfer_file_to_matrix,
+    unicode_custom_emoji_map,
+)
 from .parallel_file_transfer import parallel_transfer_to_telegram
 from .recursive_dict import recursive_del, recursive_get, recursive_set
+from .tl_json import parse_tl_json
@@ -15,12 +15,13 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations

-from typing import Optional, Union
+from typing import NamedTuple, Optional, Union
 from io import BytesIO
 from sqlite3 import IntegrityError
 import asyncio
 import logging
-import tempfile
+import pickle
+import pkgutil
 import time

 from asyncpg import UniqueViolationError
@@ -31,6 +32,7 @@ from telethon.errors import (
    LocationInvalidError,
    SecurityError,
 )
+from telethon.tl.functions.messages import GetCustomEmojiDocumentsRequest
 from telethon.tl.types import (
    Document,
    InputDocumentFileLocation,
@@ -41,26 +43,23 @@ from telethon.tl.types import (
    PhotoSize,
    TypePhotoSize,
 )
-import magic

 from mautrix.appservice import IntentAPI
+from mautrix.util import ffmpeg, magic, variation_selector

+from .. import abstract_user as au
 from ..db import TelegramFile as DBTelegramFile
 from ..tgclient import MautrixTelegramClient
 from ..util import sane_mimetypes
 from .parallel_file_transfer import parallel_transfer_to_matrix
 from .tgs_converter import convert_tgs_to
+from .webm_converter import convert_webm_to

 try:
    from PIL import Image
 except ImportError:
    Image = None

-try:
-    from moviepy.editor import VideoFileClip
-except ImportError:
-    VideoFileClip = None
-
 try:
    from mautrix.crypto.attachments import encrypt_attachment
 except ImportError:
@@ -98,36 +97,23 @@ def convert_image(
        return source_mime, file, None, None


-def _read_video_thumbnail(
-    data: bytes,
-    video_ext: str = "mp4",
-    frame_ext: str = "png",
-    max_size: tuple[int, int] = (1024, 720),
-) -> tuple[bytes, int, int]:
-    with tempfile.NamedTemporaryFile(prefix="mxtg_video_", suffix=f".{video_ext}") as file:
-        # We don't have any way to read the video from memory, so save it to disk.
-        file.write(data)
-
-        # Read temp file and get frame
-        frame = VideoFileClip(file.name).get_frame(0)
-
-    # Convert to png and save to BytesIO
-    image = Image.fromarray(frame).convert("RGBA")
-
-    thumbnail_file = BytesIO()
-    if max_size:
-        image.thumbnail(max_size, Image.ANTIALIAS)
-    image.save(thumbnail_file, frame_ext)
-
-    w, h = image.size
-    return thumbnail_file.getvalue(), w, h
+async def _read_video_thumbnail(data: bytes, mime_type: str) -> tuple[bytes, int, int]:
+    first_frame = await ffmpeg.convert_bytes(
+        data,
+        output_extension=".png",
+        output_args=("-update", "1", "-frames:v", "1"),
+        input_mime=mime_type,
+        logger=log,
+    )
+    width, height = Image.open(BytesIO(first_frame)).size
+    return first_frame, width, height


 def _location_to_id(location: TypeLocation) -> str:
    if isinstance(location, Document):
-        return f"{location.id}-{location.access_hash}"
+        return str(location.id)
    elif isinstance(location, (InputDocumentFileLocation, InputPhotoFileLocation)):
-        return f"{location.id}-{location.access_hash}-{location.thumb_size}"
+        return f"{location.id}-{location.thumb_size}"
    elif isinstance(location, InputFileLocation):
        return f"{location.volume_id}-{location.local_id}"
    elif isinstance(location, InputPeerPhotoFileLocation):
@@ -146,7 +132,7 @@ async def transfer_thumbnail_to_matrix(
    height: int | None = None,
    async_upload: bool = False,
 ) -> DBTelegramFile | None:
-    if not Image or not VideoFileClip:
+    if not Image or not ffmpeg.ffmpeg_path:
        return None

    loc_id = _location_to_id(thumbnail_loc)
@@ -155,6 +141,8 @@ async def transfer_thumbnail_to_matrix(

    if custom_data:
        loc_id += "-mau_custom_thumbnail"
+    if encrypt:
+        loc_id += "-encrypted"

    db_file = await DBTelegramFile.get(loc_id)
    if db_file:
@@ -163,16 +151,18 @@ async def transfer_thumbnail_to_matrix(
    video_ext = sane_mimetypes.guess_extension(mime_type)
    if custom_data:
        file = custom_data
-    elif VideoFileClip and video_ext and video:
+    elif video_ext and video:
+        log.debug(f"Generating thumbnail for video {loc_id} with ffmpeg")
        try:
-            file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
-        except OSError:
+            file, width, height = await _read_video_thumbnail(video, mime_type=mime_type)
+        except Exception:
+            log.warning(f"Failed to generate thumbnail for {loc_id}", exc_info=True)
            return None
        mime_type = "image/png"
    else:
        file = await client.download_file(thumbnail_loc)
        width, height = None, None
-        mime_type = magic.from_buffer(file, mime=True)
+        mime_type = magic.mimetype(file)

    decryption_info = None
    upload_mime_type = mime_type
@@ -207,9 +197,71 @@ async def transfer_thumbnail_to_matrix(

 transfer_locks: dict[str, asyncio.Lock] = {}

+unicode_custom_emoji_map = pickle.loads(
+    pkgutil.get_data("mautrix_telegram", "unicodemojipack.pickle")
+)
+reverse_unicode_custom_emoji_map = {
+    doc_id: emoji for emoji, doc_id in unicode_custom_emoji_map.items()
+}
+
 TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]


+class UnicodeCustomEmoji(NamedTuple):
+    emoji: str
+
+
+async def transfer_custom_emojis_to_matrix(
+    source: au.AbstractUser, emoji_ids: list[int], client: MautrixTelegramClient | None = None
+) -> dict[int, DBTelegramFile | UnicodeCustomEmoji]:
+    if not client:
+        client = source.client
+    emoji_ids = set(emoji_ids)
+    existing_unicode = {}
+    for emoji_id in emoji_ids:
+        try:
+            existing_unicode[emoji_id] = UnicodeCustomEmoji(
+                variation_selector.add(reverse_unicode_custom_emoji_map[emoji_id])
+            )
+        except KeyError:
+            pass
+    emoji_ids -= existing_unicode.keys()
+    if not emoji_ids:
+        return existing_unicode
+    existing = await DBTelegramFile.get_many([str(id) for id in emoji_ids])
+    file_map = {int(file.id): file for file in existing} | existing_unicode
+    not_existing_ids = list(emoji_ids - file_map.keys())
+    if not_existing_ids:
+        log.debug(f"Transferring custom emojis through {source.mxid}: {not_existing_ids}")
+
+        documents: list[Document] = await client(
+            GetCustomEmojiDocumentsRequest(document_id=not_existing_ids)
+        )
+
+        tgs_args = source.config["bridge.animated_emoji"]
+        webm_convert = tgs_args["target"]
+
+        transfer_sema = asyncio.Semaphore(5)
+
+        async def transfer(document: Document) -> None:
+            async with transfer_sema:
+                file_map[document.id] = await transfer_file_to_matrix(
+                    client,
+                    source.bridge.az.intent,
+                    document,
+                    is_sticker=True,
+                    tgs_convert=tgs_args,
+                    webm_convert=webm_convert,
+                    filename=f"emoji-{document.id}",
+                    # Emojis are used as inline images and can't be encrypted
+                    encrypt=False,
+                    async_upload=source.config["homeserver.async_media"],
+                )
+
+        await asyncio.gather(*[transfer(doc) for doc in documents])
+    return file_map
+
+
 async def transfer_file_to_matrix(
    client: MautrixTelegramClient,
    intent: IntentAPI,
@@ -218,6 +270,7 @@ async def transfer_file_to_matrix(
    *,
    is_sticker: bool = False,
    tgs_convert: dict | None = None,
+    webm_convert: str | None = None,
    filename: str | None = None,
    encrypt: bool = False,
    parallel_id: int | None = None,
@@ -226,6 +279,8 @@ async def transfer_file_to_matrix(
    location_id = _location_to_id(location)
    if not location_id:
        return None
+    if encrypt:
+        location_id += "-encrypted"

    db_file = await DBTelegramFile.get(location_id)
    if db_file:
@@ -245,6 +300,7 @@ async def transfer_file_to_matrix(
            thumbnail,
            is_sticker,
            tgs_convert,
+            webm_convert,
            filename,
            encrypt,
            parallel_id,
@@ -260,6 +316,7 @@ async def _unlocked_transfer_file_to_matrix(
    thumbnail: TypeThumbnail,
    is_sticker: bool,
    tgs_convert: dict | None,
+    webm_convert: str | None,
    filename: str | None,
    encrypt: bool,
    parallel_id: int | None,
@@ -276,10 +333,10 @@ async def _unlocked_transfer_file_to_matrix(
            client, intent, loc_id, location, filename, encrypt, parallel_id
        )
        mime_type = location.mime_type
-        file = None
+        unencrypted_file = None
    else:
        try:
-            file = await client.download_file(location)
+            unencrypted_file = file = await client.download_file(location)
        except (LocationInvalidError, FileIdInvalidError):
            return None
        except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
@@ -287,13 +344,10 @@ async def _unlocked_transfer_file_to_matrix(
            return None

        width, height = None, None
-        mime_type = magic.from_buffer(file, mime=True)
+        mime_type = magic.mimetype(file)

        image_converted = False
-        # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
-        is_tgs = mime_type == "application/gzip" or (
-            mime_type == "application/octet-stream" and magic.from_buffer(file).startswith("gzip")
-        )
+        is_tgs = mime_type == "application/gzip"
        if is_sticker and tgs_convert and is_tgs:
            converted_anim = await convert_tgs_to(
                file, tgs_convert["target"], **tgs_convert["args"]
@@ -303,6 +357,12 @@ async def _unlocked_transfer_file_to_matrix(
            width, height = converted_anim.width, converted_anim.height
            image_converted = mime_type != "application/gzip"
            thumbnail = None
+        elif is_sticker and webm_convert and webm_convert != "webm" and mime_type == "video/webm":
+            converted_anim = await convert_webm_to(file, webm_convert)
+            mime_type = converted_anim.mime
+            file = converted_anim.data
+            image_converted = mime_type != "video/webm"
+            thumbnail = None

        decryption_info = None
        upload_mime_type = mime_type
@@ -324,34 +384,37 @@ async def _unlocked_transfer_file_to_matrix(
            width=width,
            height=height,
        )
-    if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
-        if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
-            thumbnail = thumbnail.location
-        try:
+    try:
+        if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
+            if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
+                thumbnail = thumbnail.location
+            try:
+                db_file.thumbnail = await transfer_thumbnail_to_matrix(
+                    client,
+                    intent,
+                    thumbnail,
+                    video=unencrypted_file,
+                    mime_type=mime_type,
+                    encrypt=encrypt,
+                    async_upload=async_upload,
+                )
+            except FileIdInvalidError:
+                log.warning(f"Failed to transfer thumbnail {thumbnail!s}", exc_info=True)
+        elif converted_anim and converted_anim.thumbnail_data:
            db_file.thumbnail = await transfer_thumbnail_to_matrix(
                client,
                intent,
-                thumbnail,
-                video=file,
-                mime_type=mime_type,
+                location,
+                video=None,
                encrypt=encrypt,
+                custom_data=converted_anim.thumbnail_data,
+                mime_type=converted_anim.thumbnail_mime,
+                width=converted_anim.width,
+                height=converted_anim.height,
                async_upload=async_upload,
            )
-        except FileIdInvalidError:
-            log.warning(f"Failed to transfer thumbnail for {thumbnail!s}", exc_info=True)
-    elif converted_anim and converted_anim.thumbnail_data:
-        db_file.thumbnail = await transfer_thumbnail_to_matrix(
-            client,
-            intent,
-            location,
-            video=None,
-            encrypt=encrypt,
-            custom_data=converted_anim.thumbnail_data,
-            mime_type=converted_anim.thumbnail_mime,
-            width=converted_anim.width,
-            height=converted_anim.height,
-            async_upload=async_upload,
-        )
+    except Exception:
+        log.exception(f"Failed to transfer thumbnail for {loc_id}")

    try:
        await db_file.insert()
@@ -99,7 +99,7 @@ if lottieconverter:
    converters["png"] = tgs_to_png
    converters["gif"] = tgs_to_gif

-if lottieconverter and ffmpeg:
+if lottieconverter and ffmpeg.ffmpeg_path:

    async def tgs_to_webm(
        file: bytes, width: int, height: int, fps: int = 30, **_: Any
@@ -126,7 +126,33 @@ if lottieconverter and ffmpeg:
                log.error(str(e))
        return ConvertedSticker("application/gzip", file)

+    async def tgs_to_webp(
+        file: bytes, width: int, height: int, fps: int = 30, **_: Any
+    ) -> ConvertedSticker:
+        with tempfile.TemporaryDirectory(prefix="tgs_") as tmpdir:
+            file_template = tmpdir + "/out_"
+            try:
+                await _run_lottieconverter(
+                    args=("-", file_template, "pngs", f"{width}x{height}", str(fps)),
+                    input_data=file,
+                )
+                first_frame_name = min(os.listdir(tmpdir))
+                with open(f"{tmpdir}/{first_frame_name}", "rb") as first_frame_file:
+                    first_frame_data = first_frame_file.read()
+                webp_data = await ffmpeg.convert_path(
+                    input_args=("-framerate", str(fps), "-pattern_type", "glob"),
+                    input_file=f"{file_template}*.png",
+                    output_args=("-c:v", "libwebp_anim", "-pix_fmt", "yuva420p", "-f", "webp"),
+                    output_path_override="-",
+                    output_extension=None,
+                )
+                return ConvertedSticker("image/webp", webp_data, "image/png", first_frame_data)
+            except ffmpeg.ConverterError as e:
+                log.error(str(e))
+        return ConvertedSticker("application/gzip", file)
+
    converters["webm"] = tgs_to_webm
+    converters["webp"] = tgs_to_webp


 async def convert_tgs_to(
@@ -0,0 +1,39 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from telethon.tl.types import (
+    JsonArray,
+    JsonBool,
+    JsonNull,
+    JsonNumber,
+    JsonObject,
+    JsonObjectValue,
+    JsonString,
+    TypeJSONValue,
+)
+
+from mautrix.types import JSON
+
+
+def parse_tl_json(val: TypeJSONValue) -> JSON:
+    if isinstance(val, JsonObject):
+        return {entry.key: parse_tl_json(entry.value) for entry in val.value}
+    elif isinstance(val, JsonArray):
+        return [parse_tl_json(item) for item in val.value]
+    elif isinstance(val, (JsonBool, JsonNumber, JsonString)):
+        return val.value
+    elif isinstance(val, JsonNull):
+        return None
+    raise ValueError(f"Unsupported type {type(val)} in TL JSON object")
@@ -0,0 +1,52 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import logging
+
+from mautrix.util import ffmpeg
+
+from .tgs_converter import ConvertedSticker
+
+log: logging.Logger = logging.getLogger("mau.util.webm")
+
+
+converter_args = {
+    "gif": {
+        "output_args": ("-vf", "split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse"),
+    },
+    "png": {
+        "input_args": ("-ss", "0"),
+        "output_args": ("-frames:v", "1"),
+    },
+    "webp": {},
+}
+
+
+async def convert_webm_to(file: bytes, convert_to: str) -> ConvertedSticker:
+    if convert_to in ("png", "gif", "webp"):
+        try:
+            converted_data = await ffmpeg.convert_bytes(
+                data=file,
+                output_extension=f".{convert_to}",
+                **converter_args[convert_to],
+            )
+            return ConvertedSticker(f"image/{convert_to}", converted_data)
+        except ffmpeg.ConverterError as e:
+            log.error(str(e))
+    elif convert_to != "disable":
+        log.warning(f"Unable to convert webm animated sticker, type {convert_to} not supported")
+    return ConvertedSticker("video/webm", file)
@@ -38,6 +38,7 @@ from telethon.errors import (
 )

 from mautrix.bridge import InvalidAccessToken, OnlyLoginSelf
+from mautrix.util import background_task
 from mautrix.util.format_duration import format_duration

 from ...commands.telegram.auth import enter_password
@@ -128,7 +129,7 @@ class AuthAPI(abc.ABC):
                status=200,
                message=(
                    "Code requested successfully. Check your SMS "
-                    "or Telegram client and enter the code below."
+                    "or Telegram app and enter the code below."
                ),
            )
        except PhoneNumberInvalidError:
@@ -199,7 +200,7 @@ class AuthAPI(abc.ABC):
        existing_user = await User.get_by_tgid(user_info.id)
        if existing_user and existing_user != user:
            await existing_user.log_out()
-        asyncio.create_task(user.post_login(user_info, first_login=True))
+        background_task.create(user.post_login(user_info, first_login=True))
        if user.command_status and user.command_status["action"] == "Login":
            user.command_status = None

@@ -279,6 +280,14 @@ class AuthAPI(abc.ABC):
                errcode="phone_code_expired",
                error="Phone code expired.",
            )
+        except PhoneNumberUnoccupiedError:
+            return self.get_login_response(
+                mxid=user.mxid,
+                state="code",
+                status=403,
+                errcode="phone_number_unoccupied",
+                error="That phone number has not been registered.",
+            )
        except SessionPasswordNeededError:
            if not password_in_data:
                if user.command_status and user.command_status["action"] == "Login":
@@ -333,8 +342,16 @@ class AuthAPI(abc.ABC):
                errcode="password_invalid",
                error="Incorrect password.",
            )
-        except Exception:
+        except Exception as e:
            self.log.exception("Error sending password")
+            if isinstance(e, ValueError) and "You must provide a phone and a code" in str(e):
+                return self.get_login_response(
+                    mxid=user.mxid,
+                    state="request",
+                    status=400,
+                    errcode="phone_code_not_entered",
+                    error="Please request a new phone code and enter it first.",
+                )
            return self.get_login_response(
                mxid=user.mxid,
                state="password",
@@ -17,16 +17,22 @@ from __future__ import annotations

 from typing import TYPE_CHECKING, Awaitable, Callable
 import asyncio
+import datetime
 import json
 import logging

 from aiohttp import web
+from telethon.errors import SessionPasswordNeededError
+from telethon.tl.custom import QRLogin
+from telethon.tl.functions.messages import GetAllStickersRequest
 from telethon.tl.types import ChannelForbidden, ChatForbidden, TypeChat, User as TLUser
 from telethon.utils import get_peer_id, resolve_id

 from mautrix.appservice import AppService
+from mautrix.client import Client
 from mautrix.errors import IntentError, MatrixRequestError
 from mautrix.types import UserID
+from mautrix.util import background_task

 from ...commands.portal.util import get_initial_state, user_has_power_level
 from ...portal import Portal
@@ -53,7 +59,7 @@ class ProvisioningAPI(AuthAPI):

        self.app = web.Application(loop=bridge.loop, middlewares=[self.error_middleware])

-        portal_prefix = "/v1/portal/{mxid:![^/]+}"
+        portal_prefix = "/v1/portal/{mxid}"
        self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal_by_mxid)
        self.app.router.add_route("GET", "/v1/portal/{tgid:-[0-9]+}", self.get_portal_by_tgid)
        self.app.router.add_route(
@@ -62,19 +68,27 @@ class ProvisioningAPI(AuthAPI):
        self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat)
        self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat)

-        user_prefix = "/v1/user/{mxid:@[^:]*:[^/]+}"
+        user_prefix = "/v1/user/{mxid}"
        self.app.router.add_route("GET", f"{user_prefix}", self.get_user_info)
        self.app.router.add_route("GET", f"{user_prefix}/chats", self.get_chats)
        self.app.router.add_route("GET", f"{user_prefix}/contacts", self.get_contacts)
+        self.app.router.add_route(
+            "GET", f"{user_prefix}/resolve_identifier/{{identifier}}", self.resolve_identifier
+        )
        self.app.router.add_route("POST", f"{user_prefix}/pm/{{identifier}}", self.start_dm)

+        self.app.router.add_route("GET", f"{user_prefix}/stickersets", self.get_stickersets)
+
+        self.app.router.add_route("POST", f"{user_prefix}/retry_takeout", self.retry_takeout)
+
        self.app.router.add_route("POST", f"{user_prefix}/logout", self.logout)
+        self.app.router.add_route("GET", f"{user_prefix}/login/qr", self.login_qr)
        self.app.router.add_route("POST", f"{user_prefix}/login/bot_token", self.send_bot_token)
        self.app.router.add_route("POST", f"{user_prefix}/login/request_code", self.request_code)
        self.app.router.add_route("POST", f"{user_prefix}/login/send_code", self.send_code)
        self.app.router.add_route("POST", f"{user_prefix}/login/send_password", self.send_password)

-        self.app.router.add_route("GET", "/bridge", self.bridge_info)
+        self.app.router.add_route("GET", "/v1/bridge", self.bridge_info)

    async def get_portal_by_mxid(self, request: web.Request) -> web.Response:
        err = self.check_authorization(request)
@@ -214,7 +228,7 @@ class ProvisioningAPI(AuthAPI):
            portal.photo_id = ""
            await portal.save()

-        asyncio.create_task(portal.update_matrix_room(user, entity, levels=levels))
+        background_task.create(portal.update_matrix_room(user, entity, levels=levels))

        return web.Response(status=202, body="{}")

@@ -335,7 +349,7 @@ class ProvisioningAPI(AuthAPI):
                self.log.exception("Failed to disconnect chat")
                return self.get_error_response(500, "exception", "Failed to disconnect chat")
        else:
-            asyncio.create_task(coro)
+            background_task.create(coro)
        return web.json_response({}, status=200 if sync else 202)

    async def get_user_info(self, request: web.Request) -> web.Response:
@@ -401,40 +415,80 @@ class ProvisioningAPI(AuthAPI):
            return err
        return web.json_response(data=await user.sync_contacts())

-    async def start_dm(self, request: web.Request) -> web.Response:
+    async def _resolve_id(
+        self, request: web.Request
+    ) -> tuple[Portal | None, User | None, TLUser | None, web.Response | None]:
        data, user, err = await self.get_user_request_info(request, expect_logged_in=True)
        if err is not None:
-            return err
+            return None, user, None, err
        try:
            identifier: str | int = request.match_info["identifier"]
            if isinstance(identifier, str) and identifier.isdecimal():
                identifier = int(identifier)
            target = await user.client.get_entity(identifier)
        except ValueError:
-            return web.json_response(
-                {
-                    "error": "Invalid user identifier or user not found.",
-                    "errcode": "M_NOT_FOUND",
-                },
-                status=404,
+            return (
+                None,
+                user,
+                None,
+                web.json_response(
+                    {
+                        "error": "Invalid user identifier or user not found.",
+                        "errcode": "M_NOT_FOUND",
+                    },
+                    status=404,
+                ),
            )

        if not target:
-            return web.json_response(
-                {
-                    "error": "User not found.",
-                    "errcode": "M_NOT_FOUND",
-                },
-                status=404,
+            return (
+                None,
+                user,
+                None,
+                web.json_response(
+                    {
+                        "error": "User not found.",
+                        "errcode": "M_NOT_FOUND",
+                    },
+                    status=404,
+                ),
            )
        elif not isinstance(target, TLUser):
-            return web.json_response(
-                {
-                    "error": "Identifier is not a user.",
-                },
-                status=400,
+            return (
+                None,
+                user,
+                None,
+                web.json_response(
+                    {
+                        "error": "Identifier is not a user.",
+                        "errcode": "FI.MAU.TELEGRAM_ID_NOT_USER",
+                    },
+                    status=400,
+                ),
            )
        portal = await Portal.get_by_entity(target, tg_receiver=user.tgid)
+        return portal, user, target, None
+
+    async def resolve_identifier(self, request: web.Request) -> web.Response:
+        portal, user, target, err = await self._resolve_id(request)
+        if err is not None:
+            return err
+        puppet = await portal.get_dm_puppet()
+        await puppet.update_info(user, target)
+        return web.json_response(
+            {
+                "room_id": portal.mxid,
+                "just_created": False,
+                "id": portal.tgid,
+                "contact_info": puppet.contact_info,
+            },
+            status=200,
+        )
+
+    async def start_dm(self, request: web.Request) -> web.Response:
+        portal, user, target, err = await self._resolve_id(request)
+        if err is not None:
+            return err
        puppet = await portal.get_dm_puppet()
        if portal.mxid:
            just_created = False
@@ -451,6 +505,78 @@ class ProvisioningAPI(AuthAPI):
            status=201 if just_created else 200,
        )

+    async def get_stickersets(self, request: web.Request) -> web.Response:
+        _, user, err = await self.get_user_request_info(
+            request, expect_logged_in=True, want_data=False
+        )
+        if err is not None:
+            return err
+        result = await user.client(GetAllStickersRequest(0))
+        resp = []
+        for stickerset in result.sets:
+            resp.append(stickerset.short_name)
+        return web.json_response(resp, status=200)
+
+    async def retry_takeout(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(
+            request, expect_logged_in=True, want_data=False
+        )
+        if err is not None:
+            return err
+        if not user.takeout_requested:
+            return web.json_response(
+                {
+                    "error": "There was no takeout requested",
+                },
+                status=400,
+            )
+        user.takeout_retry_immediate.set()
+        return web.json_response({}, status=200)
+
+    async def login_qr(self, request: web.Request) -> web.Response:
+        _, user, err = await self.get_user_request_info(request, websocket=True)
+        if err is not None:
+            return err
+
+        await user.ensure_started(even_if_no_session=True)
+        qr_login = QRLogin(user.client, ignored_ids=[])
+
+        ws = web.WebSocketResponse(protocols=["net.maunium.telegram.login"])
+        await ws.prepare(request)
+
+        retries = 0
+        user_info = None
+        while retries < 4:
+            try:
+                await qr_login.recreate()
+                await ws.send_json(
+                    {
+                        "code": qr_login.url,
+                        "timeout": int(
+                            (
+                                qr_login.expires - datetime.datetime.now(tz=datetime.timezone.utc)
+                            ).total_seconds()
+                        ),
+                    }
+                )
+                user_info = await qr_login.wait()
+                break
+            except asyncio.TimeoutError:
+                retries += 1
+            except SessionPasswordNeededError:
+                await ws.send_json({"success": False, "error": "password-needed"})
+                await ws.close()
+                return ws
+        else:
+            await ws.send_json({"success": False, "error": "timeout"})
+            await ws.close()
+            return ws
+
+        await self.postprocess_login(user, user_info)
+        await ws.send_json({"success": True})
+        await ws.close()
+        return ws
+
    async def send_bot_token(self, request: web.Request) -> web.Response:
        data, user, err = await self.get_user_request_info(request)
        if err is not None:
@@ -576,6 +702,15 @@ class ProvisioningAPI(AuthAPI):
            )
        return None

+    def check_websocket_authorization(self, request: web.Request) -> web.Response | None:
+        auth_parts = request.headers.get("Sec-WebSocket-Protocol").split(",")
+        for part in auth_parts:
+            if part.strip() == f"net.maunium.telegram.auth-{self.secret}":
+                return None
+        return self.get_error_response(
+            error="Shared secret is not valid.", errcode="shared_secret_invalid", status=401
+        )
+
    @staticmethod
    async def get_data(request: web.Request) -> dict | None:
        try:
@@ -596,6 +731,12 @@ class ProvisioningAPI(AuthAPI):
            return None, self.get_login_response(
                error="User ID not given.", errcode="mxid_empty", status=400
            )
+        try:
+            Client.parse_user_id(mxid)
+        except ValueError:
+            return None, self.get_login_response(
+                error="Invalid user ID", errcode="mxid_invalid", status=400
+            )

        user = await User.get_and_start_by_mxid(mxid, even_if_no_session=True)
        if require_puppeting and not user.puppet_whitelisted:
@@ -624,8 +765,12 @@ class ProvisioningAPI(AuthAPI):
        expect_logged_in: bool | None = False,
        require_puppeting: bool = False,
        want_data: bool = True,
+        websocket: bool = False,
    ) -> tuple[dict | None, User | None, web.Response | None]:
-        err = self.check_authorization(request)
+        if not websocket:
+            err = self.check_authorization(request)
+        else:
+            err = self.check_websocket_authorization(request)
        if err is not None:
            return None, None, err

@@ -2,8 +2,7 @@
 # Uncommented lines after the group definition insert things into that group.

 #/speedups
-cryptg>=0.1,<0.3
-cchardet
+cryptg>=0.1,<0.5
 aiodns
 brotli

@@ -11,14 +10,12 @@ brotli
 pillow>=4,<10
 qrcode>=6,<8

-#/hq_thumbnails
-moviepy>=1,<2

 #/formattednumbers
 phonenumbers>=8,<9

 #/metrics
-prometheus_client>=0.6,<0.15
+prometheus_client>=0.6,<0.17

 #/e2be
 python-olm>=3,<4
@@ -26,4 +23,4 @@ pycryptodome>=3,<4
 unpaddedbase64>=1,<3

 #/sqlite
-aiosqlite>=0.16,<0.18
+aiosqlite>=0.16,<0.19
@@ -3,10 +3,9 @@ python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.16.0,<0.17
-#telethon>=1.24,<1.25
-# Fork to make session storage async and update to layer 138
-tulir-telethon==1.25.0a7
-asyncpg>=0.20,<0.26
+mautrix>=0.19.4,<0.20
+#telethon>=1.25.4,<1.27
+tulir-telethon==1.28.0a3
+asyncpg>=0.20,<0.28
 mako>=1,<2
 setuptools
@@ -66,7 +66,7 @@ setuptools.setup(
    ],
    package_data={"mautrix_telegram": [
        "web/public/*.mako", "web/public/*.png", "web/public/*.css",
-        "example-config.yaml",
+        "example-config.yaml", "unicodemojipack.pickle",
    ]},
    data_files=[
        (".", ["mautrix_telegram/example-config.yaml"]),