Bump version to 0.12.0

Don't check whether User is channel, add peer property to User
Update mautrix-python again
2022-08-26 16:22:06 +03:00 · 2022-08-24 21:13:11 +03:00 · 2022-08-22 17:48:10 +03:00 · 2022-08-22 13:00:08 +03:00 · 2022-08-22 12:57:41 +03:00 · 2022-08-17 15:20:38 +03:00
71 changed files with 3960 additions and 2264 deletions
@@ -17,5 +17,5 @@ max_line_length = 99
 [*.{yaml,yml,py}]
 indent_style = space

-[{.gitlab-ci.yml,.pre-commit-config.yaml}]
+[{.gitlab-ci.yml,.pre-commit-config.yaml,mautrix_telegram/web/provisioning/spec.yaml}]
 indent_size = 2
@@ -0,0 +1,7 @@
+---
+name: Bug report
+about: If something is definitely wrong in the bridge (rather than just a setup issue),
+  file a bug report. Remember to include relevant logs.
+labels: bug
+
+---
@@ -0,0 +1,7 @@
+contact_links:
+  - name: Troubleshooting docs & FAQ
+    url: https://docs.mau.fi/bridges/general/troubleshooting.html
+    about: Check this first if you're having problems setting up the bridge.
+  - name: Support room
+    url: https://matrix.to/#/#telegram:maunium.net
+    about: For setup issues not answered by the troubleshooting docs, ask in the Matrix room.
@@ -0,0 +1,6 @@
+---
+name: Enhancement request
+about: Submit a feature request or other suggestion
+labels: enhancement
+
+---
@@ -16,7 +16,7 @@ jobs:
    - uses: psf/black@stable
      with:
        src: "./mautrix_telegram"
-        version: "22.1.0"
+        version: "22.3.0"
    - name: pre-commit
      run: |
        pip install pre-commit
@@ -1,66 +1,3 @@
-image: docker:stable
-
-stages:
- build
- manifest
-
-default:
-  before_script:
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-
-build amd64:
-  stage: build
-  tags:
-  - amd64
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  after_script:
-  - |
-    if [[ "$CI_COMMIT_BRANCH" == "master" && "$CI_JOB_STATUS" == "success" ]]; then
-      apk add --update curl jq
-      rm -rf /var/cache/apk/*
-
-      jq -n '
-         {
-           password:  env.BEEPER_DEV_ADMIN_NIGHTLY_PASS,
-           bridge:    env.BEEPER_BRIDGE_TYPE,
-           image:     "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
-           channel:   "STABLE"
-         }
-        ' | curl "$BEEPER_DEV_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
-
-      jq -n '
-         {
-           password:    env.BEEPER_PROD_ADMIN_NIGHTLY_PASS,
-           bridge:      env.BEEPER_BRIDGE_TYPE,
-           image:       "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
-           channel:     "INTERNAL",
-           deployNext:  true
-         }
-         ' | curl "$BEEPER_PROD_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
-    fi
-
-build arm64:
-  stage: build
-  tags:
-  - arm64
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-
-manifest:
-  stage: manifest
-  before_script:
-  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
-  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+include:
+- project: 'mautrix/ci'
+  file: '/python.yml'
@@ -7,17 +7,14 @@ repos:
      - id: end-of-file-fixer
      - id: check-yaml
      - id: check-added-large-files
-  # TODO convert to use the upstream psf/black when
-  # https://github.com/psf/black/issues/2493 gets fixed
-  - repo: local
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
    hooks:
      - id: black
-        name: black
-        entry: black --check
-        language: system
-        files: ^mautrix_telegram/.*\.py$
+        language_version: python3
+        files: ^mautrix_telegram/.*\.pyi?$
  - repo: https://github.com/PyCQA/isort
    rev: 5.10.1
    hooks:
      - id: isort
-        files: ^mautrix_telegram/.*$
+        files: ^mautrix_telegram/.*\.pyi?$
@@ -1,3 +1,108 @@
+# v0.12.0 (2022-08-26)
+
+**N.B.** This release requires a homeserver with Matrix v1.1 support, which
+bumps up the minimum homeserver versions to Synapse 1.54 and Dendrite 0.8.7.
+Minimum Conduit version remains at 0.4.0.
+
+### Added
+* Added provisioning API for resolving Telegram identifiers (like usernames).
+* Added support for bridging Telegram custom emojis to Matrix.
+* Added option to not bridge chats with lots of members.
+* Added option to include captions in the same message as the media to
+  implement [MSC2530]. Sending captions the same way is also supported and
+  enabled by default.
+* Added commands to kick or ban relaybot users from Telegram.
+* Added support for Telegram's disappearing messages.
+* Added support for bridging forwarded messages as forwards on Telegram.
+  * Forwarding is not allowed in relay mode as the bot wouldn't be able to
+    specify who sent the message.
+  * Matrix doesn't have real forwarding (there's no forwarding metadata), so
+    only messages bridged from Telegram can be forwarded.
+  * Double puppeted messages from Telegram currently can't be forwarded without
+    removing the `fi.mau.double_puppet_source` key from the content.
+  * If forwarding fails (e.g. due to it being blocked in the source chat), the
+    bridge will automatically fall back to sending it as a normal new message.
+* Added options to make encryption more secure.
+  * The `encryption` -> `verification_levels` config options can be used to
+    make the bridge require encrypted messages to come from cross-signed
+    devices, with trust-on-first-use validation of the cross-signing master
+    key.
+  * The `encryption` -> `require` option can be used to make the bridge ignore
+    any unencrypted messages.
+  * Key rotation settings can be configured with the `encryption` -> `rotation`
+    config.
+
+### Improved
+* Improved handling the bridge user leaving chats on Telegram, and new users
+  being added on Telegram.
+* Improved animated sticker conversion options: added support for animated webp
+  and added option to convert video stickers (webm) to the specified image
+  format.
+* Audio and video metadata is now bridged properly to Telegram.
+* Added database index on Telegram usernames (used when bridging username
+  @-mentions in messages).
+* Changed `/login/send_code` provisioning API to return a proper error when the
+  phone number is not registered on Telegram.
+  * The same login code can be used for registering an account, but registering
+    is not currently supported in the provisioning API.
+* Removed `plaintext_highlights` config option (the code using it was already
+  removed in v0.11.0).
+* Enabled appservice ephemeral events by default for new installations.
+  * Existing bridges can turn it on by enabling `ephemeral_events` and disabling
+    `sync_with_custom_puppets` in the config, then regenerating the registration
+    file.
+* Updated to API layer 144 so that Telegram would send new message types like
+  premium stickers to the bridge.
+* Updated Docker image to Alpine 3.16 and made it smaller.
+
+### Fixed
+* Fixed command prefix in game and poll messages (thanks to [@cynhr] in [#804]).
+
+[MSC2530]: https://github.com/matrix-org/matrix-spec-proposals/pull/2530
+[@cynhr]: https://github.com/cynhr
+[#804]: https://github.com/mautrix/telegram/pull/804
+
+# v0.11.3 (2022-04-17)
+
+**N.B.** This release drops support for old homeservers which don't support the
+new `/v3` API endpoints. Synapse 1.48+, Dendrite 0.6.5+ and Conduit 0.4.0+ are
+supported. Legacy `r0` API support can be temporarily re-enabled with `pip install mautrix==0.16.0`.
+However, this option will not be available in future releases.
+
+### Added
+* Added `list-invite-links` command to list invite links in a chat.
+* Added option to use [MSC2246] async media uploads.
+* Provisioning API for listing contacts and starting private chats.
+
+### Improved
+* Dropped Python 3.7 support.
+* Telegram->Matrix message formatter will now replace `t.me/c/chatid/messageid`
+  style links with a link to the bridged Matrix event (in addition to the
+  previously supported `t.me/username/messageid` links).
+* Updated formatting converter to keep newlines in code blocks as `\n` instead
+  of converting them to `<br/>`.
+* Removed `max_document_size` option. The bridge will now fetch the max size
+  automatically using the media repo config endpoint.
+* Removed redundant `msgtype` field in sticker events sent to Matrix.
+* Disabled file logging in Docker image by default.
+  * If you want to enable it, set the `filename` in the file log handler to a
+    path that is writable, then add `"file"` back to `logging.root.handlers`.
+* Reactions are now marked as read when bridging read receipts from Matrix.
+
+### Fixed
+* Fixed `!tg bridge` throwing error if the parameter is not an integer
+* Fixed `!tg bridge` failing if the command had been previously run with an
+  incorrectly prefixed chat ID (e.g. `!tg bridge -1234567` followed by
+  `!tg bridge -1001234567`).
+* Fixed `bridge_matrix_leave` config option not actually being used correctly.
+* Fixed public channel mentions always bridging into a user mention on Matrix
+  rather than a room mention.
+  * The bridge will now make room mentions if the portal exists and fall back
+    to user mentions otherwise.
+* Fixed newlines being lost in unformatted forwarded messages.
+
+[MSC2246]: https://github.com/matrix-org/matrix-spec-proposals/pull/2246
+
 # v0.11.2 (2022-02-14)

 **N.B.** This will be the last release to support Python 3.7. Future versions
@@ -229,8 +334,8 @@ path.
 * Bridging events of a user whose power level is malformed (i.e. a string
  instead of an integer) now works.

-[MSC2409]: https://github.com/matrix-org/matrix-doc/pull/2409
-[MSC2778]: https://github.com/matrix-org/matrix-doc/pull/2778
+[MSC2409]: https://github.com/matrix-org/matrix-spec-proposals/pull/2409
+[MSC2778]: https://github.com/matrix-org/matrix-spec-proposals/pull/2778

 # v0.8.2 (2020-07-27)

@@ -278,7 +383,7 @@ update (v0.5.8) and a fix to the Docker image.
 * Fixed `sync_direct_chats` option creating non-working portals.
 * Fixed video thumbnailing sometimes leaving behind downloaded videos in `/tmp`.

-[MSC2346]: https://github.com/matrix-org/matrix-doc/pull/2346
+[MSC2346]: https://github.com/matrix-org/matrix-spec-proposals/pull/2346

 ## rc1 (2020-04-25)

@@ -1,22 +1,25 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.15
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.16

 ARG TARGETARCH=amd64

 RUN apk add --no-cache \
      python3 py3-pip py3-setuptools py3-wheel \
-      py3-virtualenv \
      py3-pillow \
      py3-aiohttp \
      py3-magic \
      py3-ruamel.yaml \
      py3-commonmark \
-      py3-prometheus-client \
+      py3-phonenumbers \
+      py3-mako \
+      #py3-prometheus-client \ (pulls in twisted unnecessarily)
      # Indirect dependencies
      py3-idna \
+      py3-rsa \
      #moviepy
        py3-decorator \
        py3-tqdm \
        py3-requests \
+        #py3-proglog \
        #imageio
          py3-numpy \
      #py3-telethon \ (outdated)
@@ -25,7 +28,7 @@ RUN apk add --no-cache \
        py3-pyaes \
        # cryptg
          py3-cffi \
-	  py3-qrcode \
+          py3-qrcode \
      py3-brotli \
      # Other dependencies
      ffmpeg \
@@ -46,13 +49,15 @@ COPY requirements.txt /opt/mautrix-telegram/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
 WORKDIR /opt/mautrix-telegram
 RUN apk add --virtual .build-deps python3-dev libffi-dev build-base \
- && pip3 install -r requirements.txt -r optional-requirements.txt \
- && apk del .build-deps
+ && pip3 install /cryptg-*.whl \
+ && pip3 install --no-cache-dir -r requirements.txt -r optional-requirements.txt \
+ && apk del .build-deps \
+ && rm -f /cryptg-*.whl

 COPY . /opt/mautrix-telegram
-RUN apk add git && pip3 install .[all] && apk del git \
+RUN apk add git && pip3 install --no-cache-dir .[all] && apk del git \
  # This doesn't make the image smaller, but it's needed so that the `version` command works properly
-  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
+  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram .git build

 VOLUME /data
 ENV UID=1337 GID=1337 \
@@ -16,8 +16,8 @@ All setup and usage instructions are located on
 [docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
 Some quick links:

-* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
-  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
+* [Bridge setup](https://docs.mau.fi/bridges/python/setup.html?bridge=telegram)
+  (or [with Docker](https://docs.mau.fi/bridges/general/docker-setup.html?bridge=telegram))
 * Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
  [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
  [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)
@@ -24,6 +24,7 @@
 * Telegram → Matrix
  * [x] Message content (text, formatting, files, etc..)
  * [ ] Advanced message content/media
+    * [x] Custom emojis
    * [x] Polls
 	* [x] Games
 	* [ ] Buttons
@@ -54,7 +55,7 @@
  * [x] Automatic portal creation
    * [x] At startup
    * [x] When receiving invite or message
-  * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
+  * [x] Portal creation by inviting Matrix puppet of Telegram user to new room
  * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
  * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
  * [ ] ‡ Calls (hard, not yet supported by Telethon)
@@ -1,3 +1,3 @@
 pre-commit>=2.10.1,<3
 isort>=5.10.1,<6
-black==22.1.0
+black>=22.3,<23
@@ -2,7 +2,13 @@

 # Define functions.
 function fixperms {
-	chown -R $UID:$GID /data /opt/mautrix-telegram
+	chown -R $UID:$GID /data
+
+	# /opt/mautrix-telegram is read-only, so disable file logging if it's pointing there.
+	if [[ "$(yq e '.logging.handlers.file.filename' /data/config.yaml)" == "./mautrix-telegram.log" ]]; then
+		yq -I4 e -i 'del(.logging.root.handlers[] | select(. == "file"))' /data/config.yaml
+		yq -I4 e -i 'del(.logging.handlers.file)' /data/config.yaml
+	fi
 }

 cd /opt/mautrix-telegram
@@ -18,7 +24,10 @@ if [ ! -f /data/config.yaml ]; then
 fi

 if [ ! -f /data/registration.yaml ]; then
-	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
+	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml || exit $?
+	echo "Didn't find a registration file."
+	echo "Generated one for you."
+	echo "See https://docs.mau.fi/bridges/general/registering-appservices.html on how to use it."
 	fixperms
 	exit
 fi
@@ -1,2 +1,2 @@
-__version__ = "0.11.2"
+__version__ = "0.12.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"
@@ -103,7 +103,9 @@ class TelegramBridge(Bridge):
    def prepare_stop(self) -> None:
        for puppet in Puppet.by_custom_mxid.values():
            puppet.stop()
-        self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
+        self.add_shutdown_actions(user.stop() for user in User.by_tgid.values())
+        if self.bot:
+            self.add_shutdown_actions(self.bot.stop())

    async def get_user(self, user_id: UserID, create: bool = True) -> User | None:
        user = await User.get_by_mxid(user_id, create=create)
@@ -38,6 +38,7 @@ from telethon.tl.types import (
    PeerChat,
    PeerUser,
    TypeUpdate,
+    UpdateChannel,
    UpdateChannelUserTyping,
    UpdateChatParticipantAdmin,
    UpdateChatParticipants,
@@ -57,6 +58,7 @@ from telethon.tl.types import (
    UpdateReadChannelInbox,
    UpdateReadHistoryInbox,
    UpdateReadHistoryOutbox,
+    UpdateShort,
    UpdateShortChatMessage,
    UpdateShortMessage,
    UpdateUserName,
@@ -223,11 +225,24 @@ class AbstractUser(ABC):
            connection=connection,
            proxy=proxy,
            raise_last_call_error=True,
+            catch_up=self.config["telegram.catch_up"],
+            sequential_updates=self.config["telegram.sequential_updates"],
            loop=self.loop,
            base_logger=base_logger,
+            update_error_callback=self._telethon_update_error_callback,
        )
        self.client.add_event_handler(self._update_catch)

+    async def _telethon_update_error_callback(self, err: Exception) -> None:
+        if self.config["telegram.exit_on_update_error"]:
+            self.log.critical(f"Stopping due to update handling error {type(err).__name__}")
+            self.bridge.manual_stop(50)
+        else:
+            self.log.info("Recreating Telethon update loop in 60 seconds")
+            await asyncio.sleep(60)
+            self.log.debug("Now recreating Telethon update loop")
+            self.client._updates_handle = self.loop.create_task(self.client._update_loop())
+
    @abstractmethod
    async def update(self, update: TypeUpdate) -> bool:
        return False
@@ -297,6 +312,8 @@ class AbstractUser(ABC):
    # region Telegram update handling

    async def _update(self, update: TypeUpdate) -> None:
+        if isinstance(update, UpdateShort):
+            update = update.update
        asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
        if isinstance(
            update,
@@ -338,6 +355,8 @@ class AbstractUser(ABC):
            await self.update_pinned_dialogs(update)
        elif isinstance(update, UpdateNotifySettings):
            await self.update_notify_settings(update)
+        elif isinstance(update, UpdateChannel):
+            await self.update_channel(update)
        else:
            self.log.trace("Unhandled update: %s", update)

@@ -568,6 +587,36 @@ class AbstractUser(ABC):
            return
        await portal.handle_telegram_reactions(self, TelegramID(update.msg_id), update.reactions)

+    async def update_channel(self, update: UpdateChannel) -> None:
+        portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        if not portal:
+            return
+        if getattr(update, "mau_telethon_is_leave", False):
+            self.log.debug("UpdateChannel has mau_telethon_is_leave, leaving portal")
+            await portal.delete_telegram_user(self.tgid, sender=None)
+        elif chan := getattr(update, "mau_channel", None):
+            if not portal.mxid:
+                asyncio.create_task(self._delayed_create_channel(chan))
+            else:
+                self.log.debug("Updating channel info with data fetched by Telethon")
+                await portal.update_info(self, chan)
+                await portal.invite_to_matrix(self.mxid)
+
+    async def _delayed_create_channel(self, chan: Channel) -> None:
+        self.log.debug("Waiting 5 seconds before handling UpdateChannel for non-existent portal")
+        await asyncio.sleep(5)
+        portal = await po.Portal.get_by_tgid(TelegramID(chan.id))
+        if portal.mxid:
+            self.log.debug(
+                "Portal started existing after waiting 5 seconds, dropping UpdateChannel"
+            )
+            return
+        else:
+            self.log.info(
+                "Creating Matrix room with data fetched by Telethon due to UpdateChannel"
+            )
+            await portal.create_matrix_room(self, chan)
+
    async def update_message(self, original_update: UpdateMessage) -> None:
        update, sender, portal = await self.get_message_details(original_update)
        if not portal:
@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,8 +13,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Tuple
+from __future__ import annotations
+
+from typing import Awaitable, Callable, Literal
 import logging
+import time

 from telethon.errors import ChannelInvalidError, ChannelPrivateError
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
@@ -35,31 +38,60 @@ from telethon.tl.types import (
    PeerChannel,
    PeerChat,
    PeerUser,
+    TypeChannelParticipant,
+    TypeChatParticipant,
+    TypeInputPeer,
    TypePeer,
    UpdateNewChannelMessage,
    UpdateNewMessage,
    User,
 )
+from telethon.utils import add_surrogate, del_surrogate

-from mautrix.types import UserID
+from mautrix.errors import MBadState, MForbidden
+from mautrix.types import RoomID, UserID

 from . import portal as po, puppet as pu, user as u
 from .abstract_user import AbstractUser
-from .db import BotChat
+from .db import BotChat, Message as DBMessage
 from .types import TelegramID

 ReplyFunc = Callable[[str], Awaitable[Message]]
+BanFunc = Callable[[RoomID, UserID, str], Awaitable[None]]
+TelegramAdminPermission = Literal[
+    "change_info",
+    "post_messages",
+    "edit_messages",
+    "delete_messages",
+    "ban_users",
+    "invite_users",
+    "pin_messages",
+    "add_admins",
+    "anonymous",
+    "manage_call",
+    "other",
+]


 class Bot(AbstractUser):
    log: logging.Logger = logging.getLogger("mau.user.bot")

    token: str
-    chats: Dict[int, str]
-    tg_whitelist: List[int]
+    chats: dict[int, str]
+    tg_whitelist: list[int]
    whitelist_group_admins: bool
-    _me_info: Optional[User]
-    _me_mxid: Optional[UserID]
+    _me_info: User | None
+    _me_mxid: UserID | None
+    _admin_cache: dict[
+        tuple[int, int],
+        tuple[ChatParticipantAdmin | ChatParticipantCreator | None, float],
+    ]
+    required_permissions: dict[str, TelegramAdminPermission] = {
+        "portal": None,
+        "invite": "invite_users",
+        "mxban": "ban_users",
+        "mxkick": "ban_users",
+    }

    def __init__(self, token: str) -> None:
        super().__init__()
@@ -73,6 +105,7 @@ class Bot(AbstractUser):
        self.is_relaybot = True
        self.is_bot = True
        self.chats = {}
+        self._admin_cache = {}
        self.tg_whitelist = []
        self.whitelist_group_admins = (
            self.config["bridge.relaybot.whitelist_group_admins"] or False
@@ -80,7 +113,7 @@ class Bot(AbstractUser):
        self._me_info = None
        self._me_mxid = None

-    async def get_me(self, use_cache: bool = True) -> Tuple[User, UserID]:
+    async def get_me(self, use_cache: bool = True) -> tuple[User, UserID]:
        if not use_cache or not self._me_mxid:
            self._me_info = await self.client.get_me()
            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
@@ -98,7 +131,7 @@ class Bot(AbstractUser):
            if isinstance(user_id, int):
                self.tg_whitelist.append(user_id)

-    async def start(self, delete_unless_authenticated: bool = False) -> "Bot":
+    async def start(self, delete_unless_authenticated: bool = False) -> Bot:
        self.chats = {chat.id: chat.type for chat in await BotChat.all()}
        await super().start(delete_unless_authenticated)
        if not await self.is_logged_in():
@@ -148,7 +181,44 @@ class Bot(AbstractUser):
            pass
        await BotChat.delete_by_id(chat_id)

-    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
+    async def _get_admin_participant(
+        self, chat: TypePeer | TypeInputPeer, tgid: TelegramID
+    ) -> TypeChatParticipant | TypeChannelParticipant | None:
+        chan_id = chat.channel_id if isinstance(chat, PeerChannel) else chat.chat_id
+        try:
+            cached, created = self._admin_cache[chan_id, tgid]
+            if created + 60 < time.time():
+                return cached
+        except KeyError:
+            pass
+        if isinstance(chat, PeerChannel):
+            p = await self.client(GetParticipantRequest(chat, tgid))
+            pcp = p.participant
+            self._admin_cache[chat.channel_id, tgid] = (pcp, time.time())
+            return pcp
+        elif isinstance(chat, PeerChat):
+            chat = await self.client(GetFullChatRequest(chat.chat_id))
+            participants = chat.full_chat.participants.participants
+            for p in participants:
+                self._admin_cache[chat.channel_id, tgid] = (p, time.time())
+                if p.user_id == tgid:
+                    return p
+        return None
+
+    @staticmethod
+    def _has_participant_permission(
+        pcp: TypeChatParticipant | TypeChannelParticipant | None,
+        permission: TelegramAdminPermission | None,
+    ) -> bool:
+        if isinstance(pcp, (ChannelParticipantCreator, ChannelParticipantAdmin)):
+            return permission is None or getattr(pcp.admin_rights, permission, False)
+        elif isinstance(pcp, (ChatParticipantCreator, ChatParticipantAdmin)):
+            return True
+        return False
+
+    async def _can_use_commands(
+        self, chat: TypePeer, tgid: TelegramID, permission: TelegramAdminPermission | None = None
+    ) -> bool:
        if tgid in self.tg_whitelist:
            return True

@@ -158,22 +228,20 @@ class Bot(AbstractUser):
            return True

        if self.whitelist_group_admins:
-            if isinstance(chat, PeerChannel):
-                p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(
-                    p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin)
-                )
-            elif isinstance(chat, PeerChat):
-                chat = await self.client(GetFullChatRequest(chat.chat_id))
-                participants = chat.full_chat.participants.participants
-                for p in participants:
-                    if p.user_id == tgid:
-                        return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+            pcp = await self._get_admin_participant(chat, tgid)
+            return self._has_participant_permission(pcp, permission)
        return False

-    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
-        # FIXME event.from_id is not int
-        if not await self._can_use_commands(event.to_id, TelegramID(event.from_id)):
+    async def check_can_use_command(self, event: Message, reply: ReplyFunc, command: str) -> bool:
+        if command not in self.required_permissions:
+            # Unknown command
+            return False
+        elif not isinstance(event.from_id, PeerUser):
+            await reply("Channels can't use commands")
+            return False
+        elif not await self._can_use_commands(
+            event.to_id, TelegramID(event.from_id.user_id), self.required_permissions[command]
+        ):
            await reply("You do not have the permission to use that command.")
            return False
        return True
@@ -193,6 +261,8 @@ class Bot(AbstractUser):
                )
            else:
                return await reply("Portal is not public. Use `/invite <mxid>` to get an invite.")
+        else:
+            return await reply("Couldn't create portal room")

    async def handle_command_invite(
        self, portal: po.Portal, reply: ReplyFunc, mxid_input: UserID
@@ -213,9 +283,59 @@ class Bot(AbstractUser):
                f"Just invite [{displayname}](tg://user?id={user.tgid})"
            )
        else:
-            await portal.invite_to_matrix(user.mxid)
+            try:
+                await portal.invite_to_matrix(user.mxid)
+            except MBadState:
+                try:
+                    await portal.main_intent.unban_user(
+                        portal.mxid, user.mxid, reason="Invited from Telegram"
+                    )
+                except Exception:
+                    return await reply(f"Failed to unban `{user.mxid}` from the portal.")
+                await portal.invite_to_matrix(user.mxid)
+                return await reply(f"Unbanned and invited `{user.mxid}` to the portal.")
            return await reply(f"Invited `{user.mxid}` to the portal.")

+    async def handle_command_ban(
+        self,
+        message: Message,
+        portal: po.Portal,
+        reply: ReplyFunc,
+        reason: str,
+        action: Literal["kick", "ban"] = "ban",
+    ) -> Message:
+        if not message.reply_to:
+            return await reply("You must reply to a relaybot message when using that command")
+        reply_to_id = TelegramID(message.reply_to.reply_to_msg_id)
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        msg = await DBMessage.get_one_by_tgid(reply_to_id, tg_space)
+        if not msg or msg.sender != self.tgid or not msg.sender_mxid:
+            return await reply("Target message is not a relayed message")
+        puppet = await pu.Puppet.get_by_peer(message.from_id)
+        actioned = "Banned" if action == "ban" else "Kicked"
+        try:
+            intent = puppet.intent_for(portal)
+            func: BanFunc = intent.ban_user if action == "ban" else intent.kick_user
+            await func(portal.mxid, msg.sender_mxid, reason)
+        except MForbidden as e:
+            self.log.warning(
+                f"Failed to {action} {msg.sender_mxid} from {portal.mxid} as {puppet.mxid}: {e}, "
+                f"falling back to bridge bot"
+            )
+            reason_prefix = f"{actioned} by {puppet.displayname or puppet.tgid}"
+            reason = f"{reason_prefix}: {reason}" if reason else reason_prefix
+            try:
+                func: BanFunc = (
+                    self.az.intent.ban_user if action == "ban" else self.az.intent.kick_user
+                )
+                await func(portal.mxid, msg.sender_mxid, reason)
+            except MForbidden as e:
+                self.log.warning(
+                    f"Failed to {action} {msg.sender_mxid} from {portal.mxid} as bridge bot: {e}"
+                )
+                return await reply(f"Failed to {action} `{msg.sender_mxid}`")
+        return await reply(f"Successfully {actioned.lower()} `{msg.sender_mxid}`")
+
    @staticmethod
    def handle_command_id(message: Message, reply: ReplyFunc) -> Awaitable[Message]:
        # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
@@ -233,53 +353,46 @@ class Bot(AbstractUser):
        else:
            return reply("Failed to find chat ID.")

-    def match_command(self, text: str, command: str) -> bool:
-        text = text.lower()
-        command = f"/{command.lower()}"
-        command_targeted = f"{command}@{self.tg_username.lower()}"
+    def parse_command(self, message: Message) -> tuple[str | None, str | None]:
+        if not message.entities or len(message.entities) < 1 or not message.message:
+            return None, None
+        cmd_entity = message.entities[0]
+        if not isinstance(cmd_entity, MessageEntityBotCommand) or cmd_entity.offset != 0:
+            return None, None
+        surrogated_text = add_surrogate(message.message)
+        command: str = del_surrogate(surrogated_text[: cmd_entity.length]).lower()
+        rest_of_message: str = ""
+        if len(surrogated_text) > cmd_entity.length + 1:
+            rest_of_message: str = del_surrogate(surrogated_text[cmd_entity.length + 1 :])
+        command, *target = command.split("@", 1)
+        if not command.startswith("/"):
+            return None, None
+        elif target and target[0] != self.tg_username.lower():
+            return None, None
+        return command[1:], rest_of_message

-        is_plain_command = text == command or text == command_targeted
-        if is_plain_command:
-            return True
-
-        is_arg_command = text.startswith(command + " ") or text.startswith(command_targeted + " ")
-        if is_arg_command:
-            return True
-
-        return False
-
-    async def handle_command(self, message: Message) -> None:
+    async def handle_command(self, message: Message, command: str, args: str) -> None:
        def reply(reply_text: str) -> Awaitable[Message]:
            return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)

-        text = message.message
-
-        if self.match_command(text, "start"):
+        if command == "start":
            pcm = self.config["bridge.relaybot.private_chat.message"]
            if pcm:
                await reply(pcm)
-            return
-        elif self.match_command(text, "id"):
+        elif command == "id":
            await self.handle_command_id(message, reply)
-            return
-        elif message.is_private:
-            return
-
-        portal = await po.Portal.get_by_entity(message.to_id)
-
-        is_portal_cmd = self.match_command(text, "portal")
-        is_invite_cmd = self.match_command(text, "invite")
-        if is_portal_cmd or is_invite_cmd:
-            if not await self.check_can_use_commands(message, reply):
+        elif not message.is_private:
+            if not await self.check_can_use_command(message, reply, command):
                return
-            if is_portal_cmd:
+            portal = await po.Portal.get_by_entity(message.to_id)
+            if command == "portal":
                await self.handle_command_portal(portal, reply)
-            elif is_invite_cmd:
-                try:
-                    mxid = text[text.index(" ") + 1 :]
-                except ValueError:
-                    mxid = ""
-                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
+            elif command == "invite":
+                await self.handle_command_invite(portal, reply, mxid_input=UserID(args))
+            elif command == "mxban":
+                await self.handle_command_ban(message, portal, reply, reason=args)
+            elif command == "mxkick":
+                await self.handle_command_ban(message, portal, reply, reason=args, action="kick")

    async def handle_service_message(self, message: MessageService) -> None:
        to_peer = message.to_id
@@ -308,15 +421,10 @@ class Bot(AbstractUser):
            await self.handle_service_message(update.message)
            return False

-        is_command = (
-            isinstance(update.message, Message)
-            and update.message.entities
-            and len(update.message.entities) > 0
-            and isinstance(update.message.entities[0], MessageEntityBotCommand)
-            and update.message.entities[0].offset == 0
-        )
-        if is_command:
-            await self.handle_command(update.message)
+        if isinstance(update.message, Message):
+            command, args = self.parse_command(update.message)
+            if command:
+                await self.handle_command(update.message, command, args)
        return False

    def is_in_chat(self, peer_id) -> bool:
@@ -137,9 +137,9 @@ class CommandHandler(BaseCommandHandler):

    async def get_permission_error(self, evt: CommandEvent) -> str | None:
        if self.needs_puppeting and not evt.sender.puppet_whitelisted:
-            return "This command requires puppeting privileges."
+            return "That command is limited to users with puppeting privileges."
        elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
-            return "This command requires Matrix puppeting privileges."
+            return "That command is limited to users with full puppeting privileges."
        return await super().get_permission_error(evt)

    def has_permission(self, key: HelpCacheKey) -> bool:
@@ -81,5 +81,5 @@ async def enter_matrix_token(evt: CommandEvent) -> EventID:
    except InvalidAccessToken:
        return await evt.reply("Failed to verify access token.")
    return await evt.reply(
-        "Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}."
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}."
    )
@@ -59,17 +59,22 @@ async def bridge(evt: CommandEvent) -> EventID:

    # The /id bot command provides the prefixed ID, so we assume
    tgid_str = evt.args[0]
-    if tgid_str.startswith("-100"):
-        tgid = TelegramID(int(tgid_str[4:]))
-        peer_type = "channel"
-    elif tgid_str.startswith("-"):
-        tgid = TelegramID(-int(tgid_str))
-        peer_type = "chat"
-    else:
+    tgid = None
+    try:
+        if tgid_str.startswith("-100"):
+            tgid = TelegramID(int(tgid_str[4:]))
+            peer_type = "channel"
+        elif tgid_str.startswith("-"):
+            tgid = TelegramID(-int(tgid_str))
+            peer_type = "chat"
+    except ValueError:
+        # Invalid integer
+        pass
+    if not tgid:
        return await evt.reply(
            "That doesn't seem like a prefixed Telegram chat ID.\n\n"
-            "If you did not get the ID using the `/id` bot command, please "
-            "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+            "If you did not get the ID using the `/id` bot command, please prefix"
+            "channel/supergroup IDs with `-100` and non-super group IDs with `-`.\n\n"
            "Bridging private chats to existing rooms is not allowed."
        )

@@ -80,7 +85,7 @@ async def bridge(evt: CommandEvent) -> EventID:
            "If you're the bridge admin, try "
            "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first."
        )
-    if portal.mxid:
+    elif portal.mxid:
        has_portal_message = (
            "That Telegram chat already has a portal at "
            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). "
@@ -96,7 +101,7 @@ async def bridge(evt: CommandEvent) -> EventID:
            "mxid": portal.mxid,
            "bridge_to_mxid": room_id,
            "tgid": portal.tgid,
-            "peer_type": portal.peer_type,
+            "peer_type": peer_type,
            "force_use_bot": force_use_bot,
        }
        return await evt.reply(
@@ -112,7 +117,7 @@ async def bridge(evt: CommandEvent) -> EventID:
        "action": "Room bridging",
        "bridge_to_mxid": room_id,
        "tgid": portal.tgid,
-        "peer_type": portal.peer_type,
+        "peer_type": peer_type,
        "force_use_bot": force_use_bot,
    }
    return await evt.reply(
@@ -163,6 +168,18 @@ async def confirm_bridge(evt: CommandEvent) -> EventID | None:
    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]

    if "mxid" in status:
+        if portal.peer_type != status["peer_type"]:
+            evt.log.warning(
+                "Portal %d in database has mismatching peer type %s (expected %s),"
+                " trusting database as a room already existed",
+                portal.tgid,
+                portal.peer_type,
+                status["peer_type"],
+            )
+            await evt.reply(
+                "Mismatching peer type in command and portal table, "
+                "trusting portal as room already existed"
+            )
        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
        if not ok:
            return None
@@ -181,6 +198,19 @@ async def confirm_bridge(evt: CommandEvent) -> EventID | None:
            "Please use `$cmdprefix+sp continue` to confirm the bridging or "
            "`$cmdprefix+sp cancel` to cancel."
        )
+    elif portal.peer_type != status["peer_type"]:
+        evt.log.warning(
+            "Portal %d in database has mismatching peer type %s (expected %s),"
+            " trusting new peer type as there's no existing room",
+            portal.tgid,
+            portal.peer_type,
+            status["peer_type"],
+        )
+        await evt.reply(
+            "Mismatching peer type in command and portal table, "
+            "trusting you as portal room doesn't exist"
+        )
+        portal.peer_type = status["peer_type"]

    evt.sender.command_status = None
    async with portal._room_create_lock:
@@ -221,7 +251,7 @@ async def _locked_confirm_bridge(
    await portal.save()
    await portal.update_bridge_info()

-    asyncio.create_task(portal.update_matrix_room(user, entity, direct=False, levels=levels))
+    asyncio.create_task(portal.update_matrix_room(user, entity, levels=levels))

    await warn_missing_power(levels, evt)

@@ -29,6 +29,7 @@ from .. import SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler

@command_handler(
    needs_auth=False,
+    needs_puppeting=False,
    help_section=SECTION_PORTAL_MANAGEMENT,
    help_text="View or change per-portal settings.",
    help_args="<`help`|_subcommand_> [...]",
@@ -98,7 +99,7 @@ def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
                "exceptions": evt.config["bridge.bridge_notices.exceptions"],
            },
            "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
-            "inline_images": evt.config["bridge.inline_images"],
+            "caption_in_message": evt.config["bridge.caption_in_message"],
            "message_formats": evt.config["bridge.message_formats"],
            "emote_format": evt.config["bridge.emote_format"],
            "state_event_formats": evt.config["bridge.state_event_formats"],
@@ -81,4 +81,3 @@ async def create(evt: CommandEvent) -> EventID:
    except ValueError as e:
        await portal.delete()
        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
@@ -25,12 +25,22 @@ from telethon.errors import (
    UsernameNotModifiedError,
    UsernameOccupiedError,
 )
+from telethon.helpers import add_surrogate
 from telethon.tl.functions.channels import GetFullChannelRequest
-from telethon.tl.functions.messages import GetFullChatRequest
+from telethon.tl.functions.messages import GetExportedChatInvitesRequest, GetFullChatRequest
+from telethon.tl.types import (
+    ChatInviteExported,
+    InputMessageEntityMentionName,
+    InputUserSelf,
+    MessageEntityMention,
+    TypeInputPeer,
+    TypeInputUser,
+)
+from telethon.tl.types.messages import ExportedChatInvites

 from mautrix.types import EventID

-from ... import portal as po
+from ... import formatter as fmt, portal as po, puppet as pu
 from .. import SECTION_MISC, SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
 from .util import user_has_power_level

@@ -101,30 +111,37 @@ async def get_id(evt: CommandEvent) -> EventID:


 invite_link_usage = (
-    "**Usage:** `$cmdprefix+sp invite-link [--uses=<amount>] [--expire=<delta>]`"
+    "**Usage:** `$cmdprefix+sp invite-link "
+    "[--uses=<amount>] [--expire=<delta>] [--request-needed] -- [title]`"
    "\n\n"
    "* `--uses`: the number of times the invite link can be used."
    "            Defaults to unlimited.\n"
    "* `--expire`: the duration after which the link will expire."
-    "              A number suffixed with d(ay), h(our), m(inute) or s(econd)"
+    "              A number suffixed with d(ay), h(our), m(inute) or s(econd)\n"
+    "* `--request-needed`: should the link require admins to approve joins?\n"
+    "* `title`: a description of the link (only shown to admins)."
 )


 def _parse_flag(args: list[str]) -> tuple[str, str]:
    arg = args.pop(0).lower()
+    if arg == "--":
+        return "", ""
+    value = ""
    if arg.startswith("--"):
-        value_start = arg.index("=")
-        if value_start:
+        value_start = arg.find("=")
+        if value_start > 0:
            flag = arg[2:value_start]
            value = arg[value_start + 1 :]
        else:
            flag = arg[2:]
-            value = args.pop(0).lower()
+            if arg not in ("request", "request-needed"):
+                value = args.pop(0).lower()
    elif arg.startswith("-"):
        flag = arg[1]
        if len(arg) > 3 and arg[2] == "=":
            value = arg[3:]
-        else:
+        elif arg != "r":
            value = args.pop(0).lower()
    else:
        raise ValueError("invalid flag")
@@ -159,18 +176,24 @@ def _parse_delta(value: str) -> timedelta | None:
@command_handler(
    help_section=SECTION_PORTAL_MANAGEMENT,
    help_text="Get a Telegram invite link to the current chat.",
-    help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>]",
+    help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>] [--request-needed] -- [title]",
 )
 async def invite_link(evt: CommandEvent) -> EventID:
+    if not evt.is_portal:
+        return await evt.reply("This is not a portal room.")
+
    # TODO once we switch to Python 3.9 minimum, use argparse with exit_on_error=False
    uses = None
    expire = None
+    request_needed = False
    while evt.args:
        try:
            flag, value = _parse_flag(evt.args)
        except (ValueError, IndexError):
            return await evt.reply(invite_link_usage)
-        if flag in ("uses", "u"):
+        if not flag:
+            break
+        elif flag in ("uses", "u"):
            try:
                uses = int(value)
            except ValueError:
@@ -180,23 +203,90 @@ async def invite_link(evt: CommandEvent) -> EventID:
            if not expire_delta:
                await evt.reply("Invalid format for expiry time delta")
            expire = datetime.now() + expire_delta
+        elif flag in ("request", "request-needed", "r"):
+            request_needed = True
+    title = " ".join(evt.args)

-    portal = await po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
+    if evt.portal.peer_type == "user":
        return await evt.reply("You can't invite users to private chats.")

    try:
-        link = await portal.get_invite_link(evt.sender, uses=uses, expire=expire)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
+        link = await evt.portal.get_invite_link(
+            evt.sender, uses=uses, expire=expire, request_needed=request_needed, title=title
+        )
+        return await evt.reply(f"Invite link to {evt.portal.title}: {link}")
    except ValueError as e:
        return await evt.reply(e.args[0])
    except ChatAdminRequiredError:
        return await evt.reply("You don't have the permission to create an invite link.")


+async def _format_invite_link(link: ChatInviteExported) -> str:
+    desc = f"* {link.link}"
+    if link.title:
+        desc += f" - {link.title}"
+    if link.expire_date:
+        desc += f"  \n  Expires at {link.expire_date.isoformat()}"
+    if link.usage_limit:
+        desc += f"  \n  Used {link.usage or 0} out of {link.usage_limit} times"
+    elif link.usage:
+        desc += f"  \n  Used {link.usage} times"
+    else:
+        desc += "  \n  Never used"
+    if link.request_needed:
+        desc += "  \n  Join requests enabled - using link requires admin approval"
+    return desc
+
+
+async def _hacky_find_mention(evt: CommandEvent) -> TypeInputUser | TypeInputPeer | None:
+    if len(evt.args) == 0:
+        return None
+    text, entities = await fmt.matrix_to_telegram(
+        evt.sender.client, text=evt.content.body, html=evt.content.formatted_body
+    )
+    for entity in entities:
+        if isinstance(entity, MessageEntityMention):
+            admin_username = add_surrogate(text)[entity.offset + 1 : entity.offset + entity.length]
+            return await evt.sender.client.get_input_entity(admin_username)
+        elif isinstance(entity, InputMessageEntityMentionName):
+            return entity.user_id
+    return None
+
+
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="List existing Telegram invite links to the current chat.",
+    help_args="[creator]",
+)
+async def list_invite_links(evt: CommandEvent) -> EventID:
+    admin_id = InputUserSelf()
+    try:
+        admin_id = await _hacky_find_mention(evt) or InputUserSelf()
+    except Exception:
+        pass
+    resp: ExportedChatInvites = await evt.sender.client(
+        GetExportedChatInvitesRequest(
+            peer=await evt.portal.get_input_entity(evt.sender),
+            admin_id=admin_id,
+            limit=100,
+        )
+    )
+    if resp.count == 0:
+        if isinstance(admin_id, InputUserSelf):
+            return await evt.reply("You haven't created any invite links to the current chat")
+        else:
+            return await evt.reply("That user hasn't created any invite links to the current chat")
+    formatted_links = "\n".join([await _format_invite_link(link) for link in resp.invites])
+    if isinstance(admin_id, InputUserSelf):
+        await evt.reply(f"Your links to this chat:\n\n{formatted_links}")
+    else:
+        puppet = await pu.Puppet.get_by_peer(admin_id)
+        await evt.reply(
+            f"[{puppet.displayname}](https://matrix.to/#/{puppet.mxid})'s links to this chat:\n\n"
+            f"{formatted_links}"
+        )
+
+
@command_handler(
    help_section=SECTION_PORTAL_MANAGEMENT,
    help_text="Upgrade a normal Telegram group to a supergroup.",
@@ -68,5 +68,5 @@ async def user_has_power_level(
        await intent.get_power_levels(room_id)
    except MatrixRequestError:
        return False
-    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
+    event_type = EventType.find(f"fi.mau.telegram.{event}", t_class=EventType.Class.STATE)
    return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)
@@ -38,6 +38,7 @@ from telethon.errors import (
 from telethon.tl.types import User

 from mautrix.client import Client
+from mautrix.errors import MForbidden
 from mautrix.types import (
    EventID,
    ImageInfo,
@@ -215,6 +216,10 @@ async def login_qr(evt: CommandEvent) -> EventID:
            return await evt.reply(
                "Your account has two-factor authentication. Please send your password here."
            )
+        try:
+            await evt.main_intent.redact(evt.room_id, qr_event_id, reason="QR code scanned")
+        except Exception:
+            pass
    else:
        timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
        timeout.set_edit(qr_event_id)
@@ -377,6 +382,7 @@ async def enter_password(evt: CommandEvent) -> EventID | None:
            "This bridge instance does not allow in-Matrix login. "
            "Please use `$cmdprefix+sp login` to get login instructions"
        )
+    await evt.redact()
    try:
        await _sign_in(
            evt,
@@ -26,6 +26,7 @@ from telethon.errors import (
    EmoticonInvalidError,
    InviteHashExpiredError,
    InviteHashInvalidError,
+    InviteRequestSentError,
    OptionsTooMuchError,
    TakeoutInitDelayError,
    UserAlreadyParticipantError,
@@ -65,6 +66,7 @@ from ...types import TelegramID

@command_handler(
    needs_auth=False,
+    needs_puppeting=False,
    help_section=SECTION_MISC,
    help_args="<_caption_>",
    help_text="Set a caption for the next image you send",
@@ -171,6 +173,8 @@ async def _join(
            return (await evt.sender.client(ImportChatInviteRequest(identifier))), None
        except UserAlreadyParticipantError:
            return None, await evt.reply("You are already in that chat.")
+        except InviteRequestSentError:
+            return None, await evt.reply("Invite request sent successfully.")
    else:
        channel = await evt.sender.client.get_entity(identifier)
        if not channel:
@@ -230,7 +234,10 @@ async def join(evt: CommandEvent) -> EventID | None:
                    updates.stringify(),
                )
                raise e
-            return await evt.reply(f"Created room for {portal.title}")
+            if portal.mxid:
+                return await evt.reply(f"Created room for {portal.title}")
+            else:
+                return await evt.reply(f"Couldn't create room for {portal.title}")
    return None


@@ -84,6 +84,10 @@ class Config(BaseBridgeConfig):

        copy("appservice.provisioning.enabled")
        copy("appservice.provisioning.prefix")
+        if base["appservice.provisioning.prefix"].endswith("/v1"):
+            base["appservice.provisioning.prefix"] = base["appservice.provisioning.prefix"][
+                : -len("/v1")
+            ]
        copy("appservice.provisioning.shared_secret")
        if base["appservice.provisioning.shared_secret"] == "generate":
            base["appservice.provisioning.shared_secret"] = self._new_token()
@@ -107,6 +111,7 @@ class Config(BaseBridgeConfig):
        copy("bridge.allow_avatar_remove")

        copy("bridge.max_initial_member_sync")
+        copy("bridge.max_member_count")
        copy("bridge.sync_channel_members")
        copy("bridge.skip_deleted_members")
        copy("bridge.startup_sync")
@@ -120,12 +125,12 @@ class Config(BaseBridgeConfig):
        copy("bridge.max_telegram_delete")
        copy("bridge.sync_matrix_state")
        copy("bridge.allow_matrix_login")
-        copy("bridge.plaintext_highlights")
        copy("bridge.public_portals")
        copy("bridge.sync_with_custom_puppets")
        copy("bridge.sync_direct_chat_list")
        copy("bridge.double_puppet_server_map")
        copy("bridge.double_puppet_allow_discovery")
+        copy("bridge.create_group_on_invite")
        if "bridge.login_shared_secret" in self:
            base["bridge.login_shared_secret_map"] = {
                base["homeserver.domain"]: self["bridge.login_shared_secret"]
@@ -134,25 +139,24 @@ class Config(BaseBridgeConfig):
            copy("bridge.login_shared_secret_map")
        copy("bridge.telegram_link_preview")
        copy("bridge.invite_link_resolve")
-        copy("bridge.inline_images")
+        copy("bridge.caption_in_message")
        copy("bridge.image_as_file_size")
        copy("bridge.image_as_file_pixels")
-        copy("bridge.max_document_size")
        copy("bridge.parallel_file_transfer")
        copy("bridge.federate_rooms")
        copy("bridge.animated_sticker.target")
+        copy("bridge.animated_sticker.convert_from_webm")
        copy("bridge.animated_sticker.args.width")
        copy("bridge.animated_sticker.args.height")
        copy("bridge.animated_sticker.args.fps")
-        copy("bridge.encryption.allow")
-        copy("bridge.encryption.default")
-        copy("bridge.encryption.database")
-        copy("bridge.encryption.key_sharing.allow")
-        copy("bridge.encryption.key_sharing.require_cross_signing")
-        copy("bridge.encryption.key_sharing.require_verification")
+        copy("bridge.animated_emoji.target")
+        copy("bridge.animated_emoji.args.width")
+        copy("bridge.animated_emoji.args.height")
+        copy("bridge.animated_emoji.args.fps")
        copy("bridge.private_chat_portal_meta")
        copy("bridge.delivery_receipts")
        copy("bridge.delivery_error_reports")
+        copy("bridge.message_status_events")
        copy("bridge.resend_bridge_info")
        copy("bridge.mute_bridging")
        copy("bridge.pinned_tag")
@@ -160,6 +164,7 @@ class Config(BaseBridgeConfig):
        copy("bridge.tag_only_on_create")
        copy("bridge.bridge_matrix_leave")
        copy("bridge.kick_on_logout")
+        copy("bridge.always_read_joined_telegram_notice")
        copy("bridge.backfill.invite_own_puppet")
        copy("bridge.backfill.takeout_limit")
        copy("bridge.backfill.initial_limit")
@@ -224,6 +229,10 @@ class Config(BaseBridgeConfig):
        copy("telegram.api_hash")
        copy("telegram.bot_token")

+        copy("telegram.catch_up")
+        copy("telegram.sequential_updates")
+        copy("telegram.exit_on_update_error")
+
        copy("telegram.connection.timeout")
        copy("telegram.connection.retries")
        copy("telegram.connection.retry_delay")
@@ -20,8 +20,8 @@ from typing import TYPE_CHECKING, ClassVar
 from asyncpg import Record
 from attr import dataclass

-from mautrix.types import EventID, RoomID
-from mautrix.util.async_db import Database
+from mautrix.types import EventID, RoomID, UserID
+from mautrix.util.async_db import Database, Scheme

 from ..types import TelegramID

@@ -39,6 +39,8 @@ class Message:
    edit_index: int
    redacted: bool = False
    content_hash: bytes | None = None
+    sender_mxid: UserID | None = None
+    sender: TelegramID | None = None

    @classmethod
    def _from_row(cls, row: Record | None) -> Message | None:
@@ -46,7 +48,19 @@ class Message:
            return None
        return cls(**row)

-    columns: ClassVar[str] = "mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash"
+    columns: ClassVar[str] = ", ".join(
+        (
+            "mxid",
+            "mx_room",
+            "tgid",
+            "tg_space",
+            "edit_index",
+            "redacted",
+            "content_hash",
+            "sender_mxid",
+            "sender",
+        )
+    )

    @classmethod
    async def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> list[Message]:
@@ -76,7 +90,7 @@ class Message:
    async def get_first_by_tgids(
        cls, tgids: list[TelegramID], tg_space: TelegramID
    ) -> list[Message]:
-        if cls.db.scheme == "postgres":
+        if cls.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
            q = (
                f"SELECT {cls.columns} FROM message"
                " WHERE tgid=ANY($1) AND tg_space=$2 AND edit_index=0"
@@ -123,7 +137,7 @@ class Message:
    async def get_by_mxids(
        cls, mxids: list[EventID], mx_room: RoomID, tg_space: TelegramID
    ) -> list[Message]:
-        if cls.db.scheme == "postgres":
+        if cls.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
            q = (
                f"SELECT {cls.columns} FROM message"
                " WHERE mxid=ANY($1) AND mx_room=$2 AND tg_space=$3"
@@ -158,12 +172,16 @@ class Message:
            self.edit_index,
            self.redacted,
            self.content_hash,
+            self.sender_mxid,
+            self.sender,
        )

    async def insert(self) -> None:
        q = """
-            INSERT INTO message (mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash)
-            VALUES ($1, $2, $3, $4, $5, $6, $7)
+            INSERT INTO message (
+                mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash,
+                sender_mxid, sender
+            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
        """
        await self.db.execute(q, *self._values)

@@ -22,7 +22,7 @@ from asyncpg import Record
 from attr import dataclass
 import attr

-from mautrix.types import ContentURI, EventID, RoomID
+from mautrix.types import BatchID, ContentURI, EventID, RoomID
 from mautrix.util.async_db import Database

 from ..types import TelegramID
@@ -44,6 +44,9 @@ class Portal:
    mxid: RoomID | None
    avatar_url: ContentURI | None
    encrypted: bool
+    first_event_id: EventID | None
+    next_batch_id: BatchID | None
+    base_insertion_id: EventID | None

    sponsored_event_id: EventID | None
    sponsored_event_ts: int | None
@@ -67,10 +70,29 @@ class Portal:
        data["local_config"] = json.loads(data.pop("config", None) or "{}")
        return cls(**data)

-    columns: ClassVar[str] = (
-        "tgid, tg_receiver, peer_type, megagroup, mxid, avatar_url, encrypted, sponsored_event_id,"
-        "sponsored_event_ts, sponsored_msg_random_id, username, title, about, photo_id, "
-        "name_set, avatar_set, config"
+    columns: ClassVar[str] = ", ".join(
+        (
+            "tgid",
+            "tg_receiver",
+            "peer_type",
+            "megagroup",
+            "mxid",
+            "avatar_url",
+            "encrypted",
+            "first_event_id",
+            "next_batch_id",
+            "base_insertion_id",
+            "sponsored_event_id",
+            "sponsored_event_ts",
+            "sponsored_msg_random_id",
+            "username",
+            "title",
+            "about",
+            "photo_id",
+            "name_set",
+            "avatar_set",
+            "config",
+        )
    )

    @classmethod
@@ -112,6 +134,9 @@ class Portal:
            self.mxid,
            self.avatar_url,
            self.encrypted,
+            self.first_event_id,
+            self.next_batch_id,
+            self.base_insertion_id,
            self.sponsored_event_id,
            self.sponsored_event_ts,
            self.sponsored_msg_random_id,
@@ -128,9 +153,11 @@ class Portal:
    async def save(self) -> None:
        q = """
        UPDATE portal
-        SET mxid=$4, avatar_url=$5, encrypted=$6, sponsored_event_id=$7, sponsored_event_ts=$8,
-            sponsored_msg_random_id=$9, username=$10, title=$11, about=$12, photo_id=$13,
-            name_set=$14, avatar_set=$15, megagroup=$16, config=$17
+        SET mxid=$4, avatar_url=$5, encrypted=$6,
+            first_event_id=$7, next_batch_id=$8, base_insertion_id=$9,
+            sponsored_event_id=$10, sponsored_event_ts=$11, sponsored_msg_random_id=$12,
+            username=$13, title=$14, about=$15, photo_id=$16, name_set=$17, avatar_set=$18,
+            megagroup=$19, config=$20
        WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)
        """
        await self.db.execute(q, *self._values)
@@ -149,9 +176,11 @@ class Portal:
        q = """
        INSERT INTO portal (
            tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,
+            first_event_id, base_insertion_id, next_batch_id,
            sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,
            username, title, about, photo_id, name_set, avatar_set, megagroup, config
-        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
+                  $19, $20)
        """
        await self.db.execute(q, *self._values)

@@ -43,6 +43,7 @@ class Puppet:
    displayname_quality: int
    disable_updates: bool
    username: str | None
+    phone: str | None
    photo_id: str | None
    avatar_url: ContentURI | None
    name_set: bool
@@ -65,7 +66,7 @@ class Puppet:

    columns: ClassVar[str] = (
        "id, is_registered, displayname, displayname_source, displayname_contact, "
-        "displayname_quality, disable_updates, username, photo_id, avatar_url, "
+        "displayname_quality, disable_updates, username, phone, photo_id, avatar_url, "
        "name_set, avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url"
    )

@@ -89,11 +90,6 @@ class Puppet:
        q = f"SELECT {cls.columns} FROM puppet WHERE lower(username)=$1"
        return cls._from_row(await cls.db.fetchrow(q, username.lower()))

-    @classmethod
-    async def find_by_displayname(cls, displayname: str) -> Puppet | None:
-        q = f"SELECT {cls.columns} FROM puppet WHERE displayname=$1"
-        return cls._from_row(await cls.db.fetchrow(q, displayname))
-
    @property
    def _values(self):
        return (
@@ -105,6 +101,7 @@ class Puppet:
            self.displayname_quality,
            self.disable_updates,
            self.username,
+            self.phone,
            self.photo_id,
            self.avatar_url,
            self.name_set,
@@ -121,9 +118,9 @@ class Puppet:
        q = """
        UPDATE puppet
        SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,
-            displayname_quality=$6, disable_updates=$7, username=$8, photo_id=$9,
-            avatar_url=$10, name_set=$11, avatar_set=$12, is_bot=$13, is_channel=$14,
-            custom_mxid=$15, access_token=$16, next_batch=$17, base_url=$18
+            displayname_quality=$6, disable_updates=$7, username=$8, phone=$9, photo_id=$10,
+            avatar_url=$11, name_set=$12, avatar_set=$13, is_bot=$14, is_channel=$15,
+            custom_mxid=$16, access_token=$17, next_batch=$18, base_url=$19
        WHERE id=$1
        """
        await self.db.execute(q, *self._values)
@@ -132,8 +129,9 @@ class Puppet:
        q = """
        INSERT INTO puppet (
            id, is_registered, displayname, displayname_source, displayname_contact,
-            displayname_quality, disable_updates, username, photo_id, avatar_url, name_set,
+            displayname_quality, disable_updates, username, phone, photo_id, avatar_url, name_set,
            avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url
-        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
+                  $19)
        """
        await self.db.execute(q, *self._values)
@@ -17,10 +17,11 @@ from __future__ import annotations

 from typing import TYPE_CHECKING, ClassVar

+from asyncpg import Record
 from attr import dataclass

 from mautrix.types import ContentURI, EncryptedFile
-from mautrix.util.async_db import Database
+from mautrix.util.async_db import Database, Scheme

 fake_db = Database.create("") if TYPE_CHECKING else None

@@ -40,28 +41,47 @@ class TelegramFile:
    decryption_info: EncryptedFile | None
    thumbnail: TelegramFile | None = None

+    columns: ClassVar[str] = (
+        "id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail, "
+        "decryption_info"
+    )
+
    @classmethod
-    async def get(cls, loc_id: str, *, _thumbnail: bool = False) -> TelegramFile | None:
-        q = (
-            "SELECT id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail,"
-            "       decryption_info "
-            "FROM telegram_file WHERE id=$1"
-        )
-        row = await cls.db.fetchrow(q, loc_id)
+    def _from_row(cls, row: Record | None) -> TelegramFile | None:
        if row is None:
            return None
        data = {**row}
-        thumbnail_id = data.pop("thumbnail", None)
-        if _thumbnail:
-            # Don't allow more than one level of recursion
-            thumbnail_id = None
+        data.pop("thumbnail", None)
        decryption_info = data.pop("decryption_info", None)
        return cls(
            **data,
-            thumbnail=(await cls.get(thumbnail_id, _thumbnail=True)) if thumbnail_id else None,
+            thumbnail=None,
            decryption_info=EncryptedFile.parse_json(decryption_info) if decryption_info else None,
        )

+    @classmethod
+    async def get_many(cls, loc_ids: list[str]) -> list[TelegramFile]:
+        if cls.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+            q = f"SELECT {cls.columns} FROM telegram_file WHERE id=ANY($1)"
+            rows = await cls.db.fetch(q, loc_ids)
+        else:
+            tgid_placeholders = ("?," * len(loc_ids)).rstrip(",")
+            q = f"SELECT {cls.columns} FROM telegram_file WHERE id IN ({tgid_placeholders})"
+            rows = await cls.db.fetch(q, *loc_ids)
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def get(cls, loc_id: str, *, _thumbnail: bool = False) -> TelegramFile | None:
+        q = f"SELECT {cls.columns} FROM telegram_file WHERE id=$1"
+        row = await cls.db.fetchrow(q, loc_id)
+        file = cls._from_row(row)
+        if file is None:
+            return None
+        thumbnail_id = row.get("thumbnail", None)
+        if thumbnail_id and not _thumbnail:
+            file.thumbnail = await cls.get(thumbnail_id, _thumbnail=True)
+        return file
+
    async def insert(self) -> None:
        q = (
            "INSERT INTO telegram_file (id, mxc, mime_type, was_converted, size, width, height, "
@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations

-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING, ClassVar, Iterable
 import asyncio
 import datetime

@@ -24,7 +24,7 @@ from telethon.crypto import AuthKey
 from telethon.sessions import MemorySession
 from telethon.tl.types import PeerChannel, PeerChat, PeerUser, updates

-from mautrix.util.async_db import Database
+from mautrix.util.async_db import Database, Scheme

 fake_db = Database.create("") if TYPE_CHECKING else None

@@ -124,18 +124,42 @@ class PgSession(MemorySession):
        return updates.State(row["pts"], row["qts"], date, row["seq"], row["unread_count"])

    async def set_update_state(self, entity_id: int, row: updates.State) -> None:
-        q = (
-            "INSERT INTO telethon_update_state"
-            "    (session_id, entity_id, pts, qts, date, seq, unread_count) "
-            "VALUES ($1, $2, $3, $4, $5, $6, $7)"
-            "ON CONFLICT (session_id, entity_id) DO UPDATE"
-            "    SET pts=$3, qts=$4, date=$5, seq=$6, unread_count=$7"
-        )
+        q = """
+        INSERT INTO telethon_update_state(session_id, entity_id, pts, qts, date, seq, unread_count)
+        VALUES ($1, $2, $3, $4, $5, $6, $7)
+        ON CONFLICT (session_id, entity_id) DO UPDATE SET
+            pts=excluded.pts, qts=excluded.qts, date=excluded.date, seq=excluded.seq,
+            unread_count=excluded.unread_count
+        """
        ts = row.date.timestamp()
        await self.db.execute(
            q, self.session_id, entity_id, row.pts, row.qts, ts, row.seq, row.unread_count
        )

+    async def delete_update_state(self, entity_id: int) -> None:
+        q = "DELETE FROM telethon_update_state WHERE session_id=$1 AND entity_id=$2"
+        await self.db.execute(q, self.session_id, entity_id)
+
+    async def get_update_states(self) -> Iterable[tuple[int, updates.State], ...]:
+        q = (
+            "SELECT entity_id, pts, qts, date, seq, unread_count FROM telethon_update_state "
+            "WHERE session_id=$1"
+        )
+        rows = await self.db.fetch(q, self.session_id)
+        return (
+            (
+                row["entity_id"],
+                updates.State(
+                    row["pts"],
+                    row["qts"],
+                    datetime.datetime.utcfromtimestamp(row["date"]),
+                    row["seq"],
+                    row["unread_count"],
+                ),
+            )
+            for row in rows
+        )
+
    def _entity_values_to_row(
        self, id: int, hash: int, username: str | None, phone: str | int | None, name: str | None
    ) -> tuple[str, int, int, str | None, str | None, str | None]:
@@ -153,7 +177,7 @@ class PgSession(MemorySession):
        ] = self._entities_to_rows(tlo)
        if not rows:
            return
-        if self.db.scheme == "postgres":
+        if self.db.scheme == Scheme.POSTGRES:
            q = (
                "INSERT INTO telethon_entities (session_id, id, hash, username, phone, name) "
                "VALUES ($1, unnest($2::bigint[]), unnest($3::bigint[]), "
@@ -176,32 +200,31 @@ class PgSession(MemorySession):
    async def _select_entity(
        self, constraint: str, *args: str | int | tuple[int, ...]
    ) -> tuple[int, int] | None:
-        row = await self.db.fetchrow(
-            f"SELECT id, hash FROM telethon_entities WHERE {constraint}", *args
-        )
+        q = f"SELECT id, hash FROM telethon_entities WHERE session_id=$1 AND {constraint}"
+        row = await self.db.fetchrow(q, self.session_id, *args)
        if row is None:
            return None
        return row["id"], row["hash"]

    async def get_entity_rows_by_phone(self, key: str | int) -> tuple[int, int] | None:
-        return await self._select_entity("phone=$1", str(key))
+        return await self._select_entity("phone=$2", str(key))

    async def get_entity_rows_by_username(self, key: str) -> tuple[int, int] | None:
-        return await self._select_entity("username=$1", key)
+        return await self._select_entity("username=$2", key)

    async def get_entity_rows_by_name(self, key: str) -> tuple[int, int] | None:
-        return await self._select_entity("name=$1", key)
+        return await self._select_entity("name=$2", key)

    async def get_entity_rows_by_id(self, key: int, exact: bool = True) -> tuple[int, int] | None:
        if exact:
-            return await self._select_entity("id=$1", key)
+            return await self._select_entity("id=$2", key)

        ids = (
            utils.get_peer_id(PeerUser(key)),
            utils.get_peer_id(PeerChat(key)),
            utils.get_peer_id(PeerChannel(key)),
        )
-        if self.db.scheme == "postgres":
-            return await self._select_entity("id=ANY($1)", ids)
+        if self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+            return await self._select_entity("id=ANY($2)", ids)
        else:
-            return await self._select_entity(f"id IN ($1, $2, $3)", *ids)
+            return await self._select_entity(f"id IN ($2, $3, $4)", *ids)
@@ -9,4 +9,10 @@ from . import (
    v04_disappearing_messages,
    v05_channel_ghosts,
    v06_puppet_avatar_url,
+    v07_puppet_phone_number,
+    v08_portal_first_event,
+    v09_puppet_username_index,
+    v10_more_backfill_fields,
+    v11_backfill_queue,
+    v12_message_sender,
 )
@@ -0,0 +1,207 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+latest_version = 10
+
+
+async def create_latest_tables(conn: Connection) -> int:
+    await conn.execute(
+        """CREATE TABLE "user" (
+            mxid TEXT   PRIMARY KEY,
+            tgid BIGINT UNIQUE,
+            tg_username    TEXT,
+            tg_phone       TEXT,
+            is_bot         BOOLEAN NOT NULL DEFAULT false,
+            saved_contacts INTEGER NOT NULL DEFAULT 0
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE portal (
+            tgid        BIGINT,
+            tg_receiver BIGINT,
+            peer_type   TEXT NOT NULL,
+            mxid        TEXT UNIQUE,
+            avatar_url  TEXT,
+            encrypted   BOOLEAN NOT NULL DEFAULT false,
+            username    TEXT,
+            title       TEXT,
+            about       TEXT,
+            photo_id    TEXT,
+            name_set    BOOLEAN NOT NULL DEFAULT false,
+            avatar_set  BOOLEAN NOT NULL DEFAULT false,
+            megagroup   BOOLEAN,
+            config      jsonb,
+
+            first_event_id    TEXT,
+            next_batch_id     TEXT,
+            base_insertion_id TEXT,
+
+            sponsored_event_id     TEXT,
+            sponsored_event_ts     BIGINT,
+            sponsored_msg_random_id bytea,
+
+            PRIMARY KEY (tgid, tg_receiver)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE message (
+            mxid         TEXT   NOT NULL,
+            mx_room      TEXT   NOT NULL,
+            tgid         BIGINT,
+            tg_space     BIGINT,
+            edit_index   INTEGER,
+            redacted     BOOLEAN NOT NULL DEFAULT false,
+            content_hash bytea,
+            PRIMARY KEY (tgid, tg_space, edit_index),
+            UNIQUE (mxid, mx_room, tg_space)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE reaction (
+            mxid      TEXT NOT NULL,
+            mx_room   TEXT NOT NULL,
+            msg_mxid  TEXT NOT NULL,
+            tg_sender BIGINT,
+            reaction  TEXT NOT NULL,
+
+            PRIMARY KEY (msg_mxid, mx_room, tg_sender),
+            UNIQUE (mxid, mx_room)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE disappearing_message (
+            room_id             TEXT,
+            event_id            TEXT,
+            expiration_seconds  BIGINT,
+            expiration_ts       BIGINT,
+
+            PRIMARY KEY (room_id, event_id)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE puppet (
+            id BIGINT PRIMARY KEY,
+
+            is_registered BOOLEAN NOT NULL DEFAULT false,
+
+            displayname         TEXT,
+            displayname_source  BIGINT,
+            displayname_contact BOOLEAN NOT NULL DEFAULT true,
+            displayname_quality INTEGER NOT NULL DEFAULT 0,
+            disable_updates     BOOLEAN NOT NULL DEFAULT false,
+            username            TEXT,
+            phone               TEXT,
+            photo_id            TEXT,
+            avatar_url          TEXT,
+            name_set            BOOLEAN NOT NULL DEFAULT false,
+            avatar_set          BOOLEAN NOT NULL DEFAULT false,
+            is_bot              BOOLEAN,
+            is_channel          BOOLEAN NOT NULL DEFAULT false,
+
+            access_token TEXT,
+            custom_mxid  TEXT,
+            next_batch   TEXT,
+            base_url     TEXT
+        )"""
+    )
+    await conn.execute("CREATE INDEX puppet_username_idx ON puppet(LOWER(username))")
+    await conn.execute(
+        """CREATE TABLE telegram_file (
+            id              TEXT PRIMARY KEY,
+            mxc             TEXT NOT NULL,
+            mime_type       TEXT,
+            was_converted   BOOLEAN NOT NULL DEFAULT false,
+            timestamp       BIGINT  NOT NULL DEFAULT 0,
+            size            BIGINT,
+            width           INTEGER,
+            height          INTEGER,
+            thumbnail       TEXT,
+            decryption_info jsonb,
+            FOREIGN KEY (thumbnail) REFERENCES telegram_file(id)
+                ON UPDATE CASCADE ON DELETE SET NULL
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE bot_chat (
+            id   BIGINT PRIMARY KEY,
+            type TEXT NOT NULL
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE user_portal (
+            "user"          BIGINT,
+            portal          BIGINT,
+            portal_receiver BIGINT,
+            PRIMARY KEY ("user", portal, portal_receiver),
+            FOREIGN KEY ("user") REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (portal, portal_receiver) REFERENCES portal(tgid, tg_receiver)
+                 ON DELETE CASCADE ON UPDATE CASCADE
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE contact (
+            "user"  BIGINT,
+            contact BIGINT,
+            PRIMARY KEY ("user", contact),
+            FOREIGN KEY ("user")  REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (contact) REFERENCES puppet(id)   ON DELETE CASCADE ON UPDATE CASCADE
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_sessions (
+            session_id     TEXT PRIMARY KEY,
+            dc_id          INTEGER,
+            server_address TEXT,
+            port           INTEGER,
+            auth_key       bytea
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_entities (
+            session_id TEXT,
+            id         BIGINT,
+            hash       BIGINT NOT NULL,
+            username   TEXT,
+            phone      TEXT,
+            name       TEXT,
+            PRIMARY KEY (session_id, id)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_sent_files (
+            session_id TEXT,
+            md5_digest bytea,
+            file_size  INTEGER,
+            type       INTEGER,
+            id         BIGINT,
+            hash       BIGINT,
+            PRIMARY KEY (session_id, md5_digest, file_size, type)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_update_state (
+            session_id   TEXT,
+            entity_id    BIGINT,
+            pts          BIGINT,
+            qts          BIGINT,
+            date         BIGINT,
+            seq          BIGINT,
+            unread_count INTEGER,
+            PRIMARY KEY (session_id, entity_id)
+        )"""
+    )
+    return latest_version
@@ -15,29 +15,38 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations

-from asyncpg import Connection
+from mautrix.util.async_db import Connection, Scheme

 from . import upgrade_table
+from .v00_latest_revision import create_latest_tables, latest_version

 legacy_version_query = "SELECT version_num FROM alembic_version"
 last_legacy_version = "bfc0a39bfe02"


 def table_exists(scheme: str, name: str) -> str:
-    if scheme == "sqlite":
+    if scheme == Scheme.SQLITE:
        return f"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='{name}')"
-    elif scheme == "postgres":
-        return f"SELECT EXISTS(SELECT FROM information_schema.tables WHERE table_name='{name}')"
+    elif scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+        return f"SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name='{name}')"
    raise RuntimeError("unsupported database scheme")


-@upgrade_table.register(description="Initial asyncpg revision")
-async def upgrade_v1(conn: Connection, scheme: str) -> None:
+async def first_upgrade_target(conn: Connection, scheme: str) -> int:
+    is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
+    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to latest.
+    # If it's a new db, we'll create the latest tables directly (see create_latest_tables call).
+    return 1 if is_legacy else latest_version
+
+
+@upgrade_table.register(description="Initial asyncpg revision", upgrades_to=first_upgrade_target)
+async def upgrade_v1(conn: Connection, scheme: str) -> int:
    is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
    if is_legacy:
        await migrate_legacy_to_v1(conn, scheme)
+        return 1
    else:
-        await create_v1_tables(conn)
+        return await create_latest_tables(conn)


 async def drop_constraints(conn: Connection, table: str, contype: str) -> None:
@@ -178,151 +187,3 @@ async def varchar_to_text(conn: Connection) -> None:
    for table, columns in columns_to_adjust.items():
        for column in columns:
            await conn.execute(f'ALTER TABLE "{table}" ALTER COLUMN {column} TYPE TEXT')
-
-
-async def create_v1_tables(conn: Connection) -> None:
-    await conn.execute(
-        """CREATE TABLE "user" (
-            mxid TEXT   PRIMARY KEY,
-            tgid BIGINT UNIQUE,
-            tg_username    TEXT,
-            tg_phone       TEXT,
-            is_bot         BOOLEAN NOT NULL DEFAULT false,
-            saved_contacts INTEGER NOT NULL DEFAULT 0
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE portal (
-            tgid        BIGINT,
-            tg_receiver BIGINT,
-            peer_type   TEXT NOT NULL,
-            mxid        TEXT UNIQUE,
-            avatar_url  TEXT,
-            encrypted   BOOLEAN NOT NULL DEFAULT false,
-            username    TEXT,
-            title       TEXT,
-            about       TEXT,
-            photo_id    TEXT,
-            megagroup   BOOLEAN,
-            config      jsonb,
-            PRIMARY KEY (tgid, tg_receiver)
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE message (
-            mxid       TEXT,
-            mx_room    TEXT,
-            tgid       BIGINT NOT NULL,
-            tg_space   BIGINT NOT NULL,
-            edit_index INTEGER NOT NULL,
-            redacted   BOOLEAN NOT NULL DEFAULT false,
-            PRIMARY KEY (tgid, tg_space, edit_index),
-            UNIQUE (mxid, mx_room, tg_space)
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE puppet (
-            id BIGINT PRIMARY KEY,
-
-            is_registered BOOLEAN NOT NULL DEFAULT false,
-
-            displayname         TEXT,
-            displayname_source  BIGINT,
-            displayname_contact BOOLEAN NOT NULL DEFAULT true,
-            displayname_quality INTEGER NOT NULL DEFAULT 0,
-            disable_updates     BOOLEAN NOT NULL DEFAULT false,
-            username            TEXT,
-            photo_id            TEXT,
-            is_bot              BOOLEAN,
-
-            access_token TEXT,
-            custom_mxid  TEXT,
-            next_batch   TEXT,
-            base_url     TEXT
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telegram_file (
-            id              TEXT PRIMARY KEY,
-            mxc             TEXT NOT NULL,
-            mime_type       TEXT,
-            was_converted   BOOLEAN NOT NULL DEFAULT false,
-            timestamp       BIGINT  NOT NULL DEFAULT 0,
-            size            BIGINT,
-            width           INTEGER,
-            height          INTEGER,
-            thumbnail       TEXT,
-            decryption_info jsonb,
-            FOREIGN KEY (thumbnail) REFERENCES telegram_file(id)
-                ON UPDATE CASCADE ON DELETE SET NULL
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE bot_chat (
-            id   BIGINT PRIMARY KEY,
-            type TEXT NOT NULL
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE user_portal (
-            "user"          BIGINT,
-            portal          BIGINT,
-            portal_receiver BIGINT,
-            PRIMARY KEY ("user", portal, portal_receiver),
-            FOREIGN KEY ("user") REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
-            FOREIGN KEY (portal, portal_receiver) REFERENCES portal(tgid, tg_receiver)
-                 ON DELETE CASCADE ON UPDATE CASCADE
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE contact (
-            "user"  BIGINT,
-            contact BIGINT,
-            PRIMARY KEY ("user", contact),
-            FOREIGN KEY ("user")  REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
-            FOREIGN KEY (contact) REFERENCES puppet(id)   ON DELETE CASCADE ON UPDATE CASCADE
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telethon_sessions (
-            session_id     TEXT PRIMARY KEY,
-            dc_id          INTEGER,
-            server_address TEXT,
-            port           INTEGER,
-            auth_key       bytea
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telethon_entities (
-            session_id TEXT,
-            id         BIGINT,
-            hash       BIGINT NOT NULL,
-            username   TEXT,
-            phone      TEXT,
-            name       TEXT,
-            PRIMARY KEY (session_id, id)
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telethon_sent_files (
-            session_id TEXT,
-            md5_digest bytea,
-            file_size  INTEGER,
-            type       INTEGER,
-            id         BIGINT,
-            hash       BIGINT,
-            PRIMARY KEY (session_id, md5_digest, file_size, type)
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telethon_update_state (
-            session_id   TEXT,
-            entity_id    BIGINT,
-            pts          BIGINT,
-            qts          BIGINT,
-            date         BIGINT,
-            seq          BIGINT,
-            unread_count INTEGER,
-            PRIMARY KEY (session_id, entity_id)
-        )"""
-    )
@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from asyncpg import Connection
+from mautrix.util.async_db import Connection

 from . import upgrade_table

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from asyncpg import Connection
+from mautrix.util.async_db import Connection

 from . import upgrade_table

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from asyncpg import Connection
+from mautrix.util.async_db import Connection

 from . import upgrade_table

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from asyncpg import Connection
+from mautrix.util.async_db import Connection, Scheme

 from . import upgrade_table

@@ -21,5 +21,5 @@ from . import upgrade_table
@upgrade_table.register(description="Add separate ghost users for channel senders")
 async def upgrade_v5(conn: Connection, scheme: str) -> None:
    await conn.execute("ALTER TABLE puppet ADD COLUMN is_channel BOOLEAN NOT NULL DEFAULT false")
-    if scheme == "postgres":
+    if scheme == Scheme.POSTGRES:
        await conn.execute("ALTER TABLE puppet ALTER COLUMN is_channel DROP DEFAULT")
@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from asyncpg import Connection
+from mautrix.util.async_db import Connection

 from . import upgrade_table

@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store phone number in puppet table")
+async def upgrade_v7(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE puppet ADD COLUMN phone TEXT")
@@ -0,0 +1,24 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Track first event ID in portals for infinite backfilling")
+async def upgrade_v8(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN first_event_id TEXT")
+    await conn.execute("ALTER TABLE portal ADD COLUMN next_batch_id TEXT")
@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add index to puppet username column")
+async def upgrade_v9(conn: Connection) -> None:
+    await conn.execute("CREATE INDEX puppet_username_idx ON puppet(LOWER(username))")
@@ -0,0 +1,23 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add more portal columns related to infinite backfill")
+async def upgrade_v10(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN base_insertion_id TEXT")
@@ -0,0 +1,45 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan, Sumner Evans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add the backfill queue table")
+async def upgrade_v11(conn: Connection, scheme: Scheme) -> None:
+    gen = ""
+    if scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+        gen = "GENERATED ALWAYS AS IDENTITY"
+    await conn.execute(
+        f"""
+        CREATE TABLE backfill_queue (
+            queue_id            INTEGER PRIMARY KEY {gen},
+            user_mxid           TEXT,
+            priority            INTEGER NOT NULL,
+            portal_tgid         BIGINT,
+            portal_tg_receiver  BIGINT,
+            messages_per_batch  INTEGER NOT NULL,
+            post_batch_delay    INTEGER NOT NULL,
+            max_batches         INTEGER NOT NULL,
+            dispatch_time       TIMESTAMP,
+            completed_at        TIMESTAMP,
+            cooldown_timeout    TIMESTAMP,
+            FOREIGN KEY (user_mxid) REFERENCES "user"(mxid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (portal_tgid, portal_tg_receiver)
+                REFERENCES portal(tgid, tg_receiver) ON DELETE CASCADE
+        )
+        """
+    )
@@ -0,0 +1,24 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store sender in message table")
+async def upgrade_v12(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE message ADD COLUMN sender_mxid TEXT")
+    await conn.execute("ALTER TABLE message ADD COLUMN sender BIGINT")
@@ -21,7 +21,7 @@ from asyncpg import Record
 from attr import dataclass

 from mautrix.types import UserID
-from mautrix.util.async_db import Database
+from mautrix.util.async_db import Database, Scheme

 from ..types import TelegramID

@@ -104,7 +104,7 @@ class User:
        records = [(self.tgid, puppet_id) for puppet_id in puppets]
        async with self.db.acquire() as conn, conn.transaction():
            await conn.execute('DELETE FROM contact WHERE "user"=$1', self.tgid)
-            if self.db.scheme == "postgres":
+            if self.db.scheme == Scheme.POSTGRES:
                await conn.copy_records_to_table("contact", records=records, columns=columns)
            else:
                q = 'INSERT INTO contact ("user", contact) VALUES ($1, $2)'
@@ -120,7 +120,7 @@ class User:
        records = [(self.tgid, tgid, tg_receiver) for tgid, tg_receiver in portals]
        async with self.db.acquire() as conn, conn.transaction():
            await conn.execute('DELETE FROM user_portal WHERE "user"=$1', self.tgid)
-            if self.db.scheme == "postgres":
+            if self.db.scheme == Scheme.POSTGRES:
                await conn.copy_records_to_table("user_portal", records=records, columns=columns)
            else:
                q = 'INSERT INTO user_portal ("user", portal, portal_receiver) VALUES ($1, $2, $3)'
@@ -16,6 +16,9 @@ homeserver:
    status_endpoint: null
    # Endpoint for reporting per-message status.
    message_send_checkpoint_endpoint: null
+    # Whether asynchronous uploads via MSC2246 should be enabled for media.
+    # Requires a media repo that supports MSC2246.
+    async_media: false

 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
@@ -64,7 +67,7 @@ appservice:
        # Whether or not the provisioning API should be enabled.
        enabled: true
        # The prefix to use in the provisioning API endpoints.
-        prefix: /_matrix/provision/v1
+        prefix: /_matrix/provision
        # The shared secret to authorize users of the API.
        # Set to "generate" to generate and save a new token.
        shared_secret: generate
@@ -81,7 +84,7 @@ appservice:
    # Whether or not to receive ephemeral events via appservice transactions.
    # Requires MSC2409 support (i.e. Synapse 1.22+).
    # You should disable bridge -> sync_with_custom_puppets when this is enabled.
-    ephemeral_events: false
+    ephemeral_events: true

    # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
    as_token: "This value is generated when generating the registration"
@@ -145,15 +148,19 @@ bridge:
    # will not send any more members.
    # -1 means no limit (which means it's limited to 10000 by the server)
    max_initial_member_sync: 100
+    # Maximum number of participants in chats to bridge. Only applies when the portal is being created.
+    # If there are more members when trying to create a room, the room creation will be cancelled.
+    # -1 means no limit (which means all chats can be bridged)
+    max_member_count: -1
    # Whether or not to sync the member list in channels.
    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
    # list regardless of this setting.
-    sync_channel_members: true
+    sync_channel_members: false
    # Whether or not to skip deleted members when syncing members.
    skip_deleted_members: true
    # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
    # their Telegram account at startup.
-    startup_sync: true
+    startup_sync: false
    # Number of most recently active dialogs to check when syncing chats.
    # Set to 0 to remove limit.
    sync_update_limit: 0
@@ -171,15 +178,11 @@ bridge:
    # Allow logging in within Matrix. If false, users can only log in using login-qr or the
    # out-of-Matrix login website (see appservice.public config section)
    allow_matrix_login: true
-    # Whether or not to bridge plaintext highlights.
-    # Only enable this if your displayname_template has some static part that the bridge can use to
-    # reliably identify what is a plaintext highlight.
-    plaintext_highlights: false
    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
-    public_portals: true
+    public_portals: false
    # Whether or not to use /sync to get presence, read receipts and typing notifications
    # when double puppeting is enabled
-    sync_with_custom_puppets: true
+    sync_with_custom_puppets: false
    # Whether or not to update the m.direct account data event when double puppeting is enabled.
    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
    # and is therefore prone to race conditions.
@@ -203,15 +206,13 @@ bridge:
    # Whether or not the !tg join command should do a HTTP request
    # to resolve redirects in invite links.
    invite_link_resolve: false
-    # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Element iOS/Android).
-    inline_images: false
+    # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552.
+    # This is currently not supported in most clients.
+    caption_in_message: false
    # Maximum size of image in megabytes before sending to Telegram as a document.
    image_as_file_size: 10
-    # Maximum number of pixels in an image before sending to Telegram as a document. Defaults to 1280x1280 = 1638400.
-    image_as_file_pixels: 1638400
-    # Maximum size of Telegram documents in megabytes to bridge.
-    max_document_size: 100
+    # Maximum number of pixels in an image before sending to Telegram as a document. Defaults to 4096x4096 = 16777216.
+    image_as_file_pixels: 16777216
    # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
    # streaming from/to Matrix and using many connections for Telegram.
    # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
@@ -227,12 +228,24 @@ bridge:
        # png - converts to non-animated png (fastest),
        # gif - converts to animated gif
        # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support
+        # webp - converts to animated webp, requires ffmpeg executable with webp codec/container support
        target: gif
+        # Should video stickers be converted to the specified format as well?
+        convert_from_webm: false
        # Arguments for converter. All converters take width and height.
        args:
            width: 256
            height: 256
-            fps: 25 # only for webm and gif (2, 5, 10, 20 or 25 recommended)
+            fps: 25 # only for webm, webp and gif (2, 5, 10, 20 or 25 recommended)
+    # Settings for converting animated emoji.
+    # Same as animated_sticker, but webm is not supported as the target
+    # (because inline images can only contain images, not videos).
+    animated_emoji:
+        target: webp
+        args:
+            width: 64
+            height: 64
+            fps: 25
    # End-to-bridge encryption support options.
    #
    # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info.
@@ -242,20 +255,46 @@ bridge:
        # Default to encryption, force-enable encryption in all portals the bridge creates
        # This will cause the bridge bot to be in private chats for the encryption to work properly.
        default: false
-        # Database for the encryption data. If set to `default`, will use the appservice database.
-        database: default
-        # Options for automatic key sharing.
-        key_sharing:
-            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
-            # You must use a client that supports requesting keys from other users to use this feature.
-            allow: false
-            # Require the requesting device to have a valid cross-signing signature?
-            # This doesn't require that the bridge has verified the device, only that the user has verified it.
-            # Not yet implemented.
-            require_cross_signing: false
-            # Require devices to be verified by the bridge?
-            # Verification by the bridge is not yet implemented.
-            require_verification: true
+        # Require encryption, drop any unencrypted messages.
+        require: false
+        # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
+        # You must use a client that supports requesting keys from other users to use this feature.
+        allow_key_sharing: false
+        # What level of device verification should be required from users?
+        #
+        # Valid levels:
+        #   unverified - Send keys to all device in the room.
+        #   cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys.
+        #   cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes).
+        #   cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot.
+        #                           Note that creating user signatures from the bridge bot is not currently possible.
+        #   verified - Require manual per-device verification
+        #              (currently only possible by modifying the `trust` column in the `crypto_device` database table).
+        verification_levels:
+            # Minimum level for which the bridge should send keys to when bridging messages from Telegram to Matrix.
+            receive: unverified
+            # Minimum level that the bridge should accept for incoming Matrix messages.
+            send: unverified
+            # Minimum level that the bridge should require for accepting key requests.
+            share: cross-signed-tofu
+        # Options for Megolm room key rotation. These options allow you to
+        # configure the m.room.encryption event content. See:
+        # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for
+        # more information about that event.
+        rotation:
+            # Enable custom Megolm room key rotation settings. Note that these
+            # settings will only apply to rooms created after this option is
+            # set.
+            enable_custom: false
+            # The maximum number of milliseconds a session should be used
+            # before changing it. The Matrix spec recommends 604800000 (a week)
+            # as the default.
+            milliseconds: 604800000
+            # The maximum number of messages that should be sent with a given a
+            # session before changing it. The Matrix spec recommends 100 as the
+            # default.
+            messages: 100
+
    # Whether or not to explicitly set the avatar and room name for private
    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
    private_chat_portal_meta: false
@@ -264,6 +303,8 @@ bridge:
    delivery_receipts: false
    # Whether or not delivery errors should be reported as messages in the Matrix room.
    delivery_error_reports: false
+    # Whether the bridge should send the message status as a custom com.beeper.message_send_status event.
+    message_status_events: false
    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
    # This field will automatically be changed back to false after it,
    # except if the config file is not writable.
@@ -281,6 +322,11 @@ bridge:
    bridge_matrix_leave: true
    # Should the user be kicked out of all portals when logging out of the bridge?
    kick_on_logout: true
+    # Should the "* user joined Telegram" notice always be marked as read automatically?
+    always_read_joined_telegram_notice: true
+    # Should the bridge auto-create a group chat on Telegram when a ghost is invited to a room?
+    # Requires the user to have sufficient power level and double puppeting enabled.
+    create_group_on_invite: true
    # Settings for backfilling messages from Telegram.
    backfill:
        # Whether or not the Telegram ghosts of logged in Matrix users should be
@@ -452,6 +498,12 @@ telegram:
    # (Optional) Create your own bot at https://t.me/BotFather
    bot_token: disabled

+    # Should the bridge request missed updates from Telegram when restarting?
+    catch_up: true
+    # Should incoming updates be handled sequentially to make sure order is preserved on Matrix?
+    sequential_updates: true
+    exit_on_update_error: false
+
    # Telethon connection options.
    connection:
        # The timeout in seconds to be used when connecting.
@@ -477,7 +529,7 @@ telegram:
    # Device info sent to Telegram.
    device_info:
        # "auto" = OS name+version.
-        device_model: auto
+        device_model: mautrix-telegram
        # "auto" = Telethon version.
        system_version: auto
        # "auto" = mautrix-telegram version.
@@ -1,2 +1,2 @@
 from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram
-from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
+from .from_telegram import telegram_to_matrix
@@ -18,7 +18,7 @@ from __future__ import annotations
 import re

 from telethon import TelegramClient
-from telethon.helpers import add_surrogate, del_surrogate
+from telethon.helpers import add_surrogate, del_surrogate, strip_text
 from telethon.tl.types import MessageEntityItalic, TypeMessageEntity

 from mautrix.types import MessageEventContent, RoomID
@@ -73,8 +73,8 @@ async def _matrix_html_to_telegram(
        html = not_command_regex.sub(r"\1", html)

        parsed = await MatrixParser(client).parse(add_surrogate(html))
-        text = del_surrogate(parsed.text.strip())
-        text, entities = _cut_long_message(text, parsed.telegram_entities)
+        text, entities = _cut_long_message(parsed.text, parsed.telegram_entities)
+        text = del_surrogate(strip_text(text, entities))

        return text, entities
    except Exception as e:
@@ -28,6 +28,7 @@ from telethon.tl.types import (
    MessageEntityBotCommand,
    MessageEntityCashtag,
    MessageEntityCode,
+    MessageEntityCustomEmoji,
    MessageEntityEmail,
    MessageEntityHashtag,
    MessageEntityItalic,
@@ -48,42 +49,19 @@ from telethon.tl.types import (
    TypeMessageEntity,
 )

-from mautrix.appservice import IntentAPI
-from mautrix.types import (
-    EventType,
-    Format,
-    MessageType,
-    RelatesTo,
-    RelationType,
-    TextMessageEventContent,
-)
+from mautrix.types import Format, MessageType, TextMessageEventContent

 from .. import abstract_user as au, portal as po, puppet as pu, user as u
-from ..db import Message as DBMessage
+from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
 from ..types import TelegramID
+from ..util.file_transfer import transfer_custom_emojis_to_matrix

 log: logging.Logger = logging.getLogger("mau.fmt.tg")


-async def telegram_reply_to_matrix(evt: Message, source: au.AbstractUser) -> RelatesTo | None:
-    if evt.reply_to:
-        space = (
-            evt.peer_id.channel_id
-            if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
-            else source.tgid
-        )
-        msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
-        if msg:
-            return RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
-    return None
-
-
 async def _add_forward_header(
    source: au.AbstractUser, content: TextMessageEventContent, fwd_from: MessageFwdHeader
 ) -> None:
-    if not content.formatted_body or content.format != Format.HTML:
-        content.format = Format.HTML
-        content.formatted_body = escape(content.body)
    fwd_from_html, fwd_from_text = None, None
    if isinstance(fwd_from.from_id, PeerUser):
        user = await u.User.get_by_tgid(TelegramID(fwd_from.from_id.user_id))
@@ -139,6 +117,7 @@ async def _add_forward_header(
        fwd_from_text = "unknown source"
        fwd_from_html = f"unknown source"

+    content.ensure_has_html()
    content.body = "\n".join([f"> {line}" for line in content.body.split("\n")])
    content.body = f"Forwarded from {fwd_from_text}:\n{content.body}"
    content.formatted_body = (
@@ -147,76 +126,53 @@ async def _add_forward_header(
    )


-async def _add_reply_header(
-    source: au.AbstractUser, content: TextMessageEventContent, evt: Message, main_intent: IntentAPI
+class ReuploadedCustomEmoji(MessageEntityCustomEmoji):
+    file: DBTelegramFile
+
+    def __init__(self, parent: MessageEntityCustomEmoji, file: DBTelegramFile) -> None:
+        super().__init__(parent.offset, parent.length, parent.document_id)
+        self.file = file
+
+
+async def _convert_custom_emoji(
+    source: au.AbstractUser, entities: list[TypeMessageEntity]
 ) -> None:
-    space = (
-        evt.peer_id.channel_id
-        if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
-        else source.tgid
-    )
-
-    msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
-    if not msg:
-        return
-
-    content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
-
-    try:
-        event = await main_intent.get_event(msg.mx_room, msg.mxid)
-        if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
-            event = await source.bridge.matrix.e2ee.decrypt(event)
-        if isinstance(event.content, TextMessageEventContent):
-            event.content.trim_reply_fallback()
-        puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
-        content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
-    except Exception:
-        log.exception("Failed to get event to add reply fallback")
+    emoji_ids = [
+        entity.document_id for entity in entities if isinstance(entity, MessageEntityCustomEmoji)
+    ]
+    custom_emojis = await transfer_custom_emojis_to_matrix(source, emoji_ids)
+    if len(custom_emojis) > 0:
+        for i, entity in enumerate(entities):
+            if isinstance(entity, MessageEntityCustomEmoji):
+                entities[i] = ReuploadedCustomEmoji(entity, custom_emojis[entity.document_id])


 async def telegram_to_matrix(
    evt: Message | SponsoredMessage,
    source: au.AbstractUser,
-    main_intent: IntentAPI | None = None,
-    prefix_text: str | None = None,
-    prefix_html: str | None = None,
    override_text: str = None,
    override_entities: list[TypeMessageEntity] = None,
-    no_reply_fallback: bool = False,
    require_html: bool = False,
 ) -> TextMessageEventContent:
    content = TextMessageEventContent(
        msgtype=MessageType.TEXT,
-        body=add_surrogate(override_text or evt.message),
+        body=override_text or evt.message,
    )
    entities = override_entities or evt.entities
    if entities:
+        await _convert_custom_emoji(source, entities)
        content.format = Format.HTML
        html = await _telegram_entities_to_matrix_catch(add_surrogate(content.body), entities)
-        content.formatted_body = del_surrogate(html).replace("\n", "<br/>")
-
-    def force_html():
-        if not content.formatted_body:
-            content.format = Format.HTML
-            content.formatted_body = escape(content.body).replace("\n", "<br/>")
+        content.formatted_body = del_surrogate(html)

    if require_html:
-        force_html()
-
-    if prefix_html:
-        force_html()
-        content.formatted_body = prefix_html + content.formatted_body
-    if prefix_text:
-        content.body = prefix_text + content.body
+        content.ensure_has_html()

    if getattr(evt, "fwd_from", None):
        await _add_forward_header(source, content, evt.fwd_from)

-    if getattr(evt, "reply_to", None) and not no_reply_fallback:
-        await _add_reply_header(source, content, evt, main_intent)
-
    if isinstance(evt, Message) and evt.post and evt.post_author:
-        force_html()
+        content.ensure_has_html()
        content.body += f"\n- {evt.post_author}"
        content.formatted_body += f"<br/><i>- <u>{evt.post_author}</u></i>"

@@ -233,31 +189,63 @@ async def _telegram_entities_to_matrix_catch(text: str, entities: list[TypeMessa
    return "[failed conversion in _telegram_entities_to_matrix]"


+def within_surrogate(text, index):
+    """
+    `True` if ``index`` is within a surrogate (before and after it, not at!).
+    """
+    return (
+        1 < index < len(text)  # in bounds
+        and "\ud800" <= text[index - 1] <= "\udbff"  # current is low surrogate
+        and "\udc00" <= text[index] <= "\udfff"  # previous is high surrogate
+    )
+
+
 async def _telegram_entities_to_matrix(
-    text: str, entities: list[TypeMessageEntity], offset: int = 0, length: int = None
+    text: str,
+    entities: list[TypeMessageEntity | ReuploadedCustomEmoji],
+    offset: int = 0,
+    length: int = None,
+    in_codeblock: bool = False,
 ) -> str:
+    def text_to_html(
+        val: str, _in_codeblock: bool = in_codeblock, escape_html: bool = True
+    ) -> str:
+        if escape_html:
+            val = escape(val)
+        if not _in_codeblock:
+            val = val.replace("\n", "<br/>")
+        return val
+
    if not entities:
-        return escape(text)
+        return text_to_html(text)
    if length is None:
        length = len(text)
    html = []
    last_offset = 0
    for i, entity in enumerate(entities):
-        if entity.offset > offset + length:
+        if entity.offset >= offset + length:
            break
        relative_offset = entity.offset - offset
        if relative_offset > last_offset:
-            html.append(escape(text[last_offset:relative_offset]))
+            html.append(text_to_html(text[last_offset:relative_offset]))
        elif relative_offset < last_offset:
            continue

+        while within_surrogate(text, relative_offset):
+            relative_offset += 1
+        while within_surrogate(text, relative_offset + entity.length):
+            entity.length += 1
+
        skip_entity = False
+        is_code_entity = isinstance(entity, (MessageEntityCode, MessageEntityPre))
        entity_text = await _telegram_entities_to_matrix(
            text=text[relative_offset : relative_offset + entity.length],
            entities=entities[i + 1 :],
            offset=entity.offset,
            length=entity.length,
+            in_codeblock=is_code_entity,
        )
+        entity_text = text_to_html(entity_text, is_code_entity, escape_html=False)
        entity_type = type(entity)

        if entity_type == MessageEntityBold:
@@ -285,9 +273,16 @@ async def _telegram_entities_to_matrix(
        elif entity_type == MessageEntityEmail:
            html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
        elif entity_type in (MessageEntityTextUrl, MessageEntityUrl):
-            skip_entity = await _parse_url(
+            await _parse_url(
                html, entity_text, entity.url if entity_type == MessageEntityTextUrl else None
            )
+        elif entity_type == MessageEntityCustomEmoji:
+            html.append(entity_text)
+        elif entity_type == ReuploadedCustomEmoji:
+            html.append(
+                f'<img data-mx-emoticon data-mau-animated-emoji src="{escape(entity.file.mxc)}" '
+                f'height="32" width="32" alt="{entity_text}" title="{entity_text}"/>'
+            )
        elif entity_type in (
            MessageEntityBotCommand,
            MessageEntityHashtag,
@@ -300,7 +295,7 @@ async def _telegram_entities_to_matrix(
        else:
            skip_entity = True
        last_offset = relative_offset + (0 if skip_entity else entity.length)
-    html.append(escape(text[last_offset:]))
+    html.append(text_to_html(text[last_offset:]))

    return "".join(html)

@@ -316,12 +311,24 @@ def _parse_pre(html: list[str], entity_text: str, language: str) -> bool:
 async def _parse_mention(html: list[str], entity_text: str) -> bool:
    username = entity_text[1:]

+    mxid = None
+    portal = None
+    # This is a bit complicated because public channels have both Puppet and Portal instances.
+    # Basically the currently intended output is:
+    # User/bot mention (bridge user)          -> real user mention
+    # User/bot mention (normal Telegram user) -> ghost user mention
+    # Public channel with existing portal     -> room mention
+    # Public channel without portal           -> ghost user mention
+    # Other chat                              -> room mention
    user = await u.User.find_by_username(username) or await pu.Puppet.find_by_username(username)
    if user:
+        if isinstance(user, pu.Puppet) and user.is_channel:
+            portal = await po.Portal.get_by_tgid(user.tgid)
        mxid = user.mxid
    else:
        portal = await po.Portal.find_by_username(username)
-        mxid = portal.alias or portal.mxid if portal else None
+    if portal and (portal.mxid or not user):
+        mxid = portal.alias or portal.mxid

    if mxid:
        html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
@@ -345,11 +352,15 @@ async def _parse_name_mention(html: list[str], entity_text: str, user_id: Telegr


 message_link_regex = re.compile(
-    r"https?://t(?:elegram)?\.(?:me|dog)/([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})"
+    r"https?://t(?:elegram)?\.(?:me|dog)"
+    # /username or /c/id
+    r"/([A-Za-z][A-Za-z0-9_]{3,31}[A-Za-z0-9]|[Cc]/[0-9]{1,20})"
+    # /messageid
+    r"/([0-9]{1,20})"
 )


-async def _parse_url(html: list[str], entity_text: str, url: str) -> bool:
+async def _parse_url(html: list[str], entity_text: str, url: str) -> None:
    url = escape(url) if url else entity_text
    if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
        url = "http://" + url
@@ -359,11 +370,13 @@ async def _parse_url(html: list[str], entity_text: str, url: str) -> bool:
        group, msgid_str = message_link_match.groups()
        msgid = int(msgid_str)

-        portal = await po.Portal.find_by_username(group)
+        if group.lower().startswith("c/"):
+            portal = await po.Portal.get_by_tgid(TelegramID(int(group[2:])))
+        else:
+            portal = await po.Portal.find_by_username(group)
        if portal:
            message = await DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
            if message:
                url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"

    html.append(f"<a href='{url}'>{entity_text}</a>")
-    return False
@@ -42,6 +42,8 @@ from mautrix.types import (
 )

 from . import commands as com, portal as po, puppet as pu, user as u
+from .commands.portal.util import get_initial_state, user_has_power_level, warn_missing_power
+from .types import TelegramID

 if TYPE_CHECKING:
    from .__main__ import TelegramBridge
@@ -61,99 +63,83 @@ class MatrixHandler(BaseMatrixHandler):

        self._previously_typing = {}

-    async def handle_puppet_invite(
-        self, room_id: RoomID, puppet: pu.Puppet, inviter: u.User, event_id: EventID
+    async def handle_puppet_group_invite(
+        self,
+        room_id: RoomID,
+        puppet: pu.Puppet,
+        invited_by: u.User,
+        evt: StateEvent,
+        members: list[UserID],
    ) -> None:
-        intent = puppet.default_mxid_intent
-        self.log.debug(f"{inviter.mxid} invited puppet for {puppet.tgid} to {room_id}")
-        if puppet.is_channel:
-            self.log.debug(f"Rejecting invite for {puppet.tgid} to {room_id}: puppet is a channel")
-            await intent.leave_room(room_id, reason="Channels can't be invited to chats")
-            return
-
-        if not await inviter.is_logged_in():
-            self.log.debug(f"Rejecting invite for {puppet.tgid} to {room_id}: user not logged in")
-            await intent.leave_room(
-                room_id,
-                reason="Only users who are logged into the bridge can invite Telegram ghosts.",
-            )
-            return
-
-        portal = await po.Portal.get_by_mxid(room_id)
-        if portal:
-            if portal.peer_type == "user":
-                await intent.error_and_leave(
-                    room_id, text="You can not invite additional users to private chats."
-                )
-                return
-            await portal.invite_telegram(inviter, puppet)
-            await intent.join_room(room_id)
-            return
-        try:
-            members = await intent.get_room_members(room_id)
-        except MatrixError:
-            self.log.exception(f"Failed to get members after joining {room_id} as {intent.mxid}")
-            return
-        if self.az.bot_mxid not in members:
-            if len(members) > 2:
-                await intent.error_and_leave(
+        double_puppet = await pu.Puppet.get_by_custom_mxid(invited_by.mxid)
+        if (
+            not double_puppet
+            or self.az.bot_mxid in members
+            or not self.config["bridge.create_group_on_invite"]
+        ):
+            if self.az.bot_mxid not in members:
+                await puppet.default_mxid_intent.leave_room(
                    room_id,
-                    text=None,
-                    html=(
-                        f"Please invite "
-                        f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
-                        f"first if you want to create a Telegram chat."
-                    ),
+                    reason="This ghost does not join multi-user rooms without the bridge bot.",
                )
-                return
-
-            await intent.join_room(room_id)
-            portal = await po.Portal.get_by_tgid(
-                puppet.tgid, tg_receiver=inviter.tgid, peer_type="user"
-            )
-            if portal.mxid:
-                try:
-                    await portal.invite_to_matrix(inviter.mxid)
-                    await intent.send_notice(
-                        room_id,
-                        text=f"You already have a private chat with me: {portal.mxid}",
-                        html=(
-                            "You already have a private chat with me: "
-                            f"<a href='https://matrix.to/#/{portal.mxid}'>Link to room</a>"
-                        ),
-                    )
-                    await intent.leave_room(room_id)
-                    return
-                except MatrixError:
-                    pass
-            portal.mxid = room_id
-            e2be_ok = await portal.check_dm_encryption()
-            await portal.save()
-            await inviter.register_portal(portal)
-            if e2be_ok is True:
-                evt_type, content = await self.e2ee.encrypt(
-                    room_id,
-                    EventType.ROOM_MESSAGE,
-                    TextMessageEventContent(
-                        msgtype=MessageType.NOTICE,
-                        body=(
-                            "Portal to private chat created and end-to-bridge encryption enabled."
-                        ),
-                    ),
-                )
-                await intent.send_message_event(room_id, evt_type, content)
            else:
-                message = "Portal to private chat created."
-                if e2be_ok is False:
-                    message += "\n\nWarning: Failed to enable end-to-bridge encryption"
-                await intent.send_notice(room_id, message)
-            await portal.update_bridge_info()
-        else:
-            await intent.join_room(room_id)
-            await intent.send_notice(
-                room_id,
-                "This puppet will remain inactive until a Telegram chat is created for this room.",
+                await puppet.default_mxid_intent.send_notice(
+                    room_id,
+                    "This ghost will remain inactive "
+                    "until a Telegram chat is created for this room.",
+                )
+            return
+        elif not await user_has_power_level(
+            evt.room_id, double_puppet.intent, invited_by, "bridge"
+        ):
+            await puppet.default_mxid_intent.leave_room(
+                room_id, reason="You do not have the permissions to bridge this room."
            )
+            return
+
+        await double_puppet.intent.invite_user(room_id, self.az.bot_mxid)
+
+        title, about, levels, encrypted = await get_initial_state(double_puppet.intent, room_id)
+        if not title:
+            await puppet.default_mxid_intent.leave_room(
+                room_id, reason="Please set a title before inviting Telegram ghosts."
+            )
+            return
+
+        portal = po.Portal(
+            tgid=TelegramID(0),
+            tg_receiver=TelegramID(0),
+            peer_type="channel",
+            mxid=evt.room_id,
+            title=title,
+            about=about,
+            encrypted=encrypted,
+        )
+        await portal.az.intent.ensure_joined(room_id)
+        levels = await portal.az.intent.get_power_levels(room_id)
+        invited_by_level = levels.get_user_level(invited_by.mxid)
+        if invited_by_level > levels.get_user_level(self.az.bot_mxid):
+            levels.users[self.az.bot_mxid] = 100 if invited_by_level >= 100 else invited_by_level
+            await double_puppet.intent.set_power_levels(room_id, levels)
+
+        invites, errors = await portal.get_telegram_users_in_matrix_room(
+            invited_by, pre_create=True
+        )
+        if len(errors) > 0:
+            error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
+            await portal.az.intent.send_notice(
+                room_id,
+                f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
+                "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
+                "those users.",
+            )
+
+        try:
+            await portal.create_telegram_chat(invited_by, invites=invites, supergroup=True)
+        except ValueError as e:
+            await portal.delete()
+            await portal.az.intent.send_notice(room_id, e.args[0])
+            return

    async def handle_invite(
        self, room_id: RoomID, user_id: UserID, inviter: u.User, event_id: EventID
@@ -163,9 +149,13 @@ class MatrixHandler(BaseMatrixHandler):
            return
        await user.ensure_started()
        portal = await po.Portal.get_by_mxid(room_id)
-        if user and await user.has_full_access(allow_bot=True):
-            if portal and portal.allow_bridging:
-                await portal.invite_telegram(inviter, user)
+        if (
+            user
+            and portal
+            and await user.has_full_access(allow_bot=True)
+            and portal.allow_bridging
+        ):
+            await portal.handle_matrix_invite(inviter, user)

    async def handle_join(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None:
        user = await u.User.get_and_start_by_mxid(user_id)
@@ -1,5 +1,5 @@
 from .deduplication import PortalDedup
-from .media_fallback import make_contact_event_content, make_dice_event_content
+from .message_convert import ConvertedMessage, TelegramMessageConverter
 from .participants import get_users
 from .power_levels import get_base_power_levels, participants_to_power_levels
 from .send_lock import PortalReactionLock, PortalSendLock
@@ -62,7 +62,6 @@ media_content_table = {


 class PortalDedup:
-    pre_db_check: bool = False
    cache_queue_length: int = 256

    _dedup: deque[bytes | int]
@@ -97,13 +96,13 @@ class PortalDedup:
                )
                yield media_hash_func(event.media)

-    def _hash_event(self, event: TypeMessage) -> bytes:
+    def hash_event(self, event: TypeMessage) -> bytes:
        return hashlib.sha256(
            "-".join(str(a) for a in self._hash_content(event)).encode("utf-8")
        ).digest()

    def check_action(self, event: TypeMessage) -> bool:
-        dedup_id = self._hash_event(event) if self._always_force_hash else event.id
+        dedup_id = self.hash_event(event) if self._always_force_hash else event.id
        if dedup_id in self._dedup_action:
            return True

@@ -117,7 +116,7 @@ class PortalDedup:
        expected_mxid: DedupMXID | None = None,
        force_hash: bool = False,
    ) -> tuple[bytes, DedupMXID | None]:
-        evt_hash = self._hash_event(event)
+        evt_hash = self.hash_event(event)
        dedup_id = evt_hash if self._always_force_hash or force_hash else event.id
        try:
            found_mxid = self._dedup_mxid[dedup_id]
@@ -134,7 +133,7 @@ class PortalDedup:
    def check(
        self, event: TypeMessage, mxid: DedupMXID = None, force_hash: bool = False
    ) -> tuple[bytes, DedupMXID | None]:
-        evt_hash = self._hash_event(event)
+        evt_hash = self.hash_event(event)
        dedup_id = evt_hash if self._always_force_hash or force_hash else event.id
        if dedup_id in self._dedup:
            return evt_hash, self._dedup_mxid[dedup_id]
@@ -1,133 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-import html
-
-from telethon.tl.types import MessageMediaContact, MessageMediaDice, PeerUser
-
-from mautrix.types import Format, MessageType, TextMessageEventContent
-
-from .. import abstract_user as au, puppet as pu
-from ..types import TelegramID
-
-try:
-    import phonenumbers
-except ImportError:
-    phonenumbers = None
-
-
-def _format_dice(roll: MessageMediaDice) -> str:
-    if roll.emoticon == "\U0001F3B0":
-        emojis = {
-            0: "\U0001F36B",  # "🍫",
-            1: "\U0001F352",  # "🍒",
-            2: "\U0001F34B",  # "🍋",
-            3: "7\ufe0f\u20e3",  # "7️⃣",
-        }
-        res = roll.value - 1
-        slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
-        return f"{slot1} {slot2} {slot3} ({roll.value})"
-    elif roll.emoticon == "\u26BD":
-        results = {
-            1: "miss",
-            2: "hit the woodwork",
-            3: "goal",  # seems to go in through the center
-            4: "goal",
-            5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
-        }
-    elif roll.emoticon == "\U0001F3B3":
-        results = {
-            1: "miss",
-            2: "1 pin down",
-            3: "3 pins down, split",
-            4: "4 pins down, split",
-            5: "5 pins down",
-            6: "strike 🎉",
-        }
-    # elif roll.emoticon == "\U0001F3C0":
-    #     results = {
-    #         2: "rolled off",
-    #         3: "stuck",
-    #     }
-    # elif roll.emoticon == "\U0001F3AF":
-    #     results = {
-    #         1: "bounced off",
-    #         2: "outer rim",
-    #
-    #         6: "bullseye",
-    #     }
-    else:
-        return str(roll.value)
-    return f"{results[roll.value]} ({roll.value})"
-
-
-def make_dice_event_content(roll: MessageMediaDice) -> TextMessageEventContent:
-    emoji_text = {
-        "\U0001F3AF": " Dart throw",
-        "\U0001F3B2": " Dice roll",
-        "\U0001F3C0": " Basketball throw",
-        "\U0001F3B0": " Slot machine",
-        "\U0001F3B3": " Bowling",
-        "\u26BD": " Football kick",
-    }
-    text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {_format_dice(roll)}"
-    content = TextMessageEventContent(
-        msgtype=MessageType.TEXT, format=Format.HTML, body=text, formatted_body=f"<h4>{text}</h4>"
-    )
-    content["net.maunium.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
-    return content
-
-
-async def make_contact_event_content(
-    source: au.AbstractUser, contact: MessageMediaContact
-) -> TextMessageEventContent:
-    name = " ".join(x for x in [contact.first_name, contact.last_name] if x)
-    formatted_phone = f"+{contact.phone_number}"
-    if phonenumbers is not None:
-        try:
-            parsed = phonenumbers.parse(formatted_phone)
-            fmt = phonenumbers.PhoneNumberFormat.INTERNATIONAL
-            formatted_phone = phonenumbers.format_number(parsed, fmt)
-        except phonenumbers.NumberParseException:
-            pass
-    content = TextMessageEventContent(
-        msgtype=MessageType.TEXT,
-        body=f"Shared contact info for {name}: {formatted_phone}",
-    )
-    content["net.maunium.telegram.contact"] = {
-        "user_id": contact.user_id,
-        "first_name": contact.first_name,
-        "last_name": contact.last_name,
-        "phone_number": contact.phone_number,
-        "vcard": contact.vcard,
-    }
-
-    puppet = await pu.Puppet.get_by_tgid(TelegramID(contact.user_id))
-    if not puppet.displayname:
-        try:
-            entity = await source.client.get_entity(PeerUser(contact.user_id))
-            await puppet.update_info(source, entity)
-        except Exception as e:
-            source.log.warning(f"Failed to sync puppet info of received contact: {e}")
-    else:
-        content.format = Format.HTML
-        content.formatted_body = (
-            f"Shared contact info for "
-            f"<a href='https://matrix.to/#/{puppet.mxid}'>{html.escape(name)}</a>: "
-            f"{html.escape(formatted_phone)}"
-        )
-    return content
@@ -0,0 +1,780 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Any, NamedTuple
+import base64
+import codecs
+import html
+import mimetypes
+import unicodedata
+
+from attr import dataclass
+from telethon.tl.types import (
+    Document,
+    DocumentAttributeAnimated,
+    DocumentAttributeAudio,
+    DocumentAttributeFilename,
+    DocumentAttributeImageSize,
+    DocumentAttributeSticker,
+    DocumentAttributeVideo,
+    Game,
+    InputPhotoFileLocation,
+    Message,
+    MessageEntityPre,
+    MessageMediaContact,
+    MessageMediaDice,
+    MessageMediaDocument,
+    MessageMediaGame,
+    MessageMediaGeo,
+    MessageMediaGeoLive,
+    MessageMediaPhoto,
+    MessageMediaPoll,
+    MessageMediaUnsupported,
+    MessageMediaVenue,
+    MessageMediaWebPage,
+    PeerChannel,
+    PeerUser,
+    Photo,
+    PhotoCachedSize,
+    PhotoEmpty,
+    PhotoSize,
+    PhotoSizeEmpty,
+    PhotoSizeProgressive,
+    Poll,
+    TypeDocumentAttribute,
+    TypePhotoSize,
+    WebPage,
+)
+from telethon.utils import decode_waveform
+
+from mautrix.appservice import IntentAPI
+from mautrix.types import (
+    EventType,
+    Format,
+    ImageInfo,
+    LocationMessageEventContent,
+    MediaMessageEventContent,
+    MessageEventContent,
+    MessageType,
+    TextMessageEventContent,
+    ThumbnailInfo,
+)
+from mautrix.util.logging import TraceLogger
+
+from .. import abstract_user as au, formatter, matrix as m, portal as po, puppet as pu, util
+from ..config import Config
+from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
+from ..types import TelegramID
+from ..util import sane_mimetypes
+
+try:
+    import phonenumbers
+except ImportError:
+    phonenumbers = None
+
+
+@dataclass
+class ConvertedMessage:
+    content: MessageEventContent
+    caption: MessageEventContent | None = None
+    type: EventType = EventType.ROOM_MESSAGE
+    disappear_seconds: int | None = None
+    disappear_start_immediately: bool = False
+
+
+class DocAttrs(NamedTuple):
+    name: str | None
+    mime_type: str | None
+    is_sticker: bool
+    sticker_alt: str | None
+    width: int
+    height: int
+    is_gif: bool
+    is_audio: bool
+    is_voice: bool
+    duration: int
+    waveform: bytes
+
+
+BEEPER_LINK_PREVIEWS_KEY = "com.beeper.linkpreviews"
+BEEPER_IMAGE_ENCRYPTION_KEY = "beeper:image:encryption"
+
+
+class TelegramMessageConverter:
+    portal: po.Portal
+    matrix: m.MatrixHandler
+    config: Config
+    command_prefix: str
+    log: TraceLogger
+
+    def __init__(self, portal: po.Portal) -> None:
+        self.portal = portal
+        self.matrix = portal.matrix
+        self.config = portal.config
+        self.command_prefix = self.config["bridge.command_prefix"]
+        self.log = portal.log.getChild("msg_conv")
+
+        self._media_converters = {
+            MessageMediaPhoto: self._convert_photo,
+            MessageMediaDocument: self._convert_document,
+            MessageMediaGeo: self._convert_location,
+            MessageMediaGeoLive: self._convert_location,
+            MessageMediaVenue: self._convert_location,
+            MessageMediaPoll: self._convert_poll,
+            MessageMediaDice: self._convert_dice,
+            MessageMediaUnsupported: self._convert_unsupported,
+            MessageMediaGame: self._convert_game,
+            MessageMediaContact: self._convert_contact,
+        }
+        self._allowed_media = tuple(self._media_converters.keys())
+
+    async def convert(
+        self,
+        source: au.AbstractUser,
+        intent: IntentAPI,
+        is_bot: bool,
+        evt: Message,
+        no_reply_fallback: bool = False,
+    ) -> ConvertedMessage | None:
+        if hasattr(evt, "media") and isinstance(evt.media, self._allowed_media):
+            convert_media = self._media_converters[type(evt.media)]
+            converted = await convert_media(source=source, intent=intent, evt=evt)
+        elif evt.message:
+            converted = await self._convert_text(source, intent, is_bot, evt)
+        else:
+            self.log.debug("Unhandled Telegram message %d", evt.id)
+            return
+        if converted:
+            if evt.ttl_period and not converted.disappear_seconds:
+                converted.disappear_seconds = evt.ttl_period
+                converted.disappear_start_immediately = True
+            converted.content.external_url = self._get_external_url(evt)
+            converted.content["fi.mau.telegram.source"] = {
+                "space": self.portal.tgid if self.portal.peer_type == "channel" else source.tgid,
+                "chat_id": self.portal.tgid,
+                "peer_type": self.portal.peer_type,
+                "id": evt.id,
+            }
+            if converted.caption:
+                converted.caption["fi.mau.telegram.source"] = converted.content[
+                    "fi.mau.telegram.source"
+                ]
+                converted.caption.external_url = converted.content.external_url
+                if self.portal.get_config("caption_in_message"):
+                    self._caption_to_message(converted)
+            await self._set_reply(source, evt, converted.content, no_fallback=no_reply_fallback)
+        return converted
+
+    @staticmethod
+    def _caption_to_message(converted: ConvertedMessage) -> None:
+        content, caption = converted.content, converted.caption
+        converted.caption = None
+
+        content["filename"] = content.body
+        content["org.matrix.msc1767.caption"] = {
+            "org.matrix.msc1767.text": caption.body,
+        }
+        content.body = caption.body
+        if caption.format == Format.HTML:
+            content["org.matrix.msc1767.caption"][
+                "org.matrix.msc1767.html"
+            ] = caption.formatted_body
+            content["formatted_body"] = caption.formatted_body
+            content["format"] = Format.HTML.value
+
+    def _get_external_url(self, evt: Message) -> str | None:
+        if self.portal.peer_type == "channel" and self.portal.username is not None:
+            return f"https://t.me/{self.portal.username}/{evt.id}"
+        elif self.portal.peer_type != "user":
+            return f"https://t.me/c/{self.portal.tgid}/{evt.id}"
+        return None
+
+    @staticmethod
+    def _int_to_bytes(i: int) -> bytes:
+        return codecs.decode(f"{i:010x}", "hex")
+
+    def _encode_msgid(self, source: au.AbstractUser, evt: Message) -> str:
+        if self.portal.peer_type == "channel":
+            play_id = b"c" + self._int_to_bytes(self.portal.tgid) + self._int_to_bytes(evt.id)
+        elif self.portal.peer_type == "chat":
+            play_id = (
+                b"g"
+                + self._int_to_bytes(self.portal.tgid)
+                + self._int_to_bytes(evt.id)
+                + self._int_to_bytes(source.tgid)
+            )
+        elif self.portal.peer_type == "user":
+            play_id = b"u" + self._int_to_bytes(self.portal.tgid) + self._int_to_bytes(evt.id)
+        else:
+            raise ValueError("Portal has invalid peer type")
+        return base64.b64encode(play_id).decode("utf-8").rstrip("=")
+
+    async def _set_reply(
+        self,
+        source: au.AbstractUser,
+        evt: Message,
+        content: MessageEventContent,
+        no_fallback: bool = False,
+    ) -> None:
+        if not evt.reply_to:
+            return
+        space = (
+            evt.peer_id.channel_id
+            if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
+            else source.tgid
+        )
+        msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
+        if not msg or msg.mx_room != self.portal.mxid:
+            return
+        elif not isinstance(content, TextMessageEventContent) or no_fallback:
+            # Not a text message, just set the reply metadata and return
+            content.set_reply(msg.mxid)
+            return
+
+        # Text message, try to fetch original message to generate reply fallback.
+        try:
+            event = await self.portal.main_intent.get_event(msg.mx_room, msg.mxid)
+            if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
+                event = await source.bridge.matrix.e2ee.decrypt(event)
+            if isinstance(event.content, TextMessageEventContent):
+                event.content.trim_reply_fallback()
+            puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
+            content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
+        except Exception:
+            self.log.exception("Failed to get event to add reply fallback")
+            content.set_reply(msg.mxid)
+
+    @staticmethod
+    def _photo_size_key(photo: TypePhotoSize) -> int:
+        if isinstance(photo, PhotoSize):
+            return photo.size
+        elif isinstance(photo, PhotoSizeProgressive):
+            return max(photo.sizes)
+        elif isinstance(photo, PhotoSizeEmpty):
+            return 0
+        else:
+            return len(photo.bytes)
+
+    @classmethod
+    def get_largest_photo_size(
+        cls, photo: Photo | Document
+    ) -> tuple[InputPhotoFileLocation | None, TypePhotoSize | None]:
+        if (
+            not photo
+            or isinstance(photo, PhotoEmpty)
+            or (isinstance(photo, Document) and not photo.thumbs)
+        ):
+            return None, None
+
+        largest = max(
+            photo.thumbs if isinstance(photo, Document) else photo.sizes, key=cls._photo_size_key
+        )
+        return (
+            InputPhotoFileLocation(
+                id=photo.id,
+                access_hash=photo.access_hash,
+                file_reference=photo.file_reference,
+                thumb_size=largest.type,
+            ),
+            largest,
+        )
+
+    async def _webpage_to_beeper_link_preview(
+        self, source: au.AbstractUser, intent: IntentAPI, webpage: WebPage
+    ) -> dict[str, Any]:
+        beeper_link_preview: dict[str, Any] = {
+            "matched_url": webpage.url,
+            "og:title": webpage.title,
+            "og:url": webpage.url,
+            "og:description": webpage.description,
+        }
+
+        # Upload an image corresponding to the link preview if it exists.
+        if webpage.photo:
+            loc, largest_size = self.get_largest_photo_size(webpage.photo)
+            if loc is None:
+                return beeper_link_preview
+            beeper_link_preview["og:image:height"] = largest_size.h
+            beeper_link_preview["og:image:width"] = largest_size.w
+            file = await util.transfer_file_to_matrix(
+                source.client,
+                intent,
+                loc,
+                encrypt=self.portal.encrypted,
+                async_upload=self.config["homeserver.async_media"],
+            )
+
+            if file.decryption_info:
+                beeper_link_preview[BEEPER_IMAGE_ENCRYPTION_KEY] = file.decryption_info.serialize()
+            else:
+                beeper_link_preview["og:image"] = file.mxc
+
+        return beeper_link_preview
+
+    async def _convert_text(
+        self, source: au.AbstractUser, intent: IntentAPI, is_bot: bool, evt: Message
+    ) -> ConvertedMessage:
+        content = await formatter.telegram_to_matrix(evt, source)
+        if is_bot and self.portal.get_config("bot_messages_as_notices"):
+            content.msgtype = MessageType.NOTICE
+
+        if (
+            hasattr(evt, "media")
+            and isinstance(evt.media, MessageMediaWebPage)
+            and isinstance(evt.media.webpage, WebPage)
+        ):
+            content[BEEPER_LINK_PREVIEWS_KEY] = [
+                await self._webpage_to_beeper_link_preview(source, intent, evt.media.webpage)
+            ]
+
+        return ConvertedMessage(content=content)
+
+    async def _convert_photo(
+        self, source: au.AbstractUser, intent: IntentAPI, evt: Message
+    ) -> ConvertedMessage | None:
+        media: MessageMediaPhoto = evt.media
+        if media.photo is None and media.ttl_seconds:
+            return ConvertedMessage(
+                content=TextMessageEventContent(
+                    msgtype=MessageType.NOTICE, body="Photo has expired"
+                )
+            )
+        loc, largest_size = self.get_largest_photo_size(media.photo)
+        if loc is None:
+            return ConvertedMessage(
+                content=TextMessageEventContent(
+                    msgtype=MessageType.TEXT,
+                    body="Failed to bridge image",
+                )
+            )
+        file = await util.transfer_file_to_matrix(
+            source.client,
+            intent,
+            loc,
+            encrypt=self.portal.encrypted,
+            async_upload=self.config["homeserver.async_media"],
+        )
+        if not file:
+            return None
+        info = ImageInfo(
+            height=largest_size.h,
+            width=largest_size.w,
+            orientation=0,
+            mimetype=file.mime_type,
+            size=self._photo_size_key(largest_size),
+        )
+        ext = sane_mimetypes.guess_extension(file.mime_type)
+        name = f"disappearing_image{ext}" if media.ttl_seconds else f"image{ext}"
+        content = MediaMessageEventContent(
+            msgtype=MessageType.IMAGE,
+            info=info,
+            body=name,
+        )
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+        caption_content = await formatter.telegram_to_matrix(evt, source) if evt.message else None
+        return ConvertedMessage(
+            content=content,
+            caption=caption_content,
+            disappear_seconds=media.ttl_seconds,
+        )
+
+    async def _convert_document(
+        self, source: au.AbstractUser, intent: IntentAPI, evt: Message
+    ) -> ConvertedMessage | None:
+        document = evt.media.document
+
+        attrs = _parse_document_attributes(document.attributes)
+
+        if document.size > self.matrix.media_config.upload_size:
+            name = attrs.name or ""
+            caption = f"\n{evt.message}" if evt.message else ""
+            return ConvertedMessage(
+                content=TextMessageEventContent(
+                    msgtype=MessageType.NOTICE, body=f"Too large file {name}{caption}"
+                )
+            )
+
+        thumb_loc, thumb_size = self.get_largest_photo_size(document)
+        if thumb_size and not isinstance(thumb_size, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb_size)}")
+            thumb_loc = None
+            thumb_size = None
+        parallel_id = source.tgid if self.config["bridge.parallel_file_transfer"] else None
+        tgs_convert = self.config["bridge.animated_sticker"]
+        file = await util.transfer_file_to_matrix(
+            source.client,
+            intent,
+            document,
+            thumb_loc,
+            is_sticker=attrs.is_sticker,
+            tgs_convert=tgs_convert,
+            webm_convert=tgs_convert["target"] if tgs_convert["convert_from_webm"] else None,
+            filename=attrs.name,
+            parallel_id=parallel_id,
+            encrypt=self.portal.encrypted,
+            async_upload=self.config["homeserver.async_media"],
+        )
+        if not file:
+            return None
+
+        info, name = _parse_document_meta(evt, file, attrs, thumb_size)
+
+        event_type = EventType.ROOM_MESSAGE
+        # Elements only support images as stickers, so send animated webm stickers as m.video
+        if attrs.is_sticker and file.mime_type.startswith("image/"):
+            event_type = EventType.STICKER
+            # Tell clients to render the stickers as 256x256 if they're bigger
+            if info.width > 256 or info.height > 256:
+                if info.width > info.height:
+                    info.height = int(info.height / (info.width / 256))
+                    info.width = 256
+                else:
+                    info.width = int(info.width / (info.height / 256))
+                    info.height = 256
+            if info.thumbnail_info:
+                info.thumbnail_info.width = info.width
+                info.thumbnail_info.height = info.height
+        if attrs.is_gif or (attrs.is_sticker and info.mimetype == "video/webm"):
+            if attrs.is_gif:
+                info["fi.mau.telegram.gif"] = True
+            else:
+                info["fi.mau.telegram.animated_sticker"] = True
+            info["fi.mau.loop"] = True
+            info["fi.mau.autoplay"] = True
+            info["fi.mau.hide_controls"] = True
+            info["fi.mau.no_audio"] = True
+        if not name:
+            ext = sane_mimetypes.guess_extension(file.mime_type)
+            name = "unnamed_file" + ext
+
+        content = MediaMessageEventContent(
+            body=name,
+            info=info,
+            msgtype={
+                "video/": MessageType.VIDEO,
+                "audio/": MessageType.AUDIO,
+                "image/": MessageType.IMAGE,
+            }.get(info.mimetype[:6], MessageType.FILE),
+        )
+        if event_type == EventType.STICKER:
+            content.msgtype = None
+        if attrs.is_audio:
+            content["org.matrix.msc1767.audio"] = {"duration": attrs.duration * 1000}
+            if attrs.waveform:
+                content["org.matrix.msc1767.audio"]["waveform"] = [x << 5 for x in attrs.waveform]
+            if attrs.is_voice:
+                content["org.matrix.msc3245.voice"] = {}
+        if file.decryption_info:
+            content.file = file.decryption_info
+        else:
+            content.url = file.mxc
+
+        caption_content = await formatter.telegram_to_matrix(evt, source) if evt.message else None
+
+        return ConvertedMessage(
+            type=event_type,
+            content=content,
+            caption=caption_content,
+            disappear_seconds=evt.media.ttl_seconds,
+        )
+
+    @staticmethod
+    async def _convert_location(evt: Message, **_) -> ConvertedMessage:
+        long = evt.media.geo.long
+        lat = evt.media.geo.lat
+        long_char = "E" if long > 0 else "W"
+        lat_char = "N" if lat > 0 else "S"
+        geo = f"{round(lat, 6)},{round(long, 6)}"
+
+        body = f"{round(abs(lat), 4)}° {lat_char}, {round(abs(long), 4)}° {long_char}"
+        url = f"https://maps.google.com/?q={geo}"
+
+        if isinstance(evt.media, MessageMediaGeoLive):
+            note = "Live Location (see your Telegram client for live updates)"
+        elif isinstance(evt.media, MessageMediaVenue):
+            note = evt.media.title
+        else:
+            note = "Location"
+
+        content = LocationMessageEventContent(
+            msgtype=MessageType.LOCATION,
+            geo_uri=f"geo:{geo}",
+            body=f"{note}: {body}\n{url}",
+        )
+        content["format"] = str(Format.HTML)
+        content["formatted_body"] = f"{note}: <a href='{url}'>{body}</a>"
+        content["org.matrix.msc3488.location"] = {
+            "uri": content.geo_uri,
+            "description": note,
+        }
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_unsupported(source: au.AbstractUser, evt: Message, **_) -> ConvertedMessage:
+        override_text = (
+            "This message is not supported on your version of Mautrix-Telegram. "
+            "Please check https://github.com/mautrix/telegram or ask your "
+            "bridge administrator about possible updates."
+        )
+        content = await formatter.telegram_to_matrix(evt, source, override_text=override_text)
+        content.msgtype = MessageType.NOTICE
+        content["fi.mau.telegram.unsupported"] = True
+        return ConvertedMessage(content=content)
+
+    async def _convert_poll(self, source: au.AbstractUser, evt: Message, **_) -> ConvertedMessage:
+        poll: Poll = evt.media.poll
+        poll_id = self._encode_msgid(source, evt)
+
+        _n = 0
+
+        def n() -> int:
+            nonlocal _n
+            _n += 1
+            return _n
+
+        text_answers = "\n".join(f"{n()}. {answer.text}" for answer in poll.answers)
+        html_answers = "\n".join(f"<li>{answer.text}</li>" for answer in poll.answers)
+        vote_command = f"{self.command_prefix} vote {poll_id}"
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT,
+            format=Format.HTML,
+            body=(
+                f"Poll: {poll.question}\n{text_answers}\n"
+                f"Vote with {vote_command} <choice number>"
+            ),
+            formatted_body=(
+                f"<strong>Poll</strong>: {poll.question}<br/>\n"
+                f"<ol>{html_answers}</ol>\n"
+                f"Vote with <code>{vote_command} &lt;choice number&gt;</code>"
+            ),
+        )
+
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_dice(evt: Message, **_) -> ConvertedMessage:
+        roll: MessageMediaDice = evt.media
+        emoji_text = {
+            "\U0001F3AF": " Dart throw",
+            "\U0001F3B2": " Dice roll",
+            "\U0001F3C0": " Basketball throw",
+            "\U0001F3B0": " Slot machine",
+            "\U0001F3B3": " Bowling",
+            "\u26BD": " Football kick",
+        }
+        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {_format_dice(roll)}"
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT,
+            format=Format.HTML,
+            body=text,
+            formatted_body=f"<h4>{text}</h4>",
+        )
+        content["fi.mau.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
+        return ConvertedMessage(content=content)
+
+    async def _convert_game(self, source: au.AbstractUser, evt: Message, **_) -> ConvertedMessage:
+        game: Game = evt.media.game
+        play_id = self._encode_msgid(source, evt)
+        command = f"{self.command_prefix} play {play_id}"
+        override_text = f"Run {command} in your bridge management room to play {game.title}"
+        override_entities = [
+            MessageEntityPre(offset=len("Run "), length=len(command), language="")
+        ]
+
+        content = await formatter.telegram_to_matrix(
+            evt, source, override_text=override_text, override_entities=override_entities
+        )
+        content.msgtype = MessageType.NOTICE
+        content["fi.mau.telegram.game"] = play_id
+
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_contact(source: au.AbstractUser, evt: Message, **_) -> ConvertedMessage:
+        contact: MessageMediaContact = evt.media
+        name = " ".join(x for x in [contact.first_name, contact.last_name] if x)
+        formatted_phone = f"+{contact.phone_number}"
+        if phonenumbers is not None:
+            try:
+                parsed = phonenumbers.parse(formatted_phone)
+                fmt = phonenumbers.PhoneNumberFormat.INTERNATIONAL
+                formatted_phone = phonenumbers.format_number(parsed, fmt)
+            except phonenumbers.NumberParseException:
+                pass
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT,
+            body=f"Shared contact info for {name}: {formatted_phone}",
+        )
+        content["fi.mau.telegram.contact"] = {
+            "user_id": contact.user_id,
+            "first_name": contact.first_name,
+            "last_name": contact.last_name,
+            "phone_number": contact.phone_number,
+            "vcard": contact.vcard,
+        }
+
+        puppet = await pu.Puppet.get_by_tgid(TelegramID(contact.user_id))
+        if not puppet.displayname:
+            try:
+                entity = await source.client.get_entity(PeerUser(contact.user_id))
+                await puppet.update_info(source, entity)
+            except Exception as e:
+                source.log.warning(f"Failed to sync puppet info of received contact: {e}")
+        else:
+            content.format = Format.HTML
+            content.formatted_body = (
+                f"Shared contact info for "
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>{html.escape(name)}</a>: "
+                f"{html.escape(formatted_phone)}"
+            )
+        return ConvertedMessage(content=content)
+
+
+def _parse_document_attributes(attributes: list[TypeDocumentAttribute]) -> DocAttrs:
+    name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
+    is_gif, is_audio, is_voice, duration, waveform = False, False, False, 0, bytes()
+    for attr in attributes:
+        if isinstance(attr, DocumentAttributeFilename):
+            name = name or attr.file_name
+            mime_type, _ = mimetypes.guess_type(attr.file_name)
+        elif isinstance(attr, DocumentAttributeSticker):
+            is_sticker = True
+            sticker_alt = attr.alt
+        elif isinstance(attr, DocumentAttributeAnimated):
+            is_gif = True
+        elif isinstance(attr, DocumentAttributeVideo):
+            width, height = attr.w, attr.h
+        elif isinstance(attr, DocumentAttributeImageSize):
+            width, height = attr.w, attr.h
+        elif isinstance(attr, DocumentAttributeAudio):
+            is_audio = True
+            is_voice = attr.voice or False
+            duration = attr.duration
+            waveform = decode_waveform(attr.waveform) if attr.waveform else b""
+
+    return DocAttrs(
+        name,
+        mime_type,
+        is_sticker,
+        sticker_alt,
+        width,
+        height,
+        is_gif,
+        is_audio,
+        is_voice,
+        duration,
+        waveform,
+    )
+
+
+def _parse_document_meta(
+    evt: Message, file: DBTelegramFile, attrs: DocAttrs, thumb_size: TypePhotoSize
+) -> tuple[ImageInfo, str]:
+    document = evt.media.document
+    name = attrs.name
+    if attrs.is_sticker:
+        alt = attrs.sticker_alt
+        if len(alt) > 0:
+            try:
+                name = f"{alt} ({unicodedata.name(alt[0]).lower()})"
+            except ValueError:
+                name = alt
+
+    generic_types = ("text/plain", "application/octet-stream")
+    if file.mime_type in generic_types and document.mime_type not in generic_types:
+        mime_type = document.mime_type or file.mime_type
+    elif file.mime_type == "application/ogg":
+        mime_type = "audio/ogg"
+    else:
+        mime_type = file.mime_type or document.mime_type
+    info = ImageInfo(size=file.size, mimetype=mime_type)
+
+    if attrs.mime_type and not file.was_converted:
+        file.mime_type = attrs.mime_type or file.mime_type
+    if file.width and file.height:
+        info.width, info.height = file.width, file.height
+    elif attrs.width and attrs.height:
+        info.width, info.height = attrs.width, attrs.height
+
+    if file.thumbnail:
+        if file.thumbnail.decryption_info:
+            info.thumbnail_file = file.thumbnail.decryption_info
+        else:
+            info.thumbnail_url = file.thumbnail.mxc
+        info.thumbnail_info = ThumbnailInfo(
+            mimetype=file.thumbnail.mime_type,
+            height=file.thumbnail.height or thumb_size.h,
+            width=file.thumbnail.width or thumb_size.w,
+            size=file.thumbnail.size,
+        )
+    elif attrs.is_sticker:
+        # This is a hack for bad clients like Element iOS that require a thumbnail
+        info.thumbnail_info = ImageInfo.deserialize(info.serialize())
+        if file.decryption_info:
+            info.thumbnail_file = file.decryption_info
+        else:
+            info.thumbnail_url = file.mxc
+
+    return info, name
+
+
+def _format_dice(roll: MessageMediaDice) -> str:
+    if roll.emoticon == "\U0001F3B0":
+        emojis = {
+            0: "\U0001F36B",  # "🍫",
+            1: "\U0001F352",  # "🍒",
+            2: "\U0001F34B",  # "🍋",
+            3: "7\ufe0f\u20e3",  # "7️⃣",
+        }
+        res = roll.value - 1
+        slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
+        return f"{slot1} {slot2} {slot3} ({roll.value})"
+    elif roll.emoticon == "\u26BD":
+        results = {
+            1: "miss",
+            2: "hit the woodwork",
+            3: "goal",  # seems to go in through the center
+            4: "goal",
+            5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
+        }
+    elif roll.emoticon == "\U0001F3B3":
+        results = {
+            1: "miss",
+            2: "1 pin down",
+            3: "3 pins down, split",
+            4: "4 pins down, split",
+            5: "5 pins down",
+            6: "strike 🎉",
+        }
+    # elif roll.emoticon == "\U0001F3C0":
+    #     results = {
+    #         2: "rolled off",
+    #         3: "stuck",
+    #     }
+    # elif roll.emoticon == "\U0001F3AF":
+    #     results = {
+    #         1: "bounced off",
+    #         2: "outer rim",
+    #
+    #         6: "bullseye",
+    #     }
+    else:
+        return str(roll.value)
+    return f"{results[roll.value]} ({roll.value})"
@@ -80,16 +80,16 @@ def get_base_power_levels(
        levels.events_default = overrides.get(
            "events_default",
            50
-            if (
-                portal.peer_type == "channel"
-                and not entity.megagroup
-                or entity.default_banned_rights.send_messages
-            )
+            if portal.peer_type == "channel" and not entity.megagroup or dbr.send_messages
            else 0,
        )
    for evt_type, value in overrides.get("events", {}).items():
        levels.events[EventType.find(evt_type)] = value
-    levels.users = overrides.get("users", {})
+    userlevel_overrides = overrides.get("users", {})
+    bot_level = levels.get_user_level(portal.main_intent.mxid)
+    for user, user_level in levels.users.items():
+        if user_level < bot_level:
+            levels.users[user] = userlevel_overrides.get(user, 0)
    if portal.main_intent.mxid not in levels.users:
        levels.users[portal.main_intent.mxid] = 100
    return levels
@@ -83,7 +83,7 @@ async def make_sponsored_message_content(
    else:
        sponsor_name = sponsor_name_html = "unknown entity"

-    content["net.maunium.telegram.sponsored"] = sponsored_meta
+    content["fi.mau.telegram.sponsored"] = sponsored_meta
    content.formatted_body += (
        f"<br/><br/>Sponsored message from {sponsor_name_html} "
        f"- <a href='{content.external_url}'>{action}</a>"
@@ -24,6 +24,7 @@ from telethon.tl.types import (
    ChatPhoto,
    ChatPhotoEmpty,
    InputPeerPhotoFileLocation,
+    InputPeerUser,
    PeerChannel,
    PeerChat,
    PeerUser,
@@ -72,6 +73,7 @@ class Puppet(DBPuppet, BasePuppet):
        displayname_quality: int = 0,
        disable_updates: bool = False,
        username: str | None = None,
+        phone: str | None = None,
        photo_id: str | None = None,
        avatar_url: ContentURI | None = None,
        name_set: bool = False,
@@ -92,6 +94,7 @@ class Puppet(DBPuppet, BasePuppet):
            displayname_quality=displayname_quality,
            disable_updates=disable_updates,
            username=username,
+            phone=phone,
            photo_id=photo_id,
            avatar_url=avatar_url,
            name_set=name_set,
@@ -128,6 +131,16 @@ class Puppet(DBPuppet, BasePuppet):
            PeerChannel(channel_id=self.tgid) if self.is_channel else PeerUser(user_id=self.tgid)
        )

+    @property
+    def contact_info(self) -> dict:
+        return {
+            "name": self.displayname,
+            "username": self.username,
+            "phone": f"+{self.phone.lstrip('+')}" if self.phone else None,
+            "is_bot": self.is_bot,
+            "avatar_url": self.avatar_url,
+        }
+
    @property
    def plain_displayname(self) -> str:
        return self.displayname_template.parse(self.displayname) or self.displayname
@@ -252,6 +265,10 @@ class Puppet(DBPuppet, BasePuppet):
            self.username = info.username
            changed = True

+        if getattr(info, "phone", None) and self.phone != info.phone:
+            self.phone = info.phone
+            changed = True
+
        if not self.disable_updates:
            try:
                changed = await self.update_displayname(source, info) or changed
@@ -359,6 +376,7 @@ class Puppet(DBPuppet, BasePuppet):
                    location=InputPeerPhotoFileLocation(
                        peer=await self.get_input_entity(source), photo_id=photo.photo_id, big=True
                    ),
+                    async_upload=self.config["homeserver.async_media"],
                )
                if not file:
                    return False
@@ -388,7 +406,7 @@ class Puppet(DBPuppet, BasePuppet):
    @classmethod
    @async_getter_lock
    async def get_by_tgid(
-        cls, tgid: TelegramID, *, create: bool = True, is_channel: bool = False
+        cls, tgid: TelegramID, /, *, create: bool = True, is_channel: bool = False
    ) -> Puppet | None:
        if tgid is None:
            return None
@@ -413,7 +431,7 @@ class Puppet(DBPuppet, BasePuppet):

    @staticmethod
    def get_id_from_peer(peer: TypePeer | User | Channel) -> TelegramID:
-        if isinstance(peer, PeerUser):
+        if isinstance(peer, (PeerUser, InputPeerUser)):
            return TelegramID(peer.user_id)
        elif isinstance(peer, PeerChannel):
            return TelegramID(peer.channel_id)
@@ -441,7 +459,7 @@ class Puppet(DBPuppet, BasePuppet):

    @classmethod
    @async_getter_lock
-    async def get_by_custom_mxid(cls, mxid: UserID) -> Puppet | None:
+    async def get_by_custom_mxid(cls, mxid: UserID, /) -> Puppet | None:
        try:
            return cls.by_custom_mxid[mxid]
        except KeyError:
@@ -494,23 +512,4 @@ class Puppet(DBPuppet, BasePuppet):

        return None

-    @classmethod
-    async def find_by_displayname(cls, displayname: str) -> Puppet | None:
-        if not displayname:
-            return None
-
-        for _, puppet in cls.by_tgid.items():
-            if puppet.displayname and puppet.displayname == displayname:
-                return puppet
-
-        puppet = cast(cls, await super().find_by_displayname(displayname))
-        if puppet:
-            try:
-                return cls.by_tgid[puppet.tgid]
-            except KeyError:
-                puppet._add_to_cache()
-                return puppet
-
-        return None
-
    # endregion
@@ -26,10 +26,12 @@ from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.updates import GetStateRequest
 from telethon.tl.functions.users import GetUsersRequest
 from telethon.tl.types import (
+    Channel,
    Chat,
    ChatForbidden,
    InputUserSelf,
    NotifyPeer,
+    PeerUser,
    TypeUpdate,
    UpdateFolderPeers,
    UpdateNewChannelMessage,
@@ -129,6 +131,10 @@ class User(DBUser, AbstractUser, BaseUser):
    def human_tg_id(self) -> str:
        return f"@{self.tg_username}" if self.tg_username else f"+{self.tg_phone}" or None

+    @property
+    def peer(self) -> PeerUser | None:
+        return PeerUser(user_id=self.tgid) if self.tgid else None
+
    # TODO replace with proper displayname getting everywhere
    @property
    def displayname(self) -> str:
@@ -269,6 +275,13 @@ class User(DBUser, AbstractUser, BaseUser):
            return None
        return await pu.Puppet.get_by_tgid(self.tgid)

+    async def get_portal_with(self, puppet: pu.Puppet, create: bool = True) -> po.Portal | None:
+        if not self.tgid:
+            return None
+        return await po.Portal.get_by_tgid(
+            puppet.tgid, tg_receiver=self.tgid, peer_type="user" if create else None
+        )
+
    async def stop(self) -> None:
        if self._track_connection_task:
            self._track_connection_task.cancel()
@@ -372,7 +385,7 @@ class User(DBUser, AbstractUser, BaseUser):
        if not self.config["bridge.kick_on_logout"]:
            return
        portals = await self.get_cached_portals()
-        for _, portal in portals.values():
+        for portal in portals.values():
            if not portal or portal.deleted or not portal.mxid or portal.has_bot:
                continue
            if portal.peer_type == "user":
@@ -462,17 +475,22 @@ class User(DBUser, AbstractUser, BaseUser):
        if active and tag_info is None:
            tag_info = RoomTagInfo(order=0.5)
            tag_info[DOUBLE_PUPPET_SOURCE_KEY] = self.bridge.name
+            self.log.debug("Adding tag {tag} to {portal.mxid}/{portal.tgid}")
            await puppet.intent.set_room_tag(portal.mxid, tag, tag_info)
        elif (
            not active and tag_info and tag_info.get(DOUBLE_PUPPET_SOURCE_KEY) == self.bridge.name
        ):
+            self.log.debug("Removing tag {tag} from {portal.mxid}/{portal.tgid}")
            await puppet.intent.remove_room_tag(portal.mxid, tag)

-    async def _mute_room(cls, puppet: pu.Puppet, portal: po.Portal, mute_until: datetime) -> None:
-        if not cls.config["bridge.mute_bridging"] or not portal or not portal.mxid:
+    async def _mute_room(self, puppet: pu.Puppet, portal: po.Portal, mute_until: datetime) -> None:
+        if not self.config["bridge.mute_bridging"] or not portal or not portal.mxid:
            return
        now = datetime.utcnow().replace(tzinfo=timezone.utc)
        if mute_until is not None and mute_until > now:
+            self.log.debug(
+                f"Muting {portal.mxid}/{portal.tgid} (muted until {mute_until} on Telegram)"
+            )
            await puppet.intent.set_push_rule(
                PushRuleScope.GLOBAL,
                PushRuleKind.ROOM,
@@ -484,6 +502,7 @@ class User(DBUser, AbstractUser, BaseUser):
                await puppet.intent.remove_push_rule(
                    PushRuleScope.GLOBAL, PushRuleKind.ROOM, portal.mxid
                )
+                self.log.debug(f"Unmuted {portal.mxid}/{portal.tgid}")
            except MNotFound:
                pass

@@ -646,20 +665,28 @@ class User(DBUser, AbstractUser, BaseUser):
            acc = (acc * 20261 + contact) & 0xFFFFFFFF
        return acc & 0x7FFFFFFF

-    async def sync_contacts(self) -> None:
+    async def sync_contacts(self, get_info: bool = False) -> dict[TelegramID, dict]:
        existing_contacts = await self.get_contacts()
        contact_hash = self._hash_contacts(self.saved_contacts, existing_contacts)
        response = await self.client(GetContactsRequest(hash=contact_hash))
        if isinstance(response, ContactsNotModified):
-            return
+            if get_info:
+                return {
+                    tgid: (await pu.Puppet.get_by_tgid(tgid)).contact_info
+                    for tgid in existing_contacts
+                }
+            return {}
        self.log.debug(f"Updating contacts of {self.name}...")
        if self.saved_contacts != response.saved_count:
            self.saved_contacts = response.saved_count
            await self.save()
+        contacts = {}
        for user in response.users:
-            puppet = await pu.Puppet.get_by_tgid(user.id)
+            puppet: pu.Puppet = await pu.Puppet.get_by_tgid(user.id)
            await puppet.update_info(self, user)
-        await self.set_contacts(user.id for user in response.users)
+            contacts[user.id] = puppet.contact_info
+        await self.set_contacts(contacts.keys())
+        return contacts

    # endregion
    # region Class instance lookup
@@ -689,7 +716,7 @@ class User(DBUser, AbstractUser, BaseUser):
    @classmethod
    @async_getter_lock
    async def get_by_mxid(
-        cls, mxid: UserID, *, check_db: bool = True, create: bool = True
+        cls, mxid: UserID, /, *, check_db: bool = True, create: bool = True
    ) -> User | None:
        if not mxid or pu.Puppet.get_id_from_mxid(mxid) or mxid == cls.az.bot_mxid:
            return None
@@ -717,7 +744,7 @@ class User(DBUser, AbstractUser, BaseUser):

    @classmethod
    @async_getter_lock
-    async def get_by_tgid(cls, tgid: TelegramID) -> User | None:
+    async def get_by_tgid(cls, tgid: TelegramID, /) -> User | None:
        try:
            return cls.by_tgid[tgid]
        except KeyError:
@@ -31,6 +31,7 @@ from telethon.errors import (
    LocationInvalidError,
    SecurityError,
 )
+from telethon.tl.functions.messages import GetCustomEmojiDocumentsRequest
 from telethon.tl.types import (
    Document,
    InputDocumentFileLocation,
@@ -45,11 +46,13 @@ import magic

 from mautrix.appservice import IntentAPI

+from .. import abstract_user as au
 from ..db import TelegramFile as DBTelegramFile
 from ..tgclient import MautrixTelegramClient
 from ..util import sane_mimetypes
 from .parallel_file_transfer import parallel_transfer_to_matrix
 from .tgs_converter import convert_tgs_to
+from .webm_converter import convert_webm_to

 try:
    from PIL import Image
@@ -125,9 +128,9 @@ def _read_video_thumbnail(

 def _location_to_id(location: TypeLocation) -> str:
    if isinstance(location, Document):
-        return f"{location.id}-{location.access_hash}"
+        return str(location.id)
    elif isinstance(location, (InputDocumentFileLocation, InputPhotoFileLocation)):
-        return f"{location.id}-{location.access_hash}-{location.thumb_size}"
+        return f"{location.id}-{location.thumb_size}"
    elif isinstance(location, InputFileLocation):
        return f"{location.volume_id}-{location.local_id}"
    elif isinstance(location, InputPeerPhotoFileLocation):
@@ -144,6 +147,7 @@ async def transfer_thumbnail_to_matrix(
    custom_data: bytes | None = None,
    width: int | None = None,
    height: int | None = None,
+    async_upload: bool = False,
 ) -> DBTelegramFile | None:
    if not Image or not VideoFileClip:
        return None
@@ -154,6 +158,8 @@ async def transfer_thumbnail_to_matrix(

    if custom_data:
        loc_id += "-mau_custom_thumbnail"
+    if encrypt:
+        loc_id += "-encrypted"

    db_file = await DBTelegramFile.get(loc_id)
    if db_file:
@@ -178,7 +184,7 @@ async def transfer_thumbnail_to_matrix(
    if encrypt:
        file, decryption_info = encrypt_attachment(file)
        upload_mime_type = "application/octet-stream"
-    content_uri = await intent.upload_media(file, upload_mime_type)
+    content_uri = await intent.upload_media(file, upload_mime_type, async_upload=async_upload)
    if decryption_info:
        decryption_info.url = content_uri

@@ -209,6 +215,44 @@ transfer_locks: dict[str, asyncio.Lock] = {}
 TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]


+async def transfer_custom_emojis_to_matrix(
+    source: au.AbstractUser, emoji_ids: list[int]
+) -> dict[int, DBTelegramFile]:
+    emoji_ids = set(emoji_ids)
+    existing = await DBTelegramFile.get_many([str(id) for id in emoji_ids])
+    file_map = {int(file.id): file for file in existing}
+    not_existing_ids = list(emoji_ids - file_map.keys())
+    if not_existing_ids:
+        log.debug(f"Transferring custom emojis through {source.mxid}: {not_existing_ids}")
+
+        documents: list[Document] = await source.client(
+            GetCustomEmojiDocumentsRequest(document_id=not_existing_ids)
+        )
+
+        tgs_args = source.config["bridge.animated_emoji"]
+        webm_convert = tgs_args["target"]
+
+        transfer_sema = asyncio.Semaphore(5)
+
+        async def transfer(document: Document) -> None:
+            async with transfer_sema:
+                file_map[document.id] = await transfer_file_to_matrix(
+                    source.client,
+                    source.bridge.az.intent,
+                    document,
+                    is_sticker=True,
+                    tgs_convert=tgs_args,
+                    webm_convert=webm_convert,
+                    filename=f"emoji-{document.id}",
+                    # Emojis are used as inline images and can't be encrypted
+                    encrypt=False,
+                    async_upload=source.config["homeserver.async_media"],
+                )
+
+        await asyncio.gather(*[transfer(doc) for doc in documents])
+    return file_map
+
+
 async def transfer_file_to_matrix(
    client: MautrixTelegramClient,
    intent: IntentAPI,
@@ -217,13 +261,17 @@ async def transfer_file_to_matrix(
    *,
    is_sticker: bool = False,
    tgs_convert: dict | None = None,
+    webm_convert: str | None = None,
    filename: str | None = None,
    encrypt: bool = False,
    parallel_id: int | None = None,
+    async_upload: bool = False,
 ) -> DBTelegramFile | None:
    location_id = _location_to_id(location)
    if not location_id:
        return None
+    if encrypt:
+        location_id += "-encrypted"

    db_file = await DBTelegramFile.get(location_id)
    if db_file:
@@ -243,9 +291,11 @@ async def transfer_file_to_matrix(
            thumbnail,
            is_sticker,
            tgs_convert,
+            webm_convert,
            filename,
            encrypt,
            parallel_id,
+            async_upload=async_upload,
        )


@@ -257,9 +307,11 @@ async def _unlocked_transfer_file_to_matrix(
    thumbnail: TypeThumbnail,
    is_sticker: bool,
    tgs_convert: dict | None,
+    webm_convert: str | None,
    filename: str | None,
    encrypt: bool,
    parallel_id: int | None,
+    async_upload: bool = False,
 ) -> DBTelegramFile | None:
    db_file = await DBTelegramFile.get(loc_id)
    if db_file:
@@ -299,13 +351,19 @@ async def _unlocked_transfer_file_to_matrix(
            width, height = converted_anim.width, converted_anim.height
            image_converted = mime_type != "application/gzip"
            thumbnail = None
+        elif is_sticker and webm_convert and webm_convert != "webm" and mime_type == "video/webm":
+            converted_anim = await convert_webm_to(file, webm_convert)
+            mime_type = converted_anim.mime
+            file = converted_anim.data
+            image_converted = mime_type != "video/webm"
+            thumbnail = None

        decryption_info = None
        upload_mime_type = mime_type
        if encrypt and encrypt_attachment:
            file, decryption_info = encrypt_attachment(file)
            upload_mime_type = "application/octet-stream"
-        content_uri = await intent.upload_media(file, upload_mime_type)
+        content_uri = await intent.upload_media(file, upload_mime_type, async_upload=async_upload)
        if decryption_info:
            decryption_info.url = content_uri

@@ -325,7 +383,13 @@ async def _unlocked_transfer_file_to_matrix(
            thumbnail = thumbnail.location
        try:
            db_file.thumbnail = await transfer_thumbnail_to_matrix(
-                client, intent, thumbnail, video=file, mime_type=mime_type, encrypt=encrypt
+                client,
+                intent,
+                thumbnail,
+                video=file,
+                mime_type=mime_type,
+                encrypt=encrypt,
+                async_upload=async_upload,
            )
        except FileIdInvalidError:
            log.warning(f"Failed to transfer thumbnail for {thumbnail!s}", exc_info=True)
@@ -340,6 +404,7 @@ async def _unlocked_transfer_file_to_matrix(
            mime_type=converted_anim.thumbnail_mime,
            width=converted_anim.width,
            height=converted_anim.height,
+            async_upload=async_upload,
        )

    try:
@@ -99,7 +99,7 @@ if lottieconverter:
    converters["png"] = tgs_to_png
    converters["gif"] = tgs_to_gif

-if lottieconverter and ffmpeg:
+if lottieconverter and ffmpeg.ffmpeg_path:

    async def tgs_to_webm(
        file: bytes, width: int, height: int, fps: int = 30, **_: Any
@@ -126,7 +126,33 @@ if lottieconverter and ffmpeg:
                log.error(str(e))
        return ConvertedSticker("application/gzip", file)

+    async def tgs_to_webp(
+        file: bytes, width: int, height: int, fps: int = 30, **_: Any
+    ) -> ConvertedSticker:
+        with tempfile.TemporaryDirectory(prefix="tgs_") as tmpdir:
+            file_template = tmpdir + "/out_"
+            try:
+                await _run_lottieconverter(
+                    args=("-", file_template, "pngs", f"{width}x{height}", str(fps)),
+                    input_data=file,
+                )
+                first_frame_name = min(os.listdir(tmpdir))
+                with open(f"{tmpdir}/{first_frame_name}", "rb") as first_frame_file:
+                    first_frame_data = first_frame_file.read()
+                webp_data = await ffmpeg.convert_path(
+                    input_args=("-framerate", str(fps), "-pattern_type", "glob"),
+                    input_file=f"{file_template}*.png",
+                    output_args=("-c:v", "libwebp_anim", "-pix_fmt", "yuva420p", "-f", "webp"),
+                    output_path_override="-",
+                    output_extension=None,
+                )
+                return ConvertedSticker("image/webp", webp_data, "image/png", first_frame_data)
+            except ffmpeg.ConverterError as e:
+                log.error(str(e))
+        return ConvertedSticker("application/gzip", file)
+
    converters["webm"] = tgs_to_webm
+    converters["webp"] = tgs_to_webp


 async def convert_tgs_to(
@@ -0,0 +1,52 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import logging
+
+from mautrix.util import ffmpeg
+
+from .tgs_converter import ConvertedSticker
+
+log: logging.Logger = logging.getLogger("mau.util.webm")
+
+
+converter_args = {
+    "gif": {
+        "output_args": ("-vf", "split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse"),
+    },
+    "png": {
+        "input_args": ("-ss", "0"),
+        "output_args": ("-frames:v", "1"),
+    },
+    "webp": {},
+}
+
+
+async def convert_webm_to(file: bytes, convert_to: str) -> ConvertedSticker:
+    if convert_to in ("png", "gif", "webp"):
+        try:
+            converted_data = await ffmpeg.convert_bytes(
+                data=file,
+                output_extension=f".{convert_to}",
+                **converter_args[convert_to],
+            )
+            return ConvertedSticker(f"image/{convert_to}", converted_data)
+        except ffmpeg.ConverterError as e:
+            log.error(str(e))
+    elif convert_to != "disable":
+        log.warning(f"Unable to convert webm animated sticker, type {convert_to} not supported")
+    return ConvertedSticker("video/webm", file)
@@ -279,6 +279,14 @@ class AuthAPI(abc.ABC):
                errcode="phone_code_expired",
                error="Phone code expired.",
            )
+        except PhoneNumberUnoccupiedError:
+            return self.get_login_response(
+                mxid=user.mxid,
+                state="code",
+                status=403,
+                errcode="phone_number_unoccupied",
+                error="That phone number has not been registered.",
+            )
        except SessionPasswordNeededError:
            if not password_in_data:
                if user.command_status and user.command_status["action"] == "Login":
@@ -21,7 +21,7 @@ import json
 import logging

 from aiohttp import web
-from telethon.tl.types import ChannelForbidden, ChatForbidden, TypeChat
+from telethon.tl.types import ChannelForbidden, ChatForbidden, TypeChat, User as TLUser
 from telethon.utils import get_peer_id, resolve_id

 from mautrix.appservice import AppService
@@ -53,18 +53,23 @@ class ProvisioningAPI(AuthAPI):

        self.app = web.Application(loop=bridge.loop, middlewares=[self.error_middleware])

-        portal_prefix = "/portal/{mxid:![^/]+}"
+        portal_prefix = "/v1/portal/{mxid:![^/]+}"
        self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal_by_mxid)
-        self.app.router.add_route("GET", "/portal/{tgid:-[0-9]+}", self.get_portal_by_tgid)
+        self.app.router.add_route("GET", "/v1/portal/{tgid:-[0-9]+}", self.get_portal_by_tgid)
        self.app.router.add_route(
            "POST", portal_prefix + "/connect/{chat_id:-[0-9]+}", self.connect_chat
        )
        self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat)
        self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat)

-        user_prefix = "/user/{mxid:@[^:]*:[^/]+}"
+        user_prefix = "/v1/user/{mxid:@[^:]*:[^/]+}"
        self.app.router.add_route("GET", f"{user_prefix}", self.get_user_info)
        self.app.router.add_route("GET", f"{user_prefix}/chats", self.get_chats)
+        self.app.router.add_route("GET", f"{user_prefix}/contacts", self.get_contacts)
+        self.app.router.add_route(
+            "GET", f"{user_prefix}/resolve_identifier/{{identifier}}", self.resolve_identifier
+        )
+        self.app.router.add_route("POST", f"{user_prefix}/pm/{{identifier}}", self.start_dm)

        self.app.router.add_route("POST", f"{user_prefix}/logout", self.logout)
        self.app.router.add_route("POST", f"{user_prefix}/login/bot_token", self.send_bot_token)
@@ -72,7 +77,7 @@ class ProvisioningAPI(AuthAPI):
        self.app.router.add_route("POST", f"{user_prefix}/login/send_code", self.send_code)
        self.app.router.add_route("POST", f"{user_prefix}/login/send_password", self.send_password)

-        self.app.router.add_route("GET", "/bridge", self.bridge_info)
+        self.app.router.add_route("GET", "/v1/bridge", self.bridge_info)

    async def get_portal_by_mxid(self, request: web.Request) -> web.Response:
        err = self.check_authorization(request)
@@ -212,7 +217,7 @@ class ProvisioningAPI(AuthAPI):
            portal.photo_id = ""
            await portal.save()

-        asyncio.create_task(portal.update_matrix_room(user, entity, direct=False, levels=levels))
+        asyncio.create_task(portal.update_matrix_room(user, entity, levels=levels))

        return web.Response(status=202, body="{}")

@@ -393,6 +398,102 @@ class ProvisioningAPI(AuthAPI):
                ]
            )

+    async def get_contacts(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request, expect_logged_in=True)
+        if err is not None:
+            return err
+        return web.json_response(data=await user.sync_contacts())
+
+    async def _resolve_id(
+        self, request: web.Request
+    ) -> tuple[Portal | None, User | None, TLUser | None, web.Response | None]:
+        data, user, err = await self.get_user_request_info(request, expect_logged_in=True)
+        if err is not None:
+            return None, user, None, err
+        try:
+            identifier: str | int = request.match_info["identifier"]
+            if isinstance(identifier, str) and identifier.isdecimal():
+                identifier = int(identifier)
+            target = await user.client.get_entity(identifier)
+        except ValueError:
+            return (
+                None,
+                user,
+                None,
+                web.json_response(
+                    {
+                        "error": "Invalid user identifier or user not found.",
+                        "errcode": "M_NOT_FOUND",
+                    },
+                    status=404,
+                ),
+            )
+
+        if not target:
+            return (
+                None,
+                user,
+                None,
+                web.json_response(
+                    {
+                        "error": "User not found.",
+                        "errcode": "M_NOT_FOUND",
+                    },
+                    status=404,
+                ),
+            )
+        elif not isinstance(target, TLUser):
+            return (
+                None,
+                user,
+                None,
+                web.json_response(
+                    {
+                        "error": "Identifier is not a user.",
+                        "errcode": "FI.MAU.TELEGRAM_ID_NOT_USER",
+                    },
+                    status=400,
+                ),
+            )
+        portal = await Portal.get_by_entity(target, tg_receiver=user.tgid)
+        return portal, user, target, None
+
+    async def resolve_identifier(self, request: web.Request) -> web.Response:
+        portal, user, target, err = await self._resolve_id(request)
+        if err is not None:
+            return err
+        puppet = await portal.get_dm_puppet()
+        await puppet.update_info(user, target)
+        return web.json_response(
+            {
+                "room_id": portal.mxid,
+                "just_created": False,
+                "id": portal.tgid,
+                "contact_info": puppet.contact_info,
+            },
+            status=200,
+        )
+
+    async def start_dm(self, request: web.Request) -> web.Response:
+        portal, user, target, err = await self._resolve_id(request)
+        if err is not None:
+            return err
+        puppet = await portal.get_dm_puppet()
+        if portal.mxid:
+            just_created = False
+        else:
+            await portal.create_matrix_room(user, target, [user.mxid])
+            just_created = True
+        return web.json_response(
+            {
+                "room_id": portal.mxid,
+                "just_created": just_created,
+                "id": portal.tgid,
+                "contact_info": puppet.contact_info,
+            },
+            status=201 if just_created else 200,
+        )
+
    async def send_bot_token(self, request: web.Request) -> web.Response:
        data, user, err = await self.get_user_request_info(request)
        if err is not None:
@@ -574,7 +675,7 @@ class ProvisioningAPI(AuthAPI):
        data = None
        if want_data and (request.method == "POST" or request.method == "PUT"):
            data = await self.get_data(request)
-            if not data:
+            if data is None:
                return (
                    None,
                    None,
@@ -2,7 +2,7 @@
 # Uncommented lines after the group definition insert things into that group.

 #/speedups
-cryptg>=0.1,<0.3
+cryptg>=0.1,<0.4
 cchardet
 aiodns
 brotli
@@ -18,7 +18,7 @@ moviepy>=1,<2
 phonenumbers>=8,<9

 #/metrics
-prometheus_client>=0.6,<0.14
+prometheus_client>=0.6,<0.15

 #/e2be
 python-olm>=3,<4
@@ -4,9 +4,9 @@ force_to_top = "typing"
 from_first = true
 combine_as_imports = true
 known_first_party = "mautrix"
+known_third_party = "telethon"
 line_length = 99

 [tool.black]
 line-length = 99
 target-version = ["py38"]
-required-version = "22.1.0"
@@ -3,10 +3,9 @@ python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.14.9,<0.15
+mautrix>=0.17.8,<0.18
 #telethon>=1.24,<1.25
-# Fork to make session storage async and update to layer 138
-tulir-telethon==1.25.0a5
-asyncpg>=0.20,<0.26
+tulir-telethon==1.25.0a20
+asyncpg>=0.20,<0.27
 mako>=1,<2
 setuptools
@@ -51,7 +51,7 @@ setuptools.setup(

    install_requires=install_requires,
    extras_require=extras_require,
-    python_requires="~=3.7",
+    python_requires="~=3.8",

    classifiers=[
        "Development Status :: 4 - Beta",
Author	SHA1	Message	Date
Tulir Asokan	0289f4b524	Bump version to 0.12.0	2022-08-26 16:22:06 +03:00
Malte E	85b8f5def7	Don't check whether User is channel, add peer property to User	2022-08-24 21:13:11 +03:00
Tulir Asokan	f012cb790f	Update mautrix-python again	2022-08-22 17:48:10 +03:00
Tulir Asokan	05476d7435	Update mautrix-python	2022-08-22 13:00:08 +03:00
Tulir Asokan	583427da05	Enable appservice ephemeral events by default	2022-08-22 12:57:41 +03:00
Tulir Asokan	e3a067c27a	Update mautrix-python	2022-08-17 15:20:38 +03:00
Tulir Asokan	b3ed4cf657	Fix handling messages with no sender	2022-08-17 15:14:07 +03:00
Tulir Asokan	952c81eadc	Update mautrix-python	2022-08-15 11:40:28 +03:00
Tulir Asokan	cc29ce19ca	Add missing parameter when handling Matrix files	2022-08-15 11:09:10 +03:00
Tulir Asokan	941aa5f9d8	Fix mistake in mark_disappearing	2022-08-14 14:28:23 +03:00
Tulir Asokan	15e5cc8da1	Add command to kick relaybot users from Telegram	2022-08-14 14:20:43 +03:00
Tulir Asokan	2cf9205cda	Add command to ban relaybot users from Telegram Fixes #357 Closes #819	2022-08-14 14:07:48 +03:00
Tulir Asokan	2ec89bc57e	Add keywords to mark_matrix_handled calls	2022-08-14 13:47:00 +03:00
Tulir Asokan	89294c57d8	Store message sender in database	2022-08-14 13:44:59 +03:00
Tulir Asokan	624c72fa99	Merge remote-tracking branch 'zsinskri/delivery-receipts'	2022-08-14 12:52:33 +03:00
Tulir Asokan	34af580846	Move misc things from infinite backfill PR	2022-08-14 12:50:28 +03:00
Tulir Asokan	910a681f4b	Mark key parameters as positional-only in async getter lock methods	2022-08-14 12:49:45 +03:00
Tulir Asokan	c4c225343c	Add backfill queue table	2022-08-14 12:49:13 +03:00
Tulir Asokan	f13a9d0e96	Add support for disappearing messages	2022-08-14 01:49:39 +03:00
Tulir Asokan	c54ae9548f	Add support for converting video stickers to images	2022-08-14 00:53:21 +03:00
Tulir Asokan	1216607763	Add custom attribute for custom emojis	2022-08-12 22:45:52 +03:00
Tulir Asokan	ecd4d5c338	Limit number of custom emoji being transferred simultaneously	2022-08-12 22:14:53 +03:00
Tulir Asokan	a5fe05cff2	Add support for converting animated stickers to webp	2022-08-12 22:07:52 +03:00
Tulir Asokan	76eafbf48c	Add basic support for bridging custom emojis from Telegram	2022-08-12 21:35:50 +03:00
Tulir Asokan	473ab17fe7	Update Telethon and strip empty entities when sending to Telegram	2022-08-02 13:46:06 +03:00
Tulir Asokan	bea9bc4ec0	Mention forwarding limitations in changelog. Closes #818	2022-07-29 12:24:41 +03:00
Tulir Asokan	5df1e84fae	Update mautrix-python	2022-07-29 12:23:46 +03:00
Tulir Asokan	8665871502	Fix some issues with auto-creating groups	2022-07-18 13:01:50 +03:00
Zsin Skri	ef57f1021c	Revert "Don't send delivery receipts to unencrypted private chat portals. Fixes #483 " This reverts commit `a4595b427d`. Commit `a4595b4` avoids sending delivery receipts to rooms that do not contain the bridge bot. That was necessary as trying to send a read marker would automatically attempt to join the bridge bot to the room. That join without invite would fail, hence #483. But since https://github.com/mautrix/python/commit/f272f16a1d151a1c6612c9349776eda985c8ea3e we no longer attempt to join the sender of read receipts, fixing #483 without necessarily sacrificing the delivery receipt functionality. Thus: - `a4595b4` is no longer necessary, its original purpose is fulfilled by f272f16. - `a4595b4` prevents delivery receipts from working in unencrypted rooms. - This reverts `a4595b4`, thus enabling delivery receipts in unencrypted rooms.	2022-07-17 20:57:17 +02:00
Tulir Asokan	b6312f306a	Move config check when handling ghost invites	2022-07-17 16:09:02 +03:00
Tulir Asokan	70b73868c7	Merge remote-tracking branch 'maltee1/auto_create_group'	2022-07-17 16:04:56 +03:00
Tulir Asokan	0717b4a290	Disable public portals by default	2022-07-17 16:04:20 +03:00
Tulir Asokan	a9b6539910	Update changelog	2022-07-13 14:24:13 +03:00
Tulir Asokan	49520bb8a3	Try to avoid race conditions with supergroup upgrades	2022-07-13 14:20:39 +03:00
Tulir Asokan	7abe19aec9	Add another backfill column	2022-07-13 12:17:35 +03:00
Malte E	3dd0c51be7	add config option, update roadmap	2022-07-12 21:37:41 +02:00
Malte E	565bb87470	implement handle_puppet_group_invite to auto-create groups	2022-07-12 21:02:14 +02:00
Tulir Asokan	9188251501	Add status field to message status events	2022-07-12 15:05:06 +03:00
Tulir Asokan	cb11e147ce	Add support for Matrix -> Telegram captions with MSC2530	2022-07-12 11:35:51 +03:00
Tulir Asokan	eb1190359d	Update asyncpg	2022-07-10 18:20:53 +03:00
Tulir Asokan	cdfc6fd007	Remove noisy error on ignored messages	2022-07-10 18:20:46 +03:00
Tulir Asokan	df9b7d343e	Add support for forwarding messages	2022-07-07 13:02:01 +03:00
Tulir Asokan	f26973f46c	Update changelog and mautrix-python	2022-07-06 19:37:29 +03:00
Tulir Asokan	2335431060	Update mautrix-python again	2022-07-05 20:06:29 +03:00
Tulir Asokan	8fd97af0a9	Update mautrix-python	2022-07-05 13:26:03 +03:00
Tulir Asokan	3ea491d379	Fix handling location messages	2022-07-04 10:44:37 +03:00
Tulir Asokan	3bd7d846f4	Update mautrix-python	2022-06-28 19:35:23 +03:00
Tulir Asokan	99344c38a4	Create room on UpdateChannel	2022-06-28 19:20:18 +03:00
Tulir Asokan	d917499d1f	Fix check for using double puppeted leaves	2022-06-28 19:14:37 +03:00
Tulir Asokan	98da5fecc3	Use wildcard for cryptg wheel name	2022-06-27 21:41:59 +03:00
Tulir Asokan	6b0ece5da1	Update cryptg	2022-06-27 21:39:18 +03:00
Tulir Asokan	448b149e8e	Make docker image smaller	2022-06-27 21:14:14 +03:00
Tulir Asokan	120514125f	Update changelog	2022-06-27 20:52:47 +03:00
Tulir Asokan	cd4b4365bd	Update Docker image to Alpine 3.16	2022-06-27 15:59:11 +03:00
Tulir Asokan	8f68801aa9	Maybe improve channel leave handling	2022-06-27 15:59:11 +03:00
Sumner Evans	1d0e8c7e0c	Merge pull request #810 from mautrix/mautrix-0.16.10 deps/mautrix: update to v0.16.10	2022-06-24 11:42:23 -06:00
Sumner Evans	3ff43165c2	deps/mautrix: update to v0.16.10 Signed-off-by: Sumner Evans <sumner@beeper.com>	2022-06-24 11:33:49 -06:00
Tulir Asokan	1fdbdb654a	Update Telethon for API layer update	2022-06-24 11:17:24 +03:00
Tulir Asokan	0e024b3b7c	Fetch participant count if it's not included in the entity	2022-06-23 23:59:44 +03:00
Tulir Asokan	e1a5e30a75	Remove legacy crypto database from example config It doesn't seem to be used anywhere	2022-06-23 09:58:10 +03:00
Sumner Evans	05d4923db9	encryption: add ability to control rotation settings Signed-off-by: Sumner Evans <sumner@beeper.com>	2022-06-23 09:58:10 +03:00
Tulir Asokan	f18713cd5e	Update changelog	2022-06-22 13:28:16 +03:00
Tulir Asokan	ef05875bfd	Remove plaintext_highlights config option The code using it was removed in v0.11.0, so it hasn't actually worked for a while now.	2022-06-22 12:40:30 +03:00
Tulir Asokan	59d85a1e16	Fix incorrect method call	2022-06-22 12:35:58 +03:00
Tulir Asokan	7eec0d1ed3	Add index on puppet username. Fixes #799	2022-06-22 12:30:07 +03:00
Tulir Asokan	f917ee189d	Don't require puppeting for caption and config commands Fixes #790	2022-06-22 12:30:01 +03:00
Tulir Asokan	ab2e38b33b	Merge remote-tracking branch 'ofalvai/patch-1'	2022-06-22 12:10:52 +03:00
Tulir Asokan	38af35e776	Merge remote-tracking branch 'cynhr/cmd-prefix'	2022-06-22 12:10:44 +03:00
Tulir Asokan	c6adb87aea	Set sync_channel_members to false by default	2022-06-22 12:09:38 +03:00
Tulir Asokan	e8eef1c31e	Add option to not bridge chats with too many members	2022-06-22 12:05:48 +03:00
Tulir Asokan	bac3abcb4c	Update mautrix-python	2022-06-21 15:42:27 +03:00
Tulir Asokan	c682bdc01e	Update dependencies	2022-06-19 15:05:59 +03:00
Cy Nhr	50cd878f13	Include command prefix in game and poll messages Game and poll messages send by the bridge to matrix each include a command the receiver might want to run (to play the game or to vote in the poll). But these command suggestions did always include the "!tg" command prefix, even if the command prefix was changed to a different value in the config. That could lead to the bridge ignoring the exact command it suggested earlier. With this commit, these messages contain the correct command prefix as defined in the config so that the command suggestions can be executed by the user without manually correcting the prefix.	2022-06-18 20:23:59 +02:00
Tulir Asokan	ea49ba8be2	Move CI script to mautrix/ci repo	2022-06-18 14:23:01 +03:00
Tulir Asokan	b60056c560	Add missing prefix to bridge info endpoint	2022-06-18 09:56:54 +03:00
Tulir Asokan	820210dc44	Fix bridging polls from Telegram	2022-06-02 19:40:23 +03:00
Tulir Asokan	7d998dca3f	Add support for custom message bridging status events	2022-06-01 15:36:22 +03:00
Tulir Asokan	037d93471d	Catch PhoneNumberUnoccupied in /login/send_code provisioning API	2022-05-30 22:18:28 +03:00
Tulir Asokan	5cb2b871cd	Fix sticker event type	2022-05-29 00:35:25 +03:00
Tulir Asokan	44f2c648a8	Add config option to exit if telethon update loop fails	2022-05-26 17:37:21 +03:00
Tulir Asokan	0ae8a5877e	Rename db upgrade	2022-05-26 17:28:44 +03:00
Tulir Asokan	18f6622340	Separate Telegram message conversion code from Matrix sending	2022-05-26 15:46:20 +03:00
Tulir Asokan	591e79f5a0	Enable catch_up and sequential_updates by default	2022-05-25 16:49:59 +03:00
Tulir Asokan	d898486b49	Add first_event_id and next_batch_id columns for portals	2022-05-25 14:56:41 +03:00
Tulir Asokan	74e0aee421	Update Telethon a third time	2022-05-23 17:58:34 +03:00
Tulir Asokan	07f32e1256	Update Telethon again	2022-05-23 14:59:36 +03:00
Tulir Asokan	ea680cf871	Update Telethon	2022-05-23 14:22:11 +03:00
Tulir Asokan	e89c75c6cd	Don't try to stop relaybot if it's not enabled	2022-05-23 10:46:00 +03:00
Tulir Asokan	59d052afd2	Update Telethon	2022-05-20 21:55:22 +03:00
Tulir Asokan	9383249ade	Stop relaybot connection cleanly	2022-05-20 18:44:36 +03:00
Tulir Asokan	0a4f30bf02	Update setup doc links	2022-05-20 15:10:30 +03:00
Tulir Asokan	190f452910	Fix some bugs and update Telethon	2022-05-20 14:24:28 +03:00
Tulir Asokan	3c59a1af97	Adjust logs slightly	2022-05-20 12:28:39 +03:00
Tulir Asokan	11ff628ef8	Always check database before handling message	2022-05-20 12:02:32 +03:00
Tulir Asokan	908e600dc9	Switch /resolve_identifier to GET	2022-05-19 18:22:04 +03:00
Tulir Asokan	eb43fde3e4	Add provisioning API for resolving identifiers	2022-05-19 13:15:44 +03:00
Tulir Asokan	e6ef40e51d	Update Telethon	2022-05-19 13:15:39 +03:00
Tulir Asokan	7feea5aa6d	Redact QR code after login	2022-05-16 19:13:06 +03:00
Lonami	d084cca983	Add get_update_states to telethon_session (#795 ) This is needed for an upcoming patch in order to properly catch up on all channels the client is in.	2022-05-16 19:09:39 +03:00
Tulir Asokan	d9018868a1	Use new helper method to redact command	2022-05-10 17:27:03 +03:00
Tulir Asokan	72360457ef	Bridge audio and video metadata properly	2022-05-10 17:13:14 +03:00
Tulir Asokan	0e4c1b71e6	Redact 2fa password when using in-Matrix login	2022-05-10 17:04:39 +03:00
Olivér Falvai	575b761f77	Increase `image_as_file_pixels` default value	2022-05-07 12:23:01 +02:00
Tulir Asokan	68e950a6bc	Add issue templates	2022-04-20 14:02:36 +03:00
Sumner Evans	ba5bbebb3e	Merge pull request #788 from mautrix/dev-update-stable-and-nightly ci: automatically update both STABLE and NIGHTLY on dev environment	2022-04-19 08:56:58 -06:00
Sumner Evans	cb38896593	ci: automatically update both STABLE and NIGHTLY on dev environment	2022-04-18 19:23:04 -06:00
Tulir Asokan	21c6a7d87f	Bump version to 0.11.3	2022-04-17 13:30:38 +03:00
Tulir Asokan	7c2a569235	Remove some unused fields	2022-04-13 14:43:53 +03:00
Tulir Asokan	1f5b91cbec	Update mautrix-python	2022-04-09 20:52:45 +03:00
Tulir Asokan	937f37eff0	Don't print generated registration message if config is invalid	2022-04-09 20:46:25 +03:00
Tulir Asokan	4f9f74204a	Update dependencies	2022-04-08 18:06:24 +03:00
Tulir Asokan	ed6735f10f	Fix creating new database	2022-04-06 19:04:12 +03:00
Tulir Asokan	5acd3cf007	Move API version number to endpoint definition	2022-04-06 14:33:03 +03:00
Tulir Asokan	279b997bd3	Add contacts and PM endpoints to OpenAPI spec	2022-04-06 14:29:50 +03:00
Tulir Asokan	4eb6095822	Update provisioning API spec to OpenAPI 3.1.0	2022-04-06 14:06:10 +03:00
Tulir Asokan	da5b8556f2	Add phone number field for puppets	2022-04-06 12:49:01 +03:00
Tulir Asokan	261f99ac82	Add provisioning API for listing contacts and starting DMs	2022-04-06 12:40:55 +03:00
Tulir Asokan	61f3c39cc2	Mark reactions as read when reading from Matrix	2022-04-01 15:56:16 +03:00
Tulir Asokan	39ab1d0c22	Fix another bug	2022-03-31 01:58:40 +03:00
Tulir Asokan	8abb9c3884	Fix bugs in Telegram entity parser	2022-03-31 01:53:51 +03:00
Tulir Asokan	58f8ee2ee2	Add config option to mark joined Telegram notices as read automatically	2022-03-30 11:58:40 +03:00
Tulir Asokan	474bcc9544	Update and unpin black	2022-03-28 22:29:22 +03:00
Tulir Asokan	a3f4e25101	Fix some bugs and add command to list invite links	2022-03-28 15:49:08 +03:00
Tulir Asokan	8befb664b6	Handle accepted into group action messages	2022-03-28 15:06:35 +03:00
Tulir Asokan	819dd1bcff	Allow generating invite links that need join approval	2022-03-28 15:03:22 +03:00
Tulir Asokan	2b8b853fec	Add proper message when requesting to join via invite link	2022-03-28 15:03:05 +03:00
Tulir Asokan	c536c4a265	Update changelog	2022-03-27 23:39:46 +03:00
Tulir Asokan	f13acfe825	Clarify that supergroups are channels in !tg bridge	2022-03-27 23:39:46 +03:00
Sumner Evans	8e763ba067	Merge pull request #775 from mautrix/sumner/bri-2582 async media: add ability to upload media asynchronously	2022-03-27 12:31:39 -06:00
Sumner Evans	8d7cfd8e46	parallel transfer: disable async_upload	2022-03-27 12:26:44 -06:00
Sumner Evans	601058d61c	async media: add ability to upload media asynchronously Co-authored-by: Tulir Asokan <tulir@maunium.net>	2022-03-27 12:26:44 -06:00
Tulir Asokan	f8596ef368	Use new ensure_has_html method instead of duplicating code	2022-03-23 19:51:01 +02:00
Tulir Asokan	7f0494d52d	Merge remote-tracking branch 'origin/sumner/bri-2496'	2022-03-22 16:29:48 +02:00
Sumner Evans	828478514b	Merge pull request #772 from mautrix/fix-kick-from-portals user: fix bug in kick_from_portals	2022-03-22 08:00:02 -06:00
Tulir Asokan	146f5437d1	Drop Python 3.7 support	2022-03-22 13:44:52 +02:00
Tulir Asokan	c28760f2a8	Adjust permission error messages	2022-03-22 13:44:52 +02:00
Tulir Asokan	04f30f6f29	Update mautrix-python	2022-03-22 13:44:52 +02:00
Tulir Asokan	caa1d3565b	Update changelog	2022-03-22 13:44:52 +02:00
Sumner Evans	1a7a020bb2	backfill: set timestamp on backfilled reactions to message timestamp	2022-03-22 00:48:12 -06:00
Sumner Evans	077ab2bb38	user: fix bug in kick_from_portals	2022-03-22 00:46:32 -06:00
Sumner Evans	6f491bf7d1	Merge pull request #771 from ProkopRandacek/master Add missing f in front of the f-string	2022-03-21 10:51:51 -06:00
Prokop Randacek	9b80c21d0a	add missing F	2022-03-21 10:11:45 +01:00
Tulir Asokan	e9dc76a860	Fix public channel mentions always using user instead of portal mxid	2022-03-15 16:32:21 +02:00
Tulir Asokan	9e73324a20	Fix bridge_matrix_leave config option	2022-03-14 12:00:14 +02:00
Tulir Asokan	7df93485d8	Remove extra parameter in call	2022-03-11 12:02:02 +02:00
Tulir Asokan	9018cea5ae	Update changelog	2022-03-07 18:52:15 +02:00
Tulir Asokan	32e023231d	Catch invalid integers passed to !tg create	2022-03-05 20:16:04 +02:00
Tulir Asokan	4766d14359	Move DM creation code to mautrix-python	2022-03-04 16:12:02 +02:00
Tulir Asokan	526b99ec04	Disable file logging in Docker by default To enable it, use a custom path that points at a writable volume	2022-03-04 10:57:08 +02:00
Nick Mills-Barrett	da132438bd	Only change the data directory ownership on Docker start	2022-03-03 18:17:39 +02:00
Tulir Asokan	54176ba2db	Fix self parameter name in _mute_room. Fixes #764	2022-03-02 14:33:09 +02:00
Tulir Asokan	1eca3c2ffd	Check peer_type in database when manually bridging portal	2022-03-02 14:33:06 +02:00
Tulir Asokan	98142f28cd	Improve logging of backfill count	2022-02-28 12:36:43 +02:00
Tulir Asokan	2cf7fc7059	Improve backfilling to fetch less redundant messages	2022-02-28 12:26:24 +02:00
Tulir Asokan	a34a18c6cc	Deduplicate user joined telegram messages	2022-02-28 11:59:44 +02:00
Tulir Asokan	fa738fbadf	Fix condition	2022-02-26 17:20:22 +02:00
Tulir Asokan	9ea0516166	Log when tagging and muting rooms	2022-02-25 19:35:05 +02:00
Tulir Asokan	b760aadb01	Add custom flag for force sending images as document	2022-02-25 12:38:01 +02:00
Tulir Asokan	24162e14ac	Remove msgtype in stickers	2022-02-23 14:36:53 +02:00
Tulir Asokan	9ea495324d	Don't try to set room state in non-existent portals	2022-02-23 12:46:16 +02:00
Tulir Asokan	437e86a15b	Keep newlines as-is in code blocks	2022-02-23 12:44:56 +02:00
Tulir Asokan	d9e0b75e9b	Update mautrix-python again	2022-02-22 13:53:43 +02:00
Tulir Asokan	9606518ba7	Update mautrix-python again	2022-02-22 12:40:16 +02:00
Tulir Asokan	e2774b830f	Update mautrix-python version	2022-02-22 11:58:27 +02:00
Tulir Asokan	951d82ad27	Remove max_document_size option and use media repo config directly	2022-02-20 13:47:40 +02:00
Tulir Asokan	4a55cf589c	Add initial db upgrade that jumps to latest version	2022-02-19 00:19:49 +02:00
Tulir Asokan	b07d80d876	Add support for converting t.me/c/<id>/<msgid> links	2022-02-18 17:22:26 +02:00