Bump version to 0.15.0

* Add quick 5 retries when connecting to Telegram

* Fix attempt initialisation

Co-authored-by: Tulir Asokan <tulir@maunium.net>

* Log only when retrying

Co-authored-by: Tulir Asokan <tulir@maunium.net>

---------

Co-authored-by: Tulir Asokan <tulir@maunium.net>

.gitignore

@@ -10,12 +10,19 @@ __pycache__
 /*.egg-info
 /.eggs
 
-/config.yaml
-/registration.yaml
+*.yaml
+!.pre-commit-config.yaml
+!example-config.yaml
+!/mautrix_telegram/web/provisioning/spec.yaml
+!/.github/workflows/*.yaml
+
+/start
+/mautrix
+/telethon
+
 *.log*
 *.db
+*.db-*
 /*.pickle
 *.bak
-/*.session
-/*.session-journal
 /*.json

CHANGELOG.md

@@ -1,3 +1,78 @@
+# v0.15.0 (2023-11-26)
+
+* Removed support for MSC2716 backfilling.
+* Added `add-contact` and `delete-contact` commands.
+* Updated Telegram API layer to 166.
+  * Includes receiving view-once media, blockquotes, quote replies and other
+    such things
+* Fixed AuthKeyNotFound errors not being handled and causing users to get stuck
+  in a non-logged-in state.
+
+# v0.14.2 (2023-09-19)
+
+* **Security:** Updated Pillow to 10.0.1.
+* Added support for double puppeting with arbitrary `as_token`s.
+  See [docs](https://docs.mau.fi/bridges/general/double-puppeting.html#appservice-method-new) for more info.
+* Added support for sending webm and tgs files as stickers.
+* Updated to Telegram API layer 161.
+* Fixed cached usernames for Telegram users being cleared incorrectly, leading
+  to mentions not being bridged as usernames.
+* Fixed reaction bridging failing if the server running the bridge was rebooted
+  less than 12 hours ago.
+
+# v0.14.1 (2023-06-26)
+
+### Added
+* Added option to delete megolm sessions that were received before the
+  automatic ratcheting options were introduced.
+* Added config option to use IPv6 for Telegram connection
+  (thanks to [@exciler] in [#920]).
+
+### Improved
+* Dropped support for Python 3.8.
+* Updated Docker image to Alpine 3.18.
+* Added timeout for forward backfills to prevent it from getting stuck
+  permanently.
+
+### Fixed
+* Fixed `bridge.filter.users` config option not being read correctly.
+* Fixed proxy support to use python-socks instead of pysocks.
+
+[@exciler]: https://github.com/exciler
+[#920]: https://github.com/mautrix/telegram/pull/920
+
+# v0.14.0 (2023-05-26)
+
+### Added
+* Added fallback messages for calls and premium gifts.
+* Added options to automatically ratchet/delete megolm sessions to minimize
+  access to old messages.
+* Added option to not set room name/avatar even in encrypted rooms.
+* Implemented appservice pinging using MSC2659.
+* Added option to disable or filter bridging direct chats
+  (thanks to [@Steffo99] in [#892]).
+* Added options to specify different limits for forward and catchup backfilling
+  depending on chat type.
+
+### Improved
+* Improved handling logouts and certain connection errors.
+* Changed reaction bridging to preserve timestamps.
+* Disabled creating portals for DMs that don't have any messages when
+  `sync_direct_chats` is enabled.
+
+### Fixed
+* Fixed syncing mute status when portal is created through incoming message
+  rather than in startup sync.
+* Fixed bridge incorrectly trusting member list and kicking users when
+  supergroup has member list hidden.
+* Fixed sending messages after creating groups from Matrix using relaybot
+  instead of puppet (thanks to [@maltee1] in [#902]).
+
+[@Steffo99]: https://github.com/Steffo99
+[@maltee1]: https://github.com/maltee1
+[#892]: https://github.com/mautrix/telegram/pull/892
+[#902]: https://github.com/mautrix/telegram/pull/902
+
 # v0.13.0 (2023-02-26)
 
 ### Added

Dockerfile

@@ -1,8 +1,8 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.17
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.18
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
-      py3-pillow \
+      #py3-pillow \
       py3-aiohttp \
       py3-magic \
       py3-ruamel.yaml \
@@ -14,8 +14,6 @@ RUN apk add --no-cache \
       py3-idna \
       py3-rsa \
       #py3-telethon \ (outdated)
-        # Optional for socks proxies
-        py3-pysocks \
         py3-pyaes \
         # cryptg
           py3-cffi \
@@ -34,7 +32,9 @@ RUN apk add --no-cache \
       bash \
       curl \
       jq \
-      yq
+      yq \
+  # Temporarily install pillow from edge repo to get up-to-date version
+  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
 
 COPY requirements.txt /opt/mautrix-telegram/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.13.0"
+__version__ = "0.15.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -39,6 +39,8 @@ from .abstract_user import AbstractUser  # isort: skip
 class TelegramBridge(Bridge):
     module = "mautrix_telegram"
     name = "mautrix-telegram"
+    beeper_service_name = "telegram"
+    beeper_network_name = "telegram"
     command = "python -m mautrix-telegram"
     description = "A Matrix-Telegram puppeting bridge."
     repo_url = "https://github.com/mautrix/telegram"
@@ -50,6 +52,7 @@ class TelegramBridge(Bridge):
 
     config: Config
     bot: Bot | None
+    matrix: MatrixHandler
     public_website: PublicBridgeWebsite | None
     provisioning_api: ProvisioningAPI | None
 
@@ -101,8 +104,6 @@ class TelegramBridge(Bridge):
         self.log.info("Finished re-sending bridge info state events")
 
     def prepare_stop(self) -> None:
-        for puppet in Puppet.by_custom_mxid.values():
-            puppet.stop()
         self.add_shutdown_actions(user.stop() for user in User.by_tgid.values())
         if self.bot:
             self.add_shutdown_actions(self.bot.stop())

mautrix_telegram/abstract_user.py

@@ -38,6 +38,7 @@ from telethon.tl.types import (
     PeerChannel,
     PeerChat,
     PeerUser,
+    PhoneCallRequested,
     TypeUpdate,
     UpdateChannel,
     UpdateChannelUserTyping,
@@ -54,6 +55,7 @@ from telethon.tl.types import (
     UpdateNewChannelMessage,
     UpdateNewMessage,
     UpdateNotifySettings,
+    UpdatePhoneCall,
     UpdatePinnedChannelMessages,
     UpdatePinnedDialogs,
     UpdatePinnedMessages,
@@ -206,6 +208,8 @@ class AbstractUser(ABC):
         sysversion = self.config["telegram.device_info.system_version"]
         appversion = self.config["telegram.device_info.app_version"]
         connection, proxy = self._proxy_settings
+        if proxy:
+            self.log.debug(f"Using proxy setting: {proxy}")
 
         assert isinstance(session, Session)
 
@@ -233,6 +237,7 @@ class AbstractUser(ABC):
             loop=self.loop,
             base_logger=base_logger,
             update_error_callback=self._telethon_update_error_callback,
+            use_ipv6=self.config["telegram.connection.use_ipv6"],
         )
         self.client.add_event_handler(self._update_catch)
 
@@ -301,7 +306,18 @@ class AbstractUser(ABC):
     async def start(self, delete_unless_authenticated: bool = False) -> AbstractUser:
         if not self.client:
             await self._init_client()
-        await self.client.connect()
+        attempts = 1
+        while True:
+            try:
+                await self.client.connect()
+            except Exception:
+                attempts += 1
+                if attempts > 10:
+                    raise
+                self.log.exception("Exception connecting to Telegram, retrying in 5s...")
+                await asyncio.sleep(5)
+            else:
+                break
         self.log.debug(f"{'Bot' if self.is_relaybot else self.mxid} connected: {self.connected}")
         return self
 
@@ -343,6 +359,8 @@ class AbstractUser(ABC):
             await self.delete_message(update)
         elif isinstance(update, UpdateDeleteChannelMessages):
             await self.delete_channel_message(update)
+        elif isinstance(update, UpdatePhoneCall):
+            await self.update_phone_call(update)
         elif isinstance(update, UpdateMessageReactions):
             await self.update_reactions(update)
         elif isinstance(update, (UpdateChatUserTyping, UpdateChannelUserTyping, UpdateUserTyping)):
@@ -449,6 +467,7 @@ class AbstractUser(ABC):
             return
 
         if not portal or not portal.mxid:
+            # TODO This explodes on channels because the field is channel_id
             self.log.debug(f"Dropping own read receipt in unknown chat ({update.peer})")
             return
 
@@ -617,6 +636,19 @@ class AbstractUser(ABC):
             return
         await portal.handle_telegram_reactions(self, TelegramID(update.msg_id), update.reactions)
 
+    async def update_phone_call(self, update: UpdatePhoneCall) -> None:
+        self.log.debug("Phone call update %s", update)
+        if not isinstance(update.phone_call, PhoneCallRequested):
+            return
+        tgid = TelegramID(update.phone_call.participant_id)
+        if tgid == self.tgid:
+            tgid = update.phone_call.admin_id
+        portal = await po.Portal.get_by_tgid(tgid, tg_receiver=self.tgid, peer_type="user")
+        if not portal or not portal.mxid or not portal.allow_bridging:
+            return
+        sender = await pu.Puppet.get_by_tgid(TelegramID(update.phone_call.admin_id))
+        await portal.handle_telegram_direct_call(self, sender, update)
+
     async def update_channel(self, update: UpdateChannel) -> None:
         portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if not portal:
@@ -659,7 +691,15 @@ class AbstractUser(ABC):
         if not portal:
             return
         elif portal and not portal.allow_bridging:
-            self.log.debug(f"Ignoring message in portal {portal.tgid_log} (bridging disallowed)")
+            self.log.debug(
+                f"Ignoring message {update.id} in portal {portal.tgid_log} (bridging disallowed)"
+            )
+            return
+
+        if not portal.mxid and getattr(original_update, "mau_left_channel", False):
+            self.log.debug(
+                f"Ignoring message {update.id} in portal {portal.tgid_log} because user isn't in the chat"
+            )
             return
 
         if self.is_relaybot:
@@ -683,6 +723,22 @@ class AbstractUser(ABC):
             self.log.debug("Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
             return
 
+        task = self._call_portal_message_handler(update, original_update, portal, sender)
+        if portal.backfill_lock.locked:
+            self.log.debug(
+                f"{portal.tgid_log} is backfill locked, moving incoming message to async task"
+            )
+            background_task.create(task)
+        else:
+            await task
+
+    async def _call_portal_message_handler(
+        self,
+        update: UpdateMessageContent,
+        original_update: UpdateMessage,
+        portal: po.Portal,
+        sender: pu.Puppet,
+    ) -> None:
         await portal.backfill_lock.wait(f"update {update.id}")
 
         if isinstance(update, MessageService):

mautrix_telegram/bot.py

@@ -395,7 +395,7 @@ class Bot(AbstractUser):
         def reply(reply_text: str) -> Awaitable[Message]:
             return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)
 
-        if command == "start":
+        if command == "start" and message.is_private:
             pcm = self.config["bridge.relaybot.private_chat.message"]
             if pcm:
                 await reply(pcm)

mautrix_telegram/commands/handler.py

@@ -159,6 +159,7 @@ def command_handler(
     needs_admin: bool = False,
     management_only: bool = False,
     name: str | None = None,
+    aliases: list[str] | None = None,
     help_text: str = "",
     help_args: str = "",
     help_section: HelpSection = None,
@@ -167,6 +168,7 @@ def command_handler(
         _func,
         _handler_class=CommandHandler,
         name=name,
+        aliases=aliases,
         help_text=help_text,
         help_args=help_args,
         help_section=help_section,

mautrix_telegram/commands/portal/admin.py

@@ -39,8 +39,6 @@ async def clear_db_cache(evt: CommandEvent) -> EventID:
         await evt.reply("Cleared portal cache")
     elif section == "puppet":
         pu.Puppet.by_tgid = {}
-        for puppet in pu.Puppet.by_custom_mxid.values():
-            puppet.stop()
         pu.Puppet.by_custom_mxid = {}
         await asyncio.gather(
             *[puppet.try_start() async for puppet in pu.Puppet.all_with_custom_mxid()]
@@ -69,8 +67,6 @@ async def reload_user(evt: CommandEvent) -> EventID:
     if not user:
         return await evt.reply("User not found")
     puppet = await pu.Puppet.get_by_custom_mxid(mxid)
-    if puppet:
-        puppet.stop()
     await user.stop()
     del u.User.by_tgid[user.tgid]
     del u.User.by_mxid[user.mxid]

mautrix_telegram/commands/portal/bridge.py

@@ -56,7 +56,9 @@ async def bridge(evt: CommandEvent) -> EventID:
         return await evt.reply(f"{that_this} room is already a portal room.")
 
     if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
+        return await evt.reply(
+            f"You do not have the permissions to bridge {that_this.lower()} room."
+        )
 
     # The /id bot command provides the prefixed ID, so we assume
     tgid_str = evt.args[0]

mautrix_telegram/commands/portal/create_chat.py

@@ -65,19 +65,11 @@ async def create(evt: CommandEvent) -> EventID:
         about=about,
         encrypted=encrypted,
     )
-    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender, pre_create=True)
-    if len(errors) > 0:
-        error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
-        await evt.reply(
-            f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
-            "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
-            "those users."
-        )
 
     await warn_missing_power(levels, evt)
 
     try:
-        await portal.create_telegram_chat(evt.sender, invites=invites, supergroup=supergroup)
+        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
     except ValueError as e:
         await portal.delete()
         return await evt.reply(e.args[0])

mautrix_telegram/commands/telegram/misc.py

@@ -18,8 +18,8 @@ from __future__ import annotations
 from typing import cast
 import base64
 import codecs
-import math
 import re
+import shlex
 
 from aiohttp import ClientSession, InvalidURL
 from telethon.errors import (
@@ -29,10 +29,10 @@ from telethon.errors import (
     InviteHashInvalidError,
     InviteRequestSentError,
     OptionsTooMuchError,
-    TakeoutInitDelayError,
     UserAlreadyParticipantError,
 )
 from telethon.tl.functions.channels import JoinChannelRequest
+from telethon.tl.functions.contacts import DeleteByPhonesRequest, ImportContactsRequest
 from telethon.tl.functions.messages import (
     CheckChatInviteRequest,
     GetBotCallbackAnswerRequest,
@@ -42,12 +42,14 @@ from telethon.tl.functions.messages import (
 from telethon.tl.patched import Message
 from telethon.tl.types import (
     InputMediaDice,
+    InputPhoneContact,
     MessageMediaGame,
     MessageMediaPoll,
     TypeInputPeer,
     TypeUpdates,
     User as TLUser,
 )
+from telethon.tl.types.contacts import ImportedContacts
 from telethon.tl.types.messages import BotCallbackAnswer
 
 from mautrix.types import EventID, Format
@@ -161,6 +163,76 @@ async def pm(evt: CommandEvent) -> EventID:
     return await evt.reply(f"Created private chat room with {displayname}")
 
 
+async def _handle_contact(source: AbstractUser, user: TLUser) -> str:
+    puppet: pu.Puppet = await pu.Puppet.get_by_tgid(user.id)
+    await puppet.update_info(source, user)
+
+    params = []
+    if user.username:
+        params.append(f"[@{user.username}](https://t.me/{user.username})")
+    if user.phone:
+        params.append(f"+{user.phone}")
+    params.append(f"ID `{user.id}`")
+    params_str = " / ".join(params)
+    return f"[{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): {params_str}"
+
+
+@command_handler(
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="<_phone_> <_first name_> <_last name_>",
+    help_text="Add a phone number to your contacts on Telegram",
+)
+async def add_contact(evt: CommandEvent) -> EventID:
+    if len(evt.args) < 3:
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp add-contact <phone> <first name> <last name>`"
+        )
+    try:
+        names = shlex.split(" ".join(evt.args[1:]))
+    except ValueError as e:
+        return await evt.reply(
+            f"Failed to parse names (use shell quoting for names with spaces): {e}"
+        )
+    if len(names) != 2:
+        return await evt.reply(
+            "Wrong number of names, must have first and last name "
+            "(use shell quoting for names with spaces)"
+        )
+    res: ImportedContacts = await evt.sender.client(
+        ImportContactsRequest(
+            contacts=[
+                InputPhoneContact(
+                    client_id=1, phone=evt.args[0], first_name=names[0], last_name=names[1]
+                )
+            ]
+        )
+    )
+    if res.retry_contacts:
+        return await evt.reply("Failed to import contacts")
+    elif not res.users:
+        return await evt.reply("Contact imported, but user not found on Telegram")
+    imported_str = "\n".join(
+        [f"* {await _handle_contact(evt.sender, user)}" for user in res.users]
+    )
+    return await evt.reply(f"Imported contacts:\n\n{imported_str}")
+
+
+@command_handler(
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="<_phones..._>",
+    help_text="Remove phone numbers from your contacts on Telegram.",
+    aliases=["remove-contact", "delete-contacts", "remove-contacts"],
+)
+async def delete_contact(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp delete-contact <phones...>`")
+    ok = await evt.sender.client(DeleteByPhonesRequest(phones=evt.args))
+    if ok:
+        return await evt.reply("Contacts deleted")
+    else:
+        return await evt.reply("Contacts not deleted?")
+
+
 async def _join(
     evt: CommandEvent, identifier: str, link_type: str
 ) -> tuple[TypeUpdates | None, EventID | None]:
@@ -440,14 +512,5 @@ async def backfill(evt: CommandEvent) -> None:
     if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
         await evt.reply("Backfilling normal groups is disabled in the bridge config")
         return
-    if portal.backfill_msc2716:
-        messages_per_batch = evt.config["bridge.backfill.incremental.messages_per_batch"]
-        batches = math.ceil(limit / messages_per_batch)
-        rounded = ""
-        if batches * messages_per_batch != limit:
-            rounded = f" (rounded message limit to {batches}*{messages_per_batch})"
-        await portal.enqueue_backfill(evt.sender, priority=0, max_batches=batches)
-        await evt.reply(f"Backfill queued{rounded}")
-    else:
-        output = await portal.forward_backfill(evt.sender, initial=False, override_limit=limit)
-        await evt.reply(output)
+    output = await portal.forward_backfill(evt.sender, initial=False, override_limit=limit)
+    await evt.reply(output)

mautrix_telegram/config.py

@@ -148,7 +148,16 @@ class Config(BaseBridgeConfig):
         copy("bridge.animated_emoji.args.width")
         copy("bridge.animated_emoji.args.height")
         copy("bridge.animated_emoji.args.fps")
-        copy("bridge.private_chat_portal_meta")
+        if isinstance(self.get("bridge.private_chat_portal_meta", "default"), bool):
+            base["bridge.private_chat_portal_meta"] = (
+                "always" if self["bridge.private_chat_portal_meta"] else "default"
+            )
+        else:
+            copy("bridge.private_chat_portal_meta")
+            if base["bridge.private_chat_portal_meta"] not in ("default", "always", "never"):
+                base["bridge.private_chat_portal_meta"] = "default"
+        copy("bridge.disable_reply_fallbacks")
+        copy("bridge.cross_room_replies")
         copy("bridge.delivery_receipts")
         copy("bridge.delivery_error_reports")
         copy("bridge.incoming_bridge_error_reports")
@@ -162,12 +171,29 @@ class Config(BaseBridgeConfig):
         copy("bridge.kick_on_logout")
         copy("bridge.always_read_joined_telegram_notice")
         copy("bridge.backfill.enable")
-        copy("bridge.backfill.msc2716")
-        copy("bridge.backfill.double_puppet_backfill")
         copy("bridge.backfill.normal_groups")
         copy("bridge.backfill.unread_hours_threshold")
-        copy("bridge.backfill.forward.initial_limit")
-        copy("bridge.backfill.forward.sync_limit")
+        if "bridge.backfill.forward" in self:
+            initial_limit = self.get("bridge.backfill.forward.initial_limit", 10)
+            sync_limit = self.get("bridge.backfill.forward.sync_limit", 100)
+            base["bridge.backfill.forward_limits.initial.user"] = initial_limit
+            base["bridge.backfill.forward_limits.initial.normal_group"] = initial_limit
+            base["bridge.backfill.forward_limits.initial.supergroup"] = initial_limit
+            base["bridge.backfill.forward_limits.initial.channel"] = initial_limit
+            base["bridge.backfill.forward_limits.sync.user"] = sync_limit
+            base["bridge.backfill.forward_limits.sync.normal_group"] = sync_limit
+            base["bridge.backfill.forward_limits.sync.supergroup"] = sync_limit
+            base["bridge.backfill.forward_limits.sync.channel"] = sync_limit
+        else:
+            copy("bridge.backfill.forward_limits.initial.user")
+            copy("bridge.backfill.forward_limits.initial.normal_group")
+            copy("bridge.backfill.forward_limits.initial.supergroup")
+            copy("bridge.backfill.forward_limits.initial.channel")
+            copy("bridge.backfill.forward_limits.sync.user")
+            copy("bridge.backfill.forward_limits.sync.normal_group")
+            copy("bridge.backfill.forward_limits.sync.supergroup")
+            copy("bridge.backfill.forward_limits.sync.channel")
+        copy("bridge.backfill.forward_timeout")
         copy("bridge.backfill.incremental.messages_per_batch")
         copy("bridge.backfill.incremental.post_batch_delay")
         copy("bridge.backfill.incremental.max_batches.user")
@@ -197,6 +223,7 @@ class Config(BaseBridgeConfig):
 
         copy("bridge.filter.mode")
         copy("bridge.filter.list")
+        copy("bridge.filter.users")
 
         copy("bridge.command_prefix")
 
@@ -241,6 +268,7 @@ class Config(BaseBridgeConfig):
         copy("telegram.connection.retry_delay")
         copy("telegram.connection.flood_sleep_threshold")
         copy("telegram.connection.request_retries")
+        copy("telegram.connection.use_ipv6")
 
         copy("telegram.device_info.device_model")
         copy("telegram.device_info.system_version")

mautrix_telegram/db/puppet.py

@@ -48,6 +48,7 @@ class Puppet:
     avatar_url: ContentURI | None
     name_set: bool
     avatar_set: bool
+    contact_info_set: bool
     is_bot: bool | None
     is_channel: bool
     is_premium: bool
@@ -68,7 +69,7 @@ class Puppet:
     columns: ClassVar[str] = (
         "id, is_registered, displayname, displayname_source, displayname_contact, "
         "displayname_quality, disable_updates, username, phone, photo_id, avatar_url, "
-        "name_set, avatar_set, is_bot, is_channel, is_premium, "
+        "name_set, avatar_set, contact_info_set, is_bot, is_channel, is_premium, "
         "custom_mxid, access_token, next_batch, base_url"
     )
 
@@ -108,6 +109,7 @@ class Puppet:
             self.avatar_url,
             self.name_set,
             self.avatar_set,
+            self.contact_info_set,
             self.is_bot,
             self.is_channel,
             self.is_premium,
@@ -122,8 +124,9 @@ class Puppet:
         UPDATE puppet
         SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,
             displayname_quality=$6, disable_updates=$7, username=$8, phone=$9, photo_id=$10,
-            avatar_url=$11, name_set=$12, avatar_set=$13, is_bot=$14, is_channel=$15,
-            is_premium=$16, custom_mxid=$17, access_token=$18, next_batch=$19, base_url=$20
+            avatar_url=$11, name_set=$12, avatar_set=$13, contact_info_set=$14, is_bot=$15,
+            is_channel=$16, is_premium=$17, custom_mxid=$18, access_token=$19, next_batch=$20,
+            base_url=$21
         WHERE id=$1
         """
         await self.db.execute(q, *self._values)
@@ -133,9 +136,9 @@ class Puppet:
         INSERT INTO puppet (
             id, is_registered, displayname, displayname_source, displayname_contact,
             displayname_quality, disable_updates, username, phone, photo_id, avatar_url, name_set,
-            avatar_set, is_bot, is_channel, is_premium, custom_mxid, access_token, next_batch,
-            base_url
+            avatar_set, contact_info_set, is_bot, is_channel, is_premium, custom_mxid,
+            access_token, next_batch, base_url
         ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
-                  $19, $20)
+                  $19, $20, $21)
         """
         await self.db.execute(q, *self._values)

mautrix_telegram/db/upgrade/__init__.py

@@ -20,4 +20,5 @@ from . import (
     v15_backfill_anchor_id,
     v16_backfill_type,
     v17_message_find_recent,
+    v18_puppet_contact_info_set,
 )

mautrix_telegram/db/upgrade/v00_latest_revision.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from mautrix.util.async_db import Connection, Scheme
 
-latest_version = 17
+latest_version = 18
 
 
 async def create_latest_tables(conn: Connection, scheme: Scheme) -> int:
@@ -113,6 +113,7 @@ async def create_latest_tables(conn: Connection, scheme: Scheme) -> int:
             avatar_url          TEXT,
             name_set            BOOLEAN NOT NULL DEFAULT false,
             avatar_set          BOOLEAN NOT NULL DEFAULT false,
+            contact_info_set    BOOLEAN NOT NULL DEFAULT false,
             is_bot              BOOLEAN,
             is_channel          BOOLEAN NOT NULL DEFAULT false,
             is_premium          BOOLEAN NOT NULL DEFAULT false,

mautrix_telegram/db/upgrade/v18_puppet_contact_info_set.py

@@ -0,0 +1,25 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add contact_info_set column to puppet table")
+async def upgrade_v18(conn: Connection) -> None:
+    await conn.execute(
+        "ALTER TABLE puppet ADD COLUMN contact_info_set BOOLEAN NOT NULL DEFAULT false"
+    )

mautrix_telegram/example-config.yaml

@@ -40,7 +40,7 @@ appservice:
 
     # The full URI to the database. SQLite and Postgres are supported.
     # Format examples:
-    #   SQLite:   sqlite:///filename.db
+    #   SQLite:   sqlite:filename.db
     #   Postgres: postgres://username:password@hostname/dbname
     database: postgres://username:password@hostname/dbname
     # Additional arguments for asyncpg.create_pool() or sqlite3.connect()
@@ -274,6 +274,27 @@ bridge:
         # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
         # You must use a client that supports requesting keys from other users to use this feature.
         allow_key_sharing: false
+        # Options for deleting megolm sessions from the bridge.
+        delete_keys:
+            # Beeper-specific: delete outbound sessions when hungryserv confirms
+            # that the user has uploaded the key to key backup.
+            delete_outbound_on_ack: false
+            # Don't store outbound sessions in the inbound table.
+            dont_store_outbound: false
+            # Ratchet megolm sessions forward after decrypting messages.
+            ratchet_on_decrypt: false
+            # Delete fully used keys (index >= max_messages) after decrypting messages.
+            delete_fully_used_on_decrypt: false
+            # Delete previous megolm sessions from same device when receiving a new one.
+            delete_prev_on_new_session: false
+            # Delete megolm sessions received from a device when the device is deleted.
+            delete_on_device_delete: false
+            # Periodically delete megolm sessions when 2x max_age has passed since receiving the session.
+            periodically_delete_expired: false
+            # Delete inbound megolm sessions that don't have the received_at field used for
+            # automatic ratcheting and expired session deletion. This is meant as a migration
+            # to delete old keys prior to the bridge update.
+            delete_outdated_inbound: false
         # What level of device verification should be required from users?
         #
         # Valid levels:
@@ -309,9 +330,20 @@ bridge:
             # default.
             messages: 100
 
-    # Whether or not to explicitly set the avatar and room name for private
-    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
-    private_chat_portal_meta: false
+            # Disable rotating keys when a user's devices change?
+            # You should not enable this option unless you understand all the implications.
+            disable_device_change_key_rotation: false
+
+    # Whether to explicitly set the avatar and room name for private chat portal rooms.
+    # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms.
+    # If set to `always`, all DM rooms will have explicit names and avatars set.
+    # If set to `never`, DM rooms will never have names and avatars set.
+    private_chat_portal_meta: default
+    # Disable generating reply fallbacks? Some extremely bad clients still rely on them,
+    # but they're being phased out and will be completely removed in the future.
+    disable_reply_fallbacks: false
+    # Should cross-chat replies from Telegram be bridged? Most servers and clients don't support this.
+    cross_room_replies: false
     # Whether or not the bridge should send a read receipt from the bridge bot when a message has
     # been sent to Telegram.
     delivery_receipts: false
@@ -347,20 +379,6 @@ bridge:
     backfill:
         # Allow backfilling at all?
         enable: true
-        # Use MSC2716 for backfilling?
-        #
-        # This requires a server with MSC2716 support, which is currently an experimental feature in Synapse.
-        # It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml.
-        msc2716: false
-        # Use double puppets for backfilling?
-        #
-        # If using MSC2716, the double puppets must be in the appservice's user ID namespace
-        # (because the bridge can't use the double puppet access token with batch sending).
-        #
-        # Even without MSC2716, bridging old messages with correct timestamps requires the double
-        # puppets to be in an appservice namespace, or the server to be modified to allow
-        # overriding timestamps anyway.
-        double_puppet_backfill: false
         # Whether or not to enable backfilling in normal groups.
         # Normal groups have numerous technical problems in Telegram, and backfilling normal groups
         # will likely cause problems if there are multiple Matrix users in the group.
@@ -370,17 +388,26 @@ bridge:
         # Set to -1 to let any chat be unread.
         unread_hours_threshold: 720
 
-        # Forward backfilling limits. These apply to both MSC2716 and legacy backfill.
+        # Forward backfilling limits.
         #
         # Using a negative initial limit is not recommended, as it would try to backfill everything in a single batch.
-        # MSC2716 and the incremental settings are meant for backfilling everything incrementally rather than at once.
-        forward:
+        forward_limits:
             # Number of messages to backfill immediately after creating a portal.
-            initial_limit: 10
+            initial:
+                user: 50
+                normal_group: 100
+                supergroup: 10
+                channel: 10
             # Number of messages to backfill when syncing chats.
-            sync_limit: 100
+            sync:
+                user: 100
+                normal_group: 100
+                supergroup: 100
+                channel: 100
+        # Timeout for forward backfills in seconds. If you have a high limit, you'll have to increase this too.
+        forward_timeout: 900
 
-        # Settings for incremental backfill of history. These only apply when using MSC2716.
+        # Settings for incremental backfill of history. These only apply to Beeper, as upstream abandoned MSC2716.
         incremental:
             # Maximum number of messages to backfill per batch.
             messages_per_batch: 100
@@ -458,7 +485,6 @@ bridge:
     # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
     # `filter-mode` management commands.
     #
-    # Filters do not affect direct chats.
     # An empty blacklist will essentially disable the filter.
     filter:
         # Filter mode to use. Either "blacklist" or "whitelist".
@@ -467,6 +493,11 @@ bridge:
         mode: blacklist
         # The list of group/channel IDs to filter.
         list: []
+        # How to handle direct chats:
+        # If users is "null", direct chats will follow the previous settings.
+        # If users is "true", direct chats will always be bridged.
+        # If users is "false", direct chats will never be bridged.
+        users: true
 
     # The prefix for commands. Only required in non-management rooms.
     command_prefix: "!tg"
@@ -567,6 +598,8 @@ telegram:
         # is not recommended, since some requests can always trigger a call fail (such as searching
         # for messages).
         request_retries: 5
+        # Use IPv6 for Telethon connection
+        use_ipv6: false
 
     # Device info sent to Telegram.
     device_info:

mautrix_telegram/formatter/__init__.py

@@ -1,2 +1,2 @@
 from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram
-from .from_telegram import telegram_to_matrix
+from .from_telegram import telegram_text_to_matrix_html, telegram_to_matrix

mautrix_telegram/formatter/from_matrix/parser.py

@@ -82,14 +82,6 @@ class MatrixParser(BaseMatrixParser[TelegramMessage]):
         prefix = "#" * length + " "
         return TelegramMessage.join(children, "").prepend(prefix).format(TelegramEntityType.BOLD)
 
-    async def blockquote_to_fstring(
-        self, node: HTMLNode, ctx: RecursionContext
-    ) -> TelegramMessage:
-        msg = await self.tag_aware_parse_node(node, ctx)
-        children = msg.trim().split("\n")
-        children = [child.prepend("> ") for child in children]
-        return TelegramMessage.join(children, "\n")
-
     async def color_to_fstring(self, msg: TelegramMessage, color: str) -> TelegramMessage:
         return msg
 

mautrix_telegram/formatter/from_telegram.py

@@ -176,6 +176,21 @@ async def _convert_custom_emoji(
                 entities[i] = ReuploadedCustomEmoji(entity, custom_emojis[entity.document_id])
 
 
+async def telegram_text_to_matrix_html(
+    source: au.AbstractUser,
+    text: str,
+    entities: list[TypeMessageEntity],
+    client: MautrixTelegramClient | None = None,
+) -> str:
+    if not entities:
+        return escape(text).replace("\n", "<br/>")
+    await _convert_custom_emoji(source, entities, client=client)
+    text = add_surrogate(text)
+    html = await _telegram_entities_to_matrix_catch(text, entities)
+    html = del_surrogate(html)
+    return html
+
+
 async def telegram_to_matrix(
     evt: Message | SponsoredMessage,
     source: au.AbstractUser,
@@ -192,10 +207,10 @@ async def telegram_to_matrix(
     )
     entities = override_entities or evt.entities
     if entities:
-        await _convert_custom_emoji(source, entities, client=client)
         content.format = Format.HTML
-        html = await _telegram_entities_to_matrix_catch(add_surrogate(content.body), entities)
-        content.formatted_body = del_surrogate(html)
+        content.formatted_body = await telegram_text_to_matrix_html(
+            source, content.body, entities, client=client
+        )
 
     if require_html:
         content.ensure_has_html()
@@ -333,7 +348,11 @@ async def _telegram_entities_to_matrix(
         last_offset = relative_offset + (0 if skip_entity else entity.length)
     html.append(text_to_html(text[last_offset:]))
 
-    return "".join(html)
+    html_string = "".join(html)
+    # Remove redundant <br>'s after block tags
+    html_string = html_string.replace("</blockquote><br/>", "</blockquote>")
+    html_string = html_string.replace("</pre><br/>", "</pre>")
+    return html_string
 
 
 def _parse_pre(html: list[str], entity_text: str, language: str) -> bool:

mautrix_telegram/matrix.py

@@ -61,21 +61,6 @@ class MatrixHandler(BaseMatrixHandler):
 
         self._previously_typing = {}
 
-    async def check_versions(self) -> None:
-        await super().check_versions()
-        if self.config["bridge.backfill.msc2716"] and not (
-            support := self.versions.supports("org.matrix.msc2716")
-        ):
-            self.log.fatal(
-                "Backfilling with MSC2716 is enabled in bridge config, but "
-                + (
-                    "batch sending is not enabled on homeserver"
-                    if support is False
-                    else "homeserver does not support batch sending"
-                )
-            )
-            sys.exit(18)
-
     async def handle_puppet_group_invite(
         self,
         room_id: RoomID,
@@ -135,20 +120,8 @@ class MatrixHandler(BaseMatrixHandler):
             levels.users[self.az.bot_mxid] = 100 if invited_by_level >= 100 else invited_by_level
             await double_puppet.intent.set_power_levels(room_id, levels)
 
-        invites, errors = await portal.get_telegram_users_in_matrix_room(
-            invited_by, pre_create=True
-        )
-        if len(errors) > 0:
-            error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
-            await portal.az.intent.send_notice(
-                room_id,
-                f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
-                "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
-                "those users.",
-            )
-
         try:
-            await portal.create_telegram_chat(invited_by, invites=invites, supergroup=True)
+            await portal.create_telegram_chat(invited_by, supergroup=True)
         except ValueError as e:
             await portal.delete()
             await portal.az.intent.send_notice(room_id, e.args[0])
@@ -186,7 +159,8 @@ class MatrixHandler(BaseMatrixHandler):
             await portal.main_intent.kick_user(
                 room_id,
                 user.mxid,
-                "This chat does not have a bot relaying messages for unauthenticated users.",
+                "This chat does not have a bot on the Telegram side for relaying messages sent by"
+                " unauthenticated Matrix users.",
             )
             return
 

mautrix_telegram/portal_util/message_convert.py

@@ -34,6 +34,8 @@ from telethon.tl.types import (
     DocumentAttributeVideo,
     Game,
     InputPhotoFileLocation,
+    InputStickerSetID,
+    InputStickerSetShortName,
     Message,
     MessageEntityPre,
     MessageMediaContact,
@@ -42,11 +44,14 @@ from telethon.tl.types import (
     MessageMediaGame,
     MessageMediaGeo,
     MessageMediaGeoLive,
+    MessageMediaInvoice,
     MessageMediaPhoto,
     MessageMediaPoll,
+    MessageMediaStory,
     MessageMediaUnsupported,
     MessageMediaVenue,
     MessageMediaWebPage,
+    MessageReplyStoryHeader,
     PeerChannel,
     PeerUser,
     Photo,
@@ -104,6 +109,7 @@ class DocAttrs(NamedTuple):
     mime_type: str | None
     is_sticker: bool
     sticker_alt: str | None
+    sticker_pack_ref: dict | None
     width: int
     height: int
     is_gif: bool
@@ -142,6 +148,8 @@ class TelegramMessageConverter:
             MessageMediaUnsupported: self._convert_unsupported,
             MessageMediaGame: self._convert_game,
             MessageMediaContact: self._convert_contact,
+            MessageMediaStory: self._convert_story,
+            MessageMediaInvoice: self._convert_invoice,
         }
         self._allowed_media = tuple(self._media_converters.keys())
 
@@ -252,20 +260,48 @@ class TelegramMessageConverter:
     ) -> None:
         if not evt.reply_to:
             return
+        elif isinstance(evt.reply_to, MessageReplyStoryHeader):
+            return
+
+        if evt.reply_to.quote and content.msgtype.is_text:
+            content.ensure_has_html()
+            quote_html = await formatter.telegram_text_to_matrix_html(
+                source, evt.reply_to.quote_text, evt.reply_to.quote_entities
+            )
+            content.formatted_body = (
+                f"<blockquote data-telegram-partial-reply>{quote_html}</blockquote>"
+                f"{content.formatted_body}"
+            )
+
         space = (
             evt.peer_id.channel_id
             if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
             else source.tgid
         )
+        if evt.reply_to.reply_to_peer_id and evt.reply_to.reply_to_peer_id != evt.peer_id:
+            if not self.config["bridge.cross_room_replies"]:
+                return
+            space = (
+                evt.reply_to.reply_to_peer_id.channel_id
+                if isinstance(evt.reply_to.reply_to_peer_id, PeerChannel)
+                else source.tgid
+            )
+
         reply_to_id = TelegramID(evt.reply_to.reply_to_msg_id)
         msg = await DBMessage.get_one_by_tgid(reply_to_id, space)
-        if not msg or msg.mx_room != self.portal.mxid:
+        no_fallback = no_fallback or self.config["bridge.disable_reply_fallbacks"]
+        if not msg:
+            # TODO try to find room ID when generating deterministic ID for cross-room reply
             if deterministic_id:
                 content.set_reply(self.deterministic_event_id(space, reply_to_id))
             return
+        elif msg.mx_room != self.portal.mxid and not self.config["bridge.cross_room_replies"]:
+            return
         elif not isinstance(content, TextMessageEventContent) or no_fallback:
             # Not a text message, just set the reply metadata and return
             content.set_reply(msg.mxid)
+            if msg.mx_room != self.portal.mxid:
+                content.relates_to.in_reply_to["room_id"] = msg.mx_room
             return
 
         # Text message, try to fetch original message to generate reply fallback.
@@ -280,6 +316,8 @@ class TelegramMessageConverter:
         except Exception:
             self.log.exception("Failed to get event to add reply fallback")
             content.set_reply(msg.mxid)
+        if msg.mx_room != self.portal.mxid:
+            content.relates_to.in_reply_to["room_id"] = msg.mx_room
 
     @staticmethod
     def _photo_size_key(photo: TypePhotoSize) -> int:
@@ -428,9 +466,21 @@ class TelegramMessageConverter:
         return ConvertedMessage(
             content=content,
             caption=caption_content,
-            disappear_seconds=media.ttl_seconds,
+            disappear_seconds=self._adjust_ttl(media.ttl_seconds),
         )
 
+    @staticmethod
+    def _adjust_ttl(ttl: int | None) -> int | None:
+        if not ttl:
+            return None
+        elif ttl == 2147483647:
+            # View-once media, set low TTL
+            return 15
+        else:
+            # Increase media TTL because it's supposed to be counted from opening the media,
+            # but we can only count it from read receipt.
+            return ttl * 5
+
     async def _convert_document(
         self,
         source: au.AbstractUser,
@@ -496,6 +546,7 @@ class TelegramMessageConverter:
                 info["fi.mau.telegram.gif"] = True
             else:
                 info["fi.mau.telegram.animated_sticker"] = True
+            info["fi.mau.gif"] = True
             info["fi.mau.loop"] = True
             info["fi.mau.autoplay"] = True
             info["fi.mau.hide_controls"] = True
@@ -536,7 +587,7 @@ class TelegramMessageConverter:
             type=event_type,
             content=content,
             caption=caption_content,
-            disappear_seconds=evt.media.ttl_seconds,
+            disappear_seconds=self._adjust_ttl(evt.media.ttl_seconds),
         )
 
     @staticmethod
@@ -698,10 +749,33 @@ class TelegramMessageConverter:
             )
         return ConvertedMessage(content=content)
 
+    @staticmethod
+    async def _convert_story(
+        source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
+    ) -> ConvertedMessage:
+        content = await formatter.telegram_to_matrix(
+            evt, source, client, override_text="Stories are not yet supported"
+        )
+        content.msgtype = MessageType.NOTICE
+        content["fi.mau.telegram.unsupported"] = True
+        return ConvertedMessage(content=content)
+
+    @staticmethod
+    async def _convert_invoice(
+        source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
+    ) -> ConvertedMessage:
+        content = await formatter.telegram_to_matrix(
+            evt, source, client, override_text="Invoices are not yet supported"
+        )
+        content.msgtype = MessageType.NOTICE
+        content["fi.mau.telegram.unsupported"] = True
+        return ConvertedMessage(content=content)
+
 
 def _parse_document_attributes(attributes: list[TypeDocumentAttribute]) -> DocAttrs:
     name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
     is_gif, is_audio, is_voice, duration, waveform = False, False, False, 0, bytes()
+    sticker_pack_ref = None
     for attr in attributes:
         if isinstance(attr, DocumentAttributeFilename):
             name = name or attr.file_name
@@ -709,6 +783,13 @@ def _parse_document_attributes(attributes: list[TypeDocumentAttribute]) -> DocAt
         elif isinstance(attr, DocumentAttributeSticker):
             is_sticker = True
             sticker_alt = attr.alt
+            if isinstance(attr.stickerset, InputStickerSetID):
+                sticker_pack_ref = {
+                    "id": str(attr.stickerset.id),
+                    "access_hash": str(attr.stickerset.access_hash),
+                }
+            elif isinstance(attr.stickerset, InputStickerSetShortName):
+                sticker_pack_ref = {"short_name": attr.stickerset.short_name}
         elif isinstance(attr, DocumentAttributeAnimated):
             is_gif = True
         elif isinstance(attr, DocumentAttributeVideo):
@@ -722,17 +803,18 @@ def _parse_document_attributes(attributes: list[TypeDocumentAttribute]) -> DocAt
             waveform = decode_waveform(attr.waveform) if attr.waveform else b""
 
     return DocAttrs(
-        name,
-        mime_type,
-        is_sticker,
-        sticker_alt,
-        width,
-        height,
-        is_gif,
-        is_audio,
-        is_voice,
-        duration,
-        waveform,
+        name=name,
+        mime_type=mime_type,
+        is_sticker=is_sticker,
+        sticker_alt=sticker_alt,
+        sticker_pack_ref=sticker_pack_ref,
+        width=width,
+        height=height,
+        is_gif=is_gif,
+        is_audio=is_audio,
+        is_voice=is_voice,
+        duration=duration,
+        waveform=waveform,
     )
 
 
@@ -758,6 +840,13 @@ def _parse_document_meta(
         mime_type = file.mime_type or document.mime_type
     info = ImageInfo(size=file.size, mimetype=mime_type)
 
+    if attrs.is_sticker:
+        info["fi.mau.telegram.sticker"] = {
+            "alt": attrs.sticker_alt,
+            "id": str(document.id),
+            "pack": attrs.sticker_pack_ref,
+        }
+
     if attrs.mime_type and not file.was_converted:
         file.mime_type = attrs.mime_type or file.mime_type
     if file.width and file.height:
@@ -777,6 +866,10 @@ def _parse_document_meta(
             size=file.thumbnail.size,
         )
     elif attrs.is_sticker:
+        if not info.width or not info.height:
+            info.width = 256
+            info.height = 256
+
         # This is a hack for bad clients like Element iOS that require a thumbnail
         info.thumbnail_info = ImageInfo.deserialize(info.serialize())
         if file.decryption_info:

mautrix_telegram/puppet.py

@@ -55,6 +55,7 @@ if TYPE_CHECKING:
 
 
 class Puppet(DBPuppet, BasePuppet):
+    bridge: TelegramBridge
     config: Config
     hs_domain: str
     mxid_template: SimpleTemplate[TelegramID]
@@ -78,6 +79,7 @@ class Puppet(DBPuppet, BasePuppet):
         avatar_url: ContentURI | None = None,
         name_set: bool = False,
         avatar_set: bool = False,
+        contact_info_set: bool = False,
         is_bot: bool = False,
         is_channel: bool = False,
         is_premium: bool = False,
@@ -100,6 +102,7 @@ class Puppet(DBPuppet, BasePuppet):
             avatar_url=avatar_url,
             name_set=name_set,
             avatar_set=avatar_set,
+            contact_info_set=contact_info_set,
             is_bot=is_bot,
             is_channel=is_channel,
             is_premium=is_premium,
@@ -154,6 +157,7 @@ class Puppet(DBPuppet, BasePuppet):
 
     @classmethod
     def init_cls(cls, bridge: "TelegramBridge") -> AsyncIterable[Awaitable[None]]:
+        cls.bridge = bridge
         cls.config = bridge.config
         cls.loop = bridge.loop
         cls.mx = bridge.matrix
@@ -265,11 +269,14 @@ class Puppet(DBPuppet, BasePuppet):
             is_bot != self.is_bot or is_channel != self.is_channel or is_premium != self.is_premium
         )
 
-        self.is_bot = is_bot
+        if is_bot is not None:
+            self.is_bot = is_bot
         self.is_channel = is_channel
-        self.is_premium = is_premium
+        if is_premium is not None:
+            self.is_premium = is_premium
 
-        if self.username != info.username:
+        if self.username != info.username and (info.username or not info.min):
+            self.log.debug(f"Updating username {self.username} -> {info.username}")
             self.username = info.username
             changed = True
 
@@ -279,6 +286,8 @@ class Puppet(DBPuppet, BasePuppet):
 
         if not self.disable_updates:
             try:
+                changed = await self._update_contact_info(force=changed) or changed
+
                 changed = (
                     await self.update_displayname(source, info, client_override=client_override)
                     or changed
@@ -296,8 +305,37 @@ class Puppet(DBPuppet, BasePuppet):
             await self.update_portals_meta()
             await self.save()
 
+    async def _update_contact_info(self, force: bool = False) -> bool:
+        if not self.bridge.homeserver_software.is_hungry:
+            return False
+
+        if self.contact_info_set and not force:
+            return False
+
+        try:
+            identifiers = []
+            if self.username:
+                identifiers.append(f"telegram:{self.username}")
+            if self.phone:
+                phone = "+" + self.phone.lstrip("+")
+                identifiers.append(f"tel:{phone}")
+            await self.default_mxid_intent.beeper_update_profile(
+                {
+                    "com.beeper.bridge.identifiers": identifiers,
+                    "com.beeper.bridge.remote_id": str(self.tgid),
+                    "com.beeper.bridge.service": "telegram",
+                    "com.beeper.bridge.network": "telegram",
+                    "com.beeper.bridge.is_network_bot": self.is_bot,
+                }
+            )
+            self.contact_info_set = True
+        except Exception:
+            self.log.exception("Error updating contact info")
+            self.contact_info_set = False
+        return True
+
     async def update_portals_meta(self) -> None:
-        if not p.Portal.private_chat_portal_meta and not self.mx.e2ee:
+        if p.Portal.private_chat_portal_meta != "always" and not self.mx.e2ee:
             return
         async for portal in p.Portal.find_private_chats_with(self.tgid):
             await portal.update_info_from_puppet(self)

mautrix_telegram/tgclient.py

@@ -22,6 +22,7 @@ from telethon.tl.patched import Message
 from telethon.tl.types import (
     InputMediaUploadedDocument,
     InputMediaUploadedPhoto,
+    InputReplyToMessage,
     TypeDocumentAttribute,
     TypeInputMedia,
     TypeInputPeer,
@@ -67,6 +68,10 @@ class MautrixTelegramClient(TelegramClient):
         entity = await self.get_input_entity(entity)
         reply_to = utils.get_message_id(reply_to)
         request = SendMediaRequest(
-            entity, media, message=caption or "", entities=entities or [], reply_to_msg_id=reply_to
+            entity,
+            media,
+            message=caption or "",
+            entities=entities or [],
+            reply_to=InputReplyToMessage(reply_to_msg_id=reply_to) if reply_to else None,
         )
         return self._get_response_message(request, await self(request), entity)

mautrix_telegram/user.py

@@ -23,6 +23,7 @@ import time
 from telethon.errors import (
     AuthKeyDuplicatedError,
     AuthKeyError,
+    AuthKeyNotFound,
     RPCError,
     TakeoutInitDelayError,
     UnauthorizedError,
@@ -39,6 +40,9 @@ from telethon.tl.types import (
     ChatForbidden,
     InputUserSelf,
     Message,
+    MessageActionContactSignUp,
+    MessageActionHistoryClear,
+    MessageService,
     NotifyPeer,
     PeerUser,
     TypeUpdate,
@@ -52,6 +56,7 @@ from telethon.tl.types import (
     User as TLUser,
 )
 from telethon.tl.types.contacts import ContactsNotModified
+from telethon.tl.types.help import AppConfig
 from telethon.tl.types.messages import AvailableReactions
 
 from mautrix.appservice import DOUBLE_PUPPET_SOURCE_KEY
@@ -106,6 +111,7 @@ class User(DBUser, AbstractUser, BaseUser):
     _available_emoji_reactions_fetched: float
     _available_emoji_reactions_lock: asyncio.Lock
     _app_config: dict[str, Any] | None
+    _app_config_hash: int
 
     def __init__(
         self,
@@ -143,6 +149,7 @@ class User(DBUser, AbstractUser, BaseUser):
         self._available_emoji_reactions_fetched = 0
         self._available_emoji_reactions_lock = asyncio.Lock()
         self._app_config = None
+        self._app_config_hash = 0
 
         (
             self.relaybot_whitelisted,
@@ -209,7 +216,7 @@ class User(DBUser, AbstractUser, BaseUser):
         async with self._ensure_started_lock:
             return cast(User, await super().ensure_started(even_if_no_session))
 
-    async def on_signed_out(self, err: UnauthorizedError | AuthKeyError) -> None:
+    async def on_signed_out(self, err: UnauthorizedError | AuthKeyError | AuthKeyNotFound) -> None:
         error_code = "tg-auth-error"
         if isinstance(err, AuthKeyDuplicatedError):
             error_code = "tg-auth-key-duplicated"
@@ -230,8 +237,8 @@ class User(DBUser, AbstractUser, BaseUser):
     async def start(self, delete_unless_authenticated: bool = False) -> User:
         try:
             await super().start()
-        except AuthKeyDuplicatedError as e:
-            self.log.warning("Got AuthKeyDuplicatedError in start()")
+        except (AuthKeyDuplicatedError, AuthKeyNotFound) as e:
+            self.log.warning(f"Got {type(e).__name__} in start()")
             await self.on_signed_out(e)
             if not delete_unless_authenticated:
                 # The caller wants the client to be connected, so restart the connection.
@@ -487,13 +494,16 @@ class User(DBUser, AbstractUser, BaseUser):
         self.log.info(f"Creating portal for {portal.tgid_log} as part of backfill loop")
         try:
             await portal.create_matrix_room(
-                self, client=client, update_if_exists=False, invites=[self.mxid]
+                self,
+                client=client,
+                update_if_exists=False,
+                invites=[self.mxid],
+                from_dialog_sync=True,
             )
         except Exception:
             self.log.exception(f"Error while creating {portal.tgid_log}")
         else:
-            puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
-            await self._post_sync_dialog(portal, puppet, was_created=True, **post_sync_args)
+            await self.post_sync_dialog(portal, puppet=None, was_created=True, **post_sync_args)
 
     async def update(self, update: TypeUpdate) -> bool:
         if not self.is_bot:
@@ -535,7 +545,7 @@ class User(DBUser, AbstractUser, BaseUser):
             await self.stop()
             return None
 
-    async def update_info(self, info: TLUser = None) -> None:
+    async def update_info(self, info: TLUser | None = None) -> None:
         if not info:
             info = await self.get_me()
             if not info:
@@ -608,8 +618,11 @@ class User(DBUser, AbstractUser, BaseUser):
             await self.stop()
             await sess.delete()
 
+        # Drop LOGGED_OUT states if the user was already logged out previously
+        # and doesn't have a remote ID anymore
         # TODO send a management room notice for non-manual logouts?
-        await self.push_bridge_state(state, error=error, message=message)
+        if self.tgid or state != BridgeStateEvent.LOGGED_OUT:
+            await self.push_bridge_state(state, error=error, message=message)
         if delete:
             await self.delete()
             self.by_mxid.pop(self.mxid, None)
@@ -743,12 +756,12 @@ class User(DBUser, AbstractUser, BaseUser):
         )
         await self._mute_room(puppet, portal, update.notify_settings.mute_until.timestamp())
 
-    async def _sync_dialog(
-        self, portal: po.Portal, dialog: Dialog, should_create: bool, puppet: pu.Puppet | None
-    ) -> None:
-        was_created = False
-        post_sync_args = {
-            "last_message_ts": cast(datetime, dialog.date).timestamp(),
+    @staticmethod
+    def dialog_to_sync_args(dialog: Dialog) -> dict:
+        return {
+            "last_message_ts": (
+                cast(datetime, dialog.date).timestamp() if dialog.date else time.time()
+            ),
             "unread_count": dialog.unread_count,
             "max_read_id": dialog.dialog.read_inbox_max_id,
             "mute_until": (
@@ -759,6 +772,24 @@ class User(DBUser, AbstractUser, BaseUser):
             "pinned": dialog.pinned,
             "archived": dialog.archived,
         }
+
+    async def _sync_dialog(
+        self, portal: po.Portal, dialog: Dialog, should_create: bool, puppet: pu.Puppet | None
+    ) -> None:
+        if (
+            not portal.mxid
+            and isinstance(dialog.message, MessageService)
+            and isinstance(
+                dialog.message.action, (MessageActionContactSignUp, MessageActionHistoryClear)
+            )
+        ):
+            self.log.debug(
+                f"Not syncing {portal.tgid_log} "
+                f"(last message is a {type(dialog.message.action).__name__})"
+            )
+            return
+        was_created = False
+        post_sync_args = self.dialog_to_sync_args(dialog)
         if portal.mxid:
             self.log.debug(f"Backfilling and updating {portal.tgid_log} (dialog sync)")
             try:
@@ -772,7 +803,9 @@ class User(DBUser, AbstractUser, BaseUser):
         elif should_create:
             self.log.debug(f"Creating portal for {portal.tgid_log} immediately (dialog sync)")
             try:
-                await portal.create_matrix_room(self, dialog.entity, invites=[self.mxid])
+                await portal.create_matrix_room(
+                    self, dialog.entity, invites=[self.mxid], from_dialog_sync=True
+                )
                 was_created = True
             except Exception:
                 self.log.exception(f"Error while creating {portal.tgid_log}")
@@ -785,7 +818,7 @@ class User(DBUser, AbstractUser, BaseUser):
                 extra_data=post_sync_args,
             )
         if portal.mxid and puppet and puppet.is_real_user:
-            await self._post_sync_dialog(
+            await self.post_sync_dialog(
                 portal=portal,
                 puppet=puppet,
                 was_created=was_created,
@@ -793,10 +826,10 @@ class User(DBUser, AbstractUser, BaseUser):
             )
         self.log.debug(f"_sync_dialog finished for {portal.tgid_log}")
 
-    async def _post_sync_dialog(
+    async def post_sync_dialog(
         self,
         portal: po.Portal,
-        puppet: pu.Puppet,
+        puppet: pu.Puppet | None,
         was_created: bool,
         max_read_id: int,
         last_message_ts: float,
@@ -805,6 +838,10 @@ class User(DBUser, AbstractUser, BaseUser):
         pinned: bool,
         archived: bool,
     ) -> None:
+        if puppet is None:
+            puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+            if not puppet or not puppet.is_real_user:
+                return
         self.log.debug(
             f"Running dialog post-sync for {portal.tgid_log} with args "
             f"{was_created=}, {max_read_id=}, {last_message_ts=}, {unread_count=}, "
@@ -941,11 +978,18 @@ class User(DBUser, AbstractUser, BaseUser):
         self.log.debug("Contact syncing complete")
         return contacts
 
+    @property
+    def _available_reactions_up_to_date(self) -> bool:
+        return (
+            bool(self._available_emoji_reactions)
+            and self._available_emoji_reactions_fetched + 12 * 60 * 60 > time.monotonic()
+        )
+
     async def get_available_reactions(self) -> set[str]:
-        if self._available_emoji_reactions_fetched + 12 * 60 * 60 > time.monotonic():
+        if self._available_reactions_up_to_date:
             return self._available_emoji_reactions
         async with self._available_emoji_reactions_lock:
-            if self._available_emoji_reactions_fetched + 12 * 60 * 60 > time.monotonic():
+            if self._available_reactions_up_to_date:
                 return self._available_emoji_reactions
             self.log.debug("Fetching available emoji reactions")
             available_reactions = await self.client(
@@ -955,13 +999,18 @@ class User(DBUser, AbstractUser, BaseUser):
                 self._available_emoji_reactions = {
                     react.reaction
                     for react in available_reactions.reactions
-                    if self.is_premium or not react.premium
+                    if not react.inactive and (self.is_premium or not react.premium)
                 }
                 self._available_emoji_reactions_hash = available_reactions.hash
                 self._available_emoji_reactions_fetched = time.monotonic()
                 self.log.debug(
                     "Got available emoji reactions: %s", self._available_emoji_reactions
                 )
+            elif self._available_emoji_reactions is None:
+                self.log.warning(
+                    f"Got {available_reactions} in response to available reactions request"
+                    " even though nothing is cached"
+                )
             return self._available_emoji_reactions
 
     def tl_to_json(self) -> Any:
@@ -969,8 +1018,9 @@ class User(DBUser, AbstractUser, BaseUser):
 
     async def get_app_config(self) -> dict[str, Any]:
         if not self._app_config:
-            cfg = await self.client(GetAppConfigRequest())
-            self._app_config = util.parse_tl_json(cfg)
+            cfg: AppConfig = await self.client(GetAppConfigRequest(hash=self._app_config_hash))
+            self._app_config = util.parse_tl_json(cfg.config)
+            self._app_config_hash = cfg.hash
         return self._app_config
 
     async def get_max_reactions(self, is_premium: bool | None = None) -> int:

mautrix_telegram/web/common/auth_api.py

@@ -35,6 +35,7 @@ from telethon.errors import (
     PhoneNumberInvalidError,
     PhoneNumberUnoccupiedError,
     SessionPasswordNeededError,
+    SessionRevokedError,
 )
 
 from mautrix.bridge import InvalidAccessToken, OnlyLoginSelf
@@ -288,6 +289,17 @@ class AuthAPI(abc.ABC):
                 errcode="phone_number_unoccupied",
                 error="That phone number has not been registered.",
             )
+        except FloodWaitError as e:
+            return self.get_login_response(
+                mxid=user.mxid,
+                state="code",
+                status=429,
+                errcode="flood_wait",
+                error=(
+                    "You tried to enter your phone code too many times. "
+                    f"Please wait for {format_duration(e.seconds)} before trying again."
+                ),
+            )
         except SessionPasswordNeededError:
             if not password_in_data:
                 if user.command_status and user.command_status["action"] == "Login":
@@ -342,6 +354,28 @@ class AuthAPI(abc.ABC):
                 errcode="password_invalid",
                 error="Incorrect password.",
             )
+        except SessionRevokedError:
+            return self.get_login_response(
+                mxid=user.mxid,
+                state="request",
+                status=401,
+                errcode="session_revoked",
+                error=(
+                    "Please try again. Login cancelled because your other sessions were "
+                    "terminated via the Telegram app."
+                ),
+            )
+        except FloodWaitError as e:
+            return self.get_login_response(
+                mxid=user.mxid,
+                state="password",
+                status=429,
+                errcode="flood_wait",
+                error=(
+                    "You tried to enter your password too many times. "
+                    f"Please wait for {format_duration(e.seconds)} before trying again."
+                ),
+            )
         except Exception as e:
             self.log.exception("Error sending password")
             if isinstance(e, ValueError) and "You must provide a phone and a code" in str(e):
@@ -357,5 +391,5 @@ class AuthAPI(abc.ABC):
                 state="password",
                 status=500,
                 errcode="unknown_error",
-                error="Internal server error while sending password.",
+                error=f"Internal server error while sending password. {e}",
             )

optional-requirements.txt

@@ -7,15 +7,14 @@ aiodns
 brotli
 
 #/qr_login
-pillow>=4,<10
+pillow>=10.0.1,<11
 qrcode>=6,<8
 
-
 #/formattednumbers
 phonenumbers>=8,<9
 
 #/metrics
-prometheus_client>=0.6,<0.17
+prometheus_client>=0.6,<0.19
 
 #/e2be
 python-olm>=3,<4
@@ -23,4 +22,7 @@ pycryptodome>=3,<4
 unpaddedbase64>=1,<3
 
 #/sqlite
-aiosqlite>=0.16,<0.19
+aiosqlite>=0.16,<0.20
+
+#/proxy
+python-socks[asyncio]

requirements.txt

@@ -1,11 +1,10 @@
-ruamel.yaml>=0.15.35,<0.18
+ruamel.yaml>=0.15.35,<0.19
 python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.19.4,<0.20
-#telethon>=1.25.4,<1.27
-tulir-telethon==1.28.0a3
-asyncpg>=0.20,<0.28
+mautrix>=0.20.3,<0.21
+tulir-telethon==1.33.0a1
+asyncpg>=0.20,<0.30
 mako>=1,<2
 setuptools

setup.py

@@ -51,7 +51,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.8",
+    python_requires="~=3.9",
 
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -60,9 +60,10 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     package_data={"mautrix_telegram": [
         "web/public/*.mako", "web/public/*.png", "web/public/*.css",