Bump version to 0.11.2

link previews: support from Telegram -> Beeper

.editorconfig

@@ -8,11 +8,14 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 
+[*.md]
+trim_trailing_whitespace = false
+
 [*.py]
 max_line_length = 99
 
 [*.{yaml,yml,py}]
 indent_style = space
 
-[.gitlab-ci.yml]
+[{.gitlab-ci.yml,.pre-commit-config.yaml}]
 indent_size = 2

.github/workflows/python-lint.yml

@@ -13,6 +13,14 @@ jobs:
     - uses: isort/isort-action@master
       with:
         sortPaths: "./mautrix_telegram"
-    - uses: psf/black@21.12b0
+    - uses: psf/black@stable
       with:
         src: "./mautrix_telegram"
+        version: "22.1.0"
+    - name: pre-commit
+      run: |
+        pip install pre-commit
+        pre-commit run -av trailing-whitespace
+        pre-commit run -av end-of-file-fixer
+        pre-commit run -av check-yaml
+        pre-commit run -av check-added-large-files

.pre-commit-config.yaml

@@ -0,0 +1,23 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+      - id: trailing-whitespace
+        exclude_types: [markdown]
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+  # TODO convert to use the upstream psf/black when
+  # https://github.com/psf/black/issues/2493 gets fixed
+  - repo: local
+    hooks:
+      - id: black
+        name: black
+        entry: black --check
+        language: system
+        files: ^mautrix_telegram/.*\.py$
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        files: ^mautrix_telegram/.*$

CHANGELOG.md

@@ -1,8 +1,49 @@
-# (unreleased)
+# v0.11.2 (2022-02-14)
 
-* Added support for message reactions
+**N.B.** This will be the last release to support Python 3.7. Future versions
+will require Python 3.8 or higher. In general, the mautrix bridges will only
+support the lowest Python version in the latest Debian or Ubuntu LTS.
+
+### Added
+* Added simple fallback message for live location and venue messages from Telegram.
+* Added support for `t.me/+code` style invite links in `!tg join`.
+* Added support for showing channel profile when users send messages as a channel.
+* Added "user joined Telegram" message when Telegram auto-creates a DM chat for
+  a new user.
+
+### Improved
+* Added option for adding a random prefix to relayed user displaynames to help
+  distinguish them on the Telegram side.
+* Improved syncing profile info to room info when using encryption and/or the
+  `private_chat_profile_meta` config option.
+* Removed legacy `community_id` config option.
+
+### Fixed
+* Fixed newlines disappearing when bridging channel messages with signatures.
+* Fixed login throwing an error if a previous login code expired.
+* Fixed bug in v0.11.0 that broke `!tg create`.
+
+# v0.11.1 (2022-01-10)
+
+### Added
+* Added support for message reactions.
 * Added support for spoiler text.
 
+### Improved
+* Support for voice messages.
+* Changed color of blue text from Telegram to be more readable on dark themes.
+
+### Fixed
+* Fixed syncing contacts throwing an error for new accounts.
+* Fixed migrating pre-v0.11 legacy databases if the database schema had been
+  corrupted (e.g. by using 3rd party tools for SQLite -> Postgres migration).
+* Fixed converting animated stickers to webm with >33 FPS.
+* Fixed a bug in v0.11.0 that broke mentioning users in groups
+  (thanks to [@dfuchss] in [#724]).
+
+[@dfuchss]: https://github.com/dfuchss
+[#724]: https://github.com/mautrix/telegram/pull/724
+
 # v0.11.0 (2021-12-28)
 
 * Switched from SQLAlchemy to asyncpg/aiosqlite.

ROADMAP.md

@@ -3,6 +3,7 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message reactions
   * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
@@ -12,8 +13,8 @@
   * [x] Power level
     * [x] Normal chats
       * [ ] Non-hardcoded PL requirements
-	* [x] Supergroups/channels
-	  * [ ] Precise bridging (non-hardcoded PL requirements, bridge specific permissions, etc..)
+    * [x] Supergroups/channels
+      * [ ] Precise bridging (non-hardcoded PL requirements, bridge specific permissions, etc..)
   * [x] Membership actions (invite/kick/join/leave)
   * [x] Room metadata changes (name, topic, avatar)
   * [x] Initial room metadata
@@ -27,6 +28,7 @@
 	* [x] Games
 	* [ ] Buttons
   * [x] Message deletions
+  * [x] Message reactions
   * [x] Message edits
   * [x] Message history
     * [x] Manually (`!tg backfill`)
@@ -57,7 +59,7 @@
   * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
   * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
-  * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
+  * [x] End-to-bridge encryption in Matrix rooms (see [docs](https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html))
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

dev-requirements.txt

@@ -0,0 +1,3 @@
+pre-commit>=2.10.1,<3
+isort>=5.10.1,<6
+black==22.1.0

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.11.0"
+__version__ = "0.11.2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -86,6 +86,7 @@ class TelegramBridge(Bridge):
         Portal.init_cls(self)
         self.add_startup_actions(Puppet.init_cls(self))
         self.add_startup_actions(User.init_cls(self))
+        self.add_startup_actions(Portal.restart_scheduled_disappearing())
         if self.bot:
             self.add_startup_actions(self.bot.start())
         if self.config["bridge.resend_bridge_info"]:

mautrix_telegram/abstract_user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Type, Union
+from typing import TYPE_CHECKING, Any, Union
 from abc import ABC, abstractmethod
 import asyncio
 import logging
@@ -34,6 +34,7 @@ from telethon.tl.types import (
     Chat,
     MessageActionChannelMigrateFrom,
     MessageEmpty,
+    PeerChannel,
     PeerChat,
     PeerUser,
     TypeUpdate,
@@ -46,6 +47,7 @@ from telethon.tl.types import (
     UpdateEditChannelMessage,
     UpdateEditMessage,
     UpdateFolderPeers,
+    UpdateMessageReactions,
     UpdateNewChannelMessage,
     UpdateNewMessage,
     UpdateNotifySettings,
@@ -146,7 +148,7 @@ class AbstractUser(ABC):
         return self.client and self.client.is_connected()
 
     @property
-    def _proxy_settings(self) -> tuple[Type[Connection], tuple[Any, ...] | None]:
+    def _proxy_settings(self) -> tuple[type[Connection], tuple[Any, ...] | None]:
         proxy_type = self.config["telegram.proxy.type"].lower()
         connection = ConnectionTcpFull
         connection_data = (
@@ -312,6 +314,8 @@ class AbstractUser(ABC):
             await self.delete_message(update)
         elif isinstance(update, UpdateDeleteChannelMessages):
             await self.delete_channel_message(update)
+        elif isinstance(update, UpdateMessageReactions):
+            await self.update_reactions(update)
         elif isinstance(update, (UpdateChatUserTyping, UpdateChannelUserTyping, UpdateUserTyping)):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
@@ -382,7 +386,7 @@ class AbstractUser(ABC):
         if not message:
             return
 
-        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.peer.user_id))
+        puppet = await pu.Puppet.get_by_peer(update.peer)
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
     async def update_own_read_receipt(
@@ -441,10 +445,7 @@ class AbstractUser(ABC):
             return
 
         if isinstance(update, (UpdateChannelUserTyping, UpdateChatUserTyping)):
-            # Can typing notifications come from non-user peers?
-            if not update.from_id.user_id:
-                return
-            sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id.user_id))
+            sender = await pu.Puppet.get_by_peer(update.from_id)
 
         if not sender or not portal or not portal.mxid:
             return
@@ -453,8 +454,8 @@ class AbstractUser(ABC):
 
     async def _handle_entity_updates(self, entities: dict[int, User | Chat | Channel]) -> None:
         try:
-            users = (entity for entity in entities.values() if isinstance(entity, User))
-            puppets = ((await pu.Puppet.get_by_tgid(TelegramID(user.id)), user) for user in users)
+            users = (entity for entity in entities.values() if isinstance(entity, (User, Channel)))
+            puppets = ((await pu.Puppet.get_by_peer(user), user) for user in users)
             await asyncio.gather(
                 *[puppet.try_update_info(self, info) async for puppet, info in puppets if puppet]
             )
@@ -468,9 +469,11 @@ class AbstractUser(ABC):
             puppet.username = update.username
             if await puppet.update_displayname(self, update):
                 await puppet.save()
+                await puppet.update_portals_meta()
         elif isinstance(update, UpdateUserPhoto):
             if await puppet.update_avatar(self, update.photo):
                 await puppet.save()
+                await puppet.update_portals_meta()
         else:
             self.log.warning(f"Unexpected other user info update: {type(update)}")
 
@@ -512,8 +515,8 @@ class AbstractUser(ABC):
             portal = await po.Portal.get_by_entity(update.peer_id, tg_receiver=self.tgid)
             if update.out:
                 sender = await pu.Puppet.get_by_tgid(self.tgid)
-            elif isinstance(update.from_id, PeerUser):
-                sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id.user_id))
+            elif isinstance(update.from_id, (PeerUser, PeerChannel)):
+                sender = await pu.Puppet.get_by_peer(update.from_id)
             else:
                 sender = None
         else:
@@ -559,6 +562,12 @@ class AbstractUser(ABC):
                 await message.delete()
                 await self._try_redact(message)
 
+    async def update_reactions(self, update: UpdateMessageReactions) -> None:
+        portal = await po.Portal.get_by_entity(update.peer, tg_receiver=self.tgid)
+        if not portal or not portal.mxid or not portal.allow_bridging:
+            return
+        await portal.handle_telegram_reactions(self, TelegramID(update.msg_id), update.reactions)
+
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = await self.get_message_details(original_update)
         if not portal:

mautrix_telegram/bot.py

@@ -225,7 +225,11 @@ class Bot(AbstractUser):
         elif isinstance(message.to_id, PeerChat):
             return reply(str(-message.to_id.chat_id))
         elif isinstance(message.to_id, PeerUser):
-            return reply(f"Your user ID is {message.to_id.user_id}.")
+            return reply(
+                f"Your user ID is {message.to_id.user_id}.\n\n"
+                f"If you're trying to bridge a group chat to Matrix, you must run the command in "
+                f"the group, not here. **The ID above will not work** with `!tg bridge`."
+            )
         else:
             return reply("Failed to find chat ID.")
 

mautrix_telegram/commands/portal/create_chat.py

@@ -65,7 +65,7 @@ async def create(evt: CommandEvent) -> EventID:
         about=about,
         encrypted=encrypted,
     )
-    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender)
+    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender, pre_create=True)
     if len(errors) > 0:
         error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
         await evt.reply(

mautrix_telegram/commands/telegram/auth.py

@@ -37,6 +37,7 @@ from telethon.errors import (
 )
 from telethon.tl.types import User
 
+from mautrix.client import Client
 from mautrix.types import (
     EventID,
     ImageInfo,
@@ -230,9 +231,23 @@ async def login_qr(evt: CommandEvent) -> EventID:
 )
 async def login(evt: CommandEvent) -> EventID:
     override_sender = False
-    if len(evt.args) > 0 and evt.sender.is_admin:
-        evt.sender = await u.User.get_and_start_by_mxid(UserID(evt.args[0]))
+    if len(evt.args) > 0 and evt.sender.is_admin and evt.args[0]:
+        override_user_id = UserID(evt.args[0])
+        try:
+            Client.parse_user_id(override_user_id)
+        except ValueError:
+            return await evt.reply(
+                f"**Usage:** `$cmdprefix+sp login [override user ID]`\n\n"
+                f"{override_user_id!r} is not a valid Matrix user ID"
+            )
+        orig_user_id = evt.sender.mxid
+        evt.sender = await u.User.get_and_start_by_mxid(override_user_id)
         override_sender = True
+        if orig_user_id != evt.sender:
+            await evt.reply(
+                f"Admin override: logging in as {evt.sender.mxid} instead of {orig_user_id}"
+            )
+
     if await evt.sender.is_logged_in():
         return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 

mautrix_telegram/commands/telegram/misc.py

@@ -208,6 +208,9 @@ async def join(evt: CommandEvent) -> EventID | None:
     link_type = data["type"]
     if link_type:
         link_type = link_type.lower()
+    elif identifier.startswith("+"):
+        link_type = "joinchat"
+        identifier = identifier[1:]
     updates, _ = await _join(evt, identifier, link_type)
     if not updates:
         return None

mautrix_telegram/config.py

@@ -95,8 +95,6 @@ class Config(BaseBridgeConfig):
         if "pool_pre_ping" in base["appservice.database_opts"]:
             del base["appservice.database_opts.pool_pre_ping"]
 
-        copy("appservice.community_id")
-
         copy("metrics.enabled")
         copy("metrics.listen_port")
 
@@ -138,11 +136,14 @@ class Config(BaseBridgeConfig):
         copy("bridge.invite_link_resolve")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
+        copy("bridge.image_as_file_pixels")
         copy("bridge.max_document_size")
         copy("bridge.parallel_file_transfer")
         copy("bridge.federate_rooms")
         copy("bridge.animated_sticker.target")
-        copy("bridge.animated_sticker.args")
+        copy("bridge.animated_sticker.args.width")
+        copy("bridge.animated_sticker.args.height")
+        copy("bridge.animated_sticker.args.fps")
         copy("bridge.encryption.allow")
         copy("bridge.encryption.default")
         copy("bridge.encryption.database")
@@ -171,20 +172,16 @@ class Config(BaseBridgeConfig):
 
         copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):
-            base["bridge.bridge_notices"] = {
-                "default": self["bridge.bridge_notices"],
-                "exceptions": ["@importantbot:example.com"],
-            }
+            base["bridge.bridge_notices"]["default"] = self["bridge.bridge_notices"]
         else:
-            copy("bridge.bridge_notices")
-
-        copy("bridge.deduplication.pre_db_check")
-        copy("bridge.deduplication.cache_queue_length")
+            copy("bridge.bridge_notices.default")
+            copy("bridge.bridge_notices.exceptions")
 
         if "bridge.message_formats.m_text" in self:
             del self["bridge.message_formats"]
         copy_dict("bridge.message_formats", override_existing_map=False)
         copy("bridge.emote_format")
+        copy("bridge.relay_user_distinguishers")
 
         copy("bridge.state_event_formats.join")
         copy("bridge.state_event_formats.leave")
@@ -208,7 +205,7 @@ class Config(BaseBridgeConfig):
                 permissions[entry] = "admin"
             base["bridge.permissions"] = permissions
         else:
-            copy_dict("bridge.permissions")
+            copy_dict("bridge.permissions", override_existing_map=True)
 
         if "bridge.relaybot" not in self:
             copy("bridge.authless_relaybot_portals", "bridge.relaybot.authless_portals")

mautrix_telegram/db/__init__.py

@@ -16,9 +16,11 @@
 from mautrix.util.async_db import Database
 
 from .bot_chat import BotChat
+from .disappearing_message import DisappearingMessage
 from .message import Message
 from .portal import Portal
 from .puppet import Puppet
+from .reaction import Reaction
 from .telegram_file import TelegramFile
 from .telethon_session import PgSession
 from .upgrade import upgrade_table
@@ -26,7 +28,17 @@ from .user import User
 
 
 def init(db: Database) -> None:
-    for table in (Portal, Message, User, Puppet, TelegramFile, BotChat, PgSession):
+    for table in (
+        Portal,
+        Message,
+        Reaction,
+        User,
+        Puppet,
+        TelegramFile,
+        BotChat,
+        PgSession,
+        DisappearingMessage,
+    ):
         table.db = db
 
 
@@ -35,9 +47,11 @@ __all__ = [
     "init",
     "Portal",
     "Message",
+    "Reaction",
     "User",
     "Puppet",
     "TelegramFile",
     "BotChat",
     "PgSession",
+    "DisappearingMessage",
 ]

mautrix_telegram/db/disappearing_message.py

@@ -0,0 +1,78 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Sumner Evans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+import asyncpg
+
+from mautrix.bridge import AbstractDisappearingMessage
+from mautrix.types import EventID, RoomID
+from mautrix.util.async_db import Database
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+class DisappearingMessage(AbstractDisappearingMessage):
+    db: ClassVar[Database] = fake_db
+
+    async def insert(self) -> None:
+        q = """
+            INSERT INTO disappearing_message (room_id, event_id, expiration_seconds, expiration_ts)
+            VALUES ($1, $2, $3, $4)
+        """
+        await self.db.execute(
+            q, self.room_id, self.event_id, self.expiration_seconds, self.expiration_ts
+        )
+
+    async def update(self) -> None:
+        q = "UPDATE disappearing_message SET expiration_ts=$3 WHERE room_id=$1 AND event_id=$2"
+        await self.db.execute(q, self.room_id, self.event_id, self.expiration_ts)
+
+    async def delete(self) -> None:
+        q = "DELETE from disappearing_message WHERE room_id=$1 AND event_id=$2"
+        await self.db.execute(q, self.room_id, self.event_id)
+
+    @classmethod
+    def _from_row(cls, row: asyncpg.Record) -> DisappearingMessage:
+        return cls(**row)
+
+    @classmethod
+    async def get(cls, room_id: RoomID, event_id: EventID) -> DisappearingMessage | None:
+        q = """
+            SELECT room_id, event_id, expiration_seconds, expiration_ts FROM disappearing_message
+            WHERE room_id=$1 AND mxid=$2
+        """
+        try:
+            return cls._from_row(await cls.db.fetchrow(q, room_id, event_id))
+        except Exception:
+            return None
+
+    @classmethod
+    async def get_all_scheduled(cls) -> list[DisappearingMessage]:
+        q = """
+            SELECT room_id, event_id, expiration_seconds, expiration_ts FROM disappearing_message
+            WHERE expiration_ts IS NOT NULL
+        """
+        return [cls._from_row(r) for r in await cls.db.fetch(q)]
+
+    @classmethod
+    async def get_unscheduled_for_room(cls, room_id: RoomID) -> list[DisappearingMessage]:
+        q = """
+            SELECT room_id, event_id, expiration_seconds, expiration_ts FROM disappearing_message
+            WHERE room_id = $1 AND expiration_ts IS NULL
+        """
+        return [cls._from_row(r) for r in await cls.db.fetch(q, room_id)]

mautrix_telegram/db/message.py

@@ -38,6 +38,7 @@ class Message:
     tg_space: TelegramID
     edit_index: int
     redacted: bool = False
+    content_hash: bytes | None = None
 
     @classmethod
     def _from_row(cls, row: Record | None) -> Message | None:
@@ -45,7 +46,7 @@ class Message:
             return None
         return cls(**row)
 
-    columns: ClassVar[str] = "mxid, mx_room, tgid, tg_space, edit_index, redacted"
+    columns: ClassVar[str] = "mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash"
 
     @classmethod
     async def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> list[Message]:
@@ -142,14 +143,29 @@ class Message:
         q = "UPDATE message SET mxid=$1 WHERE mxid=$2 AND mx_room=$3"
         await cls.db.execute(q, real_mxid, temp_mxid, mx_room)
 
+    @classmethod
+    async def delete_temp_mxid(cls, temp_mxid: str, mx_room: RoomID) -> None:
+        q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2"
+        await cls.db.execute(q, temp_mxid, mx_room)
+
+    @property
+    def _values(self):
+        return (
+            self.mxid,
+            self.mx_room,
+            self.tgid,
+            self.tg_space,
+            self.edit_index,
+            self.redacted,
+            self.content_hash,
+        )
+
     async def insert(self) -> None:
-        q = (
-            "INSERT INTO message (mxid, mx_room, tgid, tg_space, edit_index, redacted) "
-            "VALUES ($1, $2, $3, $4, $5, $6)"
-        )
-        await self.db.execute(
-            q, self.mxid, self.mx_room, self.tgid, self.tg_space, self.edit_index, self.redacted
-        )
+        q = """
+            INSERT INTO message (mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash)
+            VALUES ($1, $2, $3, $4, $5, $6, $7)
+        """
+        await self.db.execute(q, *self._values)
 
     async def delete(self) -> None:
         q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2 AND tg_space=$3"

mautrix_telegram/db/portal.py

@@ -54,6 +54,8 @@ class Portal:
     title: str | None
     about: str | None
     photo_id: str | None
+    name_set: bool
+    avatar_set: bool
 
     local_config: dict[str, Any] = attr.ib(factory=lambda: {})
 
@@ -67,7 +69,8 @@ class Portal:
 
     columns: ClassVar[str] = (
         "tgid, tg_receiver, peer_type, megagroup, mxid, avatar_url, encrypted, sponsored_event_id,"
-        "sponsored_event_ts, sponsored_msg_random_id, username, title, about, photo_id, config"
+        "sponsored_event_ts, sponsored_msg_random_id, username, title, about, photo_id, "
+        "name_set, avatar_set, config"
     )
 
     @classmethod
@@ -86,10 +89,15 @@ class Portal:
         return cls._from_row(await cls.db.fetchrow(q, username.lower()))
 
     @classmethod
-    async def find_private_chats(cls, tg_receiver: TelegramID) -> list[Portal]:
+    async def find_private_chats_of(cls, tg_receiver: TelegramID) -> list[Portal]:
         q = f"SELECT {cls.columns} FROM portal WHERE tg_receiver=$1 AND peer_type='user'"
         return [cls._from_row(row) for row in await cls.db.fetch(q, tg_receiver)]
 
+    @classmethod
+    async def find_private_chats_with(cls, tgid: TelegramID) -> list[Portal]:
+        q = f"SELECT {cls.columns} FROM portal WHERE tgid=$1 AND peer_type='user'"
+        return [cls._from_row(row) for row in await cls.db.fetch(q, tgid)]
+
     @classmethod
     async def all(cls) -> list[Portal]:
         rows = await cls.db.fetch(f"SELECT {cls.columns} FROM portal")
@@ -111,17 +119,20 @@ class Portal:
             self.title,
             self.about,
             self.photo_id,
+            self.name_set,
+            self.avatar_set,
             self.megagroup,
             json.dumps(self.local_config) if self.local_config else None,
         )
 
     async def save(self) -> None:
-        q = (
-            "UPDATE portal SET mxid=$4, avatar_url=$5, encrypted=$6, sponsored_event_id=$7,"
-            "                  sponsored_event_ts=$8, sponsored_msg_random_id=$9, username=$10,"
-            "                  title=$11, about=$12, photo_id=$13, megagroup=$14, config=$15 "
-            "WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)"
-        )
+        q = """
+        UPDATE portal
+        SET mxid=$4, avatar_url=$5, encrypted=$6, sponsored_event_id=$7, sponsored_event_ts=$8,
+            sponsored_msg_random_id=$9, username=$10, title=$11, about=$12, photo_id=$13,
+            name_set=$14, avatar_set=$15, megagroup=$16, config=$17
+        WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)
+        """
         await self.db.execute(q, *self._values)
 
     async def update_id(self, id: TelegramID, peer_type: str) -> None:
@@ -135,12 +146,13 @@ class Portal:
         self.peer_type = peer_type
 
     async def insert(self) -> None:
-        q = (
-            "INSERT INTO portal (tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,"
-            "                    sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,"
-            "                    username, title, about, photo_id, megagroup, config) "
-            "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"
-        )
+        q = """
+        INSERT INTO portal (
+            tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,
+            sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,
+            username, title, about, photo_id, name_set, avatar_set, megagroup, config
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
+        """
         await self.db.execute(q, *self._values)
 
     async def delete(self) -> None:

mautrix_telegram/db/puppet.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -21,7 +21,7 @@ from asyncpg import Record
 from attr import dataclass
 from yarl import URL
 
-from mautrix.types import SyncToken, UserID
+from mautrix.types import ContentURI, SyncToken, UserID
 from mautrix.util.async_db import Database
 
 from ..types import TelegramID
@@ -44,7 +44,11 @@ class Puppet:
     disable_updates: bool
     username: str | None
     photo_id: str | None
+    avatar_url: ContentURI | None
+    name_set: bool
+    avatar_set: bool
     is_bot: bool | None
+    is_channel: bool
 
     custom_mxid: UserID | None
     access_token: str | None
@@ -61,8 +65,8 @@ class Puppet:
 
     columns: ClassVar[str] = (
         "id, is_registered, displayname, displayname_source, displayname_contact, "
-        "displayname_quality, disable_updates, username, photo_id, is_bot, "
-        "custom_mxid, access_token, next_batch, base_url"
+        "displayname_quality, disable_updates, username, photo_id, avatar_url, "
+        "name_set, avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url"
     )
 
     @classmethod
@@ -102,7 +106,11 @@ class Puppet:
             self.disable_updates,
             self.username,
             self.photo_id,
+            self.avatar_url,
+            self.name_set,
+            self.avatar_set,
             self.is_bot,
+            self.is_channel,
             self.custom_mxid,
             self.access_token,
             self.next_batch,
@@ -110,21 +118,22 @@ class Puppet:
         )
 
     async def save(self) -> None:
-        q = (
-            "UPDATE puppet "
-            "SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,"
-            "    displayname_quality=$6, disable_updates=$7, username=$8, photo_id=$9, is_bot=$10,"
-            "    custom_mxid=$11, access_token=$12, next_batch=$13, base_url=$14 "
-            "WHERE id=$1"
-        )
+        q = """
+        UPDATE puppet
+        SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,
+            displayname_quality=$6, disable_updates=$7, username=$8, photo_id=$9,
+            avatar_url=$10, name_set=$11, avatar_set=$12, is_bot=$13, is_channel=$14,
+            custom_mxid=$15, access_token=$16, next_batch=$17, base_url=$18
+        WHERE id=$1
+        """
         await self.db.execute(q, *self._values)
 
     async def insert(self) -> None:
-        q = (
-            "INSERT INTO puppet ("
-            "    id, is_registered, displayname, displayname_source, displayname_contact,"
-            "    displayname_quality, disable_updates, username, photo_id, is_bot,"
-            "    custom_mxid, access_token, next_batch, base_url"
-            ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)"
-        )
+        q = """
+        INSERT INTO puppet (
+            id, is_registered, displayname, displayname_source, displayname_contact,
+            displayname_quality, disable_updates, username, photo_id, avatar_url, name_set,
+            avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)
+        """
         await self.db.execute(q, *self._values)

mautrix_telegram/db/reaction.py

@@ -0,0 +1,91 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.types import EventID, RoomID
+from mautrix.util.async_db import Database
+
+from ..types import TelegramID
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+@dataclass
+class Reaction:
+    db: ClassVar[Database] = fake_db
+
+    mxid: EventID
+    mx_room: RoomID
+    msg_mxid: EventID
+    tg_sender: TelegramID
+    reaction: str
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> Reaction | None:
+        if row is None:
+            return None
+        return cls(**row)
+
+    columns: ClassVar[str] = "mxid, mx_room, msg_mxid, tg_sender, reaction"
+
+    @classmethod
+    async def delete_all(cls, mx_room: RoomID) -> None:
+        await cls.db.execute("DELETE FROM reaction WHERE mx_room=$1", mx_room)
+
+    @classmethod
+    async def get_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> Reaction | None:
+        q = f"SELECT {cls.columns} FROM reaction WHERE mxid=$1 AND mx_room=$2"
+        return cls._from_row(await cls.db.fetchrow(q, mxid, mx_room))
+
+    @classmethod
+    async def get_by_sender(
+        cls, mxid: EventID, mx_room: RoomID, tg_sender: TelegramID
+    ) -> Reaction | None:
+        q = f"SELECT {cls.columns} FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3"
+        return cls._from_row(await cls.db.fetchrow(q, mxid, mx_room, tg_sender))
+
+    @classmethod
+    async def get_all_by_message(cls, mxid: EventID, mx_room: RoomID) -> list[Reaction]:
+        q = f"SELECT {cls.columns} FROM reaction WHERE msg_mxid=$1 AND mx_room=$2"
+        rows = await cls.db.fetch(q, mxid, mx_room)
+        return [cls._from_row(row) for row in rows]
+
+    @property
+    def _values(self):
+        return (
+            self.mxid,
+            self.mx_room,
+            self.msg_mxid,
+            self.tg_sender,
+            self.reaction,
+        )
+
+    async def save(self) -> None:
+        q = """
+            INSERT INTO reaction (mxid, mx_room, msg_mxid, tg_sender, reaction)
+            VALUES ($1, $2, $3, $4, $5) ON CONFLICT (msg_mxid, mx_room, tg_sender)
+                DO UPDATE SET mxid=$1, reaction=$5
+        """
+        await self.db.execute(q, *self._values)
+
+    async def delete(self) -> None:
+        q = "DELETE FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3"
+        await self.db.execute(q, self.msg_mxid, self.mx_room, self.tg_sender)

mautrix_telegram/db/upgrade/__init__.py

@@ -2,4 +2,11 @@ from mautrix.util.async_db import UpgradeTable
 
 upgrade_table = UpgradeTable()
 
-from . import v01_initial_revision, v02_sponsored_events
+from . import (
+    v01_initial_revision,
+    v02_sponsored_events,
+    v03_reactions,
+    v04_disappearing_messages,
+    v05_channel_ghosts,
+    v06_puppet_avatar_url,
+)

mautrix_telegram/db/upgrade/v01_initial_revision.py

@@ -13,6 +13,8 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
 from asyncpg import Connection
 
 from . import upgrade_table
@@ -38,6 +40,16 @@ async def upgrade_v1(conn: Connection, scheme: str) -> None:
         await create_v1_tables(conn)
 
 
+async def drop_constraints(conn: Connection, table: str, contype: str) -> None:
+    q = (
+        "SELECT conname FROM pg_constraint con INNER JOIN pg_class rel ON rel.oid=con.conrelid "
+        f"WHERE rel.relname='{table}' AND contype='{contype}'"
+    )
+    names = [row["conname"] for row in await conn.fetch(q)]
+    drops = ", ".join(f"DROP CONSTRAINT {name}" for name in names)
+    await conn.execute(f"ALTER TABLE {table} {drops}")
+
+
 async def migrate_legacy_to_v1(conn: Connection, scheme: str) -> None:
     legacy_version = await conn.fetchval(legacy_version_query)
     if legacy_version != last_legacy_version:
@@ -46,37 +58,38 @@ async def migrate_legacy_to_v1(conn: Connection, scheme: str) -> None:
             "Please upgrade the old database with alembic or drop it completely first."
         )
     if scheme != "sqlite":
+        await drop_constraints(conn, "contact", contype="f")
         await conn.execute(
             """
             ALTER TABLE contact
-              DROP CONSTRAINT contact_user_fkey,
-              DROP CONSTRAINT contact_contact_fkey,
               ADD CONSTRAINT contact_user_fkey FOREIGN KEY (contact) REFERENCES puppet(id)
                 ON DELETE CASCADE ON UPDATE CASCADE,
               ADD CONSTRAINT contact_contact_fkey FOREIGN KEY ("user") REFERENCES "user"(tgid)
                 ON DELETE CASCADE ON UPDATE CASCADE
             """
         )
+        await drop_constraints(conn, "telethon_sessions", contype="p")
         await conn.execute(
             """
             ALTER TABLE telethon_sessions
-              DROP CONSTRAINT telethon_sessions_pkey,
               ADD CONSTRAINT telethon_sessions_pkey PRIMARY KEY (session_id)
             """
         )
+        await drop_constraints(conn, "telegram_file", contype="f")
         await conn.execute(
             """
             ALTER TABLE telegram_file
-              DROP CONSTRAINT fk_file_thumbnail,
               ADD CONSTRAINT fk_file_thumbnail
                 FOREIGN KEY (thumbnail) REFERENCES telegram_file(id)
                 ON UPDATE CASCADE ON DELETE SET NULL
             """
         )
+        await conn.execute("ALTER TABLE puppet ALTER COLUMN id DROP IDENTITY IF EXISTS")
         await conn.execute("ALTER TABLE puppet ALTER COLUMN id DROP DEFAULT")
-        await conn.execute("DROP SEQUENCE puppet_id_seq")
+        await conn.execute("DROP SEQUENCE IF EXISTS puppet_id_seq")
+        await conn.execute("ALTER TABLE bot_chat ALTER COLUMN id DROP IDENTITY IF EXISTS")
         await conn.execute("ALTER TABLE bot_chat ALTER COLUMN id DROP DEFAULT")
-        await conn.execute("DROP SEQUENCE bot_chat_id_seq")
+        await conn.execute("DROP SEQUENCE IF EXISTS bot_chat_id_seq")
         await conn.execute("ALTER TABLE portal ALTER COLUMN config TYPE jsonb USING config::jsonb")
         await conn.execute(
             "ALTER TABLE telegram_file ALTER COLUMN decryption_info TYPE jsonb "

mautrix_telegram/db/upgrade/v03_reactions.py

@@ -0,0 +1,39 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add support for reactions")
+async def upgrade_v3(conn: Connection, scheme: str) -> None:
+    await conn.execute(
+        """CREATE TABLE reaction (
+            mxid      TEXT NOT NULL,
+            mx_room   TEXT NOT NULL,
+            msg_mxid  TEXT NOT NULL,
+            tg_sender BIGINT,
+            reaction  TEXT NOT NULL,
+
+            PRIMARY KEY (msg_mxid, mx_room, tg_sender),
+            UNIQUE (mxid, mx_room)
+        )"""
+    )
+    if scheme != "sqlite":
+        await conn.execute("DELETE FROM message WHERE mxid IS NULL OR mx_room IS NULL")
+        await conn.execute("ALTER TABLE message ALTER COLUMN mxid SET NOT NULL")
+        await conn.execute("ALTER TABLE message ALTER COLUMN mx_room SET NOT NULL")
+    await conn.execute("ALTER TABLE message ADD COLUMN content_hash bytea")

mautrix_telegram/db/upgrade/v04_disappearing_messages.py

@@ -0,0 +1,32 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add support for disappearing messages")
+async def upgrade_v4(conn: Connection) -> None:
+    await conn.execute(
+        """CREATE TABLE disappearing_message (
+            room_id             TEXT,
+            event_id            TEXT,
+            expiration_seconds  BIGINT,
+            expiration_ts       BIGINT,
+
+            PRIMARY KEY (room_id, event_id)
+        )"""
+    )

mautrix_telegram/db/upgrade/v05_channel_ghosts.py

@@ -0,0 +1,25 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add separate ghost users for channel senders")
+async def upgrade_v5(conn: Connection, scheme: str) -> None:
+    await conn.execute("ALTER TABLE puppet ADD COLUMN is_channel BOOLEAN NOT NULL DEFAULT false")
+    if scheme == "postgres":
+        await conn.execute("ALTER TABLE puppet ALTER COLUMN is_channel DROP DEFAULT")

mautrix_telegram/db/upgrade/v06_puppet_avatar_url.py

@@ -0,0 +1,31 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store avatar mxc URI in puppet table")
+async def upgrade_v6(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE puppet ADD COLUMN avatar_url TEXT")
+    await conn.execute("ALTER TABLE puppet ADD COLUMN name_set BOOLEAN NOT NULL DEFAULT false")
+    await conn.execute("ALTER TABLE puppet ADD COLUMN avatar_set BOOLEAN NOT NULL DEFAULT false")
+    await conn.execute("UPDATE puppet SET name_set=true WHERE displayname<>''")
+    await conn.execute("UPDATE puppet SET avatar_set=true WHERE photo_id<>''")
+    await conn.execute("ALTER TABLE portal ADD COLUMN name_set BOOLEAN NOT NULL DEFAULT false")
+    await conn.execute("ALTER TABLE portal ADD COLUMN avatar_set BOOLEAN NOT NULL DEFAULT false")
+    await conn.execute("UPDATE portal SET name_set=true WHERE title<>'' AND mxid<>''")
+    await conn.execute("UPDATE portal SET avatar_set=true WHERE photo_id<>'' AND mxid<>''")

mautrix_telegram/example-config.yaml

@@ -78,12 +78,6 @@ appservice:
     bot_displayname: Telegram bridge bot
     bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
 
-    # Community ID for bridged users (changes registration file) and rooms.
-    # Must be created manually.
-    #
-    # Example: "+telegram:example.com". Set to false to disable.
-    community_id: false
-
     # Whether or not to receive ephemeral events via appservice transactions.
     # Requires MSC2409 support (i.e. Synapse 1.22+).
     # You should disable bridge -> sync_with_custom_puppets when this is enabled.
@@ -214,6 +208,8 @@ bridge:
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
+    # Maximum number of pixels in an image before sending to Telegram as a document. Defaults to 1280x1280 = 1638400.
+    image_as_file_pixels: 1638400
     # Maximum size of Telegram documents in megabytes to bridge.
     max_document_size: 100
     # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
@@ -328,19 +324,12 @@ bridge:
         # List of user IDs for whom the previous flag is flipped.
         # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
         #      notices from users listed here will be bridged.
-        exceptions:
-        - "@importantbot:example.com"
-
-    # Some config options related to Telegram message deduplication.
-    # The default values are usually fine, but some debug messages/warnings might recommend you
-    # change these.
-    deduplication:
-        # Whether or not to check the database if the message about to be sent is a duplicate.
-        pre_db_check: false
-        # The number of latest events to keep when checking for duplicates.
-        # You might need to increase this on high-traffic bridge instances.
-        cache_queue_length: 20
+        exceptions: []
 
+    # An array of possible values for the $distinguisher variable in message formats.
+    # Each user gets one of the values here, based on a hash of their user ID.
+    # If the array is empty, the $distinguisher variable will also be empty.
+    relay_user_distinguishers: ["🟦", "🟣", "🟩", "⭕️", "🔶", "⬛️", "🔵", "🟢"]
     # The formats to use when sending messages to Telegram via the relay bot.
     # Text msgtypes (m.text, m.notice and m.emote) support HTML, media msgtypes don't.
     #
@@ -348,16 +337,17 @@ bridge:
     #   $sender_displayname - The display name of the sender (e.g. Example User)
     #   $sender_username    - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
     #   $sender_mxid        - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $distinguisher      - A random string from the options in the relay_user_distinguishers array.
     #   $message            - The message content
     message_formats:
-        m.text: "<b>$sender_displayname</b>: $message"
-        m.notice: "<b>$sender_displayname</b>: $message"
-        m.emote: "* <b>$sender_displayname</b> $message"
-        m.file: "<b>$sender_displayname</b> sent a file: $message"
-        m.image: "<b>$sender_displayname</b> sent an image: $message"
-        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
-        m.video: "<b>$sender_displayname</b> sent a video: $message"
-        m.location: "<b>$sender_displayname</b> sent a location: $message"
+        m.text: "$distinguisher <b>$sender_displayname</b>: $message"
+        m.notice: "$distinguisher <b>$sender_displayname</b>: $message"
+        m.emote: "* $distinguisher <b>$sender_displayname</b> $message"
+        m.file: "$distinguisher <b>$sender_displayname</b> sent a file: $message"
+        m.image: "$distinguisher <b>$sender_displayname</b> sent an image: $message"
+        m.audio: "$distinguisher <b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "$distinguisher <b>$sender_displayname</b> sent a video: $message"
+        m.location: "$distinguisher <b>$sender_displayname</b> sent a location: $message"
     # Telegram doesn't have built-in emotes, this field specifies how m.emote's from authenticated
     # users are sent to telegram. All fields in message_formats are supported. Additionally, the
     # Telegram user info is available in the following variables:
@@ -373,9 +363,9 @@ bridge:
     #
     # Set format to an empty string to disable the messages for that event.
     state_event_formats:
-        join: "<b>$displayname</b> joined the room."
-        leave: "<b>$displayname</b> left the room."
-        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+        join: "$distinguisher <b>$displayname</b> joined the room."
+        leave: "$distinguisher <b>$displayname</b> left the room."
+        name_change: "$distinguisher <b>$prev_displayname</b> changed their name to $distinguisher <b>$displayname</b>"
 
     # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
     # `filter-mode` management commands.

mautrix_telegram/formatter/from_matrix/parser.py

@@ -20,8 +20,7 @@ import logging
 from telethon import TelegramClient
 
 from mautrix.types import RoomID, UserID
-from mautrix.util.formatter import MatrixParser as BaseMatrixParser, RecursionContext
-from mautrix.util.formatter.html_reader_htmlparser import HTMLNode, read_html
+from mautrix.util.formatter import HTMLNode, MatrixParser as BaseMatrixParser, RecursionContext
 from mautrix.util.logging import TraceLogger
 
 from ... import portal as po, puppet as pu, user as u
@@ -33,7 +32,6 @@ log: TraceLogger = logging.getLogger("mau.fmt.mx")
 class MatrixParser(BaseMatrixParser[TelegramMessage]):
     e = TelegramEntityType
     fs = TelegramMessage
-    read_html = staticmethod(read_html)
     client: TelegramClient
 
     def __init__(self, client: TelegramClient) -> None:
@@ -42,8 +40,8 @@ class MatrixParser(BaseMatrixParser[TelegramMessage]):
     async def custom_node_to_fstring(
         self, node: HTMLNode, ctx: RecursionContext
     ) -> TelegramMessage | None:
-        msg = await self.tag_aware_parse_node(node, ctx)
         if node.tag == "command":
+            msg = await self.tag_aware_parse_node(node, ctx)
             return msg.prepend("/").format(TelegramEntityType.COMMAND)
         return None
 
@@ -59,7 +57,7 @@ class MatrixParser(BaseMatrixParser[TelegramMessage]):
             displayname = user.plain_displayname or msg.text
             msg = TelegramMessage(displayname)
             try:
-                input_entity = self.client.get_input_entity(user.tgid)
+                input_entity = await self.client.get_input_entity(user.tgid)
             except (ValueError, TypeError) as e:
                 log.trace(f"Dropping mention of {user.tgid}: {e}")
             else:
@@ -95,5 +93,8 @@ class MatrixParser(BaseMatrixParser[TelegramMessage]):
     async def color_to_fstring(self, msg: TelegramMessage, color: str) -> TelegramMessage:
         return msg
 
-    async def spoiler_to_fstring(self, msg: TelegramMessage, spoiler: str) -> TelegramMessage:
+    async def spoiler_to_fstring(self, msg: TelegramMessage, reason: str) -> TelegramMessage:
+        msg = msg.format(self.e.SPOILER)
+        if reason:
+            msg = msg.prepend(f"{reason}: ")
         return msg

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -29,6 +29,7 @@ from telethon.tl.types import (
     MessageEntityMention as Mention,
     MessageEntityMentionName as MentionName,
     MessageEntityPre as Pre,
+    MessageEntitySpoiler as Spoiler,
     MessageEntityStrike as Strike,
     MessageEntityTextUrl as TextURL,
     MessageEntityUnderline as Underline,
@@ -55,6 +56,7 @@ class TelegramEntityType(Enum):
     MENTION = Mention
     MENTION_NAME = InputMentionName
     COMMAND = Command
+    SPOILER = Spoiler
 
     USER_MENTION = 1
     ROOM_MENTION = 2

mautrix_telegram/formatter/from_telegram.py

@@ -35,6 +35,7 @@ from telethon.tl.types import (
     MessageEntityMentionName,
     MessageEntityPhone,
     MessageEntityPre,
+    MessageEntitySpoiler,
     MessageEntityStrike,
     MessageEntityTextUrl,
     MessageEntityUnderline,
@@ -93,9 +94,7 @@ async def _add_forward_header(
             )
 
         if not fwd_from_text:
-            puppet = await pu.Puppet.get_by_tgid(
-                TelegramID(fwd_from.from_id.user_id), create=False
-            )
+            puppet = await pu.Puppet.get_by_peer(fwd_from.from_id, create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
                 fwd_from_html = (
@@ -199,7 +198,7 @@ async def telegram_to_matrix(
     def force_html():
         if not content.formatted_body:
             content.format = Format.HTML
-            content.formatted_body = escape(content.body)
+            content.formatted_body = escape(content.body).replace("\n", "<br/>")
 
     if require_html:
         force_html()
@@ -289,10 +288,15 @@ async def _telegram_entities_to_matrix(
             skip_entity = await _parse_url(
                 html, entity_text, entity.url if entity_type == MessageEntityTextUrl else None
             )
-        elif entity_type == MessageEntityBotCommand:
-            html.append(f"<font color='blue'>{entity_text}</font>")
-        elif entity_type in (MessageEntityHashtag, MessageEntityCashtag, MessageEntityPhone):
-            html.append(f"<font color='blue'>{entity_text}</font>")
+        elif entity_type in (
+            MessageEntityBotCommand,
+            MessageEntityHashtag,
+            MessageEntityCashtag,
+            MessageEntityPhone,
+        ):
+            html.append(f"<font color='#3771bb'>{entity_text}</font>")
+        elif entity_type == MessageEntitySpoiler:
+            html.append(f"<span data-mx-spoiler>{entity_text}</span>")
         else:
             skip_entity = True
         last_offset = relative_offset + (0 if skip_entity else entity.length)

mautrix_telegram/matrix.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,9 +15,8 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterable
+from typing import TYPE_CHECKING
 
-from mautrix.appservice import DOUBLE_PUPPET_SOURCE_KEY
 from mautrix.bridge import BaseMatrixHandler
 from mautrix.errors import MatrixError
 from mautrix.types import (
@@ -28,9 +27,8 @@ from mautrix.types import (
     MessageType,
     PresenceEvent,
     PresenceState,
+    ReactionEvent,
     ReceiptEvent,
-    ReceiptEventContent,
-    ReceiptType,
     RedactionEvent,
     RoomAvatarStateEventContent as AvatarContent,
     RoomID,
@@ -68,11 +66,19 @@ class MatrixHandler(BaseMatrixHandler):
     ) -> None:
         intent = puppet.default_mxid_intent
         self.log.debug(f"{inviter.mxid} invited puppet for {puppet.tgid} to {room_id}")
+        if puppet.is_channel:
+            self.log.debug(f"Rejecting invite for {puppet.tgid} to {room_id}: puppet is a channel")
+            await intent.leave_room(room_id, reason="Channels can't be invited to chats")
+            return
+
         if not await inviter.is_logged_in():
-            await intent.error_and_leave(
-                room_id, text="Please log in before inviting Telegram puppets."
+            self.log.debug(f"Rejecting invite for {puppet.tgid} to {room_id}: user not logged in")
+            await intent.leave_room(
+                room_id,
+                reason="Only users who are logged into the bridge can invite Telegram ghosts.",
             )
             return
+
         portal = await po.Portal.get_by_mxid(room_id)
         if portal:
             if portal.peer_type == "user":
@@ -278,6 +284,20 @@ class MatrixHandler(BaseMatrixHandler):
 
         await portal.handle_matrix_deletion(sender, evt.redacts, evt.event_id)
 
+    @staticmethod
+    async def handle_reaction(evt: ReactionEvent) -> None:
+        sender = await u.User.get_and_start_by_mxid(evt.sender)
+        if not await sender.has_full_access():
+            return
+
+        portal = await po.Portal.get_by_mxid(evt.room_id)
+        if not portal or not portal.allow_bridging:
+            return
+
+        await portal.handle_matrix_reaction(
+            sender, evt.content.relates_to.event_id, evt.content.relates_to.key, evt.event_id
+        )
+
     @staticmethod
     async def handle_power_levels(evt: StateEvent) -> None:
         portal = await po.Portal.get_by_mxid(evt.room_id)
@@ -400,6 +420,8 @@ class MatrixHandler(BaseMatrixHandler):
     async def handle_event(self, evt: Event) -> None:
         if evt.type == EventType.ROOM_REDACTION:
             await self.handle_redaction(evt)
+        elif evt.type == EventType.REACTION:
+            await self.handle_reaction(evt)
 
     async def handle_state_event(self, evt: StateEvent) -> None:
         if evt.type == EventType.ROOM_POWER_LEVELS:

mautrix_telegram/portal_util/__init__.py

@@ -2,5 +2,5 @@ from .deduplication import PortalDedup
 from .media_fallback import make_contact_event_content, make_dice_event_content
 from .participants import get_users
 from .power_levels import get_base_power_levels, participants_to_power_levels
-from .send_lock import PortalSendLock
+from .send_lock import PortalReactionLock, PortalSendLock
 from .sponsored_message import get_sponsored_message, make_sponsored_message_content

mautrix_telegram/portal_util/deduplication.py

@@ -15,20 +15,30 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import Tuple
+from typing import Any, Generator, Tuple, Union
 from collections import deque
 import hashlib
 
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
+    Message,
     MessageMediaContact,
+    MessageMediaDice,
     MessageMediaDocument,
+    MessageMediaGame,
     MessageMediaGeo,
     MessageMediaPhoto,
-    TypeMessage,
+    MessageMediaPoll,
+    MessageMediaUnsupported,
+    MessageService,
+    PeerChannel,
+    PeerChat,
+    PeerUser,
     TypeUpdates,
     UpdateNewChannelMessage,
     UpdateNewMessage,
+    UpdateShortChatMessage,
+    UpdateShortMessage,
 )
 
 from mautrix.types import EventID
@@ -37,60 +47,67 @@ from .. import portal as po
 from ..types import TelegramID
 
 DedupMXID = Tuple[EventID, TelegramID]
+TypeMessage = Union[Message, MessageService, UpdateShortMessage, UpdateShortChatMessage]
+
+media_content_table = {
+    MessageMediaContact: lambda media: [media.user_id],
+    MessageMediaDocument: lambda media: [media.document.id],
+    MessageMediaPhoto: lambda media: [media.photo.id if media.photo else 0],
+    MessageMediaGeo: lambda media: [media.geo.long, media.geo.lat],
+    MessageMediaGame: lambda media: [media.game.id],
+    MessageMediaPoll: lambda media: [media.poll.id],
+    MessageMediaDice: lambda media: [media.value, media.emoticon],
+    MessageMediaUnsupported: lambda media: ["unsupported media"],
+}
 
 
 class PortalDedup:
     pre_db_check: bool = False
-    cache_queue_length: int = 20
+    cache_queue_length: int = 256
 
-    _dedup: deque[str]
-    _dedup_mxid: dict[str, DedupMXID]
-    _dedup_action: deque[str]
+    _dedup: deque[bytes | int]
+    _dedup_mxid: dict[bytes | int, DedupMXID]
+    _dedup_action: deque[bytes | int]
     _portal: po.Portal
 
     def __init__(self, portal: po.Portal) -> None:
         self._dedup = deque()
         self._dedup_mxid = {}
-        self._dedup_action = deque()
+        self._dedup_action = deque(maxlen=self.cache_queue_length)
         self._portal = portal
 
     @property
     def _always_force_hash(self) -> bool:
         return self._portal.peer_type == "chat"
 
-    @staticmethod
-    def _hash_event(event: TypeMessage) -> str:
-        # Non-channel messages are unique per-user (wtf telegram), so we have no other choice than
-        # to deduplicate based on a hash of the message content.
-
-        # The timestamp is only accurate to the second, so we can't rely solely on that either.
+    def _hash_content(self, event: TypeMessage) -> Generator[Any, None, None]:
+        if not self._always_force_hash:
+            yield event.id
+        yield int(event.date.timestamp())
         if isinstance(event, MessageService):
-            hash_content = [event.date.timestamp(), event.from_id, event.action]
+            yield event.from_id
+            yield event.action
         else:
-            hash_content = [event.date.timestamp(), event.message.strip()]
+            yield event.message.strip()
             if event.fwd_from:
-                hash_content += [event.fwd_from.from_id]
-            elif isinstance(event, Message) and event.media:
-                try:
-                    hash_content += {
-                        MessageMediaContact: lambda media: [media.user_id],
-                        MessageMediaDocument: lambda media: [media.document.id],
-                        MessageMediaPhoto: lambda media: [media.photo.id if media.photo else 0],
-                        MessageMediaGeo: lambda media: [media.geo.long, media.geo.lat],
-                    }[type(event.media)](event.media)
-                except KeyError:
-                    pass
-        return hashlib.md5("-".join(str(a) for a in hash_content).encode("utf-8")).hexdigest()
+                yield event.fwd_from.from_id
+            if isinstance(event, Message) and event.media:
+                media_hash_func = media_content_table.get(type(event.media)) or (
+                    lambda media: ["unknown media"]
+                )
+                yield media_hash_func(event.media)
+
+    def _hash_event(self, event: TypeMessage) -> bytes:
+        return hashlib.sha256(
+            "-".join(str(a) for a in self._hash_content(event)).encode("utf-8")
+        ).digest()
 
     def check_action(self, event: TypeMessage) -> bool:
-        evt_hash = self._hash_event(event) if self._always_force_hash else event.id
-        if evt_hash in self._dedup_action:
+        dedup_id = self._hash_event(event) if self._always_force_hash else event.id
+        if dedup_id in self._dedup_action:
             return True
 
-        self._dedup_action.append(evt_hash)
-
-        if len(self._dedup_action) > self.cache_queue_length:
-            self._dedup_action.popleft()
+        self._dedup_action.appendleft(dedup_id)
         return False
 
     def update(
@@ -99,31 +116,38 @@ class PortalDedup:
         mxid: DedupMXID = None,
         expected_mxid: DedupMXID | None = None,
         force_hash: bool = False,
-    ) -> DedupMXID | None:
-        evt_hash = self._hash_event(event) if self._always_force_hash or force_hash else event.id
+    ) -> tuple[bytes, DedupMXID | None]:
+        evt_hash = self._hash_event(event)
+        dedup_id = evt_hash if self._always_force_hash or force_hash else event.id
         try:
-            found_mxid = self._dedup_mxid[evt_hash]
+            found_mxid = self._dedup_mxid[dedup_id]
         except KeyError:
-            return EventID("None"), TelegramID(0)
+            return evt_hash, None
 
         if found_mxid != expected_mxid:
-            return found_mxid
-        self._dedup_mxid[evt_hash] = mxid
-        return None
+            return evt_hash, found_mxid
+        self._dedup_mxid[dedup_id] = mxid
+        if evt_hash != dedup_id:
+            self._dedup_mxid[evt_hash] = mxid
+        return evt_hash, None
 
     def check(
         self, event: TypeMessage, mxid: DedupMXID = None, force_hash: bool = False
-    ) -> DedupMXID | None:
-        evt_hash = self._hash_event(event) if self._always_force_hash or force_hash else event.id
-        if evt_hash in self._dedup:
-            return self._dedup_mxid[evt_hash]
+    ) -> tuple[bytes, DedupMXID | None]:
+        evt_hash = self._hash_event(event)
+        dedup_id = evt_hash if self._always_force_hash or force_hash else event.id
+        if dedup_id in self._dedup:
+            return evt_hash, self._dedup_mxid[dedup_id]
 
-        self._dedup_mxid[evt_hash] = mxid
-        self._dedup.append(evt_hash)
+        self._dedup_mxid[dedup_id] = mxid
+        self._dedup.appendleft(dedup_id)
+        if evt_hash != dedup_id:
+            self._dedup_mxid[evt_hash] = mxid
+            self._dedup.appendleft(evt_hash)
 
-        if len(self._dedup) > self.cache_queue_length:
-            del self._dedup_mxid[self._dedup.popleft()]
-        return None
+        while len(self._dedup) > self.cache_queue_length:
+            del self._dedup_mxid[self._dedup.pop()]
+        return evt_hash, None
 
     def register_outgoing_actions(self, response: TypeUpdates) -> None:
         for update in response.updates:

mautrix_telegram/portal_util/send_lock.py

@@ -16,6 +16,9 @@
 from __future__ import annotations
 
 from asyncio import Lock
+from collections import defaultdict
+
+from mautrix.types import EventID
 
 from ..types import TelegramID
 
@@ -42,3 +45,13 @@ class PortalSendLock:
             return self._send_locks[user_id]
         except KeyError:
             return self._send_locks.setdefault(user_id, Lock()) if required else self._noop_lock
+
+
+class PortalReactionLock:
+    _reaction_locks: dict[EventID, Lock]
+
+    def __init__(self) -> None:
+        self._reaction_locks = defaultdict(lambda: Lock())
+
+    def __call__(self, mxid: EventID) -> Lock:
+        return self._reaction_locks[mxid]

mautrix_telegram/puppet.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -20,10 +20,18 @@ from difflib import SequenceMatcher
 import unicodedata
 
 from telethon.tl.types import (
+    Channel,
+    ChatPhoto,
+    ChatPhotoEmpty,
     InputPeerPhotoFileLocation,
+    PeerChannel,
+    PeerChat,
     PeerUser,
+    TypeChatPhoto,
     TypeInputPeer,
     TypeInputUser,
+    TypePeer,
+    TypeUserProfilePhoto,
     UpdateUserName,
     User,
     UserProfilePhoto,
@@ -33,7 +41,6 @@ from yarl import URL
 
 from mautrix.appservice import IntentAPI
 from mautrix.bridge import BasePuppet, async_getter_lock
-from mautrix.errors import MatrixError
 from mautrix.types import ContentURI, RoomID, SyncToken, UserID
 from mautrix.util.simple_template import SimpleTemplate
 
@@ -66,7 +73,11 @@ class Puppet(DBPuppet, BasePuppet):
         disable_updates: bool = False,
         username: str | None = None,
         photo_id: str | None = None,
+        avatar_url: ContentURI | None = None,
+        name_set: bool = False,
+        avatar_set: bool = False,
         is_bot: bool = False,
+        is_channel: bool = False,
         custom_mxid: UserID | None = None,
         access_token: str | None = None,
         next_batch: SyncToken | None = None,
@@ -82,7 +93,11 @@ class Puppet(DBPuppet, BasePuppet):
             disable_updates=disable_updates,
             username=username,
             photo_id=photo_id,
+            avatar_url=avatar_url,
+            name_set=name_set,
+            avatar_set=avatar_set,
             is_bot=is_bot,
+            is_channel=is_channel,
             custom_mxid=custom_mxid,
             access_token=access_token,
             next_batch=next_batch,
@@ -109,7 +124,9 @@ class Puppet(DBPuppet, BasePuppet):
 
     @property
     def peer(self) -> PeerUser:
-        return PeerUser(user_id=self.tgid)
+        return (
+            PeerChannel(channel_id=self.tgid) if self.is_channel else PeerUser(user_id=self.tgid)
+        )
 
     @property
     def plain_displayname(self) -> str:
@@ -185,9 +202,12 @@ class Puppet(DBPuppet, BasePuppet):
         return name
 
     @classmethod
-    def get_displayname(cls, info: User, enable_format: bool = True) -> tuple[str, int]:
-        fn = cls._filter_name(info.first_name)
-        ln = cls._filter_name(info.last_name)
+    def get_displayname(cls, info: User | Channel, enable_format: bool = True) -> tuple[str, int]:
+        if isinstance(info, Channel):
+            fn, ln = cls._filter_name(info.title), ""
+        else:
+            fn = cls._filter_name(info.first_name)
+            ln = cls._filter_name(info.last_name)
         data = {
             "phone number": info.phone if hasattr(info, "phone") else None,
             "username": info.username,
@@ -214,14 +234,20 @@ class Puppet(DBPuppet, BasePuppet):
 
         return (cls.displayname_template.format_full(name) if enable_format else name), quality
 
-    async def try_update_info(self, source: au.AbstractUser, info: User) -> None:
+    async def try_update_info(self, source: au.AbstractUser, info: User | Channel) -> None:
         try:
             await self.update_info(source, info)
         except Exception:
             source.log.exception(f"Failed to update info of {self.tgid}")
 
-    async def update_info(self, source: au.AbstractUser, info: User) -> None:
-        changed = False
+    async def update_info(self, source: au.AbstractUser, info: User | Channel) -> None:
+        is_bot = False if isinstance(info, Channel) else info.bot
+        is_channel = isinstance(info, Channel)
+        changed = is_bot != self.is_bot or is_channel != self.is_channel
+
+        self.is_bot = is_bot
+        self.is_channel = is_channel
+
         if self.username != info.username:
             self.username = info.username
             changed = True
@@ -233,32 +259,46 @@ class Puppet(DBPuppet, BasePuppet):
             except Exception:
                 self.log.exception(f"Failed to update info from source {source.tgid}")
 
-        self.is_bot = info.bot
-
         if changed:
+            await self.update_portals_meta()
             await self.save()
 
+    async def update_portals_meta(self) -> None:
+        if not p.Portal.private_chat_portal_meta and not self.mx.e2ee:
+            return
+        async for portal in p.Portal.find_private_chats_with(self.tgid):
+            await portal.update_info_from_puppet(self)
+
     async def update_displayname(
-        self, source: au.AbstractUser, info: User | UpdateUserName
+        self, source: au.AbstractUser, info: User | Channel | UpdateUserName
     ) -> bool:
         if self.disable_updates:
             return False
-        if source.is_relaybot or source.is_bot:
-            allow_because = "user is bot"
+        if (
+            self.displayname
+            and self.displayname.startswith("Deleted user ")
+            and not getattr(info, "deleted", False)
+        ):
+            allow_because = "target user was previously deleted"
+            self.displayname_quality = 0
+        elif source.is_relaybot or source.is_bot:
+            allow_because = "source user is a bot"
         elif self.displayname_source == source.tgid:
-            allow_because = "user is the primary source"
+            allow_because = "source user is the primary source"
+        elif isinstance(info, Channel):
+            allow_because = "target user is a channel"
         elif not isinstance(info, UpdateUserName) and not info.contact:
-            allow_because = "user is not a contact"
+            allow_because = "target user is not a contact"
         elif not self.displayname_source:
             allow_because = "no primary source set"
         elif not self.displayname:
-            allow_because = "user has no name"
+            allow_because = "target user has no name"
         else:
             return False
 
         if isinstance(info, UpdateUserName):
-            info = await source.client.get_entity(PeerUser(self.tgid))
-        if not info.contact:
+            info = await source.client.get_entity(self.peer)
+        if isinstance(info, Channel) or not info.contact:
             self.displayname_contact = False
         elif not self.displayname_contact:
             if not self.displayname:
@@ -267,7 +307,9 @@ class Puppet(DBPuppet, BasePuppet):
                 return False
 
         displayname, quality = self.get_displayname(info)
-        if displayname != self.displayname and quality >= self.displayname_quality:
+        needs_reset = displayname != self.displayname or not self.name_set
+        is_high_quality = quality >= self.displayname_quality
+        if needs_reset and is_high_quality:
             allow_because = f"{allow_because} and quality {quality} >= {self.displayname_quality}"
             self.log.debug(
                 f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
@@ -281,11 +323,10 @@ class Puppet(DBPuppet, BasePuppet):
                 await self.default_mxid_intent.set_displayname(
                     displayname[: self.config["bridge.displayname_max_length"]]
                 )
-            except MatrixError:
-                self.log.exception("Failed to set displayname")
-                self.displayname = ""
-                self.displayname_source = None
-                self.displayname_quality = 0
+                self.name_set = True
+            except Exception as e:
+                self.log.warning(f"Failed to set displayname: {e}")
+                self.name_set = False
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
@@ -293,42 +334,43 @@ class Puppet(DBPuppet, BasePuppet):
         return False
 
     async def update_avatar(
-        self, source: au.AbstractUser, photo: UserProfilePhoto | UserProfilePhotoEmpty
+        self, source: au.AbstractUser, photo: TypeUserProfilePhoto | TypeChatPhoto
     ) -> bool:
         if self.disable_updates:
             return False
 
-        if photo is None or isinstance(photo, UserProfilePhotoEmpty):
+        if photo is None or isinstance(photo, (UserProfilePhotoEmpty, ChatPhotoEmpty)):
             photo_id = ""
-        elif isinstance(photo, UserProfilePhoto):
+        elif isinstance(photo, (UserProfilePhoto, ChatPhoto)):
             photo_id = str(photo.photo_id)
         else:
             self.log.warning(f"Unknown user profile photo type: {type(photo)}")
             return False
         if not photo_id and not self.config["bridge.allow_avatar_remove"]:
             return False
-        if self.photo_id != photo_id:
+        if self.photo_id != photo_id or not self.avatar_set:
             if not photo_id:
                 self.photo_id = ""
-                try:
-                    await self.default_mxid_intent.set_avatar_url(ContentURI(""))
-                except MatrixError:
-                    self.log.exception("Failed to set avatar")
-                    self.photo_id = ""
-                return True
-
-            loc = InputPeerPhotoFileLocation(
-                peer=await self.get_input_entity(source), photo_id=photo.photo_id, big=True
-            )
-            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
-            if file:
+                self.avatar_url = None
+            elif self.photo_id != photo_id or not self.avatar_url:
+                file = await util.transfer_file_to_matrix(
+                    client=source.client,
+                    intent=self.default_mxid_intent,
+                    location=InputPeerPhotoFileLocation(
+                        peer=await self.get_input_entity(source), photo_id=photo.photo_id, big=True
+                    ),
+                )
+                if not file:
+                    return False
                 self.photo_id = photo_id
-                try:
-                    await self.default_mxid_intent.set_avatar_url(file.mxc)
-                except MatrixError:
-                    self.log.exception("Failed to set avatar")
-                    self.photo_id = ""
-                return True
+                self.avatar_url = file.mxc
+            try:
+                await self.default_mxid_intent.set_avatar_url(self.avatar_url or "")
+                self.avatar_set = True
+            except Exception as e:
+                self.log.warning(f"Failed to set avatar: {e}")
+                self.avatar_set = False
+            return True
         return False
 
     async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
@@ -345,7 +387,9 @@ class Puppet(DBPuppet, BasePuppet):
 
     @classmethod
     @async_getter_lock
-    async def get_by_tgid(cls, tgid: TelegramID, *, create: bool = True) -> Puppet | None:
+    async def get_by_tgid(
+        cls, tgid: TelegramID, *, create: bool = True, is_channel: bool = False
+    ) -> Puppet | None:
         if tgid is None:
             return None
 
@@ -360,13 +404,37 @@ class Puppet(DBPuppet, BasePuppet):
             return puppet
 
         if create:
-            puppet = cls(tgid)
+            puppet = cls(tgid, is_channel=is_channel)
             await puppet.insert()
             puppet._add_to_cache()
             return puppet
 
         return None
 
+    @staticmethod
+    def get_id_from_peer(peer: TypePeer | User | Channel) -> TelegramID:
+        if isinstance(peer, PeerUser):
+            return TelegramID(peer.user_id)
+        elif isinstance(peer, PeerChannel):
+            return TelegramID(peer.channel_id)
+        elif isinstance(peer, PeerChat):
+            return TelegramID(peer.chat_id)
+        elif isinstance(peer, (User, Channel)):
+            return TelegramID(peer.id)
+        raise TypeError(f"invalid type {type(peer).__name__!r} in _id_from_peer()")
+
+    @classmethod
+    async def get_by_peer(
+        cls, peer: TypePeer | User | Channel, *, create: bool = True
+    ) -> Puppet | None:
+        if isinstance(peer, PeerChat):
+            return None
+        return await cls.get_by_tgid(
+            cls.get_id_from_peer(peer),
+            create=create,
+            is_channel=isinstance(peer, (PeerChannel, Channel)),
+        )
+
     @classmethod
     def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Awaitable[Puppet | None]:
         return cls.get_by_tgid(cls.get_id_from_mxid(mxid), create=create)

mautrix_telegram/tgclient.py

@@ -40,7 +40,7 @@ class MautrixTelegramClient(TelegramClient):
         mime_type: str = None,
         attributes: List[TypeDocumentAttribute] = None,
         file_name: str = None,
-        max_image_size: float = 10 * 1000 ** 2,
+        max_image_size: float = 10 * 1000**2,
     ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
         file_handle = await super().upload_file(file, file_name=file_name)
 

mautrix_telegram/user.py

@@ -195,7 +195,8 @@ class User(DBUser, AbstractUser, BaseUser):
             if self.tgid:
                 await self.push_bridge_state(BridgeStateEvent.UNKNOWN_ERROR, message=str(e))
         except UnauthorizedError as e:
-            self.log.error(f"Authorization error in start(): {type(e)}: {e}")
+            if delete_unless_authenticated or self.tgid:
+                self.log.error(f"Authorization error in start(): {type(e)}: {e}")
             if self.tgid:
                 await self.push_bridge_state(
                     BridgeStateEvent.BAD_CREDENTIALS,
@@ -240,7 +241,7 @@ class User(DBUser, AbstractUser, BaseUser):
                 )
             else:
                 await self.push_bridge_state(
-                    BridgeStateEvent.UNKNOWN_ERROR, ttl=240, error="tg-not-connected"
+                    BridgeStateEvent.TRANSIENT_DISCONNECT, ttl=240, error="tg-not-connected"
                 )
 
     async def fill_bridge_state(self, state: BridgeState) -> None:
@@ -448,7 +449,7 @@ class User(DBUser, AbstractUser, BaseUser):
     async def get_direct_chats(self) -> dict[UserID, list[RoomID]]:
         return {
             pu.Puppet.get_mxid_from_id(portal.tgid): [portal.mxid]
-            async for portal in po.Portal.find_private_chats(self.tgid)
+            async for portal in po.Portal.find_private_chats_of(self.tgid)
             if portal.mxid
         }
 
@@ -655,10 +656,10 @@ class User(DBUser, AbstractUser, BaseUser):
         if self.saved_contacts != response.saved_count:
             self.saved_contacts = response.saved_count
             await self.save()
-        await self.set_contacts(user.id for user in response.users)
         for user in response.users:
             puppet = await pu.Puppet.get_by_tgid(user.id)
             await puppet.update_info(self, user)
+        await self.set_contacts(user.id for user in response.users)
 
     # endregion
     # region Class instance lookup

mautrix_telegram/util/tgs_converter.py

@@ -19,12 +19,15 @@ from __future__ import annotations
 from typing import Any, Awaitable, Callable
 import asyncio.subprocess
 import logging
+import os
 import os.path
 import shutil
 import tempfile
 
 from attr import dataclass
 
+from mautrix.util import ffmpeg
+
 log: logging.Logger = logging.getLogger("mau.util.tgs")
 
 
@@ -48,61 +51,49 @@ def abswhich(program: str | None) -> str | None:
 
 
 lottieconverter = abswhich("lottieconverter")
-ffmpeg = abswhich("ffmpeg")
+
+
+async def _run_lottieconverter(args: tuple[str, ...], input_data: bytes) -> bytes:
+    proc = await asyncio.create_subprocess_exec(
+        lottieconverter,
+        *args,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        stdin=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate(input_data)
+    if proc.returncode == 0:
+        return stdout
+    else:
+        err_text = stderr.decode("utf-8") if stderr else f"unknown ({proc.returncode})"
+        raise ffmpeg.ConverterError(f"lottieconverter error: {err_text}")
+
 
 if lottieconverter:
 
     async def tgs_to_png(file: bytes, width: int, height: int, **_: Any) -> ConvertedSticker:
         frame = 1
-        proc = await asyncio.create_subprocess_exec(
-            lottieconverter,
-            "-",
-            "-",
-            "png",
-            f"{width}x{height}",
-            str(frame),
-            stdout=asyncio.subprocess.PIPE,
-            stdin=asyncio.subprocess.PIPE,
-        )
-        stdout, stderr = await proc.communicate(file)
-        if proc.returncode == 0:
-            return ConvertedSticker("image/png", stdout)
-        else:
-            log.error(
-                "lottieconverter error: "
-                + (
-                    stderr.decode("utf-8")
-                    if stderr is not None
-                    else f"unknown ({proc.returncode})"
-                )
+        try:
+            converted_png = await _run_lottieconverter(
+                args=("-", "-", "png", f"{width}x{height}", str(frame)),
+                input_data=file,
             )
+            return ConvertedSticker("image/png", converted_png)
+        except ffmpeg.ConverterError as e:
+            log.error(str(e))
             return ConvertedSticker("application/gzip", file)
 
     async def tgs_to_gif(
         file: bytes, width: int, height: int, fps: int = 25, **_: Any
     ) -> ConvertedSticker:
-        proc = await asyncio.create_subprocess_exec(
-            lottieconverter,
-            "-",
-            "-",
-            "gif",
-            f"{width}x{height}",
-            str(fps),
-            stdout=asyncio.subprocess.PIPE,
-            stdin=asyncio.subprocess.PIPE,
-        )
-        stdout, stderr = await proc.communicate(file)
-        if proc.returncode == 0:
-            return ConvertedSticker("image/gif", stdout)
-        else:
-            log.error(
-                "lottieconverter error: "
-                + (
-                    stderr.decode("utf-8")
-                    if stderr is not None
-                    else f"unknown ({proc.returncode})"
-                )
+        try:
+            converted_gif = await _run_lottieconverter(
+                args=("-", "-", "gif", f"{width}x{height}", str(fps)),
+                input_data=file,
             )
+            return ConvertedSticker("image/gif", converted_gif)
+        except ffmpeg.ConverterError as e:
+            log.error(str(e))
             return ConvertedSticker("application/gzip", file)
 
     converters["png"] = tgs_to_png
@@ -115,62 +106,24 @@ if lottieconverter and ffmpeg:
     ) -> ConvertedSticker:
         with tempfile.TemporaryDirectory(prefix="tgs_") as tmpdir:
             file_template = tmpdir + "/out_"
-            proc = await asyncio.create_subprocess_exec(
-                lottieconverter,
-                "-",
-                file_template,
-                "pngs",
-                f"{width}x{height}",
-                str(fps),
-                stdout=asyncio.subprocess.PIPE,
-                stdin=asyncio.subprocess.PIPE,
-            )
-            _, stderr = await proc.communicate(file)
-            if proc.returncode == 0:
-                with open(f"{file_template}00.png", "rb") as first_frame_file:
+            try:
+                await _run_lottieconverter(
+                    args=("-", file_template, "pngs", f"{width}x{height}", str(fps)),
+                    input_data=file,
+                )
+                first_frame_name = min(os.listdir(tmpdir))
+                with open(f"{tmpdir}/{first_frame_name}", "rb") as first_frame_file:
                     first_frame_data = first_frame_file.read()
-                proc = await asyncio.create_subprocess_exec(
-                    ffmpeg,
-                    "-hide_banner",
-                    "-loglevel",
-                    "error",
-                    "-framerate",
-                    str(fps),
-                    "-pattern_type",
-                    "glob",
-                    "-i",
-                    file_template + "*.png",
-                    "-c:v",
-                    "libvpx-vp9",
-                    "-pix_fmt",
-                    "yuva420p",
-                    "-f",
-                    "webm",
-                    "-",
-                    stdout=asyncio.subprocess.PIPE,
-                    stdin=asyncio.subprocess.PIPE,
-                )
-                stdout, stderr = await proc.communicate()
-                if proc.returncode == 0:
-                    return ConvertedSticker("video/webm", stdout, "image/png", first_frame_data)
-                else:
-                    log.error(
-                        "ffmpeg error: "
-                        + (
-                            stderr.decode("utf-8")
-                            if stderr is not None
-                            else f"unknown ({proc.returncode})"
-                        )
-                    )
-            else:
-                log.error(
-                    "lottieconverter error: "
-                    + (
-                        stderr.decode("utf-8")
-                        if stderr is not None
-                        else f"unknown ({proc.returncode})"
-                    )
+                webm_data = await ffmpeg.convert_path(
+                    input_args=("-framerate", str(fps), "-pattern_type", "glob"),
+                    input_file=f"{file_template}*.png",
+                    output_args=("-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-f", "webm"),
+                    output_path_override="-",
+                    output_extension=None,
                 )
+                return ConvertedSticker("video/webm", webm_data, "image/png", first_frame_data)
+            except ffmpeg.ConverterError as e:
+                log.error(str(e))
         return ConvertedSticker("application/gzip", file)
 
     converters["webm"] = tgs_to_webm

mautrix_telegram/web/common/auth_api.py

@@ -126,8 +126,10 @@ class AuthAPI(abc.ABC):
                 mxid=user.mxid,
                 state="code",
                 status=200,
-                message="Code requested successfully. Check your SMS "
-                "or Telegram client and enter the code below.",
+                message=(
+                    "Code requested successfully. Check your SMS "
+                    "or Telegram client and enter the code below."
+                ),
             )
         except PhoneNumberInvalidError:
             return self.get_login_response(
@@ -167,8 +169,10 @@ class AuthAPI(abc.ABC):
                 state="request",
                 status=429,
                 errcode="phone_number_flood",
-                error="Your phone number has been temporarily blocked for flooding. "
-                "The ban is usually applied for around a day.",
+                error=(
+                    "Your phone number has been temporarily blocked for flooding. "
+                    "The ban is usually applied for around a day."
+                ),
             )
         except FloodWaitError as e:
             return self.get_login_response(
@@ -176,8 +180,10 @@ class AuthAPI(abc.ABC):
                 state="request",
                 status=429,
                 errcode="flood_wait",
-                error="Your phone number has been temporarily blocked for flooding. "
-                f"Please wait for {format_duration(e.seconds)} before trying again.",
+                error=(
+                    "Your phone number has been temporarily blocked for flooding. "
+                    f"Please wait for {format_duration(e.seconds)} before trying again."
+                ),
             )
         except Exception:
             self.log.exception("Error requesting phone code")
@@ -237,6 +243,14 @@ class AuthAPI(abc.ABC):
     async def post_login_code(
         self, user: User, code: int, password_in_data: bool
     ) -> web.Response | None:
+        if not code:
+            return self.get_login_response(
+                mxid=user.mxid,
+                state="code",
+                status=400,
+                errcode="phone_code_missing",
+                error="You must provide the code from your phone.",
+            )
         try:
             user_info = await user.client.sign_in(code=code)
             await self.postprocess_login(user, user_info)

optional-requirements.txt

@@ -8,7 +8,7 @@ aiodns
 brotli
 
 #/qr_login
-pillow>=4,<9
+pillow>=4,<10
 qrcode>=6,<8
 
 #/hq_thumbnails
@@ -18,7 +18,7 @@ moviepy>=1,<2
 phonenumbers>=8,<9
 
 #/metrics
-prometheus_client>=0.6,<0.13
+prometheus_client>=0.6,<0.14
 
 #/e2be
 python-olm>=3,<4

pyproject.toml

@@ -9,4 +9,4 @@ line_length = 99
 [tool.black]
 line-length = 99
 target-version = ["py38"]
-required-version = "21.12b0"
+required-version = "22.1.0"

requirements.txt

@@ -3,10 +3,10 @@ python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.14.0,<0.15
+mautrix>=0.14.9,<0.15
 #telethon>=1.24,<1.25
-# Fork to make session storage async
-tulir-telethon==1.25.0a1
+# Fork to make session storage async and update to layer 138
+tulir-telethon==1.25.0a5
 asyncpg>=0.20,<0.26
 mako>=1,<2
 setuptools

setup.py

@@ -1,5 +1,4 @@
 import setuptools
-import glob
 
 from mautrix_telegram.get_version import git_tag, git_revision, version, linkified_version