Bump version to 0.11.2

link previews: support from Telegram -> Beeper

.editorconfig

@@ -17,5 +17,5 @@ max_line_length = 99
 [*.{yaml,yml,py}]
 indent_style = space
 
-[.gitlab-ci.yml]
+[{.gitlab-ci.yml,.pre-commit-config.yaml}]
 indent_size = 2

.github/workflows/python-lint.yml

@@ -13,6 +13,14 @@ jobs:
     - uses: isort/isort-action@master
       with:
         sortPaths: "./mautrix_telegram"
-    - uses: psf/black@21.12b0
+    - uses: psf/black@stable
       with:
         src: "./mautrix_telegram"
+        version: "22.1.0"
+    - name: pre-commit
+      run: |
+        pip install pre-commit
+        pre-commit run -av trailing-whitespace
+        pre-commit run -av end-of-file-fixer
+        pre-commit run -av check-yaml
+        pre-commit run -av check-added-large-files

.pre-commit-config.yaml

@@ -0,0 +1,23 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+      - id: trailing-whitespace
+        exclude_types: [markdown]
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+  # TODO convert to use the upstream psf/black when
+  # https://github.com/psf/black/issues/2493 gets fixed
+  - repo: local
+    hooks:
+      - id: black
+        name: black
+        entry: black --check
+        language: system
+        files: ^mautrix_telegram/.*\.py$
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        files: ^mautrix_telegram/.*$

CHANGELOG.md

@@ -1,9 +1,39 @@
-# v0.11.1 (2021-01-10)
+# v0.11.2 (2022-02-14)
 
+**N.B.** This will be the last release to support Python 3.7. Future versions
+will require Python 3.8 or higher. In general, the mautrix bridges will only
+support the lowest Python version in the latest Debian or Ubuntu LTS.
+
+### Added
+* Added simple fallback message for live location and venue messages from Telegram.
+* Added support for `t.me/+code` style invite links in `!tg join`.
+* Added support for showing channel profile when users send messages as a channel.
+* Added "user joined Telegram" message when Telegram auto-creates a DM chat for
+  a new user.
+
+### Improved
+* Added option for adding a random prefix to relayed user displaynames to help
+  distinguish them on the Telegram side.
+* Improved syncing profile info to room info when using encryption and/or the
+  `private_chat_profile_meta` config option.
+* Removed legacy `community_id` config option.
+
+### Fixed
+* Fixed newlines disappearing when bridging channel messages with signatures.
+* Fixed login throwing an error if a previous login code expired.
+* Fixed bug in v0.11.0 that broke `!tg create`.
+
+# v0.11.1 (2022-01-10)
+
+### Added
 * Added support for message reactions.
 * Added support for spoiler text.
-* Improved support for voice messages.
-* Improved color of blue text from Telegram to be more readable on dark themes.
+
+### Improved
+* Support for voice messages.
+* Changed color of blue text from Telegram to be more readable on dark themes.
+
+### Fixed
 * Fixed syncing contacts throwing an error for new accounts.
 * Fixed migrating pre-v0.11 legacy databases if the database schema had been
   corrupted (e.g. by using 3rd party tools for SQLite -> Postgres migration).

dev-requirements.txt

@@ -0,0 +1,3 @@
+pre-commit>=2.10.1,<3
+isort>=5.10.1,<6
+black==22.1.0

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.11.1"
+__version__ = "0.11.2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -86,6 +86,7 @@ class TelegramBridge(Bridge):
         Portal.init_cls(self)
         self.add_startup_actions(Puppet.init_cls(self))
         self.add_startup_actions(User.init_cls(self))
+        self.add_startup_actions(Portal.restart_scheduled_disappearing())
         if self.bot:
             self.add_startup_actions(self.bot.start())
         if self.config["bridge.resend_bridge_info"]:

mautrix_telegram/abstract_user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Type, Union
+from typing import TYPE_CHECKING, Any, Union
 from abc import ABC, abstractmethod
 import asyncio
 import logging
@@ -34,6 +34,7 @@ from telethon.tl.types import (
     Chat,
     MessageActionChannelMigrateFrom,
     MessageEmpty,
+    PeerChannel,
     PeerChat,
     PeerUser,
     TypeUpdate,
@@ -147,7 +148,7 @@ class AbstractUser(ABC):
         return self.client and self.client.is_connected()
 
     @property
-    def _proxy_settings(self) -> tuple[Type[Connection], tuple[Any, ...] | None]:
+    def _proxy_settings(self) -> tuple[type[Connection], tuple[Any, ...] | None]:
         proxy_type = self.config["telegram.proxy.type"].lower()
         connection = ConnectionTcpFull
         connection_data = (
@@ -385,7 +386,7 @@ class AbstractUser(ABC):
         if not message:
             return
 
-        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.peer.user_id))
+        puppet = await pu.Puppet.get_by_peer(update.peer)
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
     async def update_own_read_receipt(
@@ -444,10 +445,7 @@ class AbstractUser(ABC):
             return
 
         if isinstance(update, (UpdateChannelUserTyping, UpdateChatUserTyping)):
-            # Can typing notifications come from non-user peers?
-            if not update.from_id.user_id:
-                return
-            sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id.user_id))
+            sender = await pu.Puppet.get_by_peer(update.from_id)
 
         if not sender or not portal or not portal.mxid:
             return
@@ -456,8 +454,8 @@ class AbstractUser(ABC):
 
     async def _handle_entity_updates(self, entities: dict[int, User | Chat | Channel]) -> None:
         try:
-            users = (entity for entity in entities.values() if isinstance(entity, User))
-            puppets = ((await pu.Puppet.get_by_tgid(TelegramID(user.id)), user) for user in users)
+            users = (entity for entity in entities.values() if isinstance(entity, (User, Channel)))
+            puppets = ((await pu.Puppet.get_by_peer(user), user) for user in users)
             await asyncio.gather(
                 *[puppet.try_update_info(self, info) async for puppet, info in puppets if puppet]
             )
@@ -471,9 +469,11 @@ class AbstractUser(ABC):
             puppet.username = update.username
             if await puppet.update_displayname(self, update):
                 await puppet.save()
+                await puppet.update_portals_meta()
         elif isinstance(update, UpdateUserPhoto):
             if await puppet.update_avatar(self, update.photo):
                 await puppet.save()
+                await puppet.update_portals_meta()
         else:
             self.log.warning(f"Unexpected other user info update: {type(update)}")
 
@@ -515,8 +515,8 @@ class AbstractUser(ABC):
             portal = await po.Portal.get_by_entity(update.peer_id, tg_receiver=self.tgid)
             if update.out:
                 sender = await pu.Puppet.get_by_tgid(self.tgid)
-            elif isinstance(update.from_id, PeerUser):
-                sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id.user_id))
+            elif isinstance(update.from_id, (PeerUser, PeerChannel)):
+                sender = await pu.Puppet.get_by_peer(update.from_id)
             else:
                 sender = None
         else:

mautrix_telegram/bot.py

@@ -225,7 +225,11 @@ class Bot(AbstractUser):
         elif isinstance(message.to_id, PeerChat):
             return reply(str(-message.to_id.chat_id))
         elif isinstance(message.to_id, PeerUser):
-            return reply(f"Your user ID is {message.to_id.user_id}.")
+            return reply(
+                f"Your user ID is {message.to_id.user_id}.\n\n"
+                f"If you're trying to bridge a group chat to Matrix, you must run the command in "
+                f"the group, not here. **The ID above will not work** with `!tg bridge`."
+            )
         else:
             return reply("Failed to find chat ID.")
 

mautrix_telegram/commands/portal/create_chat.py

@@ -65,7 +65,7 @@ async def create(evt: CommandEvent) -> EventID:
         about=about,
         encrypted=encrypted,
     )
-    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender)
+    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender, pre_create=True)
     if len(errors) > 0:
         error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
         await evt.reply(

mautrix_telegram/commands/telegram/auth.py

@@ -37,6 +37,7 @@ from telethon.errors import (
 )
 from telethon.tl.types import User
 
+from mautrix.client import Client
 from mautrix.types import (
     EventID,
     ImageInfo,
@@ -230,9 +231,23 @@ async def login_qr(evt: CommandEvent) -> EventID:
 )
 async def login(evt: CommandEvent) -> EventID:
     override_sender = False
-    if len(evt.args) > 0 and evt.sender.is_admin:
-        evt.sender = await u.User.get_and_start_by_mxid(UserID(evt.args[0]))
+    if len(evt.args) > 0 and evt.sender.is_admin and evt.args[0]:
+        override_user_id = UserID(evt.args[0])
+        try:
+            Client.parse_user_id(override_user_id)
+        except ValueError:
+            return await evt.reply(
+                f"**Usage:** `$cmdprefix+sp login [override user ID]`\n\n"
+                f"{override_user_id!r} is not a valid Matrix user ID"
+            )
+        orig_user_id = evt.sender.mxid
+        evt.sender = await u.User.get_and_start_by_mxid(override_user_id)
         override_sender = True
+        if orig_user_id != evt.sender:
+            await evt.reply(
+                f"Admin override: logging in as {evt.sender.mxid} instead of {orig_user_id}"
+            )
+
     if await evt.sender.is_logged_in():
         return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 

mautrix_telegram/commands/telegram/misc.py

@@ -208,6 +208,9 @@ async def join(evt: CommandEvent) -> EventID | None:
     link_type = data["type"]
     if link_type:
         link_type = link_type.lower()
+    elif identifier.startswith("+"):
+        link_type = "joinchat"
+        identifier = identifier[1:]
     updates, _ = await _join(evt, identifier, link_type)
     if not updates:
         return None

mautrix_telegram/config.py

@@ -95,8 +95,6 @@ class Config(BaseBridgeConfig):
         if "pool_pre_ping" in base["appservice.database_opts"]:
             del base["appservice.database_opts.pool_pre_ping"]
 
-        copy("appservice.community_id")
-
         copy("metrics.enabled")
         copy("metrics.listen_port")
 
@@ -138,11 +136,14 @@ class Config(BaseBridgeConfig):
         copy("bridge.invite_link_resolve")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
+        copy("bridge.image_as_file_pixels")
         copy("bridge.max_document_size")
         copy("bridge.parallel_file_transfer")
         copy("bridge.federate_rooms")
         copy("bridge.animated_sticker.target")
-        copy("bridge.animated_sticker.args")
+        copy("bridge.animated_sticker.args.width")
+        copy("bridge.animated_sticker.args.height")
+        copy("bridge.animated_sticker.args.fps")
         copy("bridge.encryption.allow")
         copy("bridge.encryption.default")
         copy("bridge.encryption.database")
@@ -180,6 +181,7 @@ class Config(BaseBridgeConfig):
             del self["bridge.message_formats"]
         copy_dict("bridge.message_formats", override_existing_map=False)
         copy("bridge.emote_format")
+        copy("bridge.relay_user_distinguishers")
 
         copy("bridge.state_event_formats.join")
         copy("bridge.state_event_formats.leave")

mautrix_telegram/db/__init__.py

@@ -16,6 +16,7 @@
 from mautrix.util.async_db import Database
 
 from .bot_chat import BotChat
+from .disappearing_message import DisappearingMessage
 from .message import Message
 from .portal import Portal
 from .puppet import Puppet
@@ -27,7 +28,17 @@ from .user import User
 
 
 def init(db: Database) -> None:
-    for table in (Portal, Message, Reaction, User, Puppet, TelegramFile, BotChat, PgSession):
+    for table in (
+        Portal,
+        Message,
+        Reaction,
+        User,
+        Puppet,
+        TelegramFile,
+        BotChat,
+        PgSession,
+        DisappearingMessage,
+    ):
         table.db = db
 
 
@@ -42,4 +53,5 @@ __all__ = [
     "TelegramFile",
     "BotChat",
     "PgSession",
+    "DisappearingMessage",
 ]

mautrix_telegram/db/disappearing_message.py

@@ -0,0 +1,78 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Sumner Evans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+import asyncpg
+
+from mautrix.bridge import AbstractDisappearingMessage
+from mautrix.types import EventID, RoomID
+from mautrix.util.async_db import Database
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+class DisappearingMessage(AbstractDisappearingMessage):
+    db: ClassVar[Database] = fake_db
+
+    async def insert(self) -> None:
+        q = """
+            INSERT INTO disappearing_message (room_id, event_id, expiration_seconds, expiration_ts)
+            VALUES ($1, $2, $3, $4)
+        """
+        await self.db.execute(
+            q, self.room_id, self.event_id, self.expiration_seconds, self.expiration_ts
+        )
+
+    async def update(self) -> None:
+        q = "UPDATE disappearing_message SET expiration_ts=$3 WHERE room_id=$1 AND event_id=$2"
+        await self.db.execute(q, self.room_id, self.event_id, self.expiration_ts)
+
+    async def delete(self) -> None:
+        q = "DELETE from disappearing_message WHERE room_id=$1 AND event_id=$2"
+        await self.db.execute(q, self.room_id, self.event_id)
+
+    @classmethod
+    def _from_row(cls, row: asyncpg.Record) -> DisappearingMessage:
+        return cls(**row)
+
+    @classmethod
+    async def get(cls, room_id: RoomID, event_id: EventID) -> DisappearingMessage | None:
+        q = """
+            SELECT room_id, event_id, expiration_seconds, expiration_ts FROM disappearing_message
+            WHERE room_id=$1 AND mxid=$2
+        """
+        try:
+            return cls._from_row(await cls.db.fetchrow(q, room_id, event_id))
+        except Exception:
+            return None
+
+    @classmethod
+    async def get_all_scheduled(cls) -> list[DisappearingMessage]:
+        q = """
+            SELECT room_id, event_id, expiration_seconds, expiration_ts FROM disappearing_message
+            WHERE expiration_ts IS NOT NULL
+        """
+        return [cls._from_row(r) for r in await cls.db.fetch(q)]
+
+    @classmethod
+    async def get_unscheduled_for_room(cls, room_id: RoomID) -> list[DisappearingMessage]:
+        q = """
+            SELECT room_id, event_id, expiration_seconds, expiration_ts FROM disappearing_message
+            WHERE room_id = $1 AND expiration_ts IS NULL
+        """
+        return [cls._from_row(r) for r in await cls.db.fetch(q, room_id)]

mautrix_telegram/db/portal.py

@@ -54,6 +54,8 @@ class Portal:
     title: str | None
     about: str | None
     photo_id: str | None
+    name_set: bool
+    avatar_set: bool
 
     local_config: dict[str, Any] = attr.ib(factory=lambda: {})
 
@@ -67,7 +69,8 @@ class Portal:
 
     columns: ClassVar[str] = (
         "tgid, tg_receiver, peer_type, megagroup, mxid, avatar_url, encrypted, sponsored_event_id,"
-        "sponsored_event_ts, sponsored_msg_random_id, username, title, about, photo_id, config"
+        "sponsored_event_ts, sponsored_msg_random_id, username, title, about, photo_id, "
+        "name_set, avatar_set, config"
     )
 
     @classmethod
@@ -86,10 +89,15 @@ class Portal:
         return cls._from_row(await cls.db.fetchrow(q, username.lower()))
 
     @classmethod
-    async def find_private_chats(cls, tg_receiver: TelegramID) -> list[Portal]:
+    async def find_private_chats_of(cls, tg_receiver: TelegramID) -> list[Portal]:
         q = f"SELECT {cls.columns} FROM portal WHERE tg_receiver=$1 AND peer_type='user'"
         return [cls._from_row(row) for row in await cls.db.fetch(q, tg_receiver)]
 
+    @classmethod
+    async def find_private_chats_with(cls, tgid: TelegramID) -> list[Portal]:
+        q = f"SELECT {cls.columns} FROM portal WHERE tgid=$1 AND peer_type='user'"
+        return [cls._from_row(row) for row in await cls.db.fetch(q, tgid)]
+
     @classmethod
     async def all(cls) -> list[Portal]:
         rows = await cls.db.fetch(f"SELECT {cls.columns} FROM portal")
@@ -111,17 +119,20 @@ class Portal:
             self.title,
             self.about,
             self.photo_id,
+            self.name_set,
+            self.avatar_set,
             self.megagroup,
             json.dumps(self.local_config) if self.local_config else None,
         )
 
     async def save(self) -> None:
-        q = (
-            "UPDATE portal SET mxid=$4, avatar_url=$5, encrypted=$6, sponsored_event_id=$7,"
-            "                  sponsored_event_ts=$8, sponsored_msg_random_id=$9, username=$10,"
-            "                  title=$11, about=$12, photo_id=$13, megagroup=$14, config=$15 "
-            "WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)"
-        )
+        q = """
+        UPDATE portal
+        SET mxid=$4, avatar_url=$5, encrypted=$6, sponsored_event_id=$7, sponsored_event_ts=$8,
+            sponsored_msg_random_id=$9, username=$10, title=$11, about=$12, photo_id=$13,
+            name_set=$14, avatar_set=$15, megagroup=$16, config=$17
+        WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)
+        """
         await self.db.execute(q, *self._values)
 
     async def update_id(self, id: TelegramID, peer_type: str) -> None:
@@ -135,12 +146,13 @@ class Portal:
         self.peer_type = peer_type
 
     async def insert(self) -> None:
-        q = (
-            "INSERT INTO portal (tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,"
-            "                    sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,"
-            "                    username, title, about, photo_id, megagroup, config) "
-            "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"
-        )
+        q = """
+        INSERT INTO portal (
+            tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,
+            sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,
+            username, title, about, photo_id, name_set, avatar_set, megagroup, config
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
+        """
         await self.db.execute(q, *self._values)
 
     async def delete(self) -> None:

mautrix_telegram/db/puppet.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -21,7 +21,7 @@ from asyncpg import Record
 from attr import dataclass
 from yarl import URL
 
-from mautrix.types import SyncToken, UserID
+from mautrix.types import ContentURI, SyncToken, UserID
 from mautrix.util.async_db import Database
 
 from ..types import TelegramID
@@ -44,7 +44,11 @@ class Puppet:
     disable_updates: bool
     username: str | None
     photo_id: str | None
+    avatar_url: ContentURI | None
+    name_set: bool
+    avatar_set: bool
     is_bot: bool | None
+    is_channel: bool
 
     custom_mxid: UserID | None
     access_token: str | None
@@ -61,8 +65,8 @@ class Puppet:
 
     columns: ClassVar[str] = (
         "id, is_registered, displayname, displayname_source, displayname_contact, "
-        "displayname_quality, disable_updates, username, photo_id, is_bot, "
-        "custom_mxid, access_token, next_batch, base_url"
+        "displayname_quality, disable_updates, username, photo_id, avatar_url, "
+        "name_set, avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url"
     )
 
     @classmethod
@@ -102,7 +106,11 @@ class Puppet:
             self.disable_updates,
             self.username,
             self.photo_id,
+            self.avatar_url,
+            self.name_set,
+            self.avatar_set,
             self.is_bot,
+            self.is_channel,
             self.custom_mxid,
             self.access_token,
             self.next_batch,
@@ -110,21 +118,22 @@ class Puppet:
         )
 
     async def save(self) -> None:
-        q = (
-            "UPDATE puppet "
-            "SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,"
-            "    displayname_quality=$6, disable_updates=$7, username=$8, photo_id=$9, is_bot=$10,"
-            "    custom_mxid=$11, access_token=$12, next_batch=$13, base_url=$14 "
-            "WHERE id=$1"
-        )
+        q = """
+        UPDATE puppet
+        SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,
+            displayname_quality=$6, disable_updates=$7, username=$8, photo_id=$9,
+            avatar_url=$10, name_set=$11, avatar_set=$12, is_bot=$13, is_channel=$14,
+            custom_mxid=$15, access_token=$16, next_batch=$17, base_url=$18
+        WHERE id=$1
+        """
         await self.db.execute(q, *self._values)
 
     async def insert(self) -> None:
-        q = (
-            "INSERT INTO puppet ("
-            "    id, is_registered, displayname, displayname_source, displayname_contact,"
-            "    displayname_quality, disable_updates, username, photo_id, is_bot,"
-            "    custom_mxid, access_token, next_batch, base_url"
-            ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)"
-        )
+        q = """
+        INSERT INTO puppet (
+            id, is_registered, displayname, displayname_source, displayname_contact,
+            displayname_quality, disable_updates, username, photo_id, avatar_url, name_set,
+            avatar_set, is_bot, is_channel, custom_mxid, access_token, next_batch, base_url
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)
+        """
         await self.db.execute(q, *self._values)

mautrix_telegram/db/upgrade/__init__.py

@@ -2,4 +2,11 @@ from mautrix.util.async_db import UpgradeTable
 
 upgrade_table = UpgradeTable()
 
-from . import v01_initial_revision, v02_sponsored_events, v03_reactions
+from . import (
+    v01_initial_revision,
+    v02_sponsored_events,
+    v03_reactions,
+    v04_disappearing_messages,
+    v05_channel_ghosts,
+    v06_puppet_avatar_url,
+)

mautrix_telegram/db/upgrade/v04_disappearing_messages.py

@@ -0,0 +1,32 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add support for disappearing messages")
+async def upgrade_v4(conn: Connection) -> None:
+    await conn.execute(
+        """CREATE TABLE disappearing_message (
+            room_id             TEXT,
+            event_id            TEXT,
+            expiration_seconds  BIGINT,
+            expiration_ts       BIGINT,
+
+            PRIMARY KEY (room_id, event_id)
+        )"""
+    )

mautrix_telegram/db/upgrade/v05_channel_ghosts.py

@@ -0,0 +1,25 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add separate ghost users for channel senders")
+async def upgrade_v5(conn: Connection, scheme: str) -> None:
+    await conn.execute("ALTER TABLE puppet ADD COLUMN is_channel BOOLEAN NOT NULL DEFAULT false")
+    if scheme == "postgres":
+        await conn.execute("ALTER TABLE puppet ALTER COLUMN is_channel DROP DEFAULT")

mautrix_telegram/db/upgrade/v06_puppet_avatar_url.py

@@ -0,0 +1,31 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store avatar mxc URI in puppet table")
+async def upgrade_v6(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE puppet ADD COLUMN avatar_url TEXT")
+    await conn.execute("ALTER TABLE puppet ADD COLUMN name_set BOOLEAN NOT NULL DEFAULT false")
+    await conn.execute("ALTER TABLE puppet ADD COLUMN avatar_set BOOLEAN NOT NULL DEFAULT false")
+    await conn.execute("UPDATE puppet SET name_set=true WHERE displayname<>''")
+    await conn.execute("UPDATE puppet SET avatar_set=true WHERE photo_id<>''")
+    await conn.execute("ALTER TABLE portal ADD COLUMN name_set BOOLEAN NOT NULL DEFAULT false")
+    await conn.execute("ALTER TABLE portal ADD COLUMN avatar_set BOOLEAN NOT NULL DEFAULT false")
+    await conn.execute("UPDATE portal SET name_set=true WHERE title<>'' AND mxid<>''")
+    await conn.execute("UPDATE portal SET avatar_set=true WHERE photo_id<>'' AND mxid<>''")

mautrix_telegram/example-config.yaml

@@ -78,12 +78,6 @@ appservice:
     bot_displayname: Telegram bridge bot
     bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
 
-    # Community ID for bridged users (changes registration file) and rooms.
-    # Must be created manually.
-    #
-    # Example: "+telegram:example.com". Set to false to disable.
-    community_id: false
-
     # Whether or not to receive ephemeral events via appservice transactions.
     # Requires MSC2409 support (i.e. Synapse 1.22+).
     # You should disable bridge -> sync_with_custom_puppets when this is enabled.
@@ -214,6 +208,8 @@ bridge:
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
+    # Maximum number of pixels in an image before sending to Telegram as a document. Defaults to 1280x1280 = 1638400.
+    image_as_file_pixels: 1638400
     # Maximum size of Telegram documents in megabytes to bridge.
     max_document_size: 100
     # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
@@ -328,9 +324,12 @@ bridge:
         # List of user IDs for whom the previous flag is flipped.
         # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
         #      notices from users listed here will be bridged.
-        exceptions:
-        - "@importantbot:example.com"
+        exceptions: []
 
+    # An array of possible values for the $distinguisher variable in message formats.
+    # Each user gets one of the values here, based on a hash of their user ID.
+    # If the array is empty, the $distinguisher variable will also be empty.
+    relay_user_distinguishers: ["🟦", "🟣", "🟩", "⭕️", "🔶", "⬛️", "🔵", "🟢"]
     # The formats to use when sending messages to Telegram via the relay bot.
     # Text msgtypes (m.text, m.notice and m.emote) support HTML, media msgtypes don't.
     #
@@ -338,16 +337,17 @@ bridge:
     #   $sender_displayname - The display name of the sender (e.g. Example User)
     #   $sender_username    - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
     #   $sender_mxid        - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $distinguisher      - A random string from the options in the relay_user_distinguishers array.
     #   $message            - The message content
     message_formats:
-        m.text: "<b>$sender_displayname</b>: $message"
-        m.notice: "<b>$sender_displayname</b>: $message"
-        m.emote: "* <b>$sender_displayname</b> $message"
-        m.file: "<b>$sender_displayname</b> sent a file: $message"
-        m.image: "<b>$sender_displayname</b> sent an image: $message"
-        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
-        m.video: "<b>$sender_displayname</b> sent a video: $message"
-        m.location: "<b>$sender_displayname</b> sent a location: $message"
+        m.text: "$distinguisher <b>$sender_displayname</b>: $message"
+        m.notice: "$distinguisher <b>$sender_displayname</b>: $message"
+        m.emote: "* $distinguisher <b>$sender_displayname</b> $message"
+        m.file: "$distinguisher <b>$sender_displayname</b> sent a file: $message"
+        m.image: "$distinguisher <b>$sender_displayname</b> sent an image: $message"
+        m.audio: "$distinguisher <b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "$distinguisher <b>$sender_displayname</b> sent a video: $message"
+        m.location: "$distinguisher <b>$sender_displayname</b> sent a location: $message"
     # Telegram doesn't have built-in emotes, this field specifies how m.emote's from authenticated
     # users are sent to telegram. All fields in message_formats are supported. Additionally, the
     # Telegram user info is available in the following variables:
@@ -363,9 +363,9 @@ bridge:
     #
     # Set format to an empty string to disable the messages for that event.
     state_event_formats:
-        join: "<b>$displayname</b> joined the room."
-        leave: "<b>$displayname</b> left the room."
-        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+        join: "$distinguisher <b>$displayname</b> joined the room."
+        leave: "$distinguisher <b>$displayname</b> left the room."
+        name_change: "$distinguisher <b>$prev_displayname</b> changed their name to $distinguisher <b>$displayname</b>"
 
     # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
     # `filter-mode` management commands.

mautrix_telegram/formatter/from_matrix/parser.py

@@ -20,8 +20,7 @@ import logging
 from telethon import TelegramClient
 
 from mautrix.types import RoomID, UserID
-from mautrix.util.formatter import MatrixParser as BaseMatrixParser, RecursionContext
-from mautrix.util.formatter.html_reader_htmlparser import HTMLNode, read_html
+from mautrix.util.formatter import HTMLNode, MatrixParser as BaseMatrixParser, RecursionContext
 from mautrix.util.logging import TraceLogger
 
 from ... import portal as po, puppet as pu, user as u
@@ -33,7 +32,6 @@ log: TraceLogger = logging.getLogger("mau.fmt.mx")
 class MatrixParser(BaseMatrixParser[TelegramMessage]):
     e = TelegramEntityType
     fs = TelegramMessage
-    read_html = staticmethod(read_html)
     client: TelegramClient
 
     def __init__(self, client: TelegramClient) -> None:

mautrix_telegram/formatter/from_telegram.py

@@ -94,9 +94,7 @@ async def _add_forward_header(
             )
 
         if not fwd_from_text:
-            puppet = await pu.Puppet.get_by_tgid(
-                TelegramID(fwd_from.from_id.user_id), create=False
-            )
+            puppet = await pu.Puppet.get_by_peer(fwd_from.from_id, create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
                 fwd_from_html = (
@@ -200,7 +198,7 @@ async def telegram_to_matrix(
     def force_html():
         if not content.formatted_body:
             content.format = Format.HTML
-            content.formatted_body = escape(content.body)
+            content.formatted_body = escape(content.body).replace("\n", "<br/>")
 
     if require_html:
         force_html()

mautrix_telegram/matrix.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -66,11 +66,19 @@ class MatrixHandler(BaseMatrixHandler):
     ) -> None:
         intent = puppet.default_mxid_intent
         self.log.debug(f"{inviter.mxid} invited puppet for {puppet.tgid} to {room_id}")
+        if puppet.is_channel:
+            self.log.debug(f"Rejecting invite for {puppet.tgid} to {room_id}: puppet is a channel")
+            await intent.leave_room(room_id, reason="Channels can't be invited to chats")
+            return
+
         if not await inviter.is_logged_in():
-            await intent.error_and_leave(
-                room_id, text="Please log in before inviting Telegram puppets."
+            self.log.debug(f"Rejecting invite for {puppet.tgid} to {room_id}: user not logged in")
+            await intent.leave_room(
+                room_id,
+                reason="Only users who are logged into the bridge can invite Telegram ghosts.",
             )
             return
+
         portal = await po.Portal.get_by_mxid(room_id)
         if portal:
             if portal.peer_type == "user":

mautrix_telegram/portal.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -106,6 +106,7 @@ from telethon.tl.types import (
     MessageActionChatEditTitle,
     MessageActionChatJoinedByLink,
     MessageActionChatMigrateTo,
+    MessageActionContactSignUp,
     MessageActionGameScore,
     MessageEntityPre,
     MessageMediaContact,
@@ -113,11 +114,14 @@ from telethon.tl.types import (
     MessageMediaDocument,
     MessageMediaGame,
     MessageMediaGeo,
+    MessageMediaGeoLive,
     MessageMediaPhoto,
     MessageMediaPoll,
     MessageMediaUnsupported,
+    MessageMediaVenue,
+    MessageMediaWebPage,
+    MessagePeerReaction,
     MessageReactions,
-    MessageUserReaction,
     PeerChannel,
     PeerChat,
     PeerUser,
@@ -144,6 +148,7 @@ from telethon.tl.types import (
     TypePhotoSize,
     TypeUser,
     TypeUserFull,
+    TypeUserProfilePhoto,
     UpdateChannelUserTyping,
     UpdateChatUserTyping,
     UpdateNewMessage,
@@ -152,6 +157,7 @@ from telethon.tl.types import (
     UserFull,
     UserProfilePhoto,
     UserProfilePhotoEmpty,
+    WebPage,
 )
 from telethon.utils import decode_waveform
 import magic
@@ -193,6 +199,7 @@ from mautrix.util.simple_template import SimpleTemplate
 from . import abstract_user as au, formatter, portal_util as putil, puppet as p, user as u, util
 from .config import Config
 from .db import (
+    DisappearingMessage,
     Message as DBMessage,
     Portal as DBPortal,
     Reaction as DBReaction,
@@ -213,6 +220,8 @@ if TYPE_CHECKING:
 
 StateBridge = EventType.find("m.bridge", EventType.Class.STATE)
 StateHalfShotBridge = EventType.find("uk.half-shot.bridge", EventType.Class.STATE)
+BEEPER_LINK_PREVIEWS_KEY = "com.beeper.linkpreviews"
+BEEPER_IMAGE_ENCRYPTION_KEY = "beeper:image:encryption"
 
 InviteList = Union[UserID, List[UserID]]
 UpdateTyping = Union[UpdateUserTyping, UpdateChatUserTyping, UpdateChannelUserTyping]
@@ -241,6 +250,7 @@ class DocAttrs(NamedTuple):
 class Portal(DBPortal, BasePortal):
     bot: "Bot"
     config: Config
+    disappearing_msg_class = DisappearingMessage
 
     # Instance cache
     by_mxid: dict[RoomID, Portal] = {}
@@ -300,6 +310,8 @@ class Portal(DBPortal, BasePortal):
         title: str | None = None,
         about: str | None = None,
         photo_id: str | None = None,
+        name_set: bool = False,
+        avatar_set: bool = False,
         local_config: dict[str, Any] | None = None,
     ) -> None:
         super().__init__(
@@ -317,8 +329,11 @@ class Portal(DBPortal, BasePortal):
             title=title,
             about=about,
             photo_id=photo_id,
+            name_set=name_set,
+            avatar_set=avatar_set,
             local_config=local_config or {},
         )
+        BasePortal.__init__(self)
         self.log = self.log.getChild(self.tgid_log if self.tgid else self.mxid)
         self._main_intent = None
         self.deleted = False
@@ -434,12 +449,11 @@ class Portal(DBPortal, BasePortal):
     # region Matrix -> Telegram metadata
 
     async def get_telegram_users_in_matrix_room(
-        self, source: u.User
+        self, source: u.User, pre_create: bool = False
     ) -> tuple[list[InputPeerUser], list[UserID]]:
         user_tgids = {}
-        user_mxids = await self.main_intent.get_room_members(
-            self.mxid, (Membership.JOIN, Membership.INVITE)
-        )
+        intent = self.az.intent if pre_create else self.main_intent
+        user_mxids = await intent.get_room_members(self.mxid, (Membership.JOIN, Membership.INVITE))
         for mxid in user_mxids:
             if mxid == self.az.bot_mxid:
                 continue
@@ -557,6 +571,8 @@ class Portal(DBPortal, BasePortal):
         await self.update_bridge_info()
 
     async def invite_telegram(self, source: u.User, puppet: p.Puppet | au.AbstractUser) -> None:
+        if puppet.is_channel:
+            raise ValueError("Can't invite channels to chats")
         if self.peer_type == "chat":
             await source.client(
                 AddChatUserRequest(chat_id=self.tgid, user_id=puppet.tgid, fwd_limit=0)
@@ -629,14 +645,7 @@ class Portal(DBPortal, BasePortal):
                 puppet = await p.Puppet.get_by_tgid(self.tgid)
             await puppet.update_info(user, entity)
             await puppet.intent_for(self).join_room(self.mxid)
-            if self.encrypted or self.private_chat_portal_meta:
-                # The bridge bot needs to join for e2ee, but that messes up the default name
-                # generation. If/when canonical DMs happen, this might not be necessary anymore.
-                changed = await self._update_title(puppet.displayname)
-                changed = await self._update_avatar(user, entity.photo) or changed
-                if changed:
-                    await self.save()
-                    await self.update_bridge_info()
+            await self.update_info_from_puppet(puppet, user, entity.photo)
 
             puppet = await p.Puppet.get_by_custom_mxid(user.mxid)
             if puppet:
@@ -650,6 +659,22 @@ class Portal(DBPortal, BasePortal):
         if self.sync_matrix_state:
             await self.main_intent.get_joined_members(self.mxid)
 
+    async def update_info_from_puppet(
+        self,
+        puppet: p.Puppet,
+        source: au.AbstractUser | None = None,
+        photo: UserProfilePhoto | None = None,
+    ) -> None:
+        if not self.encrypted and not self.private_chat_portal_meta:
+            return
+        # The bridge bot needs to join for e2ee, but that messes up the default name
+        # generation. If/when canonical DMs happen, this might not be necessary anymore.
+        changed = await self._update_avatar_from_puppet(puppet, source, photo)
+        changed = await self._update_title(puppet.displayname) or changed
+        if changed:
+            await self.save()
+            await self.update_bridge_info()
+
     async def create_matrix_room(
         self,
         user: au.AbstractUser,
@@ -795,8 +820,8 @@ class Portal(DBPortal, BasePortal):
                 "state_key": self.bridge_info_state_key,
                 "content": self.bridge_info,
             },
+            # TODO remove this once https://github.com/matrix-org/matrix-doc/pull/2346 is in spec
             {
-                # TODO remove this once https://github.com/matrix-org/matrix-doc/pull/2346 is in spec
                 "type": str(StateHalfShotBridge),
                 "state_key": self.bridge_info_state_key,
                 "content": self.bridge_info,
@@ -807,7 +832,7 @@ class Portal(DBPortal, BasePortal):
             self.encrypted = True
             initial_state.append(
                 {
-                    "type": "m.room.encryption",
+                    "type": str(EventType.ROOM_ENCRYPTION),
                     "content": {"algorithm": "m.megolm.v1.aes-sha2"},
                 }
             )
@@ -815,16 +840,18 @@ class Portal(DBPortal, BasePortal):
                 create_invites.append(self.az.bot_mxid)
         if direct and (self.encrypted or self.private_chat_portal_meta):
             self.title = puppet.displayname
-        if self.config["appservice.community_id"]:
-            initial_state.append(
-                {
-                    "type": "m.room.related_groups",
-                    "content": {"groups": [self.config["appservice.community_id"]]},
-                }
-            )
+            self.avatar_url = puppet.avatar_url
+            self.photo_id = puppet.photo_id
         creation_content = {}
         if not self.config["bridge.federate_rooms"]:
             creation_content["m.federate"] = False
+        if self.avatar_url:
+            initial_state.append(
+                {
+                    "type": str(EventType.ROOM_AVATAR),
+                    "content": {"url": self.avatar_url},
+                }
+            )
 
         with self.backfill_lock:
             room_id = await self.main_intent.create_room(
@@ -839,6 +866,8 @@ class Portal(DBPortal, BasePortal):
             )
             if not room_id:
                 raise Exception(f"Failed to create room")
+            self.name_set = bool(self.title)
+            self.avatar_set = bool(self.avatar_url)
 
             if self.encrypted and self.matrix.e2ee and direct:
                 try:
@@ -941,11 +970,12 @@ class Portal(DBPortal, BasePortal):
             if user_mxid == self.az.bot_mxid:
                 continue
 
-            puppet_id = p.Puppet.get_id_from_mxid(user_mxid)
-            if puppet_id:
-                if puppet_id in allowed_tgids:
+            puppet = await p.Puppet.get_by_mxid(user_mxid)
+            if puppet:
+                # TODO figure out when/how to clean up channels from the member list
+                if puppet.id in allowed_tgids or puppet.is_channel:
                     continue
-                if self.bot and puppet_id == self.bot.tgid:
+                if self.bot and puppet.id == self.bot.tgid:
                     await self.bot.remove_chat(self.tgid)
                 try:
                     await self.main_intent.kick_user(
@@ -1098,21 +1128,48 @@ class Portal(DBPortal, BasePortal):
     async def _update_title(
         self, title: str, sender: p.Puppet | None = None, save: bool = False
     ) -> bool:
-        if self.title == title:
+        if self.title == title and self.name_set:
             return False
 
         self.title = title
-        await self._try_set_state(
-            sender, EventType.ROOM_NAME, RoomNameStateEventContent(name=self.title)
-        )
+        try:
+            await self._try_set_state(
+                sender, EventType.ROOM_NAME, RoomNameStateEventContent(name=self.title)
+            )
+            self.name_set = True
+        except Exception as e:
+            self.log.warning(f"Failed to set room name: {e}")
+            self.name_set = False
         if save:
             await self.save()
         return True
 
+    async def _update_avatar_from_puppet(
+        self, puppet: p.Puppet, user: au.AbstractUser | None, photo: UserProfilePhoto | None
+    ) -> bool:
+        if self.photo_id == puppet.photo_id and self.avatar_set:
+            return False
+        if puppet.avatar_url:
+            self.photo_id = puppet.photo_id
+            self.avatar_url = puppet.avatar_url
+            try:
+                await self._try_set_state(
+                    None, EventType.ROOM_AVATAR, RoomAvatarStateEventContent(url=self.avatar_url)
+                )
+                self.avatar_set = True
+            except Exception as e:
+                self.log.warning(f"Failed to set room avatar: {e}")
+                self.avatar_set = False
+            return True
+        elif photo is not None and user is not None:
+            return await self._update_avatar(user, photo=photo)
+        else:
+            return False
+
     async def _update_avatar(
         self,
         user: au.AbstractUser,
-        photo: TypeChatPhoto,
+        photo: TypeChatPhoto | TypeUserProfilePhoto,
         sender: p.Puppet | None = None,
         save: bool = False,
     ) -> bool:
@@ -1135,26 +1192,27 @@ class Portal(DBPortal, BasePortal):
             and not self.config["bridge.allow_avatar_remove"]
         ):
             return False
-        if self.photo_id != photo_id:
+        if self.photo_id != photo_id or not self.avatar_set:
             if not photo_id:
-                await self._try_set_state(
-                    sender, EventType.ROOM_AVATAR, RoomAvatarStateEventContent(url=None)
-                )
                 self.photo_id = ""
                 self.avatar_url = None
-                if save:
-                    await self.save()
-                return True
-            file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
-            if file:
-                await self._try_set_state(
-                    sender, EventType.ROOM_AVATAR, RoomAvatarStateEventContent(url=file.mxc)
-                )
+            elif self.photo_id != photo_id or not self.avatar_url:
+                file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
+                if not file:
+                    return False
                 self.photo_id = photo_id
                 self.avatar_url = file.mxc
-                if save:
-                    await self.save()
-                return True
+            try:
+                await self._try_set_state(
+                    sender, EventType.ROOM_AVATAR, RoomAvatarStateEventContent(url=self.avatar_url)
+                )
+                self.avatar_set = True
+            except Exception as e:
+                self.log.warning(f"Failed to set room avatar: {e}")
+                self.avatar_set = False
+            if save:
+                await self.save()
+            return True
         return False
 
     # endregion
@@ -1187,6 +1245,7 @@ class Portal(DBPortal, BasePortal):
             "mxid": user.mxid,
             "username": user.mxid_localpart,
             "displayname": escape_html(displayname),
+            "distinguisher": self._get_distinguisher(user.mxid),
             **kwargs,
         }
         return Template(tpl).safe_substitute(tpl_args)
@@ -1403,6 +1462,28 @@ class Portal(DBPortal, BasePortal):
             # We'll just assume the user is already in the chat.
             pass
 
+    @staticmethod
+    def hash_user_id(val: UserID) -> int:
+        """
+        A simple Matrix user ID hashing algorithm that matches what Element does.
+
+        Args:
+            val: the Matrix user ID.
+
+        Returns:
+            A 32-bit hash of the user ID.
+        """
+        out = 0
+        for char in val:
+            out = (out << 5) - out + ord(char)
+            # Emulate JS's 32-bit signed bitwise OR `hash |= 0`
+            out = (out & 2**31 - 1) - (out & 2**31)
+        return abs(out)
+
+    def _get_distinguisher(self, user_id: UserID) -> str:
+        ruds = self.get_config("relay_user_distinguishers") or []
+        return ruds[self.hash_user_id(user_id) % len(ruds)] if ruds else ""
+
     async def _apply_msg_format(self, sender: u.User, content: MessageEventContent) -> None:
         if not isinstance(content, TextMessageEventContent) or content.format != Format.HTML:
             content.format = Format.HTML
@@ -1420,6 +1501,7 @@ class Portal(DBPortal, BasePortal):
             message=content.formatted_body,
             body=content.body,
             formatted_body=content.formatted_body,
+            distinguisher=self._get_distinguisher(sender.mxid),
         )
         content.formatted_body = Template(tpl).safe_substitute(tpl_args)
 
@@ -1511,7 +1593,8 @@ class Portal(DBPortal, BasePortal):
         else:
             w = h = None
         file_name = content["net.maunium.telegram.internal.filename"]
-        max_image_size = self.config["bridge.image_as_file_size"] * 1000 ** 2
+        max_image_size = self.config["bridge.image_as_file_size"] * 1000**2
+        max_image_pixels = self.config["bridge.image_as_file_pixels"]
 
         if self.config["bridge.parallel_file_transfer"] and content.url:
             file_handle, file_size = await util.parallel_transfer_to_telegram(
@@ -1544,12 +1627,14 @@ class Portal(DBPortal, BasePortal):
             file_size = len(file)
 
         file_handle.name = file_name
+        force_document = file_size >= max_image_size
 
         attributes = [DocumentAttributeFilename(file_name=file_name)]
         if w and h:
             attributes.append(DocumentAttributeImageSize(w, h))
+            force_document = force_document or w * h >= max_image_pixels
 
-        if (mime == "image/png" or mime == "image/jpeg") and file_size < max_image_size:
+        if (mime == "image/png" or mime == "image/jpeg") and not force_document:
             media = InputMediaUploadedPhoto(file_handle)
         else:
             media = InputMediaUploadedDocument(
@@ -1631,7 +1716,11 @@ class Portal(DBPortal, BasePortal):
         except (KeyError, ValueError):
             self.log.exception("Failed to parse location")
             return None
-        caption, entities = await formatter.matrix_to_telegram(client, text=content.body)
+        try:
+            caption = content["org.matrix.msc3488.location"]["description"]
+            entities = []
+        except KeyError:
+            caption, entities = await formatter.matrix_to_telegram(client, text=content.body)
         media = MessageMediaGeo(geo=GeoPoint(lat=lat, long=long, access_hash=0))
 
         async with self.send_lock(sender_id):
@@ -2081,11 +2170,8 @@ class Portal(DBPortal, BasePortal):
     async def enable_dm_encryption(self) -> bool:
         ok = await super().enable_dm_encryption()
         if ok:
-            try:
-                puppet = await p.Puppet.get_by_tgid(self.tgid)
-                await self.main_intent.set_room_name(self.mxid, puppet.displayname)
-            except Exception:
-                self.log.warning(f"Failed to set room name", exc_info=True)
+            puppet = await p.Puppet.get_by_tgid(self.tgid)
+            await self.update_info_from_puppet(puppet)
         return ok
 
     # endregion
@@ -2105,15 +2191,6 @@ class Portal(DBPortal, BasePortal):
             return f"https://t.me/c/{self.tgid}/{evt.id}"
         return None
 
-    async def _expire_telegram_photo(self, intent: IntentAPI, event_id: EventID, ttl: int) -> None:
-        try:
-            content = TextMessageEventContent(msgtype=MessageType.NOTICE, body="Photo has expired")
-            content.set_edit(event_id)
-            await asyncio.sleep(ttl)
-            await self._send_message(intent, content)
-        except Exception:
-            self.log.warning("Failed to expire Telegram photo %s", event_id, exc_info=True)
-
     async def _handle_telegram_photo(
         self, source: au.AbstractUser, intent: IntentAPI, evt: Message, relates_to: RelatesTo
     ) -> EventID | None:
@@ -2122,6 +2199,7 @@ class Portal(DBPortal, BasePortal):
             return await self._send_message(
                 intent,
                 TextMessageEventContent(msgtype=MessageType.NOTICE, body="Photo has expired"),
+                timestamp=evt.date,
             )
         loc, largest_size = self._get_largest_photo_size(media.photo)
         if loc is None:
@@ -2170,13 +2248,15 @@ class Portal(DBPortal, BasePortal):
             content.url = file.mxc
         result = await self._send_message(intent, content, timestamp=evt.date)
         if media.ttl_seconds:
-            asyncio.create_task(self._expire_telegram_photo(intent, result, media.ttl_seconds))
+            await DisappearingMessage(self.mxid, result, media.ttl_seconds).insert()
         if evt.message:
             caption_content = await formatter.telegram_to_matrix(
                 evt, source, self.main_intent, no_reply_fallback=True
             )
             caption_content.external_url = content.external_url
             result = await self._send_message(intent, caption_content, timestamp=evt.date)
+            if media.ttl_seconds:
+                await DisappearingMessage(self.mxid, result, media.ttl_seconds).insert()
         return result
 
     @staticmethod
@@ -2274,7 +2354,7 @@ class Portal(DBPortal, BasePortal):
 
         attrs = self._parse_telegram_document_attributes(document.attributes)
 
-        if document.size > self.config["bridge.max_document_size"] * 1000 ** 2:
+        if document.size > self.config["bridge.max_document_size"] * 1000**2:
             name = attrs.name or ""
             caption = f"\n{evt.message}" if evt.message else ""
             # TODO encrypt
@@ -2354,17 +2434,21 @@ class Portal(DBPortal, BasePortal):
         else:
             content.url = file.mxc
         res = await self._send_message(intent, content, event_type=event_type, timestamp=evt.date)
+        if evt.media.ttl_seconds:
+            await DisappearingMessage(self.mxid, res, evt.media.ttl_seconds).insert()
         if evt.message:
             caption_content = await formatter.telegram_to_matrix(
                 evt, source, self.main_intent, no_reply_fallback=True
             )
             caption_content.external_url = content.external_url
             res = await self._send_message(intent, caption_content, timestamp=evt.date)
+            if evt.media.ttl_seconds:
+                await DisappearingMessage(self.mxid, res, evt.media.ttl_seconds).insert()
         return res
 
-    def _handle_telegram_location(
-        self, source: au.AbstractUser, intent: IntentAPI, evt: Message, relates_to: RelatesTo
-    ) -> Awaitable[EventID]:
+    def _location_message_to_content(
+        self, evt: Message, relates_to: RelatesTo, note: str
+    ) -> LocationMessageEventContent:
         long = evt.media.geo.long
         lat = evt.media.geo.lat
         long_char = "E" if long > 0 else "W"
@@ -2377,15 +2461,66 @@ class Portal(DBPortal, BasePortal):
         content = LocationMessageEventContent(
             msgtype=MessageType.LOCATION,
             geo_uri=f"geo:{geo}",
-            body=f"Location: {body}\n{url}",
+            body=f"{note}: {body}\n{url}",
             relates_to=relates_to,
             external_url=self._get_external_url(evt),
         )
         content["format"] = str(Format.HTML)
-        content["formatted_body"] = f"Location: <a href='{url}'>{body}</a>"
+        content["formatted_body"] = f"{note}: <a href='{url}'>{body}</a>"
+        content["org.matrix.msc3488.location"] = {
+            "uri": content.geo_uri,
+            "description": note,
+        }
+        return content
 
+    def _handle_telegram_location(
+        self, source: au.AbstractUser, intent: IntentAPI, evt: Message, relates_to: RelatesTo
+    ) -> Awaitable[EventID]:
+        content = self._location_message_to_content(evt, relates_to, "Location")
         return self._send_message(intent, content, timestamp=evt.date)
 
+    def _handle_telegram_live_location(
+        self, source: au.AbstractUser, intent: IntentAPI, evt: Message, relates_to: RelatesTo
+    ) -> Awaitable[EventID]:
+        content = self._location_message_to_content(
+            evt, relates_to, "Live Location (see your Telegram client for live updates)"
+        )
+        return self._send_message(intent, content, timestamp=evt.date)
+
+    def _handle_telegram_venue(
+        self, source: au.AbstractUser, intent: IntentAPI, evt: Message, relates_to: RelatesTo
+    ) -> Awaitable[EventID]:
+        content = self._location_message_to_content(evt, relates_to, evt.media.title)
+        return self._send_message(intent, content, timestamp=evt.date)
+
+    async def _telegram_webpage_to_beeper_link_preview(
+        self, source: au.AbstractUser, intent: IntentAPI, webpage: WebPage
+    ) -> dict[str, Any]:
+        beeper_link_preview: dict[str, Any] = {
+            "matched_url": webpage.url,
+            "og:title": webpage.title,
+            "og:url": webpage.url,
+            "og:description": webpage.description,
+        }
+
+        # Upload an image corresponding to the link preview if it exists.
+        if webpage.photo:
+            loc, largest_size = self._get_largest_photo_size(webpage.photo)
+            if loc is None:
+                return beeper_link_preview
+            beeper_link_preview["og:image:height"] = largest_size.h
+            beeper_link_preview["og:image:width"] = largest_size.w
+            file = await util.transfer_file_to_matrix(
+                source.client, intent, loc, encrypt=self.encrypted
+            )
+
+            if file.decryption_info:
+                beeper_link_preview[BEEPER_IMAGE_ENCRYPTION_KEY] = file.decryption_info.serialize()
+            else:
+                beeper_link_preview["og:image"] = file.mxc
+
+        return beeper_link_preview
+
     async def _handle_telegram_text(
         self, source: au.AbstractUser, intent: IntentAPI, is_bot: bool, evt: Message
     ) -> EventID:
@@ -2395,6 +2530,18 @@ class Portal(DBPortal, BasePortal):
         if is_bot and self.get_config("bot_messages_as_notices"):
             content.msgtype = MessageType.NOTICE
         await intent.set_typing(self.mxid, is_typing=False)
+
+        if (
+            hasattr(evt, "media")
+            and isinstance(evt.media, MessageMediaWebPage)
+            and isinstance(evt.media.webpage, WebPage)
+        ):
+            content[BEEPER_LINK_PREVIEWS_KEY] = [
+                await self._telegram_webpage_to_beeper_link_preview(
+                    source, intent, evt.media.webpage
+                )
+            ]
+
         return await self._send_message(intent, content, timestamp=evt.date)
 
     async def _handle_telegram_unsupported(
@@ -2713,52 +2860,67 @@ class Portal(DBPortal, BasePortal):
             self.log.debug(f"Iterating all messages starting with {min_id} (approx: {limit})")
             messages = client.iter_messages(entity, reverse=True, min_id=min_id)
             async for message in messages:
-                sender = (
-                    await p.Puppet.get_by_tgid(TelegramID(message.from_id.user_id))
-                    if isinstance(message.from_id, PeerUser)
-                    else None
-                )
-                # TODO handle service messages?
-                await self.handle_telegram_message(source, sender, message)
+                await self._handle_telegram_backfill_message(source, message)
                 count += 1
         else:
             self.log.debug(f"Fetching up to {limit} most recent messages")
             messages = await client.get_messages(entity, limit=limit)
             for message in reversed(messages):
-                sender = (
-                    await p.Puppet.get_by_tgid(TelegramID(message.from_id.user_id))
-                    if isinstance(message.from_id, PeerUser)
-                    else None
-                )
-                await self.handle_telegram_message(source, sender, message)
+                await self._handle_telegram_backfill_message(source, message)
                 count += 1
         return count
 
-    def _split_dm_reaction_counts(self, counts: list[ReactionCount]) -> list[MessageUserReaction]:
+    async def _handle_telegram_backfill_message(
+        self, source: au.AbstractUser, msg: Message | MessageService
+    ) -> None:
+        if msg.from_id and isinstance(msg.from_id, (PeerUser, PeerChannel)):
+            sender = await p.Puppet.get_by_peer(msg.from_id)
+        elif isinstance(msg.peer_id, PeerUser):
+            if msg.out:
+                sender = await p.Puppet.get_by_tgid(source.tgid)
+            else:
+                sender = await p.Puppet.get_by_peer(msg.peer_id)
+        else:
+            sender = None
+        if isinstance(msg, MessageService):
+            if isinstance(msg.action, MessageActionContactSignUp):
+                await self.handle_telegram_joined(source, sender, msg, backfill=True)
+            else:
+                self.log.debug(
+                    f"Unhandled service message {type(msg.action).__name__} in backfill"
+                )
+        elif isinstance(msg, Message):
+            await self.handle_telegram_message(source, sender, msg)
+        else:
+            self.log.debug(f"Unhandled message type {type(msg).__name__} in backfill")
+
+    def _split_dm_reaction_counts(self, counts: list[ReactionCount]) -> list[MessagePeerReaction]:
         if len(counts) == 1:
             item = counts[0]
             if item.count == 2:
                 return [
-                    MessageUserReaction(reaction=item.reaction, user_id=self.tgid),
-                    MessageUserReaction(reaction=item.reaction, user_id=self.tg_receiver),
+                    MessagePeerReaction(reaction=item.reaction, peer_id=PeerUser(self.tgid)),
+                    MessagePeerReaction(
+                        reaction=item.reaction, peer_id=PeerUser(self.tg_receiver)
+                    ),
                 ]
             elif item.count == 1:
                 return [
-                    MessageUserReaction(
+                    MessagePeerReaction(
                         reaction=item.reaction,
-                        user_id=self.tg_receiver if item.chosen else self.tgid,
+                        peer_id=PeerUser(self.tg_receiver if item.chosen else self.tgid),
                     ),
                 ]
         elif len(counts) == 2:
             item1, item2 = counts
             return [
-                MessageUserReaction(
+                MessagePeerReaction(
                     reaction=item1.reaction,
-                    user_id=self.tg_receiver if item1.chosen else self.tgid,
+                    peer_id=PeerUser(self.tg_receiver if item1.chosen else self.tgid),
                 ),
-                MessageUserReaction(
+                MessagePeerReaction(
                     reaction=item2.reaction,
-                    user_id=self.tg_receiver if item2.chosen else self.tgid,
+                    peer_id=PeerUser(self.tg_receiver if item2.chosen else self.tgid),
                 ),
             ]
         return []
@@ -2793,7 +2955,7 @@ class Portal(DBPortal, BasePortal):
                 return
 
         total_count = sum(item.count for item in data.results)
-        recent_reactions = data.recent_reactons or []
+        recent_reactions = data.recent_reactions or []
         if not recent_reactions and total_count > 0:
             if self.peer_type == "user":
                 recent_reactions = self._split_dm_reaction_counts(data.results)
@@ -2812,9 +2974,13 @@ class Portal(DBPortal, BasePortal):
             await self._handle_telegram_reactions_locked(dbm, recent_reactions, total_count)
 
     async def _handle_telegram_reactions_locked(
-        self, msg: DBMessage, reaction_list: list[MessageUserReaction], total_count: int
+        self, msg: DBMessage, reaction_list: list[MessagePeerReaction], total_count: int
     ) -> None:
-        reactions = {reaction.user_id: reaction.reaction for reaction in reaction_list}
+        reactions = {
+            p.Puppet.get_id_from_peer(reaction.peer_id): reaction.reaction
+            for reaction in reaction_list
+            if isinstance(reaction.peer_id, (PeerUser, PeerChannel))
+        }
         is_full = len(reactions) == total_count
 
         existing_reactions = await DBReaction.get_all_by_message(msg.mxid, msg.mx_room)
@@ -2924,10 +3090,10 @@ class Portal(DBPortal, BasePortal):
 
         if sender and not sender.displayname:
             self.log.debug(
-                f"Telegram user {sender.tgid} sent a message, but doesn't have a "
-                "displayname, updating info..."
+                f"Telegram user {sender.tgid} sent a message, but doesn't have a displayname,"
+                " updating info..."
             )
-            entity = await source.client.get_entity(PeerUser(sender.tgid))
+            entity = await source.client.get_entity(sender.peer)
             await sender.update_info(source, entity)
             if not sender.displayname:
                 self.log.debug(
@@ -2939,6 +3105,8 @@ class Portal(DBPortal, BasePortal):
             MessageMediaPhoto,
             MessageMediaDocument,
             MessageMediaGeo,
+            MessageMediaGeoLive,
+            MessageMediaVenue,
             MessageMediaGame,
             MessageMediaDice,
             MessageMediaPoll,
@@ -2961,6 +3129,8 @@ class Portal(DBPortal, BasePortal):
                 MessageMediaPhoto: self._handle_telegram_photo,
                 MessageMediaDocument: self._handle_telegram_document,
                 MessageMediaGeo: self._handle_telegram_location,
+                MessageMediaGeoLive: self._handle_telegram_live_location,
+                MessageMediaVenue: self._handle_telegram_venue,
                 MessageMediaPoll: self._handle_telegram_poll,
                 MessageMediaDice: self._handle_telegram_dice,
                 MessageMediaUnsupported: self._handle_telegram_unsupported,
@@ -3074,9 +3244,38 @@ class Portal(DBPortal, BasePortal):
         elif isinstance(action, MessageActionGameScore):
             # TODO handle game score
             pass
+        elif isinstance(action, MessageActionContactSignUp):
+            await self.handle_telegram_joined(source, sender, update)
         else:
             self.log.trace("Unhandled Telegram action in %s: %s", self.title, action)
 
+    async def handle_telegram_joined(
+        self,
+        source: au.AbstractUser,
+        sender: p.Puppet,
+        update: MessageService,
+        backfill: bool = False,
+    ) -> None:
+        assert isinstance(update.action, MessageActionContactSignUp)
+        content = TextMessageEventContent(msgtype=MessageType.EMOTE, body="joined Telegram")
+        event_id = await self._send_message(
+            sender.intent_for(self), content, timestamp=update.date
+        )
+        await DBMessage(
+            tgid=TelegramID(update.id),
+            mx_room=self.mxid,
+            mxid=event_id,
+            tg_space=source.tgid,
+            edit_index=0,
+        ).insert()
+        # Automatically mark the notice as read if we're backfilling messages, mostly so that
+        # empty rooms created before the notice was added wouldn't become unread when the notice
+        # is backfilled in.
+        if backfill:
+            double_puppet = await p.Puppet.get_by_tgid(source.tgid)
+            if double_puppet and double_puppet.is_real_user:
+                await double_puppet.intent.mark_read(self.mxid, event_id)
+
     async def set_telegram_admin(self, user_id: TelegramID) -> None:
         puppet = await p.Puppet.get_by_tgid(user_id)
         user = await u.User.get_by_tgid(user_id)
@@ -3261,8 +3460,10 @@ class Portal(DBPortal, BasePortal):
             self.by_mxid[self.mxid] = self
 
     @classmethod
-    async def all(cls) -> AsyncGenerator[Portal, None]:
-        portals = await super().all()
+    async def _yield_portals(
+        cls, query: Awaitable[list[DBPortal]]
+    ) -> AsyncGenerator[Portal, None]:
+        portals = await query
         portal: cls
         for portal in portals:
             try:
@@ -3272,15 +3473,16 @@ class Portal(DBPortal, BasePortal):
                 yield portal
 
     @classmethod
-    async def find_private_chats(cls, tg_receiver: TelegramID) -> AsyncGenerator[Portal, None]:
-        portals = await super().find_private_chats(tg_receiver)
-        portal: cls
-        for portal in portals:
-            try:
-                yield cls.by_tgid[portal.tgid_full]
-            except KeyError:
-                await portal.postinit()
-                yield portal
+    def all(cls) -> AsyncGenerator[Portal, None]:
+        return cls._yield_portals(super().all())
+
+    @classmethod
+    def find_private_chats_of(cls, tg_receiver: TelegramID) -> AsyncGenerator[Portal, None]:
+        return cls._yield_portals(super().find_private_chats_of(tg_receiver))
+
+    @classmethod
+    def find_private_chats_with(cls, tgid: TelegramID) -> AsyncGenerator[Portal, None]:
+        return cls._yield_portals(super().find_private_chats_with(tgid))
 
     @classmethod
     @async_getter_lock

mautrix_telegram/puppet.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -20,10 +20,18 @@ from difflib import SequenceMatcher
 import unicodedata
 
 from telethon.tl.types import (
+    Channel,
+    ChatPhoto,
+    ChatPhotoEmpty,
     InputPeerPhotoFileLocation,
+    PeerChannel,
+    PeerChat,
     PeerUser,
+    TypeChatPhoto,
     TypeInputPeer,
     TypeInputUser,
+    TypePeer,
+    TypeUserProfilePhoto,
     UpdateUserName,
     User,
     UserProfilePhoto,
@@ -33,7 +41,6 @@ from yarl import URL
 
 from mautrix.appservice import IntentAPI
 from mautrix.bridge import BasePuppet, async_getter_lock
-from mautrix.errors import MatrixError
 from mautrix.types import ContentURI, RoomID, SyncToken, UserID
 from mautrix.util.simple_template import SimpleTemplate
 
@@ -66,7 +73,11 @@ class Puppet(DBPuppet, BasePuppet):
         disable_updates: bool = False,
         username: str | None = None,
         photo_id: str | None = None,
+        avatar_url: ContentURI | None = None,
+        name_set: bool = False,
+        avatar_set: bool = False,
         is_bot: bool = False,
+        is_channel: bool = False,
         custom_mxid: UserID | None = None,
         access_token: str | None = None,
         next_batch: SyncToken | None = None,
@@ -82,7 +93,11 @@ class Puppet(DBPuppet, BasePuppet):
             disable_updates=disable_updates,
             username=username,
             photo_id=photo_id,
+            avatar_url=avatar_url,
+            name_set=name_set,
+            avatar_set=avatar_set,
             is_bot=is_bot,
+            is_channel=is_channel,
             custom_mxid=custom_mxid,
             access_token=access_token,
             next_batch=next_batch,
@@ -109,7 +124,9 @@ class Puppet(DBPuppet, BasePuppet):
 
     @property
     def peer(self) -> PeerUser:
-        return PeerUser(user_id=self.tgid)
+        return (
+            PeerChannel(channel_id=self.tgid) if self.is_channel else PeerUser(user_id=self.tgid)
+        )
 
     @property
     def plain_displayname(self) -> str:
@@ -185,9 +202,12 @@ class Puppet(DBPuppet, BasePuppet):
         return name
 
     @classmethod
-    def get_displayname(cls, info: User, enable_format: bool = True) -> tuple[str, int]:
-        fn = cls._filter_name(info.first_name)
-        ln = cls._filter_name(info.last_name)
+    def get_displayname(cls, info: User | Channel, enable_format: bool = True) -> tuple[str, int]:
+        if isinstance(info, Channel):
+            fn, ln = cls._filter_name(info.title), ""
+        else:
+            fn = cls._filter_name(info.first_name)
+            ln = cls._filter_name(info.last_name)
         data = {
             "phone number": info.phone if hasattr(info, "phone") else None,
             "username": info.username,
@@ -214,14 +234,20 @@ class Puppet(DBPuppet, BasePuppet):
 
         return (cls.displayname_template.format_full(name) if enable_format else name), quality
 
-    async def try_update_info(self, source: au.AbstractUser, info: User) -> None:
+    async def try_update_info(self, source: au.AbstractUser, info: User | Channel) -> None:
         try:
             await self.update_info(source, info)
         except Exception:
             source.log.exception(f"Failed to update info of {self.tgid}")
 
-    async def update_info(self, source: au.AbstractUser, info: User) -> None:
-        changed = False
+    async def update_info(self, source: au.AbstractUser, info: User | Channel) -> None:
+        is_bot = False if isinstance(info, Channel) else info.bot
+        is_channel = isinstance(info, Channel)
+        changed = is_bot != self.is_bot or is_channel != self.is_channel
+
+        self.is_bot = is_bot
+        self.is_channel = is_channel
+
         if self.username != info.username:
             self.username = info.username
             changed = True
@@ -233,32 +259,46 @@ class Puppet(DBPuppet, BasePuppet):
             except Exception:
                 self.log.exception(f"Failed to update info from source {source.tgid}")
 
-        self.is_bot = info.bot
-
         if changed:
+            await self.update_portals_meta()
             await self.save()
 
+    async def update_portals_meta(self) -> None:
+        if not p.Portal.private_chat_portal_meta and not self.mx.e2ee:
+            return
+        async for portal in p.Portal.find_private_chats_with(self.tgid):
+            await portal.update_info_from_puppet(self)
+
     async def update_displayname(
-        self, source: au.AbstractUser, info: User | UpdateUserName
+        self, source: au.AbstractUser, info: User | Channel | UpdateUserName
     ) -> bool:
         if self.disable_updates:
             return False
-        if source.is_relaybot or source.is_bot:
-            allow_because = "user is bot"
+        if (
+            self.displayname
+            and self.displayname.startswith("Deleted user ")
+            and not getattr(info, "deleted", False)
+        ):
+            allow_because = "target user was previously deleted"
+            self.displayname_quality = 0
+        elif source.is_relaybot or source.is_bot:
+            allow_because = "source user is a bot"
         elif self.displayname_source == source.tgid:
-            allow_because = "user is the primary source"
+            allow_because = "source user is the primary source"
+        elif isinstance(info, Channel):
+            allow_because = "target user is a channel"
         elif not isinstance(info, UpdateUserName) and not info.contact:
-            allow_because = "user is not a contact"
+            allow_because = "target user is not a contact"
         elif not self.displayname_source:
             allow_because = "no primary source set"
         elif not self.displayname:
-            allow_because = "user has no name"
+            allow_because = "target user has no name"
         else:
             return False
 
         if isinstance(info, UpdateUserName):
-            info = await source.client.get_entity(PeerUser(self.tgid))
-        if not info.contact:
+            info = await source.client.get_entity(self.peer)
+        if isinstance(info, Channel) or not info.contact:
             self.displayname_contact = False
         elif not self.displayname_contact:
             if not self.displayname:
@@ -267,7 +307,9 @@ class Puppet(DBPuppet, BasePuppet):
                 return False
 
         displayname, quality = self.get_displayname(info)
-        if displayname != self.displayname and quality >= self.displayname_quality:
+        needs_reset = displayname != self.displayname or not self.name_set
+        is_high_quality = quality >= self.displayname_quality
+        if needs_reset and is_high_quality:
             allow_because = f"{allow_because} and quality {quality} >= {self.displayname_quality}"
             self.log.debug(
                 f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
@@ -281,11 +323,10 @@ class Puppet(DBPuppet, BasePuppet):
                 await self.default_mxid_intent.set_displayname(
                     displayname[: self.config["bridge.displayname_max_length"]]
                 )
-            except MatrixError:
-                self.log.exception("Failed to set displayname")
-                self.displayname = ""
-                self.displayname_source = None
-                self.displayname_quality = 0
+                self.name_set = True
+            except Exception as e:
+                self.log.warning(f"Failed to set displayname: {e}")
+                self.name_set = False
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
@@ -293,42 +334,43 @@ class Puppet(DBPuppet, BasePuppet):
         return False
 
     async def update_avatar(
-        self, source: au.AbstractUser, photo: UserProfilePhoto | UserProfilePhotoEmpty
+        self, source: au.AbstractUser, photo: TypeUserProfilePhoto | TypeChatPhoto
     ) -> bool:
         if self.disable_updates:
             return False
 
-        if photo is None or isinstance(photo, UserProfilePhotoEmpty):
+        if photo is None or isinstance(photo, (UserProfilePhotoEmpty, ChatPhotoEmpty)):
             photo_id = ""
-        elif isinstance(photo, UserProfilePhoto):
+        elif isinstance(photo, (UserProfilePhoto, ChatPhoto)):
             photo_id = str(photo.photo_id)
         else:
             self.log.warning(f"Unknown user profile photo type: {type(photo)}")
             return False
         if not photo_id and not self.config["bridge.allow_avatar_remove"]:
             return False
-        if self.photo_id != photo_id:
+        if self.photo_id != photo_id or not self.avatar_set:
             if not photo_id:
                 self.photo_id = ""
-                try:
-                    await self.default_mxid_intent.set_avatar_url(ContentURI(""))
-                except MatrixError:
-                    self.log.exception("Failed to set avatar")
-                    self.photo_id = ""
-                return True
-
-            loc = InputPeerPhotoFileLocation(
-                peer=await self.get_input_entity(source), photo_id=photo.photo_id, big=True
-            )
-            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
-            if file:
+                self.avatar_url = None
+            elif self.photo_id != photo_id or not self.avatar_url:
+                file = await util.transfer_file_to_matrix(
+                    client=source.client,
+                    intent=self.default_mxid_intent,
+                    location=InputPeerPhotoFileLocation(
+                        peer=await self.get_input_entity(source), photo_id=photo.photo_id, big=True
+                    ),
+                )
+                if not file:
+                    return False
                 self.photo_id = photo_id
-                try:
-                    await self.default_mxid_intent.set_avatar_url(file.mxc)
-                except MatrixError:
-                    self.log.exception("Failed to set avatar")
-                    self.photo_id = ""
-                return True
+                self.avatar_url = file.mxc
+            try:
+                await self.default_mxid_intent.set_avatar_url(self.avatar_url or "")
+                self.avatar_set = True
+            except Exception as e:
+                self.log.warning(f"Failed to set avatar: {e}")
+                self.avatar_set = False
+            return True
         return False
 
     async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
@@ -345,7 +387,9 @@ class Puppet(DBPuppet, BasePuppet):
 
     @classmethod
     @async_getter_lock
-    async def get_by_tgid(cls, tgid: TelegramID, *, create: bool = True) -> Puppet | None:
+    async def get_by_tgid(
+        cls, tgid: TelegramID, *, create: bool = True, is_channel: bool = False
+    ) -> Puppet | None:
         if tgid is None:
             return None
 
@@ -360,13 +404,37 @@ class Puppet(DBPuppet, BasePuppet):
             return puppet
 
         if create:
-            puppet = cls(tgid)
+            puppet = cls(tgid, is_channel=is_channel)
             await puppet.insert()
             puppet._add_to_cache()
             return puppet
 
         return None
 
+    @staticmethod
+    def get_id_from_peer(peer: TypePeer | User | Channel) -> TelegramID:
+        if isinstance(peer, PeerUser):
+            return TelegramID(peer.user_id)
+        elif isinstance(peer, PeerChannel):
+            return TelegramID(peer.channel_id)
+        elif isinstance(peer, PeerChat):
+            return TelegramID(peer.chat_id)
+        elif isinstance(peer, (User, Channel)):
+            return TelegramID(peer.id)
+        raise TypeError(f"invalid type {type(peer).__name__!r} in _id_from_peer()")
+
+    @classmethod
+    async def get_by_peer(
+        cls, peer: TypePeer | User | Channel, *, create: bool = True
+    ) -> Puppet | None:
+        if isinstance(peer, PeerChat):
+            return None
+        return await cls.get_by_tgid(
+            cls.get_id_from_peer(peer),
+            create=create,
+            is_channel=isinstance(peer, (PeerChannel, Channel)),
+        )
+
     @classmethod
     def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Awaitable[Puppet | None]:
         return cls.get_by_tgid(cls.get_id_from_mxid(mxid), create=create)

mautrix_telegram/tgclient.py

@@ -40,7 +40,7 @@ class MautrixTelegramClient(TelegramClient):
         mime_type: str = None,
         attributes: List[TypeDocumentAttribute] = None,
         file_name: str = None,
-        max_image_size: float = 10 * 1000 ** 2,
+        max_image_size: float = 10 * 1000**2,
     ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
         file_handle = await super().upload_file(file, file_name=file_name)
 

mautrix_telegram/user.py

@@ -195,7 +195,8 @@ class User(DBUser, AbstractUser, BaseUser):
             if self.tgid:
                 await self.push_bridge_state(BridgeStateEvent.UNKNOWN_ERROR, message=str(e))
         except UnauthorizedError as e:
-            self.log.error(f"Authorization error in start(): {type(e)}: {e}")
+            if delete_unless_authenticated or self.tgid:
+                self.log.error(f"Authorization error in start(): {type(e)}: {e}")
             if self.tgid:
                 await self.push_bridge_state(
                     BridgeStateEvent.BAD_CREDENTIALS,
@@ -240,7 +241,7 @@ class User(DBUser, AbstractUser, BaseUser):
                 )
             else:
                 await self.push_bridge_state(
-                    BridgeStateEvent.UNKNOWN_ERROR, ttl=240, error="tg-not-connected"
+                    BridgeStateEvent.TRANSIENT_DISCONNECT, ttl=240, error="tg-not-connected"
                 )
 
     async def fill_bridge_state(self, state: BridgeState) -> None:
@@ -448,7 +449,7 @@ class User(DBUser, AbstractUser, BaseUser):
     async def get_direct_chats(self) -> dict[UserID, list[RoomID]]:
         return {
             pu.Puppet.get_mxid_from_id(portal.tgid): [portal.mxid]
-            async for portal in po.Portal.find_private_chats(self.tgid)
+            async for portal in po.Portal.find_private_chats_of(self.tgid)
             if portal.mxid
         }
 

mautrix_telegram/web/common/auth_api.py

@@ -126,8 +126,10 @@ class AuthAPI(abc.ABC):
                 mxid=user.mxid,
                 state="code",
                 status=200,
-                message="Code requested successfully. Check your SMS "
-                "or Telegram client and enter the code below.",
+                message=(
+                    "Code requested successfully. Check your SMS "
+                    "or Telegram client and enter the code below."
+                ),
             )
         except PhoneNumberInvalidError:
             return self.get_login_response(
@@ -167,8 +169,10 @@ class AuthAPI(abc.ABC):
                 state="request",
                 status=429,
                 errcode="phone_number_flood",
-                error="Your phone number has been temporarily blocked for flooding. "
-                "The ban is usually applied for around a day.",
+                error=(
+                    "Your phone number has been temporarily blocked for flooding. "
+                    "The ban is usually applied for around a day."
+                ),
             )
         except FloodWaitError as e:
             return self.get_login_response(
@@ -176,8 +180,10 @@ class AuthAPI(abc.ABC):
                 state="request",
                 status=429,
                 errcode="flood_wait",
-                error="Your phone number has been temporarily blocked for flooding. "
-                f"Please wait for {format_duration(e.seconds)} before trying again.",
+                error=(
+                    "Your phone number has been temporarily blocked for flooding. "
+                    f"Please wait for {format_duration(e.seconds)} before trying again."
+                ),
             )
         except Exception:
             self.log.exception("Error requesting phone code")
@@ -237,6 +243,14 @@ class AuthAPI(abc.ABC):
     async def post_login_code(
         self, user: User, code: int, password_in_data: bool
     ) -> web.Response | None:
+        if not code:
+            return self.get_login_response(
+                mxid=user.mxid,
+                state="code",
+                status=400,
+                errcode="phone_code_missing",
+                error="You must provide the code from your phone.",
+            )
         try:
             user_info = await user.client.sign_in(code=code)
             await self.postprocess_login(user, user_info)

optional-requirements.txt

@@ -18,7 +18,7 @@ moviepy>=1,<2
 phonenumbers>=8,<9
 
 #/metrics
-prometheus_client>=0.6,<0.13
+prometheus_client>=0.6,<0.14
 
 #/e2be
 python-olm>=3,<4

pyproject.toml

@@ -9,4 +9,4 @@ line_length = 99
 [tool.black]
 line-length = 99
 target-version = ["py38"]
-required-version = "21.12b0"
+required-version = "22.1.0"

requirements.txt

@@ -3,10 +3,10 @@ python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.14.3,<0.15
+mautrix>=0.14.9,<0.15
 #telethon>=1.24,<1.25
-# Fork to make session storage async and update to layer 137
-tulir-telethon==1.25.0a3
+# Fork to make session storage async and update to layer 138
+tulir-telethon==1.25.0a5
 asyncpg>=0.20,<0.26
 mako>=1,<2
 setuptools