Bump version to 0.10.1

Upgrade mautrix to 0.10.2 and use new BridgeStateEvents

Dockerfile

@@ -1,12 +1,7 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.12
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.14
 
 ARG TARGETARCH=amd64
 
-RUN echo $'\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
-
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-virtualenv \
@@ -14,11 +9,12 @@ RUN apk add --no-cache \
       py3-aiohttp \
       py3-magic \
       py3-sqlalchemy \
-      py3-telethon-session-sqlalchemy@edge \
-      py3-alembic@edge \
+      py3-telethon-session-sqlalchemy \
+      py3-alembic \
       py3-psycopg2 \
       py3-ruamel.yaml \
-      py3-commonmark@edge \
+      py3-commonmark \
+      py3-prometheus-client \
       # Indirect dependencies
       py3-idna \
       #moviepy
@@ -27,12 +23,13 @@ RUN apk add --no-cache \
         py3-requests \
         #imageio
           py3-numpy \
-      #py3-telethon@edge \ (outdated)
+      #py3-telethon \ (outdated)
         # Optional for socks proxies
         py3-pysocks \
+        py3-pyaes \
         # cryptg
           py3-cffi \
-	  py3-qrcode@edge \
+	  py3-qrcode \
       py3-brotli \
       # Other dependencies
       ffmpeg \
@@ -40,15 +37,14 @@ RUN apk add --no-cache \
       su-exec \
       netcat-openbsd \
       # encryption
-      olm-dev \
+      py3-olm \
       py3-pycryptodome \
       py3-unpaddedbase64 \
       py3-future \
       bash \
       curl \
-      jq && \
-  curl -sLo yq https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${TARGETARCH} && \
-  chmod +x yq && mv yq /usr/bin/yq
+      jq \
+      yq
 
 COPY requirements.txt /opt/mautrix-telegram/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt

README.md

@@ -1,27 +1,27 @@
 # mautrix-telegram
-![Languages](https://img.shields.io/github/languages/top/tulir/mautrix-telegram.svg)
-[![License](https://img.shields.io/github/license/tulir/mautrix-telegram.svg)](LICENSE)
-[![Release](https://img.shields.io/github/release/tulir/mautrix-telegram/all.svg)](https://github.com/tulir/mautrix-telegram/releases)
-[![GitLab CI](https://mau.dev/tulir/mautrix-telegram/badges/master/pipeline.svg)](https://mau.dev/tulir/mautrix-telegram/container_registry)
-[![Maintainability](https://img.shields.io/codeclimate/maintainability/tulir/mautrix-telegram.svg)](https://codeclimate.com/github/tulir/mautrix-telegram)
+![Languages](https://img.shields.io/github/languages/top/mautrix/telegram.svg)
+[![License](https://img.shields.io/github/license/mautrix/telegram.svg)](LICENSE)
+[![Release](https://img.shields.io/github/release/mautrix/telegram/all.svg)](https://github.com/mautrix/telegram/releases)
+[![GitLab CI](https://mau.dev/mautrix/telegram/badges/master/pipeline.svg)](https://mau.dev/mautrix/telegram/container_registry)
 
 A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
 ## Sponsors
 * [Joel Lehtonen / Zouppen](https://github.com/zouppen)
 
-### Wiki
-All setup and usage instructions are located in the GitHub
-[wiki](https://github.com/tulir/mautrix-telegram/wiki). Some quick links:
+### Documentation
+All setup and usage instructions are located on
+[docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
+Some quick links:
 
-* [Bridge setup](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup)
-  (or [with Docker](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup-with-Docker))
-* Basic usage: [Authentication](https://github.com/tulir/mautrix-telegram/wiki/Authentication),
-  [Creating chats](https://github.com/tulir/mautrix-telegram/wiki/Creating-and-managing-chats),
-  [Relaybot setup](https://github.com/tulir/mautrix-telegram/wiki/Relay-bot)
+* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
+  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
+* Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
+  [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
+  [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)
 
 ### Features & Roadmap
-[ROADMAP.md](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+[ROADMAP.md](https://github.com/mautrix/telegram/blob/master/ROADMAP.md)
 contains a general overview of what is supported by the bridge.
 
 ## Discussion

alembic/versions/7de69cf5809e_add_matrix_redaction_state_to_message_.py

@@ -0,0 +1,25 @@
+"""Add Matrix redaction state to message table
+
+Revision ID: 7de69cf5809e
+Revises: 888275d58e57
+Create Date: 2020-12-19 12:39:57.368568
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '7de69cf5809e'
+down_revision = '888275d58e57'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('message', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('redacted', sa.Boolean(), server_default=sa.false(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table('message', schema=None) as batch_op:
+        batch_op.drop_column('redacted')

alembic/versions/990f4395afc6_store_displayname_contact_status_in_.py

@@ -0,0 +1,32 @@
+"""Store displayname contact status in puppet table
+
+Revision ID: 990f4395afc6
+Revises: 7de69cf5809e
+Create Date: 2021-01-01 11:56:54.610681
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '990f4395afc6'
+down_revision = '7de69cf5809e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('displayname_contact', sa.Boolean(), server_default=sa.true(), nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('displayname_contact')
+
+    # ### end Alembic commands ###

alembic/versions/bfc0a39bfe02_store_displayname_quality_in_puppet_.py

@@ -0,0 +1,32 @@
+"""Store displayname quality in puppet table
+
+Revision ID: bfc0a39bfe02
+Revises: ec1d3dcc77e9
+Create Date: 2021-03-23 20:03:08.825333
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'bfc0a39bfe02'
+down_revision = 'ec1d3dcc77e9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('displayname_quality', sa.Integer(), server_default='0', nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('displayname_quality')
+
+    # ### end Alembic commands ###

alembic/versions/ec1d3dcc77e9_switch_telegram_ids_to_bigints.py

@@ -0,0 +1,44 @@
+"""Switch Telegram IDs to bigints
+
+Revision ID: ec1d3dcc77e9
+Revises: 990f4395afc6
+Create Date: 2021-03-09 21:36:58.443727
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'ec1d3dcc77e9'
+down_revision = '990f4395afc6'
+branch_labels = None
+depends_on = None
+
+
+columns_to_upgrade = (
+    ("bot_chat", "id"),
+    ("message", "tgid"),
+    ("message", "tg_space"),
+    ("portal", "tgid"),
+    ("portal", "tg_receiver"),
+    ("puppet", "id"),
+    ("puppet", "displayname_source"),
+    ("user", "tgid"),
+    ("user_portal", "user"),
+    ("user_portal", "portal"),
+    ("user_portal", "portal_receiver"),
+    ("contact", "user"),
+    ("contact", "contact"),
+)
+
+
+def upgrade():
+    if op.get_context().dialect.name == "postgresql":
+        for table, column in columns_to_upgrade:
+            op.alter_column(table, column, existing_type=sa.Integer, type_=sa.BigInteger)
+
+
+def downgrade():
+    if op.get_context().dialect.name == "postgresql":
+        for table, column in columns_to_upgrade:
+            op.alter_column(table, column, existing_type=sa.BigInteger, type_=sa.Integer)

docker-run.sh

@@ -26,9 +26,6 @@ fi
 
 if [ ! -f /data/registration.yaml ]; then
 	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
-	echo "Didn't find a registration file."
-	echo "Generated one for you."
-	echo "Copy that over to synapses app service directory."
 	fixperms
 	exit
 fi

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.9.0"
+__version__ = "0.10.1"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -13,8 +13,9 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from typing import Dict, Any
 
+from telethon import __version__ as __telethon_version__
 from alchemysession import AlchemySessionContainer
 
 from mautrix.types import UserID, RoomID
@@ -23,7 +24,6 @@ from mautrix.util.db import Base
 
 from .web.provisioning import ProvisioningAPI
 from .web.public import PublicBridgeWebsite
-from .commands.manhole import ManholeState
 from .abstract_user import init as init_abstract_user
 from .bot import Bot, init as init_bot
 from .config import Config
@@ -47,7 +47,7 @@ class TelegramBridge(Bridge):
     name = "mautrix-telegram"
     command = "python -m mautrix-telegram"
     description = "A Matrix-Telegram puppeting bridge."
-    repo_url = "https://github.com/tulir/mautrix-telegram"
+    repo_url = "https://github.com/mautrix/telegram"
     real_user_content_key = "net.maunium.telegram.puppet"
     version = version
     markdown_version = linkified_version
@@ -57,7 +57,6 @@ class TelegramBridge(Bridge):
     config: Config
     session_container: AlchemySessionContainer
     bot: Bot
-    manhole: Optional[ManholeState]
 
     def prepare_db(self) -> None:
         super().prepare_db()
@@ -83,7 +82,6 @@ class TelegramBridge(Bridge):
         context = Context(self.az, self.config, self.loop, self.session_container, self, self.bot)
         self._prepare_website(context)
         self.matrix = context.mx = MatrixHandler(context)
-        self.manhole = None
 
         init_abstract_user(context)
         init_formatter(context)
@@ -107,9 +105,6 @@ class TelegramBridge(Bridge):
         for puppet in Puppet.by_custom_mxid.values():
             puppet.stop()
         self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
-        if self.manhole:
-            self.manhole.close()
-            self.manhole = None
 
     async def get_user(self, user_id: UserID, create: bool = True) -> User:
         user = User.get_by_mxid(user_id, create=create)
@@ -129,5 +124,20 @@ class TelegramBridge(Bridge):
     def is_bridge_ghost(self, user_id: UserID) -> bool:
         return bool(Puppet.get_id_from_mxid(user_id))
 
+    async def count_logged_in_users(self) -> int:
+        return len([user for user in User.by_tgid.values() if user.tgid])
+
+    async def manhole_global_namespace(self, user_id: UserID) -> Dict[str, Any]:
+        return {
+            **await super().manhole_global_namespace(user_id),
+            "User": User,
+            "Portal": Portal,
+            "Puppet": Puppet,
+        }
+
+    @property
+    def manhole_banner_program_version(self) -> str:
+        return f"{super().manhole_banner_program_version} and Telethon {__telethon_version__}"
+
 
 TelegramBridge().run()

mautrix_telegram/abstract_user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,9 +15,9 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Tuple, Optional, Union, Dict, Type, Any, TYPE_CHECKING
 from abc import ABC, abstractmethod
+import platform
 import asyncio
 import logging
-import platform
 import time
 
 from telethon.sessions import Session
@@ -25,13 +25,14 @@ from telethon.network import (ConnectionTcpMTProxyRandomizedIntermediate, Connec
                               Connection)
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
-    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdateChatPinnedMessage,
-    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
+    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdatePinnedMessages,
+    UpdatePinnedChannelMessages, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
     UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
     UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
     UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
     UpdateUserTyping, User, UserStatusOffline, UserStatusOnline, UpdateReadHistoryInbox,
-    UpdateReadChannelInbox, MessageEmpty)
+    UpdateReadChannelInbox, MessageEmpty, UpdateFolderPeers, UpdatePinnedDialogs,
+    UpdateNotifySettings, UpdateChannelUserTyping)
 
 from mautrix.types import UserID, PresenceState
 from mautrix.errors import MatrixError
@@ -57,6 +58,7 @@ MAX_DELETIONS: int = 10
 UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
+UpdateTyping = Union[UpdateUserTyping, UpdateChatUserTyping, UpdateChannelUserTyping]
 
 UPDATE_TIME = Histogram("bridge_telegram_update", "Time spent processing Telegram updates",
                         ("update_type",))
@@ -127,9 +129,9 @@ class AbstractUser(ABC):
     def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
 
-        self.session = self.session_container.new_session(self.name)
+        session = self.session_container.new_session(self.name)
         if config["telegram.server.enabled"]:
-            self.session.set_dc(config["telegram.server.dc"],
+            session.set_dc(config["telegram.server.dc"],
                                 config["telegram.server.ip"],
                                 config["telegram.server.port"])
 
@@ -143,10 +145,10 @@ class AbstractUser(ABC):
         appversion = config["telegram.device_info.app_version"]
         connection, proxy = self._proxy_settings
 
-        assert isinstance(self.session, Session)
+        assert isinstance(session, Session)
 
         self.client = MautrixTelegramClient(
-            session=self.session,
+            session=session,
 
             api_id=config["telegram.api_id"],
             api_hash=config["telegram.api_hash"],
@@ -235,8 +237,7 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
-        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
-                              loop=self.loop)
+        asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -244,7 +245,7 @@ class AbstractUser(ABC):
             await self.delete_message(update)
         elif isinstance(update, UpdateDeleteChannelMessages):
             await self.delete_channel_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
+        elif isinstance(update, (UpdateChatUserTyping, UpdateChannelUserTyping, UpdateUserTyping)):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
@@ -252,7 +253,7 @@ class AbstractUser(ABC):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
+        elif isinstance(update, (UpdatePinnedMessages, UpdatePinnedChannelMessages)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
@@ -260,17 +261,33 @@ class AbstractUser(ABC):
             await self.update_read_receipt(update)
         elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
             await self.update_own_read_receipt(update)
+        elif isinstance(update, UpdateFolderPeers):
+            await self.update_folder_peers(update)
+        elif isinstance(update, UpdatePinnedDialogs):
+            await self.update_pinned_dialogs(update)
+        elif isinstance(update, UpdateNotifySettings):
+            await self.update_notify_settings(update)
         else:
             self.log.trace("Unhandled update: %s", update)
 
-    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
-                                                         UpdateChatPinnedMessage]) -> None:
-        if isinstance(update, UpdateChatPinnedMessage):
-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
+        pass
+
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        pass
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        pass
+
+    async def update_pinned_messages(self, update: Union[UpdatePinnedMessages,
+                                                         UpdatePinnedChannelMessages]) -> None:
+        if isinstance(update, UpdatePinnedMessages):
+            portal = po.Portal.get_by_entity(update.peer, receiver_id=self.tgid)
         else:
             portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id, self.tgid)
+            await portal.receive_telegram_pin_ids(update.messages, self.tgid,
+                                                  remove=not update.pinned)
 
     @staticmethod
     async def update_participants(update: UpdateChatParticipants) -> None:
@@ -329,16 +346,27 @@ class AbstractUser(ABC):
 
         await portal.set_telegram_admin(TelegramID(update.user_id))
 
-    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+    async def update_typing(self, update: UpdateTyping) -> None:
+        sender = None
         if isinstance(update, UpdateUserTyping):
             portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
-        else:
+            sender = pu.Puppet.get(TelegramID(update.user_id))
+        elif isinstance(update, UpdateChannelUserTyping):
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update, UpdateChatUserTyping):
             portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
-
-        if not portal or not portal.mxid:
+        else:
+            return
+
+        if isinstance(update, (UpdateChannelUserTyping, UpdateChatUserTyping)):
+            # Can typing notifications come from non-user peers?
+            if not update.from_id.user_id:
+                return
+            sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
+
+        if not sender or not portal or not portal.mxid:
             return
 
-        sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
     async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
@@ -419,6 +447,8 @@ class AbstractUser(ABC):
 
         for message_id in update.messages:
             for message in DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
+                if message.redacted:
+                    continue
                 message.delete()
                 number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
                 if number_left == 0:
@@ -432,6 +462,8 @@ class AbstractUser(ABC):
 
         for message_id in update.messages:
             for message in DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
+                if message.redacted:
+                    continue
                 message.delete()
                 await self._try_redact(message)
 
@@ -468,7 +500,7 @@ class AbstractUser(ABC):
                 await self.register_portal(portal)
                 return
             self.log.trace("Handling action %s to %s by %d", update.action, portal.tgid_log,
-                           sender.id)
+                           (sender.id if sender else 0))
             return await portal.handle_telegram_action(self, sender, update)
 
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):

mautrix_telegram/bot.py

@@ -195,7 +195,7 @@ class Bot(AbstractUser):
             return await reply("That user seems to be logged in. "
                                f"Just invite [{displayname}](tg://user?id={user.tgid})")
         else:
-            await portal.main_intent.invite_user(portal.mxid, user.mxid)
+            await portal.invite_to_matrix(user.mxid)
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
     @staticmethod

mautrix_telegram/commands/__init__.py

@@ -1,7 +1,7 @@
 from .handler import (command_handler, CommandHandler, CommandProcessor, CommandEvent,
                       SECTION_AUTH, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT,
                       SECTION_MISC, SECTION_ADMIN)
-from . import portal, telegram, matrix_auth, manhole
+from . import portal, telegram, matrix_auth
 
 __all__ = ["command_handler", "CommandHandler", "CommandProcessor", "CommandEvent",
            "SECTION_AUTH", "SECTION_MISC", "SECTION_ADMIN", "SECTION_CREATING_PORTALS",

mautrix_telegram/commands/manhole.py

@@ -1,128 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Set, Callable
-import asyncio
-import sys
-import os
-
-from attr import dataclass
-
-from telethon import __version__ as __telethon_version__
-
-from mautrix import __version__ as __mautrix_version__
-from mautrix.types import UserID
-from mautrix.errors import MatrixConnectionError
-from mautrix.util.manhole import start_manhole
-
-from .. import __version__
-from . import command_handler, CommandEvent, SECTION_ADMIN
-
-
-@dataclass
-class ManholeState:
-    server: asyncio.AbstractServer
-    opened_by: UserID
-    close: Callable[[], None]
-    whitelist: Set[int]
-
-
-@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
-                 help_text="Open a manhole into the bridge.", help_args="<_uid..._>")
-async def open_manhole(evt: CommandEvent) -> None:
-    if not evt.config["manhole.enabled"]:
-        await evt.reply("The manhole has been disabled in the config.")
-        return
-    elif len(evt.args) == 0:
-        await evt.reply("**Usage:** `$cmdprefix+sp open-manhole <uid...>`")
-        return
-
-    whitelist = set()
-    whitelist_whitelist = evt.config["manhole.whitelist"]
-    for arg in evt.args:
-        try:
-            uid = int(arg)
-        except ValueError:
-            await evt.reply(f"{arg} is not an integer.")
-            return
-        if whitelist_whitelist and uid not in whitelist_whitelist:
-            await evt.reply(f"{uid} is not in the list of allowed UIDs.")
-            return
-        whitelist.add(uid)
-
-    if evt.bridge.manhole:
-        added = [uid for uid in whitelist
-                 if uid not in evt.bridge.manhole.whitelist]
-        evt.bridge.manhole.whitelist |= set(added)
-        if len(added) == 0:
-            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
-                            " and all the given UIDs are already whitelisted.")
-        else:
-            added_str = (f"{', '.join(str(uid) for uid in added[:-1])} and {added[-1]}"
-                         if len(added) > 1 else added[0])
-            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
-                            f". Added {added_str} to the whitelist.")
-            evt.log.info(f"{evt.sender.mxid} added {added_str} to the manhole whitelist.")
-        return
-
-    from ..portal import Portal
-    from ..puppet import Puppet
-    from ..user import User
-    namespace = {
-        "bridge": evt.bridge,
-        "User": User,
-        "Portal": Portal,
-        "Puppet": Puppet,
-    }
-    banner = (f"Python {sys.version} on {sys.platform}\n"
-              f"mautrix-telegram {__version__} with mautrix-python {__mautrix_version__} "
-              f"and Telethon {__telethon_version__}\n\nManhole opened by {evt.sender.mxid}\n")
-    path = evt.config["manhole.path"]
-
-    wl_list = list(whitelist)
-    whitelist_str = (f"{', '.join(str(uid) for uid in wl_list[:-1])} and {wl_list[-1]}"
-                     if len(wl_list) > 1 else wl_list[0])
-    evt.log.info(f"{evt.sender.mxid} opened a manhole with {whitelist_str} whitelisted.")
-    server, close = await start_manhole(path=path, banner=banner, namespace=namespace,
-                                        loop=evt.loop, whitelist=whitelist)
-    evt.bridge.manhole = ManholeState(server=server, opened_by=evt.sender.mxid, close=close,
-                                      whitelist=whitelist)
-    plrl = "s" if len(whitelist) != 1 else ""
-    await evt.reply(f"Opened manhole at unix://{path} with UID{plrl} {whitelist_str} whitelisted")
-    await server.wait_closed()
-    evt.bridge.manhole = None
-    try:
-        os.unlink(path)
-    except FileNotFoundError:
-        pass
-    evt.log.info(f"{evt.sender.mxid}'s manhole was closed.")
-    try:
-        await evt.reply("Your manhole was closed.")
-    except (AttributeError, MatrixConnectionError) as e:
-        evt.log.warning(f"Failed to send manhole close notification: {e}")
-
-
-@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
-                 help_text="Close an open manhole.")
-async def close_manhole(evt: CommandEvent) -> None:
-    if not evt.bridge.manhole:
-        await evt.reply("There is no open manhole.")
-        return
-
-    opened_by = evt.bridge.manhole.opened_by
-    evt.bridge.manhole.close()
-    evt.bridge.manhole = None
-    if opened_by != evt.sender.mxid:
-        await evt.reply(f"Closed manhole opened by {opened_by}")

mautrix_telegram/commands/portal/bridge.py

@@ -23,7 +23,7 @@ from mautrix.types import EventID, RoomID
 from ...types import TelegramID
 from ... import portal as po
 from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
-from .util import user_has_power_level, get_initial_state
+from .util import user_has_power_level, get_initial_state, warn_missing_power
 
 
 @command_handler(needs_auth=False, needs_puppeting=False,
@@ -186,8 +186,11 @@ async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id
      portal.encrypted) = await get_initial_state(evt.az.intent, evt.room_id)
     portal.photo_id = ""
     await portal.save()
+    await portal.update_bridge_info()
 
     asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
                           loop=evt.loop)
 
+    await warn_missing_power(levels, evt)
+
     return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/create_chat.py

@@ -18,7 +18,7 @@ from mautrix.types import EventID
 from ... import portal as po
 from ...types import TelegramID
 from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
-from .util import user_has_power_level, get_initial_state
+from .util import user_has_power_level, get_initial_state, warn_missing_power
 
 
 @command_handler(help_section=SECTION_CREATING_PORTALS,
@@ -52,8 +52,17 @@ async def create(evt: CommandEvent) -> EventID:
 
     portal = po.Portal(tgid=TelegramID(0), peer_type=type, mxid=evt.room_id,
                        title=title, about=about, encrypted=encrypted)
+    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender)
+    if len(errors) > 0:
+        error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
+        await evt.reply(f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
+                        "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
+                        "those users.")
+
+    await warn_missing_power(levels, evt)
+
     try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+        await portal.create_telegram_chat(evt.sender, invites=invites, supergroup=supergroup)
     except ValueError as e:
         await portal.delete()
         return await evt.reply(e.args[0])

mautrix_telegram/commands/portal/misc.py

@@ -13,6 +13,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple
+from datetime import timedelta, datetime
+import re
+
 from telethon.tl.functions.channels import GetFullChannelRequest
 from telethon.tl.functions.messages import GetFullChatRequest
 from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
@@ -80,9 +84,81 @@ async def get_id(evt: CommandEvent) -> EventID:
     await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
 
 
+invite_link_usage = ("**Usage:** `$cmdprefix+sp invite-link [--uses=<amount>] [--expire=<delta>]`"
+                     "\n\n"
+                     "* `--uses`: the number of times the invite link can be used."
+                     "            Defaults to unlimited.\n"
+                     "* `--expire`: the duration after which the link will expire."
+                     "              A number suffixed with d(ay), h(our), m(inute) or s(econd)")
+
+
+def _parse_flag(args: List[str]) -> Tuple[str, str]:
+    arg = args.pop(0).lower()
+    if arg.startswith("--"):
+        value_start = arg.index("=")
+        if value_start:
+            flag = arg[2:value_start]
+            value = arg[value_start+1:]
+        else:
+            flag = arg[2:]
+            value = args.pop(0).lower()
+    elif arg.startswith("-"):
+        flag = arg[1]
+        if len(arg) > 3 and arg[2] == "=":
+            value = arg[3:]
+        else:
+            value = args.pop(0).lower()
+    else:
+        raise ValueError("invalid flag")
+    return flag, value
+
+
+delta_regex = re.compile("([0-9]+)(w(?:eek)?|d(?:ay)?|h(?:our)?|m(?:in(?:ute)?)?|s(?:ec(?:ond)?)?)")
+
+
+def _parse_delta(value: str) -> Optional[timedelta]:
+    match = delta_regex.fullmatch(value)
+    if not match:
+        return None
+    number = int(match.group(1))
+    unit = match.group(2)[0]
+    if unit == "w":
+        return timedelta(weeks=number)
+    elif unit == "d":
+        return timedelta(days=number)
+    elif unit == "h":
+        return timedelta(hours=number)
+    elif unit == "m":
+        return timedelta(minutes=number)
+    elif unit == "s":
+        return timedelta(seconds=number)
+    else:
+        return None
+
+
 @command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Get a Telegram invite link to the current chat.")
+                 help_text="Get a Telegram invite link to the current chat.",
+                 help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>]")
 async def invite_link(evt: CommandEvent) -> EventID:
+    # TODO once we switch to Python 3.9 minimum, use argparse with exit_on_error=False
+    uses = None
+    expire = None
+    while evt.args:
+        try:
+            flag, value = _parse_flag(evt.args)
+        except (ValueError, IndexError):
+            return await evt.reply(invite_link_usage)
+        if flag in ("uses", "u"):
+            try:
+                uses = int(value)
+            except ValueError:
+                await evt.reply("The number of uses must be an integer")
+        elif flag in ("expire", "e"):
+            expire_delta = _parse_delta(value)
+            if not expire_delta:
+                await evt.reply("Invalid format for expiry time delta")
+            expire = datetime.now() + expire_delta
+
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
@@ -91,7 +167,7 @@ async def invite_link(evt: CommandEvent) -> EventID:
         return await evt.reply("You can't invite users to private chats.")
 
     try:
-        link = await portal.get_invite_link(evt.sender)
+        link = await portal.get_invite_link(evt.sender, uses=uses, expire=expire)
         return await evt.reply(f"Invite link to {portal.title}: {link}")
     except ValueError as e:
         return await evt.reply(e.args[0])

mautrix_telegram/commands/portal/util.py

@@ -18,14 +18,16 @@ from typing import Tuple, Optional
 from mautrix.errors import MatrixRequestError
 from mautrix.appservice import IntentAPI
 from mautrix.types import RoomID, EventType, PowerLevelStateEventContent
+from .. import CommandEvent
 
 from ... import user as u
 
 OptStr = Optional[str]
 
 
-async def get_initial_state(intent: IntentAPI, room_id: RoomID
-                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent], bool]:
+async def get_initial_state(
+    intent: IntentAPI, room_id: RoomID
+) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent], bool]:
     state = await intent.get_state(room_id)
     title: OptStr = None
     about: OptStr = None
@@ -49,6 +51,14 @@ async def get_initial_state(intent: IntentAPI, room_id: RoomID
     return title, about, levels, encrypted
 
 
+async def warn_missing_power(levels: PowerLevelStateEventContent, evt: CommandEvent) -> None:
+    if levels.get_user_level(evt.az.bot_mxid) < levels.redact:
+        await evt.reply("Warning: The bot does not have privileges to redact messages on Matrix. "
+                        "Message deletions from Telegram will not be bridged unless you give "
+                        "redaction permissions to "
+                        f"[{evt.az.bot_mxid}](https://matrix.to/#/{evt.az.bot_mxid})")
+
+
 async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.User,
                                event: str) -> bool:
     if sender.is_admin:

mautrix_telegram/commands/telegram/account.py

@@ -16,7 +16,7 @@
 from typing import Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError, AboutTooLongError)
 from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
                                            ResetAuthorizationRequest, UpdateProfileRequest)
@@ -53,6 +53,23 @@ async def username(evt: CommandEvent) -> EventID:
     else:
         await evt.reply(f"Username changed to {evt.sender.username}")
 
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new about_>",
+                 help_text="Change your Telegram about section.")
+async def about(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp about <new about>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own about section.")
+    new_about = " ".join(evt.args)
+    if new_about == "-":
+        new_about = ""
+    try:
+        await evt.sender.client(UpdateProfileRequest(about=new_about))
+    except AboutTooLongError:
+        return await evt.reply("The provided about section is too long")
+    return await evt.reply("About section updated")
 
 @command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
                  help_text="Change your Telegram displayname.")

mautrix_telegram/commands/telegram/misc.py

@@ -14,11 +14,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import List, Optional, Tuple, cast
-import logging
 import codecs
 import base64
 import re
 
+from aiohttp import ClientSession, InvalidURL
+
 from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
                              UserAlreadyParticipantError, ChatIdInvalidError,
                              TakeoutInitDelayError, EmoticonInvalidError)
@@ -115,25 +116,25 @@ async def pm(evt: CommandEvent) -> EventID:
         return await evt.reply("That doesn't seem to be a user.")
     portal = po.Portal.get_by_entity(user, evt.sender.tgid)
     await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply("Created private chat room with "
-                           f"{pu.Puppet.get_displayname(user, False)}")
+    displayname, _ = pu.Puppet.get_displayname(user, False)
+    return await evt.reply(f"Created private chat room with {displayname}")
 
 
-async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
-    if arg.startswith("joinchat/"):
-        invite_hash = arg[len("joinchat/"):]
+async def _join(evt: CommandEvent, identifier: str, link_type: str
+                ) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
+    if link_type == "joinchat":
         try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
+            await evt.sender.client(CheckChatInviteRequest(identifier))
         except InviteHashInvalidError:
             return None, await evt.reply("Invalid invite link.")
         except InviteHashExpiredError:
             return None, await evt.reply("Invite link expired.")
         try:
-            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
+            return (await evt.sender.client(ImportChatInviteRequest(identifier))), None
         except UserAlreadyParticipantError:
             return None, await evt.reply("You are already in that chat.")
     else:
-        channel = await evt.sender.client.get_entity(arg)
+        channel = await evt.sender.client.get_entity(identifier)
         if not channel:
             return None, await evt.reply("Channel/supergroup not found.")
         return await evt.sender.client(JoinChannelRequest(channel)), None
@@ -146,12 +147,26 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
 
-    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
-    arg = regex.match(evt.args[0])
+    url = evt.args[0]
+    if evt.config["bridge.invite_link_resolve"]:
+        try:
+            async with ClientSession() as sess, sess.get(url) as resp:
+                url = str(resp.url)
+        except InvalidURL:
+            return await evt.reply("That doesn't look like a Telegram invite link.")
+
+    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)"
+                       r"(?:/(?P<type>joinchat|s))?/(?P<id>[^/]+)/?", flags=re.IGNORECASE)
+    arg = regex.match(url)
     if not arg:
         return await evt.reply("That doesn't look like a Telegram invite link.")
 
-    updates, _ = await _join(evt, arg.group(1))
+    data = arg.groupdict()
+    identifier = data["id"]
+    link_type = data["type"]
+    if link_type:
+        link_type = link_type.lower()
+    updates, _ = await _join(evt, identifier, link_type)
     if not updates:
         return None
 
@@ -165,9 +180,8 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
             try:
                 await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
             except ChatIdInvalidError as e:
-                logging.getLogger("mau.commands").trace("ChatIdInvalidError while creating portal "
-                                                        "from !tg join command: %s",
-                                                        updates.stringify())
+                evt.log.trace("ChatIdInvalidError while creating portal from !tg join command: %s",
+                              updates.stringify())
                 raise e
             return await evt.reply(f"Created room for {portal.title}")
     return None

mautrix_telegram/config.py

@@ -75,10 +75,6 @@ class Config(BaseBridgeConfig):
         copy("metrics.enabled")
         copy("metrics.listen_port")
 
-        copy("manhole.enabled")
-        copy("manhole.path")
-        copy("manhole.whitelist")
-
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
@@ -114,6 +110,7 @@ class Config(BaseBridgeConfig):
         else:
             copy("bridge.login_shared_secret_map")
         copy("bridge.telegram_link_preview")
+        copy("bridge.invite_link_resolve")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
         copy("bridge.max_document_size")
@@ -131,6 +128,10 @@ class Config(BaseBridgeConfig):
         copy("bridge.delivery_receipts")
         copy("bridge.delivery_error_reports")
         copy("bridge.resend_bridge_info")
+        copy("bridge.mute_bridging")
+        copy("bridge.pinned_tag")
+        copy("bridge.archive_tag")
+        copy("bridge.tag_only_on_create")
         copy("bridge.backfill.invite_own_puppet")
         copy("bridge.backfill.takeout_limit")
         copy("bridge.backfill.initial_limit")

mautrix_telegram/db/bot_chat.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Iterable
 
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, BigInteger, String
 
 from mautrix.util.db import Base
 
@@ -25,7 +25,7 @@ from ..types import TelegramID
 # Fucking Telegram not telling bots what chats they are in 3:<
 class BotChat(Base):
     __tablename__ = "bot_chat"
-    id: TelegramID = Column(Integer, primary_key=True)
+    id: TelegramID = Column(BigInteger, primary_key=True)
     type: str = Column(String, nullable=False)
 
     @classmethod

mautrix_telegram/db/message.py

@@ -13,9 +13,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Iterator
+from typing import Optional, Iterator, List
 
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+from sqlalchemy import (Column, UniqueConstraint, BigInteger, Integer, String, Boolean, and_, func,
+                        desc, select, false)
 
 from mautrix.types import RoomID, EventID
 from mautrix.util.db import Base
@@ -28,9 +29,10 @@ class Message(Base):
 
     mxid: EventID = Column(String)
     mx_room: RoomID = Column(String)
-    tgid: TelegramID = Column(Integer, primary_key=True)
-    tg_space: TelegramID = Column(Integer, primary_key=True)
+    tgid: TelegramID = Column(BigInteger, primary_key=True)
+    tg_space: TelegramID = Column(BigInteger, primary_key=True)
     edit_index: int = Column(Integer, primary_key=True)
+    redacted: bool = Column(Boolean, server_default=false())
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"),)
 
@@ -51,6 +53,12 @@ class Message(Base):
             return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
                                            cls.c.edit_index == edit_index)
 
+    @classmethod
+    def get_first_by_tgids(cls, tgids: List[TelegramID], tg_space: TelegramID
+                           ) -> Iterator['Message']:
+        return cls._select_all(cls.c.tgid.in_(tgids), cls.c.tg_space == tg_space,
+                               cls.c.edit_index == 0)
+
     @classmethod
     def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
         rows = cls.db.execute(select([func.count(cls.c.tg_space)])
@@ -77,6 +85,12 @@ class Message(Base):
         return cls._select_one_or_none(cls.c.mxid == mxid, cls.c.mx_room == mx_room,
                                        cls.c.tg_space == tg_space)
 
+    @classmethod
+    def get_by_mxids(cls, mxids: List[EventID], mx_room: RoomID, tg_space: TelegramID
+                     ) -> Iterator['Message']:
+        return cls._select_all(cls.c.mxid.in_(mxids), cls.c.mx_room == mx_room,
+                               cls.c.tg_space == tg_space)
+
     @classmethod
     def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
                        **values) -> None:

mautrix_telegram/db/portal.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Boolean, Text, func, sql
+from sqlalchemy import Column, BigInteger, String, Boolean, Text, func, sql
 
 from mautrix.types import RoomID, ContentURI
 from mautrix.util.db import Base
@@ -27,8 +27,8 @@ class Portal(Base):
     __tablename__ = "portal"
 
     # Telegram chat information
-    tgid: TelegramID = Column(Integer, primary_key=True)
-    tg_receiver: TelegramID = Column(Integer, primary_key=True)
+    tgid: TelegramID = Column(BigInteger, primary_key=True)
+    tg_receiver: TelegramID = Column(BigInteger, primary_key=True)
     peer_type: str = Column(String, nullable=False)
     megagroup: bool = Column(Boolean)
 

mautrix_telegram/db/puppet.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Text, Boolean
+from sqlalchemy import Column, Integer, BigInteger, String, Text, Boolean
 from sqlalchemy.sql import expression, func
 
 from mautrix.types import UserID, SyncToken
@@ -27,13 +27,15 @@ from ..types import TelegramID
 class Puppet(Base):
     __tablename__ = "puppet"
 
-    id: TelegramID = Column(Integer, primary_key=True)
+    id: TelegramID = Column(BigInteger, primary_key=True)
     custom_mxid: UserID = Column(String, nullable=True)
     access_token: str = Column(String, nullable=True)
     next_batch: SyncToken = Column(String, nullable=True)
     base_url: str = Column(Text, nullable=True)
     displayname: str = Column(String, nullable=True)
-    displayname_source: TelegramID = Column(Integer, nullable=True)
+    displayname_source: TelegramID = Column(BigInteger, nullable=True)
+    displayname_contact: bool = Column(Boolean, nullable=False, server_default=expression.true())
+    displayname_quality: int = Column(Integer, nullable=False, server_default="0")
     username: str = Column(String, nullable=True)
     photo_id: str = Column(String, nullable=True)
     is_bot: bool = Column(Boolean, nullable=True)

mautrix_telegram/db/telegram_file.py

@@ -13,15 +13,17 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, cast, Dict, Any
+from typing import Optional, cast, Dict, Any, TYPE_CHECKING
 
 from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
                         TypeDecorator)
-from sqlalchemy.engine.result import RowProxy
 
 from mautrix.types import ContentURI, EncryptedFile
 from mautrix.util.db import Base
 
+if TYPE_CHECKING:
+    from sqlalchemy.engine.result import RowProxy
+
 
 class DBEncryptedFile(TypeDecorator):
     impl = Text
@@ -60,7 +62,7 @@ class TelegramFile(Base):
     thumbnail: Optional['TelegramFile'] = None
 
     @classmethod
-    def scan(cls, row: RowProxy) -> 'TelegramFile':
+    def scan(cls, row: 'RowProxy') -> 'TelegramFile':
         telegram_file = cast(TelegramFile, super().scan(row))
         if isinstance(telegram_file.thumbnail, str):
             telegram_file.thumbnail = cls.get(telegram_file.thumbnail)

mautrix_telegram/db/user.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable, Tuple
 
-from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String, func
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, BigInteger, Integer, String, func
 
 from mautrix.types import UserID
 from mautrix.util.db import Base
@@ -27,7 +27,7 @@ class User(Base):
     __tablename__ = "user"
 
     mxid: UserID = Column(String, primary_key=True)
-    tgid: Optional[TelegramID] = Column(Integer, nullable=True, unique=True)
+    tgid: Optional[TelegramID] = Column(BigInteger, nullable=True, unique=True)
     tg_username: str = Column(String, nullable=True)
     tg_phone: str = Column(String, nullable=True)
     saved_contacts: int = Column(Integer, default=0, nullable=False)
@@ -91,10 +91,10 @@ class User(Base):
 class UserPortal(Base):
     __tablename__ = "user_portal"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE",
+    user: TelegramID = Column(BigInteger, ForeignKey("user.tgid", onupdate="CASCADE",
                                                   ondelete="CASCADE"), primary_key=True)
-    portal: TelegramID = Column(Integer, primary_key=True)
-    portal_receiver: TelegramID = Column(Integer, primary_key=True)
+    portal: TelegramID = Column(BigInteger, primary_key=True)
+    portal_receiver: TelegramID = Column(BigInteger, primary_key=True)
 
     __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
                                            ("portal.tgid", "portal.tg_receiver"),
@@ -104,5 +104,5 @@ class UserPortal(Base):
 class Contact(Base):
     __tablename__ = "contact"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
-    contact: TelegramID = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
+    user: TelegramID = Column(BigInteger, ForeignKey("user.tgid"), primary_key=True)
+    contact: TelegramID = Column(BigInteger, ForeignKey("puppet.id"), primary_key=True)

mautrix_telegram/example-config.yaml

@@ -8,6 +8,12 @@ homeserver:
     # Only applies if address starts with https://
     verify_ssl: true
     asmux: false
+    # Number of retries for all HTTP requests if the homeserver isn't reachable.
+    http_retry_count: 4
+    # The URL to push real-time bridge status to.
+    # If set, the bridge will make POST requests to this URL whenever a user's Telegram connection state changes.
+    # The bridge will use the appservice as_token to authorize requests.
+    status_endpoint: null
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
@@ -194,8 +200,11 @@ bridge:
         example.com: foobar
     # Set to false to disable link previews in messages sent to Telegram.
     telegram_link_preview: true
+    # Whether or not the !tg join command should do a HTTP request
+    # to resolve redirects in invite links.
+    invite_link_resolve: false
     # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    # N.B. Inline images are not supported on all clients (e.g. Element iOS/Android).
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
@@ -204,6 +213,7 @@ bridge:
     # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
     # streaming from/to Matrix and using many connections for Telegram.
     # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
+    # This option uses internal Telethon implementation details and may break with minor updates.
     parallel_file_transfer: false
     # Whether or not created rooms should have federation enabled.
     # If false, created portal rooms will never be federated.
@@ -267,6 +277,15 @@ bridge:
     # This field will automatically be changed back to false after it,
     # except if the config file is not writable.
     resend_bridge_info: false
+    # When using double puppeting, should muted chats be muted in Matrix?
+    mute_bridging: false
+    # When using double puppeting, should pinned chats be moved to a specific tag in Matrix?
+    # The favorites tag is `m.favourite`.
+    pinned_tag: null
+    # Same as above for archived chats, the low priority tag is `m.lowpriority`.
+    archive_tag: null
+    # Whether or not mute status and tags should only be bridged when the portal room is created.
+    tag_only_on_create: true
     # Settings for backfilling messages from Telegram.
     backfill:
         # Whether or not the Telegram ghosts of logged in Matrix users should be

mautrix_telegram/formatter/from_telegram.py

@@ -79,7 +79,7 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
             try:
                 user = await source.client.get_entity(fwd_from.from_id)
                 if user:
-                    fwd_from_text = pu.Puppet.get_displayname(user, False)
+                    fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
                     fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
             except (ValueError, RPCError):
                 fwd_from_text = fwd_from_html = "unknown user"
@@ -87,7 +87,7 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
         from_id = (fwd_from.from_id.chat_id if isinstance(fwd_from.from_id, PeerChat)
                    else fwd_from.from_id.channel_id)
         portal = po.Portal.get_by_tgid(TelegramID(from_id))
-        if portal:
+        if portal and portal.title:
             fwd_from_text = portal.title
             if portal.alias:
                 fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"

mautrix_telegram/get_version.py

@@ -18,7 +18,7 @@ def run(cmd):
 if os.path.exists(".git") and shutil.which("git"):
     try:
         git_revision = run(["git", "rev-parse", "HEAD"]).strip().decode("ascii")
-        git_revision_url = f"https://github.com/tulir/mautrix-telegram/commit/{git_revision}"
+        git_revision_url = f"https://github.com/mautrix/telegram/commit/{git_revision}"
         git_revision = git_revision[:8]
     except (subprocess.SubprocessError, OSError):
         git_revision = "unknown"
@@ -33,7 +33,7 @@ else:
     git_revision_url = None
     git_tag = None
 
-git_tag_url = (f"https://github.com/tulir/mautrix-telegram/releases/tag/{git_tag}"
+git_tag_url = (f"https://github.com/mautrix/telegram/releases/tag/{git_tag}"
                if git_tag else None)
 
 if git_tag and __version__ == git_tag[1:].replace("-", ""):

mautrix_telegram/matrix.py

@@ -84,7 +84,7 @@ class MatrixHandler(BaseMatrixHandler):
             portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
             if portal.mxid:
                 try:
-                    await intent.invite_user(portal.mxid, inviter.mxid)
+                    await portal.invite_to_matrix(inviter.mxid)
                     await intent.send_notice(
                         room_id, text=f"You already have a private chat with me: {portal.mxid}",
                         html=("You already have a private chat with me: "
@@ -94,9 +94,7 @@ class MatrixHandler(BaseMatrixHandler):
                 except MatrixError:
                     pass
             portal.mxid = room_id
-            e2be_ok = None
-            if self.config["bridge.encryption.default"] and self.e2ee:
-                e2be_ok = await portal.enable_dm_encryption()
+            e2be_ok = await portal.check_dm_encryption()
             await portal.save()
             await inviter.register_portal(portal)
             if e2be_ok is True:
@@ -111,6 +109,7 @@ class MatrixHandler(BaseMatrixHandler):
                 if e2be_ok is False:
                     message += "\n\nWarning: Failed to enable end-to-bridge encryption"
                 await intent.send_notice(room_id, message)
+            await portal.update_bridge_info()
         else:
             await intent.join_room(room_id)
             await intent.send_notice(room_id, "This puppet will remain inactive until a "
@@ -283,13 +282,12 @@ class MatrixHandler(BaseMatrixHandler):
         portal = po.Portal.get_by_mxid(room_id)
         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
-            events = new_events - old_events
-            if len(events) > 0:
-                # New event pinned, set that as pinned in Telegram.
-                await portal.handle_matrix_pin(sender, EventID(events.pop()), event_id)
-            elif len(new_events) == 0:
-                # All pinned events removed, remove pinned event in Telegram.
-                await portal.handle_matrix_pin(sender, None, event_id)
+            if not new_events:
+                await portal.handle_matrix_unpin_all(sender, event_id)
+            else:
+                changes = {event_id: event_id in new_events
+                           for event_id in new_events ^ old_events}
+                await portal.handle_matrix_pin(sender, changes, event_id)
 
     @staticmethod
     async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID,

mautrix_telegram/portal/base.py

@@ -15,22 +15,23 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, Set, Iterable, TYPE_CHECKING
 from abc import ABC, abstractmethod
+from datetime import datetime
 import asyncio
 import logging
 import json
 
 from telethon.tl.functions.messages import ExportChatInviteRequest
-from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, ChatInviteEmpty, InputChannel,
+from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, InputChannel,
                                InputPeerChannel, InputPeerChat, InputPeerUser, InputUser,
                                PeerChannel, PeerChat, PeerUser, TypeChat, TypeInputPeer, TypePeer,
                                TypeUser, TypeUserFull, User, UserFull, TypeInputChannel, Photo,
                                Document, TypePhotoSize, PhotoSize, InputPhotoFileLocation,
                                TypeChatParticipant, TypeChannelParticipant, PhotoEmpty, ChatPhoto,
-                               ChatPhotoEmpty)
+                               ChatPhotoEmpty, PhotoSizeProgressive, PhotoSizeEmpty)
 
 from mautrix.errors import MatrixRequestError, IntentError
 from mautrix.appservice import AppService, IntentAPI
-from mautrix.types import (RoomID, RoomAlias, UserID, EventID, EventType, MessageEventContent,
+from mautrix.types import (RoomID, RoomAlias, UserID, EventID, EventType,
                            PowerLevelStateEventContent, ContentURI)
 from mautrix.util.simple_template import SimpleTemplate
 from mautrix.util.simple_lock import SimpleLock
@@ -104,6 +105,7 @@ class BasePortal(MautrixBasePortal, ABC):
 
     dedup: PortalDedup
     send_lock: PortalSendLock
+    _pin_lock: asyncio.Lock
 
     _db_instance: DBPortal
     _main_intent: Optional[IntentAPI]
@@ -138,6 +140,7 @@ class BasePortal(MautrixBasePortal, ABC):
 
         self.dedup = PortalDedup(self)
         self.send_lock = PortalSendLock()
+        self._pin_lock = asyncio.Lock()
 
         if tgid:
             self.by_tgid[self.tgid_full] = self
@@ -181,6 +184,10 @@ class BasePortal(MautrixBasePortal, ABC):
         elif self.peer_type == "channel":
             return PeerChannel(channel_id=self.tgid)
 
+    @property
+    def is_direct(self) -> bool:
+        return self.peer_type == "user"
+
     @property
     def has_bot(self) -> bool:
         return (bool(self.bot)
@@ -215,7 +222,18 @@ class BasePortal(MautrixBasePortal, ABC):
         return config[f"bridge.{key}"]
 
     @staticmethod
-    def _get_largest_photo_size(photo: Union[Photo, Document]
+    def _photo_size_key(photo: TypePhotoSize) -> int:
+        if isinstance(photo, PhotoSize):
+            return photo.size
+        elif isinstance(photo, PhotoSizeProgressive):
+            return max(photo.sizes)
+        elif isinstance(photo, PhotoSizeEmpty):
+            return 0
+        else:
+            return len(photo.bytes)
+
+    @classmethod
+    def _get_largest_photo_size(cls, photo: Union[Photo, Document]
                                 ) -> Tuple[Optional[InputPhotoFileLocation],
                                            Optional[TypePhotoSize]]:
         if not photo or isinstance(photo, PhotoEmpty) or (isinstance(photo, Document)
@@ -223,9 +241,7 @@ class BasePortal(MautrixBasePortal, ABC):
             return None, None
 
         largest = max(photo.thumbs if isinstance(photo, Document) else photo.sizes,
-                      key=(lambda photo2: (len(photo2.bytes)
-                                           if not isinstance(photo2, PhotoSize)
-                                           else photo2.size)))
+                      key=cls._photo_size_key)
         return InputPhotoFileLocation(
             id=photo.id,
             access_hash=photo.access_hash,
@@ -264,14 +280,14 @@ class BasePortal(MautrixBasePortal, ABC):
                     return dialog.entity
             raise
 
-    async def get_invite_link(self, user: 'u.User') -> str:
+    async def get_invite_link(self, user: 'u.User', uses: Optional[int] = None,
+                              expire: Optional[datetime] = None) -> str:
         if self.peer_type == "user":
             raise ValueError("You can't invite users to private chats.")
         if self.username:
             return f"https://t.me/{self.username}"
-        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user)))
-        if isinstance(link, ChatInviteEmpty):
-            raise ValueError("Failed to get invite link.")
+        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user),
+                                                         expire_date=expire, usage_limit=uses))
         return link.link
 
     # endregion

mautrix_telegram/portal/deduplication.py

@@ -61,9 +61,9 @@ class PortalDedup:
         if isinstance(event, MessageService):
             hash_content = [event.date.timestamp(), event.from_id, event.action]
         else:
-            hash_content = [event.date.timestamp(), event.message]
+            hash_content = [event.date.timestamp(), event.message.strip()]
             if event.fwd_from:
-                hash_content += [event.fwd_from.from_id, event.fwd_from.channel_id]
+                hash_content += [event.fwd_from.from_id]
             elif isinstance(event, Message) and event.media:
                 try:
                     hash_content += {

mautrix_telegram/portal/matrix.py

@@ -22,11 +22,10 @@ import magic
 
 from telethon.tl.functions.messages import (EditChatPhotoRequest, EditChatTitleRequest,
                                             UpdatePinnedMessageRequest, SetTypingRequest,
-                                            EditChatAboutRequest)
+                                            EditChatAboutRequest, UnpinAllMessagesRequest)
 from telethon.tl.functions.channels import EditPhotoRequest, EditTitleRequest, JoinChannelRequest
-from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError,
-                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError,
-                             RPCError)
+from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError, MessageIdInvalidError,
+                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, RPCError)
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
                                InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
@@ -113,7 +112,19 @@ class PortalMatrix(BasePortal, ABC):
         space = self.tgid if self.peer_type == "channel" else user.tgid
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
-            return
+            message = DBMessage.find_last(self.mxid, space)
+            if not message:
+                self.log.debug(f"Dropping Matrix read receipt from {user.mxid}: "
+                               f"target message {event_id} not known and last message"
+                               " in chat not found")
+                return
+            else:
+                self.log.debug(f"Matrix read receipt target {event_id} not known, marking "
+                               f"messages up to most recent ({message.mxid}/{message.tgid}) "
+                               f"as read by {user.mxid}/{user.tgid}")
+        else:
+            self.log.debug("Handling Matrix read receipt: marking messages up to "
+                           f"{message.mxid}/{message.tgid} as read by {user.mxid}/{user.tgid}")
         await user.client.send_read_acknowledge(self.peer, max_id=message.tgid,
                                                 clear_mentions=True)
 
@@ -321,13 +332,13 @@ class PortalMatrix(BasePortal, ABC):
                                       content: LocationMessageEventContent, reply_to: TelegramID
                                       ) -> None:
         try:
-            lat, long = content.geo_uri[len("geo:"):].split(",")
+            lat, long = content.geo_uri[len("geo:"):].split(";")[0].split(",")
             lat, long = float(lat), float(long)
         except (KeyError, ValueError):
             self.log.exception("Failed to parse location")
             return None
         caption, entities = await formatter.matrix_to_telegram(client, text=content.body)
-        media = MessageMediaGeo(geo=GeoPoint(lat, long, access_hash=0))
+        media = MessageMediaGeo(geo=GeoPoint(lat=lat, long=long, access_hash=0))
 
         async with self.send_lock(sender_id):
             if await self._matrix_document_edit(client, content, space, caption, media, event_id):
@@ -412,23 +423,23 @@ class PortalMatrix(BasePortal, ABC):
         else:
             self.log.trace("Unhandled Matrix event: %s", content)
 
-    async def handle_matrix_pin(self, sender: 'u.User', pinned_message: Optional[EventID],
+    async def handle_matrix_unpin_all(self, sender: 'u.User', pin_event_id: EventID) -> None:
+        await sender.client(UnpinAllMessagesRequest(peer=self.peer))
+        await self._send_delivery_receipt(pin_event_id)
+
+    async def handle_matrix_pin(self, sender: 'u.User', changes: Dict[EventID, bool],
                                 pin_event_id: EventID) -> None:
-        if self.peer_type != "chat" and self.peer_type != "channel":
-            return
-        try:
-            if not pinned_message:
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=0))
-            else:
-                tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
-                message = DBMessage.get_by_mxid(pinned_message, self.mxid, tg_space)
-                if message is None:
-                    self.log.warning(f"Could not find pinned {pinned_message} in {self.mxid}")
-                    return
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=message.tgid))
-            await self._send_delivery_receipt(pin_event_id)
-        except ChatNotModifiedError:
-            pass
+        tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
+        ids = {msg.mxid: msg.tgid
+               for msg in DBMessage.get_by_mxids(list(changes.keys()),
+                                                 mx_room=self.mxid, tg_space=tg_space)}
+        for event_id, pinned in changes.items():
+            try:
+                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=ids[event_id],
+                                                               unpin=not pinned))
+            except (ChatNotModifiedError, MessageIdInvalidError, KeyError):
+                pass
+        await self._send_delivery_receipt(pin_event_id)
 
     async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID,
                                      redaction_event_id: EventID) -> None:
@@ -436,12 +447,18 @@ class PortalMatrix(BasePortal, ABC):
         space = self.tgid if self.peer_type == "channel" else real_deleter.tgid
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
-            return
-        if message.edit_index == 0:
+            self.log.trace(f"Ignoring Matrix redaction of unknown event {event_id}")
+        elif message.redacted:
+            self.log.debug("Ignoring Matrix redaction of already redacted event "
+                           f"{message.mxid} in {message.mx_room}")
+        elif message.edit_index != 0:
+            message.edit(redacted=True)
+            self.log.debug("Ignoring Matrix redaction of edit event "
+                           f"{message.mxid} in {message.mx_room}")
+        else:
+            message.edit(redacted=True)
             await real_deleter.client.delete_messages(self.peer, [message.tgid])
             await self._send_delivery_receipt(redaction_event_id)
-        else:
-            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
 
     async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
                                            level: int) -> None:

mautrix_telegram/portal/metadata.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Optional, Iterable, Union, Dict, Any, TYPE_CHECKING
+from typing import List, Optional, Iterable, Union, Dict, Any, Tuple, TYPE_CHECKING
 from abc import ABC
 import asyncio
 
@@ -26,7 +26,8 @@ from telethon.tl.types import (
     Channel, ChatBannedRights, ChannelParticipantsRecent, ChannelParticipantsSearch, ChatPhoto,
     PhotoEmpty, InputChannel, InputUser, ChatPhotoEmpty, PeerUser, Photo, TypeChat, TypeInputPeer,
     TypeUser, User, InputPeerPhotoFileLocation, ChatParticipantAdmin, ChannelParticipantAdmin,
-    ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty)
+    ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty,
+    InputPeerUser)
 
 from mautrix.errors import MForbidden
 from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership,
@@ -58,21 +59,30 @@ class PortalMetadata(BasePortal, ABC):
 
     # region Matrix -> Telegram
 
-    async def _get_telegram_users_in_matrix_room(self) -> List[Union[InputUser, PeerUser]]:
-        user_tgids = set()
+    async def get_telegram_users_in_matrix_room(self, source: 'u.User'
+                                                ) -> Tuple[List[InputPeerUser], List[UserID]]:
+        user_tgids = {}
         user_mxids = await self.main_intent.get_room_members(self.mxid, (Membership.JOIN,
                                                                          Membership.INVITE))
-        for user_str in user_mxids:
-            user = UserID(user_str)
-            if user == self.az.bot_mxid:
+        for mxid in user_mxids:
+            if mxid == self.az.bot_mxid:
                 continue
-            mx_user = u.User.get_by_mxid(user, create=False)
+            mx_user = u.User.get_by_mxid(mxid, create=False)
             if mx_user and mx_user.tgid:
-                user_tgids.add(mx_user.tgid)
-            puppet_id = p.Puppet.get_id_from_mxid(user)
+                user_tgids[mx_user.tgid] = mxid
+            puppet_id = p.Puppet.get_id_from_mxid(mxid)
             if puppet_id:
-                user_tgids.add(puppet_id)
-        return [PeerUser(user_id) for user_id in user_tgids]
+                user_tgids[puppet_id] = mxid
+        input_users = []
+        errors = []
+        for tgid, mxid in user_tgids.items():
+            try:
+                input_users.append(await source.client.get_input_entity(tgid))
+            except ValueError as e:
+                source.log.debug(f"Failed to find the input entity for {tgid} ({mxid}) for "
+                                 f"creating a group: {e}")
+                errors.append(mxid)
+        return input_users, errors
 
     async def upgrade_telegram_chat(self, source: 'u.User') -> None:
         if self.peer_type != "chat":
@@ -116,13 +126,13 @@ class PortalMetadata(BasePortal, ABC):
         if await self._update_username(username):
             await self.save()
 
-    async def create_telegram_chat(self, source: 'u.User', supergroup: bool = False) -> None:
+    async def create_telegram_chat(self, source: 'u.User', invites: List[InputUser],
+                                   supergroup: bool = False) -> None:
         if not self.mxid:
             raise ValueError("Can't create Telegram chat for portal without Matrix room.")
         elif self.tgid:
             raise ValueError("Can't create Telegram chat for portal with existing Telegram chat.")
 
-        invites = await self._get_telegram_users_in_matrix_room()
         if len(invites) < 2:
             if self.bot is not None:
                 info, mxid = await self.bot.get_me()
@@ -160,6 +170,7 @@ class PortalMetadata(BasePortal, ABC):
             levels = self._get_base_power_levels(levels, entity)
             await self.main_intent.set_power_levels(self.mxid, levels)
         await self.handle_matrix_power_levels(source, levels.users, {}, None)
+        await self.update_bridge_info()
 
     async def invite_telegram(self, source: 'u.User',
                               puppet: Union[p.Puppet, 'AbstractUser']) -> None:
@@ -175,12 +186,27 @@ class PortalMetadata(BasePortal, ABC):
     # endregion
     # region Telegram -> Matrix
 
+    def _get_invite_content(self, double_puppet: Optional['p.Puppet']) -> Dict[str, Any]:
+        invite_content = {}
+        if double_puppet:
+            invite_content["fi.mau.will_auto_accept"] = True
+        if self.is_direct:
+            invite_content["is_direct"] = True
+        return invite_content
+
     async def invite_to_matrix(self, users: InviteList) -> None:
         if isinstance(users, list):
             for user in users:
-                await self.main_intent.invite_user(self.mxid, user, check_cache=True)
+                await self.invite_to_matrix(user)
         else:
-            await self.main_intent.invite_user(self.mxid, users, check_cache=True)
+            puppet = await p.Puppet.get_by_custom_mxid(users)
+            await self.main_intent.invite_user(self.mxid, users, check_cache=True,
+                                               extra_content=self._get_invite_content(puppet))
+            if puppet:
+                try:
+                    await puppet.intent.ensure_joined(self.mxid)
+                except Exception:
+                    self.log.exception("Failed to ensure %s is joined to portal", users)
 
     async def update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
                                  direct: bool = None, puppet: p.Puppet = None,
@@ -326,12 +352,13 @@ class PortalMetadata(BasePortal, ABC):
         if self.peer_type == "channel":
             self.megagroup = entity.megagroup
 
+        preset = RoomCreatePreset.PRIVATE
         if self.peer_type == "channel" and entity.username:
-            preset = RoomCreatePreset.PUBLIC
+            if self.public_portals:
+                preset = RoomCreatePreset.PUBLIC
             self.username = entity.username
             alias = self.alias_localpart
         else:
-            preset = RoomCreatePreset.PRIVATE
             # TODO invite link alias?
             alias = None
 
@@ -368,6 +395,7 @@ class PortalMetadata(BasePortal, ABC):
             "state_key": self.bridge_info_state_key,
             "content": self.bridge_info,
         }]
+        create_invites = []
         if config["bridge.encryption.default"] and self.matrix.e2ee:
             self.encrypted = True
             initial_state.append({
@@ -375,7 +403,7 @@ class PortalMetadata(BasePortal, ABC):
                 "content": {"algorithm": "m.megolm.v1.aes-sha2"},
             })
             if direct:
-                invites.append(self.az.bot_mxid)
+                create_invites.append(self.az.bot_mxid)
         if direct and (self.encrypted or self.private_chat_portal_meta):
             self.title = puppet.displayname
         if config["appservice.community_id"]:
@@ -389,7 +417,7 @@ class PortalMetadata(BasePortal, ABC):
 
         with self.backfill_lock:
             room_id = await self.main_intent.create_room(alias_localpart=alias, preset=preset,
-                                                         is_direct=direct, invitees=invites or [],
+                                                         is_direct=direct, invitees=create_invites,
                                                          name=self.title, topic=self.about,
                                                          initial_state=initial_state,
                                                          creation_content=creation_content)
@@ -408,6 +436,8 @@ class PortalMetadata(BasePortal, ABC):
             await self.az.state_store.set_power_levels(self.mxid, power_levels)
             await user.register_portal(self)
 
+            await self.invite_to_matrix(invites)
+
             update_room = self.loop.create_task(self.update_matrix_room(
                 user, entity, direct, puppet,
                 levels=power_levels, users=users))
@@ -450,7 +480,7 @@ class PortalMetadata(BasePortal, ABC):
             levels.kick = overrides.get("kick", 50)
             levels.redact = overrides.get("redact", 50)
             levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
-            levels.events[EventType.ROOM_ENCRYPTION] = 99
+            levels.events[EventType.ROOM_ENCRYPTION] = 50 if self.matrix.e2ee else 99
             levels.events[EventType.ROOM_TOMBSTONE] = 99
             levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
             levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0
@@ -558,13 +588,6 @@ class PortalMetadata(BasePortal, ABC):
             if user:
                 await self.invite_to_matrix(user.mxid)
 
-                puppet = await p.Puppet.get_by_custom_mxid(user.mxid)
-                if puppet:
-                    try:
-                        await puppet.intent.ensure_joined(self.mxid)
-                    except Exception:
-                        self.log.exception("Failed to ensure %s is joined to portal", user.mxid)
-
         # We can't trust the member list if any of the following cases is true:
         #  * There are close to 10 000 users, because Telegram might not be sending all members.
         #  * The member sync count is limited, because then we might ignore some members.
@@ -736,15 +759,13 @@ class PortalMetadata(BasePortal, ABC):
         if isinstance(photo, (ChatPhoto, UserProfilePhoto)):
             loc = InputPeerPhotoFileLocation(
                 peer=await self.get_input_entity(user),
-                local_id=photo.photo_big.local_id,
-                volume_id=photo.photo_big.volume_id,
+                photo_id=photo.photo_id,
                 big=True
             )
-            photo_id = (f"{loc.volume_id}-{loc.local_id}" if isinstance(photo, ChatPhoto)
-                        else photo.photo_id)
+            photo_id = str(photo.photo_id)
         elif isinstance(photo, Photo):
-            loc, largest = self._get_largest_photo_size(photo)
-            photo_id = f"{largest.location.volume_id}-{largest.location.local_id}"
+            loc, _ = self._get_largest_photo_size(photo)
+            photo_id = str(loc.id)
         elif isinstance(photo, (UserProfilePhotoEmpty, ChatPhotoEmpty, PhotoEmpty, type(None))):
             photo_id = ""
             loc = None

mautrix_telegram/portal/telegram.py

@@ -34,7 +34,8 @@ from telethon.tl.types import (
     MessageMediaPhoto, MessageMediaDice, MessageMediaGame, MessageMediaUnsupported, PeerUser,
     PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant, TypeDocumentAttribute,
     TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping, UpdateUserTyping,
-    MessageEntityPre, ChatPhotoEmpty)
+    MessageEntityPre, ChatPhotoEmpty, DocumentAttributeImageSize, DocumentAttributeAnimated,
+    UpdateChannelUserTyping, SendMessageTypingAction)
 
 from mautrix.appservice import IntentAPI
 from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType,
@@ -56,16 +57,20 @@ if TYPE_CHECKING:
 
 InviteList = Union[UserID, List[UserID]]
 TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
+UpdateTyping = Union[UpdateUserTyping, UpdateChatUserTyping, UpdateChannelUserTyping]
 DocAttrs = NamedTuple("DocAttrs", name=Optional[str], mime_type=Optional[str], is_sticker=bool,
-                      sticker_alt=Optional[str], width=int, height=int)
+                      sticker_alt=Optional[str], width=int, height=int, is_gif=bool)
 
 config: Optional['Config'] = None
 
 
 class PortalTelegram(BasePortal, ABC):
-    async def handle_telegram_typing(self, user: p.Puppet,
-                                     _: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
-        await user.intent_for(self).set_typing(self.mxid, is_typing=True)
+    async def handle_telegram_typing(self, user: p.Puppet, update: UpdateTyping) -> None:
+        if user.is_real_user:
+            # Ignore typing notifications from double puppeted users to avoid echoing
+            return
+        is_typing = isinstance(update.action, SendMessageTypingAction)
+        await user.default_mxid_intent.set_typing(self.mxid, is_typing=is_typing)
 
     def _get_external_url(self, evt: Message) -> Optional[str]:
         if self.peer_type == "channel" and self.username is not None:
@@ -109,8 +114,7 @@ class PortalTelegram(BasePortal, ABC):
             return await self._send_message(intent, content, timestamp=evt.date)
         info = ImageInfo(
             height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type,
-            size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize))
-                  else largest_size.size))
+            size=self._photo_size_key(largest_size))
         ext = sane_mimetypes.guess_extension(file.mime_type)
         name = f"disappearing_image{ext}" if media.ttl_seconds else f"image{ext}"
         await intent.set_typing(self.mxid, is_typing=False)
@@ -135,6 +139,7 @@ class PortalTelegram(BasePortal, ABC):
     @staticmethod
     def _parse_telegram_document_attributes(attributes: List[TypeDocumentAttribute]) -> DocAttrs:
         name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
+        is_gif = False
         for attr in attributes:
             if isinstance(attr, DocumentAttributeFilename):
                 name = name or attr.file_name
@@ -142,9 +147,13 @@ class PortalTelegram(BasePortal, ABC):
             elif isinstance(attr, DocumentAttributeSticker):
                 is_sticker = True
                 sticker_alt = attr.alt
+            elif isinstance(attr, DocumentAttributeAnimated):
+                is_gif = True
             elif isinstance(attr, DocumentAttributeVideo):
                 width, height = attr.w, attr.h
-        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height)
+            elif isinstance(attr, DocumentAttributeImageSize):
+                width, height = attr.w, attr.h
+        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height, is_gif)
 
     @staticmethod
     def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: DocAttrs,
@@ -185,7 +194,7 @@ class PortalTelegram(BasePortal, ABC):
                                                 width=file.thumbnail.width or thumb_size.w,
                                                 size=file.thumbnail.size)
         else:
-            # This is a hack for bad clients like Riot iOS that require a thumbnail
+            # This is a hack for bad clients like Element iOS that require a thumbnail
             if file.decryption_info:
                 info.thumbnail_file = file.decryption_info
             else:
@@ -226,9 +235,30 @@ class PortalTelegram(BasePortal, ABC):
         await intent.set_typing(self.mxid, is_typing=False)
 
         event_type = EventType.ROOM_MESSAGE
-        # Riot only supports images as stickers, so send animated webm stickers as m.video
+        # Elements only support images as stickers, so send animated webm stickers as m.video
         if attrs.is_sticker and file.mime_type.startswith("image/"):
             event_type = EventType.STICKER
+            # Tell clients to render the stickers as 256x256 if they're bigger
+            if info.width > 256 or info.height > 256:
+                if info.width > info.height:
+                    info.height = int(info.height / (info.width / 256))
+                    info.width = 256
+                else:
+                    info.width = int(info.width / (info.height / 256))
+                    info.height = 256
+            if info.thumbnail_info:
+                info.thumbnail_info.width = info.width
+                info.thumbnail_info.height = info.height
+        if attrs.is_gif or (attrs.is_sticker and info.mimetype == "video/webm"):
+            if attrs.is_gif:
+                info["fi.mau.telegram.gif"] = True
+            else:
+                info["fi.mau.telegram.animated_sticker"] = True
+            info["fi.mau.loop"] = True
+            info["fi.mau.autoplay"] = True
+            info["fi.mau.hide_controls"] = True
+            info["fi.mau.no_audio"] = True
+
         content = MediaMessageEventContent(
             body=name or "unnamed file", info=info, relates_to=relates_to,
             external_url=self._get_external_url(evt),
@@ -282,7 +312,7 @@ class PortalTelegram(BasePortal, ABC):
     async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
                                           evt: Message, relates_to: RelatesTo = None) -> EventID:
         override_text = ("This message is not supported on your version of Mautrix-Telegram. "
-                         "Please check https://github.com/tulir/mautrix-telegram or ask your "
+                         "Please check https://github.com/mautrix/telegram or ask your "
                          "bridge administrator about possible updates.")
         content = await formatter.telegram_to_matrix(
             evt, source, self.main_intent, override_text=override_text)
@@ -318,16 +348,63 @@ class PortalTelegram(BasePortal, ABC):
         await intent.set_typing(self.mxid, is_typing=False)
         return await self._send_message(intent, content, timestamp=evt.date)
 
+    @staticmethod
+    def _format_dice(roll: MessageMediaDice) -> str:
+        if roll.emoticon == "\U0001F3B0":
+            emojis = {
+                0: "\U0001F36B",  # "🍫",
+                1: "\U0001F352",  # "🍒",
+                2: "\U0001F34B",  # "🍋",
+                3: "7\ufe0f\u20e3"  # "7️⃣",
+            }
+            res = roll.value - 1
+            slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
+            return f"{slot1} {slot2} {slot3} ({roll.value})"
+        elif roll.emoticon == "\u26BD":
+            results = {
+                1: "miss",
+                2: "hit the woodwork",
+                3: "goal",  # seems to go in through the center
+                4: "goal",
+                5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
+            }
+        elif roll.emoticon == "\U0001F3B3":
+            results = {
+                1: "miss",
+                2: "1 pin down",
+                3: "3 pins down, split",
+                4: "4 pins down, split",
+                5: "5 pins down",
+                6: "strike 🎉",
+            }
+        # elif roll.emoticon == "\U0001F3C0":
+        #     results = {
+        #         2: "rolled off",
+        #         3: "stuck",
+        #     }
+        # elif roll.emoticon == "\U0001F3AF":
+        #     results = {
+        #         1: "bounced off",
+        #         2: "outer rim",
+        #
+        #         6: "bullseye",
+        #     }
+        else:
+            return str(roll.value)
+        return f"{results[roll.value]} ({roll.value})"
+
     async def handle_telegram_dice(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                    relates_to: RelatesTo) -> EventID:
         emoji_text = {
             "\U0001F3AF": " Dart throw",
             "\U0001F3B2": " Dice roll",
             "\U0001F3C0": " Basketball throw",
+            "\U0001F3B0": " Slot machine",
+            "\U0001F3B3": " Bowling",
             "\u26BD": " Football kick"
         }
         roll: MessageMediaDice = evt.media
-        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {roll.value}"
+        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {self._format_dice(roll)}"
         content = TextMessageEventContent(msgtype=MessageType.TEXT, format=Format.HTML, body=text,
                                           formatted_body=f"<h4>{text}</h4>", relates_to=relates_to,
                                           external_url=self._get_external_url(evt))
@@ -337,7 +414,7 @@ class PortalTelegram(BasePortal, ABC):
 
     @staticmethod
     def _int_to_bytes(i: int) -> bytes:
-        hex_value = "{0:010x}".format(i).encode("utf-8")
+        hex_value = f"{i:010x}".encode("utf-8")
         return codecs.decode(hex_value, "hex_codec")
 
     def _encode_msgid(self, source: 'AbstractUser', evt: Message) -> str:
@@ -575,6 +652,9 @@ class PortalTelegram(BasePortal, ABC):
                            "displayname, updating info...")
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
+            if not sender.displayname:
+                self.log.debug(f"Telegram user {sender.tgid} doesn't have a displayname even after"
+                               f" updating with data {entity!s}")
 
         allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
                          MessageMediaGame, MessageMediaDice, MessageMediaPoll,
@@ -694,13 +774,20 @@ class PortalTelegram(BasePortal, ABC):
             levels.users[puppet.mxid] = 50
         await self.main_intent.set_power_levels(self.mxid, levels)
 
-    async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
-        tg_space = receiver if self.peer_type != "channel" else self.tgid
-        message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
-        if message:
-            await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
-        else:
-            await self.main_intent.set_pinned_messages(self.mxid, [])
+    async def receive_telegram_pin_ids(self, msg_ids: List[TelegramID], receiver: TelegramID,
+                                       remove: bool) -> None:
+        async with self._pin_lock:
+            tg_space = receiver if self.peer_type != "channel" else self.tgid
+            previously_pinned = await self.main_intent.get_pinned_messages(self.mxid)
+            currently_pinned_dict = {event_id: True for event_id in previously_pinned}
+            for message in DBMessage.get_first_by_tgids(msg_ids, tg_space):
+                if remove:
+                    currently_pinned_dict.pop(message.mxid, None)
+                else:
+                    currently_pinned_dict[message.mxid] = True
+            currently_pinned = list(currently_pinned_dict.keys())
+            if currently_pinned != previously_pinned:
+                await self.main_intent.set_pinned_messages(self.mxid, currently_pinned)
 
     async def set_telegram_admins_enabled(self, enabled: bool) -> None:
         level = 50 if enabled else 10

mautrix_telegram/puppet.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Any, Dict, Iterable, Optional, Union, TYPE_CHECKING
+from typing import Awaitable, Any, Dict, Iterable, Optional, Union, Tuple, TYPE_CHECKING
 from difflib import SequenceMatcher
 import unicodedata
 import asyncio
@@ -24,10 +24,11 @@ from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser,
 from yarl import URL
 
 from mautrix.appservice import AppService, IntentAPI
-from mautrix.errors import MatrixRequestError
+from mautrix.errors import MatrixRequestError, MatrixError
 from mautrix.bridge import BasePuppet
-from mautrix.types import UserID, SyncToken, RoomID
+from mautrix.types import UserID, SyncToken, RoomID, ContentURI
 from mautrix.util.simple_template import SimpleTemplate
+from mautrix.util.logging import TraceLogger
 
 from .types import TelegramID
 from .db import Puppet as DBPuppet
@@ -43,7 +44,7 @@ config: Optional['Config'] = None
 
 
 class Puppet(BasePuppet):
-    log: logging.Logger = logging.getLogger("mau.puppet")
+    log: TraceLogger = logging.getLogger("mau.puppet")
     az: AppService
     mx: 'MatrixHandler'
     loop: asyncio.AbstractEventLoop
@@ -64,6 +65,8 @@ class Puppet(BasePuppet):
     username: Optional[str]
     displayname: Optional[str]
     displayname_source: Optional[TelegramID]
+    displayname_contact: bool
+    displayname_quality: int
     photo_id: Optional[str]
     is_bot: bool
     is_registered: bool
@@ -85,6 +88,8 @@ class Puppet(BasePuppet):
                  username: Optional[str] = None,
                  displayname: Optional[str] = None,
                  displayname_source: Optional[TelegramID] = None,
+                 displayname_contact: bool = True,
+                 displayname_quality: int = 0,
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
@@ -100,6 +105,8 @@ class Puppet(BasePuppet):
         self.username = username
         self.displayname = displayname
         self.displayname_source = displayname_source
+        self.displayname_contact = displayname_contact
+        self.displayname_quality = displayname_quality
         self.photo_id = photo_id
         self.is_bot = is_bot
         self.is_registered = is_registered
@@ -164,8 +171,10 @@ class Puppet(BasePuppet):
         return dict(access_token=self.access_token, next_batch=self._next_batch,
                     custom_mxid=self.custom_mxid, username=self.username, is_bot=self.is_bot,
                     displayname=self.displayname, displayname_source=self.displayname_source,
-                    photo_id=self.photo_id, matrix_registered=self.is_registered,
-                    disable_updates=self.disable_updates, base_url=self.base_url)
+                    displayname_contact=self.displayname_contact,
+                    displayname_quality=self.displayname_quality, photo_id=self.photo_id,
+                    matrix_registered=self.is_registered, disable_updates=self.disable_updates,
+                    base_url=str(self.base_url) if self.base_url else None)
 
     def new_db_instance(self) -> DBPuppet:
         return DBPuppet(id=self.id, **self._fields)
@@ -177,9 +186,10 @@ class Puppet(BasePuppet):
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
                       db_puppet.next_batch, db_puppet.base_url, db_puppet.username,
-                      db_puppet.displayname, db_puppet.displayname_source, db_puppet.photo_id,
-                      db_puppet.is_bot, db_puppet.matrix_registered, db_puppet.disable_updates,
-                      db_instance=db_puppet)
+                      db_puppet.displayname, db_puppet.displayname_source,
+                      db_puppet.displayname_contact, db_puppet.displayname_quality,
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
     # endregion
     # region Info updating
@@ -199,11 +209,13 @@ class Puppet(BasePuppet):
         whitespace = ("\t\n\r\v\f \u00a0\u034f\u180e\u2063\u202f\u205f\u2800\u3000\u3164\ufeff"
                       "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b"
                       "\u200c\u200d\u200e\u200f\ufe0f")
-        name = "".join(c for c in name.strip(whitespace) if unicodedata.category(c) != 'Cf')
+        allowed_other_format = ("\u200d", "\u200c")
+        name = "".join(c for c in name.strip(whitespace) if unicodedata.category(c) != 'Cf'
+                                                         or c in allowed_other_format)
         return name
 
     @classmethod
-    def get_displayname(cls, info: User, enable_format: bool = True) -> str:
+    def get_displayname(cls, info: User, enable_format: bool = True) -> Tuple[str, int]:
         fn = cls._filter_name(info.first_name)
         ln = cls._filter_name(info.last_name)
         data = {
@@ -216,19 +228,21 @@ class Puppet(BasePuppet):
         }
         preferences = config["bridge.displayname_preference"]
         name = None
+        quality = 99
         for preference in preferences:
             name = data[preference]
             if name:
                 break
+            quality -= 1
 
         if isinstance(info, User) and info.deleted:
             name = f"Deleted account {info.id}"
+            quality = 99
         elif not name:
             name = str(info.id)
+            quality = 0
 
-        if not enable_format:
-            return name
-        return cls.displayname_template.format_full(name)
+        return (cls.displayname_template.format_full(name) if enable_format else name), quality
 
     async def try_update_info(self, source: 'AbstractUser', info: User) -> None:
         try:
@@ -264,27 +278,40 @@ class Puppet(BasePuppet):
             allow_because = "user is the primary source"
         elif not isinstance(info, UpdateUserName) and not info.contact:
             allow_because = "user is not a contact"
-        elif self.displayname_source is None:
+        elif not self.displayname_source:
             allow_because = "no primary source set"
+        elif not self.displayname:
+            allow_because = "user has no name"
         else:
             return False
 
         if isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))
+        if not info.contact:
+            self.displayname_contact = False
+        elif not self.displayname_contact:
+            if not self.displayname:
+                self.displayname_contact = True
+            else:
+                return False
 
-        displayname = self.get_displayname(info)
-        if displayname != self.displayname:
+        displayname, quality = self.get_displayname(info)
+        if displayname != self.displayname and quality >= self.displayname_quality:
+            allow_because = f"{allow_because} and quality {quality} >= {self.displayname_quality}"
             self.log.debug(f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
                            f"because {allow_because}) from {self.displayname} to {displayname}")
+            self.log.trace("Displayname source data: %s", info)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            self.displayname_quality = quality
             try:
                 await self.default_mxid_intent.set_displayname(
                     displayname[:config["bridge.displayname_max_length"]])
-            except MatrixRequestError:
+            except MatrixError:
                 self.log.exception("Failed to set displayname")
                 self.displayname = ""
                 self.displayname_source = None
+                self.displayname_quality = 0
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
@@ -309,16 +336,15 @@ class Puppet(BasePuppet):
             if not photo_id:
                 self.photo_id = ""
                 try:
-                    await self.default_mxid_intent.set_avatar_url("")
-                except MatrixRequestError:
+                    await self.default_mxid_intent.set_avatar_url(ContentURI(""))
+                except MatrixError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
                 return True
 
             loc = InputPeerPhotoFileLocation(
                 peer=await self.get_input_entity(source),
-                local_id=photo.photo_big.local_id,
-                volume_id=photo.photo_big.volume_id,
+                photo_id=photo.photo_id,
                 big=True
             )
             file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
@@ -326,13 +352,13 @@ class Puppet(BasePuppet):
                 self.photo_id = photo_id
                 try:
                     await self.default_mxid_intent.set_avatar_url(file.mxc)
-                except MatrixRequestError:
+                except MatrixError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
                 return True
         return False
 
-    def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
+    async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
         portal: p.Portal = p.Portal.get_by_mxid(room_id)
         return portal and not portal.backfill_lock.locked and portal.peer_type != "user"
 

mautrix_telegram/user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,27 +15,30 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import (Awaitable, Dict, List, Iterable, NamedTuple, Optional, Tuple, Any, cast,
                     TYPE_CHECKING)
-from collections import defaultdict
+from datetime import datetime, timezone
 import logging
 import asyncio
 
-from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
+from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage,
                                UpdateShortChatMessage, UpdateShortMessage, User as TLUser, Chat,
-                               ChatForbidden)
+                               ChatForbidden, UpdateFolderPeers, UpdatePinnedDialogs,
+                               UpdateNotifySettings, NotifyPeer)
 from telethon.tl.custom import Dialog
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.account import UpdateStatusRequest
+from telethon.errors import AuthKeyDuplicatedError
 
 from mautrix.client import Client
-from mautrix.errors import MatrixRequestError
-from mautrix.types import UserID, RoomID
-from mautrix.bridge import BaseUser
+from mautrix.errors import MatrixRequestError, MNotFound
+from mautrix.types import UserID, RoomID, PushRuleScope, PushRuleKind, PushActionType, RoomTagInfo
+from mautrix.bridge import BaseUser, BridgeState
+from mautrix.util.bridge_state import BridgeStateEvent
 from mautrix.util.logging import TraceLogger
 from mautrix.util.opt_prometheus import Gauge
 
 from .types import TelegramID
-from .db import User as DBUser, Portal as DBPortal
+from .db import User as DBUser, Portal as DBPortal, Message as DBMessage
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
@@ -50,6 +53,11 @@ SearchResult = NamedTuple('SearchResult', puppet='pu.Puppet', similarity=int)
 METRIC_LOGGED_IN = Gauge('bridge_logged_in', 'Users logged into bridge')
 METRIC_CONNECTED = Gauge('bridge_connected', 'Users connected to Telegram')
 
+BridgeState.human_readable_errors.update({
+    "tg-not-connected": "Your Telegram connection failed",
+    "tg-auth-key-duplicated": "The bridge accidentally logged you out",
+})
+
 
 class User(AbstractUser, BaseUser):
     log: TraceLogger = logging.getLogger("mau.user")
@@ -72,8 +80,9 @@ class User(AbstractUser, BaseUser):
                  saved_contacts: int = 0, is_bot: bool = False,
                  db_portals: Optional[Iterable[Tuple[TelegramID, TelegramID]]] = None,
                  db_instance: Optional[DBUser] = None) -> None:
-        super().__init__()
+        AbstractUser.__init__(self)
         self.mxid = mxid
+        BaseUser.__init__(self)
         self.tgid = tgid
         self.is_bot = is_bot
         self.username = username
@@ -85,11 +94,8 @@ class User(AbstractUser, BaseUser):
         self.db_portals = db_portals or []
         self._db_instance = db_instance
         self._ensure_started_lock = asyncio.Lock()
-        self.dm_update_lock = asyncio.Lock()
-        self._metric_value = defaultdict(lambda: False)
         self._track_connection_task = None
-
-        self.command_status = None
+        self._is_backfilling = False
 
         (self.relaybot_whitelisted,
          self.whitelisted,
@@ -102,8 +108,6 @@ class User(AbstractUser, BaseUser):
         if tgid:
             self.by_tgid[tgid] = self
 
-        self.log = self.log.getChild(self.mxid)
-
     @property
     def name(self) -> str:
         return self.mxid
@@ -200,7 +204,22 @@ class User(AbstractUser, BaseUser):
             return cast(User, await super().ensure_started(even_if_no_session))
 
     async def start(self, delete_unless_authenticated: bool = False) -> 'User':
-        await super().start()
+        try:
+            await super().start()
+        except AuthKeyDuplicatedError:
+            self.log.warning("Got AuthKeyDuplicatedError in start()")
+            await self.push_bridge_state(BridgeStateEvent.BAD_CREDENTIALS,
+                                         error="tg-auth-key-duplicated")
+            await self.client.disconnect()
+            self.client.session.delete()
+            self.client = None
+            if not delete_unless_authenticated:
+                # The caller wants the client to be connected, so restart the connection.
+                await super().start()
+            return self
+        except Exception:
+            await self.push_bridge_state(BridgeStateEvent.UNKNOWN_ERROR)
+            raise
         if await self.is_logged_in():
             self.log.debug(f"Ensuring post_login() for {self.name}")
             self.loop.create_task(self.post_login())
@@ -210,19 +229,39 @@ class User(AbstractUser, BaseUser):
             self.client.session.delete()
         return self
 
+    @property
+    def _is_connected(self) -> bool:
+        return bool(self.client and self.client._sender
+                    and self.client._sender._transport_connected)
+
     async def _track_connection(self) -> None:
         self.log.debug("Starting loop to track connection state")
         while True:
             await asyncio.sleep(3)
-            connected = bool(self.client._sender._transport_connected
-                             if self.client and self.client._sender else False)
+            connected = self._is_connected
             self._track_metric(METRIC_CONNECTED, connected)
+            if connected:
+                await self.push_bridge_state(BridgeStateEvent.BACKFILLING if self._is_backfilling
+                                             else BridgeStateEvent.CONNECTED, ttl=3600)
+            else:
+                await self.push_bridge_state(BridgeStateEvent.UNKNOWN_ERROR, ttl=240,
+                                             error="tg-not-connected")
+
+    async def fill_bridge_state(self, state: BridgeState) -> None:
+        await super().fill_bridge_state(state)
+        state.remote_id = str(self.tgid)
+        state.remote_name = self.human_tg_id
+
+    async def get_puppet(self) -> Optional['pu.Puppet']:
+        if not self.tgid:
+            return None
+        return pu.Puppet.get(self.tgid)
 
     async def stop(self) -> None:
-        await super().stop()
         if self._track_connection_task:
             self._track_connection_task.cancel()
             self._track_connection_task = None
+        await super().stop()
         self._track_metric(METRIC_CONNECTED, False)
 
     async def post_login(self, info: TLUser = None, first_login: bool = False) -> None:
@@ -247,10 +286,13 @@ class User(AbstractUser, BaseUser):
 
         if not self.is_bot and config["bridge.startup_sync"]:
             try:
+                self._is_backfilling = True
                 await self.sync_dialogs()
                 await self.sync_contacts()
             except Exception:
                 self.log.exception("Failed to run post-login sync")
+            finally:
+                self._is_backfilling = False
 
     async def update(self, update: TypeUpdate) -> bool:
         if not self.is_bot:
@@ -315,6 +357,7 @@ class User(AbstractUser, BaseUser):
         self.portals = {}
         self.contacts = []
         await self.save(portals=True, contacts=True)
+        await self.push_bridge_state(BridgeStateEvent.LOGGED_OUT)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
@@ -363,12 +406,6 @@ class User(AbstractUser, BaseUser):
 
         return await self._search_remote(query), True
 
-    async def _catch(self, action: str, task: asyncio.Task) -> None:
-        try:
-            await task
-        except Exception:
-            self.log.exception(f"Error while {action}")
-
     async def get_direct_chats(self) -> Dict[UserID, List[RoomID]]:
         return {
             pu.Puppet.get_mxid_from_id(portal.tgid): [portal.mxid]
@@ -376,6 +413,101 @@ class User(AbstractUser, BaseUser):
             if portal.mxid
         }
 
+    async def _tag_room(self, puppet: pu.Puppet, portal: po.Portal, tag: str, active: bool
+                        ) -> None:
+        if not tag or not portal or not portal.mxid:
+            return
+        tag_info = await puppet.intent.get_room_tag(portal.mxid, tag)
+        if active and tag_info is None:
+            tag_info = RoomTagInfo(order=0.5)
+            tag_info[self.bridge.real_user_content_key] = True
+            await puppet.intent.set_room_tag(portal.mxid, tag, tag_info)
+        elif not active and tag_info and tag_info.get(self.bridge.real_user_content_key, False):
+            await puppet.intent.remove_room_tag(portal.mxid, tag)
+
+    @staticmethod
+    async def _mute_room(puppet: pu.Puppet, portal: po.Portal, mute_until: datetime) -> None:
+        if not config["bridge.mute_bridging"] or not portal or not portal.mxid:
+            return
+        now = datetime.utcnow().replace(tzinfo=timezone.utc)
+        if mute_until is not None and mute_until > now:
+            await puppet.intent.set_push_rule(PushRuleScope.GLOBAL, PushRuleKind.ROOM, portal.mxid,
+                                              actions=[PushActionType.DONT_NOTIFY])
+        else:
+            try:
+                await puppet.intent.remove_push_rule(PushRuleScope.GLOBAL, PushRuleKind.ROOM,
+                                                     portal.mxid)
+            except MNotFound:
+                pass
+
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        for peer in update.folder_peers:
+            portal = po.Portal.get_by_entity(peer.peer, receiver_id=self.tgid, create=False)
+            await self._tag_room(puppet, portal, config["bridge.archive_tag"],
+                                 peer.folder_id == 1)
+
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        # TODO bridge unpinning properly
+        for pinned in update.order:
+            portal = po.Portal.get_by_entity(pinned.peer, receiver_id=self.tgid, create=False)
+            await self._tag_room(puppet, portal, config["bridge.pinned_tag"], True)
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        elif not isinstance(update.peer, NotifyPeer):
+            # TODO handle global notification setting changes?
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        portal = po.Portal.get_by_entity(update.peer.peer, receiver_id=self.tgid, create=False)
+        await self._mute_room(puppet, portal, update.notify_settings.mute_until)
+
+    async def _sync_dialog(self, portal: po.Portal, dialog: Dialog, should_create: bool,
+                           puppet: Optional[pu.Puppet]) -> None:
+        was_created = False
+        if portal.mxid:
+            try:
+                await portal.backfill(self, last_id=dialog.message.id)
+            except Exception:
+                self.log.exception(f"Error while backfilling {portal.tgid_log}")
+            try:
+                await portal.update_matrix_room(self, dialog.entity)
+            except Exception:
+                self.log.exception(f"Error while updating {portal.tgid_log}")
+        elif should_create:
+            try:
+                await portal.create_matrix_room(self, dialog.entity, invites=[self.mxid])
+                was_created = True
+            except Exception:
+                self.log.exception(f"Error while creating {portal.tgid_log}")
+        if portal.mxid and puppet and puppet.is_real_user:
+            tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+            if dialog.unread_count == 0:
+                # This is usually more reliable than finding a specific message
+                # e.g. if the last read message is a service message that isn't in the message db
+                last_read = DBMessage.find_last(portal.mxid, tg_space)
+            else:
+                last_read = DBMessage.get_one_by_tgid(portal.tgid, tg_space,
+                                                      dialog.dialog.read_inbox_max_id)
+            if last_read:
+                await puppet.intent.mark_read(last_read.mx_room, last_read.mxid)
+            if was_created or not config["bridge.tag_only_on_create"]:
+                await self._mute_room(puppet, portal, dialog.dialog.notify_settings.mute_until)
+                await self._tag_room(puppet, portal, config["bridge.pinned_tag"], dialog.pinned)
+                await self._tag_room(puppet, portal, config["bridge.archive_tag"], dialog.archived)
+
     async def sync_dialogs(self) -> None:
         if self.is_bot:
             return
@@ -385,6 +517,8 @@ class User(AbstractUser, BaseUser):
         index = 0
         self.log.debug(f"Syncing dialogs (update_limit={update_limit}, "
                        f"create_limit={create_limit})")
+        await self.push_bridge_state(BridgeStateEvent.BACKFILLING)
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
         dialog: Dialog
         async for dialog in self.client.iter_dialogs(limit=update_limit, ignore_migrated=True,
                                                      archived=False):
@@ -400,17 +534,9 @@ class User(AbstractUser, BaseUser):
                 continue
             portal = po.Portal.get_by_entity(entity, receiver_id=self.tgid)
             self.portals[portal.tgid_full] = portal
-            if portal.mxid:
-                update_task = portal.update_matrix_room(self, entity)
-                backfill_task = portal.backfill(self, last_id=dialog.message.id)
-                creators.append(self._catch(f"updating {portal.tgid_log}",
-                                            self.loop.create_task(update_task)))
-                creators.append(self._catch(f"backfilling {portal.tgid_log}",
-                                            self.loop.create_task(backfill_task)))
-            elif not create_limit or index < create_limit:
-                create_task = portal.create_matrix_room(self, entity, invites=[self.mxid])
-                creators.append(self._catch(f"creating {portal.tgid_log}",
-                                            self.loop.create_task(create_task)))
+            coro = self._sync_dialog(portal=portal, dialog=dialog, puppet=puppet,
+                                     should_create=not create_limit or index < create_limit)
+            creators.append(self.loop.create_task(coro))
             index += 1
         await self.save(portals=True)
         await asyncio.gather(*creators)

mautrix_telegram/util/file_transfer.py

@@ -30,7 +30,6 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio
                              SecurityError, FileIdInvalidError)
 
 from mautrix.appservice import IntentAPI
-from mautrix.util.network_retry import call_with_net_retry
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
@@ -103,8 +102,10 @@ def _location_to_id(location: TypeLocation) -> str:
         return f"{location.id}-{location.access_hash}"
     elif isinstance(location, (InputDocumentFileLocation, InputPhotoFileLocation)):
         return f"{location.id}-{location.access_hash}-{location.thumb_size}"
-    elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
+    elif isinstance(location, InputFileLocation):
         return f"{location.volume_id}-{location.local_id}"
+    elif isinstance(location, InputPeerPhotoFileLocation):
+        return str(location.photo_id)
 
 
 async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
@@ -145,8 +146,7 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if encrypt:
         file, decryption_info = encrypt_attachment(file)
         upload_mime_type = "application/octet-stream"
-    content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
-                                            _action="upload media")
+    content_uri = await intent.upload_media(file, upload_mime_type)
     if decryption_info:
         decryption_info.url = content_uri
 
@@ -222,9 +222,9 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
 
         image_converted = False
         # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
-        is_tgs = (mime_type == "application/gzip" or (mime_type == "application/octet-stream"
-                                                      and magic.from_buffer(file).startswith(
-                "gzip")))
+        is_tgs = (mime_type == "application/gzip"
+                  or (mime_type == "application/octet-stream"
+                      and magic.from_buffer(file).startswith("gzip")))
         if is_sticker and tgs_convert and is_tgs:
             converted_anim = await convert_tgs_to(file, tgs_convert["target"],
                                                   **tgs_convert["args"])
@@ -234,21 +234,12 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
             image_converted = mime_type != "application/gzip"
             thumbnail = None
 
-        if mime_type == "image/webp":
-            new_mime_type, file, width, height = convert_image(
-                file, source_mime="image/webp", target_type="png",
-                thumbnail_to=(256, 256) if is_sticker else None)
-            image_converted = new_mime_type != mime_type
-            mime_type = new_mime_type
-            thumbnail = None
-
         decryption_info = None
         upload_mime_type = mime_type
         if encrypt and encrypt_attachment:
             file, decryption_info = encrypt_attachment(file)
             upload_mime_type = "application/octet-stream"
-        content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
-                                                _action="upload media")
+        content_uri = await intent.upload_media(file, upload_mime_type)
         if decryption_info:
             decryption_info.url = content_uri
 

mautrix_telegram/util/parallel_file_transfer.py

@@ -27,8 +27,10 @@ from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLoc
                                InputPhotoFileLocation, InputPeerPhotoFileLocation, TypeInputFile,
                                InputFileBig, InputFile)
 from telethon.tl.functions.auth import ExportAuthorizationRequest, ImportAuthorizationRequest
+from telethon.tl.functions import InvokeWithLayerRequest
 from telethon.tl.functions.upload import (GetFileRequest, SaveFilePartRequest,
                                           SaveBigFilePartRequest)
+from telethon.tl.alltlobjects import LAYER
 from telethon.network import MTProtoSender
 from telethon.crypto import AuthKey
 from telethon import utils, helpers
@@ -132,7 +134,7 @@ class ParallelTransferrer:
         self.upload_ticker = 0
 
     async def _cleanup(self) -> None:
-        await asyncio.gather(*[sender.disconnect() for sender in self.senders])
+        await asyncio.gather(*(sender.disconnect() for sender in self.senders))
         self.senders = None
 
     @staticmethod
@@ -159,9 +161,9 @@ class ParallelTransferrer:
             await self._create_download_sender(file, 0, part_size, connections * part_size,
                                                get_part_count()),
             *await asyncio.gather(
-                *[self._create_download_sender(file, i, part_size, connections * part_size,
+                *(self._create_download_sender(file, i, part_size, connections * part_size,
                                                get_part_count())
-                  for i in range(1, connections)])
+                  for i in range(1, connections)))
         ]
 
     async def _create_download_sender(self, file: TypeLocation, index: int, part_size: int,
@@ -175,8 +177,8 @@ class ParallelTransferrer:
         self.senders = [
             await self._create_upload_sender(file_id, part_count, big, 0, connections),
             *await asyncio.gather(
-                *[self._create_upload_sender(file_id, part_count, big, i, connections)
-                  for i in range(1, connections)])
+                *(self._create_upload_sender(file_id, part_count, big, i, connections)
+                  for i in range(1, connections)))
         ]
 
     async def _create_upload_sender(self, file_id: int, part_count: int, big: bool, index: int,
@@ -193,9 +195,9 @@ class ParallelTransferrer:
         if not self.auth_key:
             log.debug(f"Exporting auth to DC {self.dc_id}")
             auth = await self.client(ExportAuthorizationRequest(self.dc_id))
-            req = self.client._init_with(ImportAuthorizationRequest(
-                id=auth.id, bytes=auth.bytes
-            ))
+            self.client._init_request.query = ImportAuthorizationRequest(id=auth.id,
+                                                                         bytes=auth.bytes)
+            req = InvokeWithLayerRequest(LAYER, self.client._init_request)
             await sender.send(req)
             self.auth_key = sender.auth_key
         return sender

mautrix_telegram/web/provisioning/spec.yaml

@@ -6,11 +6,11 @@ info:
   description: The provisioning API for Mautrix-Telegram, the Matrix-Telegram puppeting/relaybot bridge.
   license:
     name: AGPLv3
-    url: https://github.com/tulir/mautrix-telegram/blob/master/LICENSE
+    url: https://github.com/mautrix/telegram/blob/master/LICENSE
 
 externalDocs:
-  description: Provisioning API wiki page on GitHub
-  url: https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API
+  description: Provisioning API docs on docs.mau.fi
+  url: https://docs.mau.fi/bridges/python/telegram/provisioning-api.html
 
 basePath: /_matrix/provision/v1
 

optional-requirements.txt

@@ -7,24 +7,21 @@ cchardet
 aiodns
 brotli
 
-#/webp_convert
-pillow>=4,<8
-
 #/qr_login
-pillow>=4,<8
+pillow>=4,<9
 qrcode>=6,<7
 
 #/hq_thumbnails
 moviepy>=1,<2
 
 #/metrics
-prometheus_client>=0.6,<0.9
+prometheus_client>=0.6,<0.12
 
 #/postgres
 psycopg2-binary>=2,<3
 
 #/e2be
-asyncpg>=0.20,<0.22
+asyncpg>=0.20,<0.25
 python-olm>=3,<4
 pycryptodome>=3,<4
 unpaddedbase64>=1,<2

requirements.txt

@@ -1,10 +1,10 @@
-SQLAlchemy>=1.2,<2
+SQLAlchemy>=1.2,<1.4
 alembic>=1,<2
-ruamel.yaml>=0.15.35,<0.17
+ruamel.yaml>=0.15.35,<0.18
 python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.8.3,<0.9
-telethon>=1.17,<1.18
+mautrix>=0.10.4,<0.11
+telethon>=1.22,<1.23
 telethon-session-sqlalchemy>=0.2.14,<0.3

setup.py

@@ -36,7 +36,7 @@ linkified_version = {linkified_version!r}
 setuptools.setup(
     name="mautrix-telegram",
     version=version,
-    url="https://github.com/tulir/mautrix-telegram",
+    url="https://github.com/mautrix/telegram",
 
     author="Tulir Asokan",
     author_email="tulir@maunium.net",
@@ -49,7 +49,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.6",
+    python_requires="~=3.7",
 
     setup_requires=["pytest-runner"],
     tests_require=["pytest", "pytest-asyncio", "pytest-mock"],