Bump version to 0.10.0

Fixes #375

Dockerfile

@@ -1,12 +1,7 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.12
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.13
 
 ARG TARGETARCH=amd64
 
-RUN echo $'\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing\n\
-@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
-
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-virtualenv \
@@ -14,11 +9,11 @@ RUN apk add --no-cache \
       py3-aiohttp \
       py3-magic \
       py3-sqlalchemy \
-      py3-telethon-session-sqlalchemy@edge \
+      py3-telethon-session-sqlalchemy \
-      py3-alembic@edge \
+      py3-alembic \
       py3-psycopg2 \
       py3-ruamel.yaml \
-      py3-commonmark@edge \
+      py3-commonmark \
       # Indirect dependencies
       py3-idna \
       #moviepy
@@ -27,12 +22,12 @@ RUN apk add --no-cache \
         py3-requests \
         #imageio
           py3-numpy \
-      #py3-telethon@edge \ (outdated)
+      #py3-telethon \ (outdated)
         # Optional for socks proxies
         py3-pysocks \
         # cryptg
           py3-cffi \
-	  py3-qrcode@edge \
+	  py3-qrcode \
       py3-brotli \
       # Other dependencies
       ffmpeg \
@@ -46,9 +41,8 @@ RUN apk add --no-cache \
       py3-future \
       bash \
       curl \
-      jq && \
+      jq \
-  curl -sLo yq https://github.com/mikefarah/yq/releases/download/3.3.2/yq_linux_${TARGETARCH} && \
+      yq
-  chmod +x yq && mv yq /usr/bin/yq
 
 COPY requirements.txt /opt/mautrix-telegram/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt

README.md

@@ -10,15 +10,16 @@ A Matrix-Telegram hybrid puppeting/relaybot bridge.
 ## Sponsors
 * [Joel Lehtonen / Zouppen](https://github.com/zouppen)
 
-### Wiki
+### Documentation
-All setup and usage instructions are located in the GitHub
+All setup and usage instructions are located on
-[wiki](https://github.com/tulir/mautrix-telegram/wiki). Some quick links:
+[docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
+Some quick links:
 
-* [Bridge setup](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup)
+* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
-  (or [with Docker](https://github.com/tulir/mautrix-telegram/wiki/Bridge-setup-with-Docker))
+  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
-* Basic usage: [Authentication](https://github.com/tulir/mautrix-telegram/wiki/Authentication),
+* Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
-  [Creating chats](https://github.com/tulir/mautrix-telegram/wiki/Creating-and-managing-chats),
+  [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
-  [Relaybot setup](https://github.com/tulir/mautrix-telegram/wiki/Relay-bot)
+  [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)
 
 ### Features & Roadmap
 [ROADMAP.md](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)

alembic/versions/7de69cf5809e_add_matrix_redaction_state_to_message_.py

@@ -0,0 +1,25 @@
+"""Add Matrix redaction state to message table
+
+Revision ID: 7de69cf5809e
+Revises: 888275d58e57
+Create Date: 2020-12-19 12:39:57.368568
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '7de69cf5809e'
+down_revision = '888275d58e57'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('message', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('redacted', sa.Boolean(), server_default=sa.false(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table('message', schema=None) as batch_op:
+        batch_op.drop_column('redacted')

alembic/versions/990f4395afc6_store_displayname_contact_status_in_.py

@@ -0,0 +1,32 @@
+"""Store displayname contact status in puppet table
+
+Revision ID: 990f4395afc6
+Revises: 7de69cf5809e
+Create Date: 2021-01-01 11:56:54.610681
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '990f4395afc6'
+down_revision = '7de69cf5809e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('displayname_contact', sa.Boolean(), server_default=sa.true(), nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('displayname_contact')
+
+    # ### end Alembic commands ###

alembic/versions/bfc0a39bfe02_store_displayname_quality_in_puppet_.py

@@ -0,0 +1,32 @@
+"""Store displayname quality in puppet table
+
+Revision ID: bfc0a39bfe02
+Revises: ec1d3dcc77e9
+Create Date: 2021-03-23 20:03:08.825333
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'bfc0a39bfe02'
+down_revision = 'ec1d3dcc77e9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('displayname_quality', sa.Integer(), server_default='0', nullable=False))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('puppet', schema=None) as batch_op:
+        batch_op.drop_column('displayname_quality')
+
+    # ### end Alembic commands ###

alembic/versions/ec1d3dcc77e9_switch_telegram_ids_to_bigints.py

@@ -0,0 +1,44 @@
+"""Switch Telegram IDs to bigints
+
+Revision ID: ec1d3dcc77e9
+Revises: 990f4395afc6
+Create Date: 2021-03-09 21:36:58.443727
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'ec1d3dcc77e9'
+down_revision = '990f4395afc6'
+branch_labels = None
+depends_on = None
+
+
+columns_to_upgrade = (
+    ("bot_chat", "id"),
+    ("message", "tgid"),
+    ("message", "tg_space"),
+    ("portal", "tgid"),
+    ("portal", "tg_receiver"),
+    ("puppet", "id"),
+    ("puppet", "displayname_source"),
+    ("user", "tgid"),
+    ("user_portal", "user"),
+    ("user_portal", "portal"),
+    ("user_portal", "portal_receiver"),
+    ("contact", "user"),
+    ("contact", "contact"),
+)
+
+
+def upgrade():
+    if op.get_context().dialect.name == "postgresql":
+        for table, column in columns_to_upgrade:
+            op.alter_column(table, column, existing_type=sa.Integer, type_=sa.BigInteger)
+
+
+def downgrade():
+    if op.get_context().dialect.name == "postgresql":
+        for table, column in columns_to_upgrade:
+            op.alter_column(table, column, existing_type=sa.BigInteger, type_=sa.Integer)

docker-run.sh

@@ -26,9 +26,6 @@ fi
 
 if [ ! -f /data/registration.yaml ]; then
 	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
-	echo "Didn't find a registration file."
-	echo "Generated one for you."
-	echo "Copy that over to synapses app service directory."
 	fixperms
 	exit
 fi

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.9.0"
+__version__ = "0.10.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/abstract_user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,9 +15,9 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Tuple, Optional, Union, Dict, Type, Any, TYPE_CHECKING
 from abc import ABC, abstractmethod
+import platform
 import asyncio
 import logging
-import platform
 import time
 
 from telethon.sessions import Session
@@ -25,13 +25,14 @@ from telethon.network import (ConnectionTcpMTProxyRandomizedIntermediate, Connec
                               Connection)
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
-    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdateChatPinnedMessage,
+    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdatePinnedMessages,
-    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
+    UpdatePinnedChannelMessages, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
     UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
     UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
     UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
     UpdateUserTyping, User, UserStatusOffline, UserStatusOnline, UpdateReadHistoryInbox,
-    UpdateReadChannelInbox, MessageEmpty)
+    UpdateReadChannelInbox, MessageEmpty, UpdateFolderPeers, UpdatePinnedDialogs,
+    UpdateNotifySettings, UpdateChannelUserTyping)
 
 from mautrix.types import UserID, PresenceState
 from mautrix.errors import MatrixError
@@ -57,6 +58,7 @@ MAX_DELETIONS: int = 10
 UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
+UpdateTyping = Union[UpdateUserTyping, UpdateChatUserTyping, UpdateChannelUserTyping]
 
 UPDATE_TIME = Histogram("bridge_telegram_update", "Time spent processing Telegram updates",
                         ("update_type",))
@@ -235,8 +237,7 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
-        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+        asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
-                              loop=self.loop)
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -244,7 +245,7 @@ class AbstractUser(ABC):
             await self.delete_message(update)
         elif isinstance(update, UpdateDeleteChannelMessages):
             await self.delete_channel_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
+        elif isinstance(update, (UpdateChatUserTyping, UpdateChannelUserTyping, UpdateUserTyping)):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
@@ -252,7 +253,7 @@ class AbstractUser(ABC):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
+        elif isinstance(update, (UpdatePinnedMessages, UpdatePinnedChannelMessages)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
@@ -260,17 +261,33 @@ class AbstractUser(ABC):
             await self.update_read_receipt(update)
         elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
             await self.update_own_read_receipt(update)
+        elif isinstance(update, UpdateFolderPeers):
+            await self.update_folder_peers(update)
+        elif isinstance(update, UpdatePinnedDialogs):
+            await self.update_pinned_dialogs(update)
+        elif isinstance(update, UpdateNotifySettings):
+            await self.update_notify_settings(update)
         else:
             self.log.trace("Unhandled update: %s", update)
 
-    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
-                                                         UpdateChatPinnedMessage]) -> None:
+        pass
-        if isinstance(update, UpdateChatPinnedMessage):
+
-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        pass
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        pass
+
+    async def update_pinned_messages(self, update: Union[UpdatePinnedMessages,
+                                                         UpdatePinnedChannelMessages]) -> None:
+        if isinstance(update, UpdatePinnedMessages):
+            portal = po.Portal.get_by_entity(update.peer, receiver_id=self.tgid)
         else:
             portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id, self.tgid)
+            await portal.receive_telegram_pin_ids(update.messages, self.tgid,
+                                                  remove=not update.pinned)
 
     @staticmethod
     async def update_participants(update: UpdateChatParticipants) -> None:
@@ -329,16 +346,27 @@ class AbstractUser(ABC):
 
         await portal.set_telegram_admin(TelegramID(update.user_id))
 
-    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+    async def update_typing(self, update: UpdateTyping) -> None:
+        sender = None
         if isinstance(update, UpdateUserTyping):
             portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
-        else:
+            sender = pu.Puppet.get(TelegramID(update.user_id))
+        elif isinstance(update, UpdateChannelUserTyping):
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update, UpdateChatUserTyping):
             portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
-
+        else:
-        if not portal or not portal.mxid:
+            return
+
+        if isinstance(update, (UpdateChannelUserTyping, UpdateChatUserTyping)):
+            # Can typing notifications come from non-user peers?
+            if not update.from_id.user_id:
+                return
+            sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
+
+        if not sender or not portal or not portal.mxid:
             return
 
-        sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
     async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
@@ -419,6 +447,8 @@ class AbstractUser(ABC):
 
         for message_id in update.messages:
             for message in DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
+                if message.redacted:
+                    continue
                 message.delete()
                 number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
                 if number_left == 0:
@@ -432,6 +462,8 @@ class AbstractUser(ABC):
 
         for message_id in update.messages:
             for message in DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
+                if message.redacted:
+                    continue
                 message.delete()
                 await self._try_redact(message)
 
@@ -468,7 +500,7 @@ class AbstractUser(ABC):
                 await self.register_portal(portal)
                 return
             self.log.trace("Handling action %s to %s by %d", update.action, portal.tgid_log,
-                           sender.id)
+                           (sender.id if sender else 0))
             return await portal.handle_telegram_action(self, sender, update)
 
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):

mautrix_telegram/commands/portal/bridge.py

@@ -186,6 +186,7 @@ async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id
      portal.encrypted) = await get_initial_state(evt.az.intent, evt.room_id)
     portal.photo_id = ""
     await portal.save()
+    await portal.update_bridge_info()
 
     asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
                           loop=evt.loop)

mautrix_telegram/commands/portal/create_chat.py

@@ -52,8 +52,14 @@ async def create(evt: CommandEvent) -> EventID:
 
     portal = po.Portal(tgid=TelegramID(0), peer_type=type, mxid=evt.room_id,
                        title=title, about=about, encrypted=encrypted)
+    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender)
+    if len(errors) > 0:
+        error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
+        await evt.reply(f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
+                        "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
+                        "those users.")
     try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+        await portal.create_telegram_chat(evt.sender, invites=invites, supergroup=supergroup)
     except ValueError as e:
         await portal.delete()
         return await evt.reply(e.args[0])

mautrix_telegram/commands/portal/misc.py

@@ -13,6 +13,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple
+from datetime import timedelta, datetime
+import re
+
 from telethon.tl.functions.channels import GetFullChannelRequest
 from telethon.tl.functions.messages import GetFullChatRequest
 from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
@@ -80,9 +84,81 @@ async def get_id(evt: CommandEvent) -> EventID:
     await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
 
 
+invite_link_usage = ("**Usage:** `$cmdprefix+sp invite-link [--uses=<amount>] [--expire=<delta>]`"
+                     "\n\n"
+                     "* `--uses`: the number of times the invite link can be used."
+                     "            Defaults to unlimited.\n"
+                     "* `--expire`: the duration after which the link will expire."
+                     "              A number suffixed with d(ay), h(our), m(inute) or s(econd)")
+
+
+def _parse_flag(args: List[str]) -> Tuple[str, str]:
+    arg = args.pop(0).lower()
+    if arg.startswith("--"):
+        value_start = arg.index("=")
+        if value_start:
+            flag = arg[2:value_start]
+            value = arg[value_start+1:]
+        else:
+            flag = arg[2:]
+            value = args.pop(0).lower()
+    elif arg.startswith("-"):
+        flag = arg[1]
+        if len(arg) > 3 and arg[2] == "=":
+            value = arg[3:]
+        else:
+            value = args.pop(0).lower()
+    else:
+        raise ValueError("invalid flag")
+    return flag, value
+
+
+delta_regex = re.compile("([0-9]+)(w(?:eek)?|d(?:ay)?|h(?:our)?|m(?:in(?:ute)?)?|s(?:ec(?:ond)?)?)")
+
+
+def _parse_delta(value: str) -> Optional[timedelta]:
+    match = delta_regex.fullmatch(value)
+    if not match:
+        return None
+    number = int(match.group(1))
+    unit = match.group(2)[0]
+    if unit == "w":
+        return timedelta(weeks=number)
+    elif unit == "d":
+        return timedelta(days=number)
+    elif unit == "h":
+        return timedelta(hours=number)
+    elif unit == "m":
+        return timedelta(minutes=number)
+    elif unit == "s":
+        return timedelta(seconds=number)
+    else:
+        return None
+
+
 @command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Get a Telegram invite link to the current chat.")
+                 help_text="Get a Telegram invite link to the current chat.",
+                 help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>]")
 async def invite_link(evt: CommandEvent) -> EventID:
+    # TODO once we switch to Python 3.9 minimum, use argparse with exit_on_error=False
+    uses = None
+    expire = None
+    while evt.args:
+        try:
+            flag, value = _parse_flag(evt.args)
+        except (ValueError, IndexError):
+            return await evt.reply(invite_link_usage)
+        if flag in ("uses", "u"):
+            try:
+                uses = int(value)
+            except ValueError:
+                await evt.reply("The number of uses must be an integer")
+        elif flag in ("expire", "e"):
+            expire_delta = _parse_delta(value)
+            if not expire_delta:
+                await evt.reply("Invalid format for expiry time delta")
+            expire = datetime.now() + expire_delta
+
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
@@ -91,7 +167,7 @@ async def invite_link(evt: CommandEvent) -> EventID:
         return await evt.reply("You can't invite users to private chats.")
 
     try:
-        link = await portal.get_invite_link(evt.sender)
+        link = await portal.get_invite_link(evt.sender, uses=uses, expire=expire)
         return await evt.reply(f"Invite link to {portal.title}: {link}")
     except ValueError as e:
         return await evt.reply(e.args[0])

mautrix_telegram/commands/telegram/account.py

@@ -16,7 +16,7 @@
 from typing import Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError, AboutTooLongError)
 from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
                                            ResetAuthorizationRequest, UpdateProfileRequest)
@@ -53,6 +53,23 @@ async def username(evt: CommandEvent) -> EventID:
     else:
         await evt.reply(f"Username changed to {evt.sender.username}")
 
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new about_>",
+                 help_text="Change your Telegram about section.")
+async def about(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp about <new about>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own about section.")
+    new_about = " ".join(evt.args)
+    if new_about == "-":
+        new_about = ""
+    try:
+        await evt.sender.client(UpdateProfileRequest(about=new_about))
+    except AboutTooLongError:
+        return await evt.reply("The provided about section is too long")
+    return await evt.reply("About section updated")
 
 @command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
                  help_text="Change your Telegram displayname.")

mautrix_telegram/commands/telegram/misc.py

@@ -14,11 +14,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import List, Optional, Tuple, cast
-import logging
 import codecs
 import base64
 import re
 
+from aiohttp import ClientSession, InvalidURL
+
 from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
                              UserAlreadyParticipantError, ChatIdInvalidError,
                              TakeoutInitDelayError, EmoticonInvalidError)
@@ -115,25 +116,25 @@ async def pm(evt: CommandEvent) -> EventID:
         return await evt.reply("That doesn't seem to be a user.")
     portal = po.Portal.get_by_entity(user, evt.sender.tgid)
     await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply("Created private chat room with "
+    displayname, _ = pu.Puppet.get_displayname(user, False)
-                           f"{pu.Puppet.get_displayname(user, False)}")
+    return await evt.reply(f"Created private chat room with {displayname}")
 
 
-async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
+async def _join(evt: CommandEvent, identifier: str, link_type: str
-    if arg.startswith("joinchat/"):
+                ) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
-        invite_hash = arg[len("joinchat/"):]
+    if link_type == "joinchat":
         try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
+            await evt.sender.client(CheckChatInviteRequest(identifier))
         except InviteHashInvalidError:
             return None, await evt.reply("Invalid invite link.")
         except InviteHashExpiredError:
             return None, await evt.reply("Invite link expired.")
         try:
-            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
+            return (await evt.sender.client(ImportChatInviteRequest(identifier))), None
         except UserAlreadyParticipantError:
             return None, await evt.reply("You are already in that chat.")
     else:
-        channel = await evt.sender.client.get_entity(arg)
+        channel = await evt.sender.client.get_entity(identifier)
         if not channel:
             return None, await evt.reply("Channel/supergroup not found.")
         return await evt.sender.client(JoinChannelRequest(channel)), None
@@ -146,12 +147,26 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
 
-    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
+    url = evt.args[0]
-    arg = regex.match(evt.args[0])
+    if evt.config["bridge.invite_link_resolve"]:
+        try:
+            async with ClientSession() as sess, sess.get(url) as resp:
+                url = str(resp.url)
+        except InvalidURL:
+            return await evt.reply("That doesn't look like a Telegram invite link.")
+
+    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)"
+                       r"(?:/(?P<type>joinchat|s))?/(?P<id>[^/]+)/?", flags=re.IGNORECASE)
+    arg = regex.match(url)
     if not arg:
         return await evt.reply("That doesn't look like a Telegram invite link.")
 
-    updates, _ = await _join(evt, arg.group(1))
+    data = arg.groupdict()
+    identifier = data["id"]
+    link_type = data["type"]
+    if link_type:
+        link_type = link_type.lower()
+    updates, _ = await _join(evt, identifier, link_type)
     if not updates:
         return None
 
@@ -165,9 +180,8 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
             try:
                 await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
             except ChatIdInvalidError as e:
-                logging.getLogger("mau.commands").trace("ChatIdInvalidError while creating portal "
+                evt.log.trace("ChatIdInvalidError while creating portal from !tg join command: %s",
-                                                        "from !tg join command: %s",
+                              updates.stringify())
-                                                        updates.stringify())
                 raise e
             return await evt.reply(f"Created room for {portal.title}")
     return None

mautrix_telegram/config.py

@@ -114,6 +114,7 @@ class Config(BaseBridgeConfig):
         else:
             copy("bridge.login_shared_secret_map")
         copy("bridge.telegram_link_preview")
+        copy("bridge.invite_link_resolve")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
         copy("bridge.max_document_size")
@@ -131,6 +132,10 @@ class Config(BaseBridgeConfig):
         copy("bridge.delivery_receipts")
         copy("bridge.delivery_error_reports")
         copy("bridge.resend_bridge_info")
+        copy("bridge.mute_bridging")
+        copy("bridge.pinned_tag")
+        copy("bridge.archive_tag")
+        copy("bridge.tag_only_on_create")
         copy("bridge.backfill.invite_own_puppet")
         copy("bridge.backfill.takeout_limit")
         copy("bridge.backfill.initial_limit")

mautrix_telegram/db/bot_chat.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Iterable
 
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, BigInteger, String
 
 from mautrix.util.db import Base
 
@@ -25,7 +25,7 @@ from ..types import TelegramID
 # Fucking Telegram not telling bots what chats they are in 3:<
 class BotChat(Base):
     __tablename__ = "bot_chat"
-    id: TelegramID = Column(Integer, primary_key=True)
+    id: TelegramID = Column(BigInteger, primary_key=True)
     type: str = Column(String, nullable=False)
 
     @classmethod

mautrix_telegram/db/message.py

@@ -13,9 +13,10 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Iterator
+from typing import Optional, Iterator, List
 
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+from sqlalchemy import (Column, UniqueConstraint, BigInteger, Integer, String, Boolean, and_, func,
+                        desc, select, false)
 
 from mautrix.types import RoomID, EventID
 from mautrix.util.db import Base
@@ -28,9 +29,10 @@ class Message(Base):
 
     mxid: EventID = Column(String)
     mx_room: RoomID = Column(String)
-    tgid: TelegramID = Column(Integer, primary_key=True)
+    tgid: TelegramID = Column(BigInteger, primary_key=True)
-    tg_space: TelegramID = Column(Integer, primary_key=True)
+    tg_space: TelegramID = Column(BigInteger, primary_key=True)
     edit_index: int = Column(Integer, primary_key=True)
+    redacted: bool = Column(Boolean, server_default=false())
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"),)
 
@@ -51,6 +53,12 @@ class Message(Base):
             return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
                                            cls.c.edit_index == edit_index)
 
+    @classmethod
+    def get_first_by_tgids(cls, tgids: List[TelegramID], tg_space: TelegramID
+                           ) -> Iterator['Message']:
+        return cls._select_all(cls.c.tgid.in_(tgids), cls.c.tg_space == tg_space,
+                               cls.c.edit_index == 0)
+
     @classmethod
     def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
         rows = cls.db.execute(select([func.count(cls.c.tg_space)])
@@ -77,6 +85,12 @@ class Message(Base):
         return cls._select_one_or_none(cls.c.mxid == mxid, cls.c.mx_room == mx_room,
                                        cls.c.tg_space == tg_space)
 
+    @classmethod
+    def get_by_mxids(cls, mxids: List[EventID], mx_room: RoomID, tg_space: TelegramID
+                     ) -> Iterator['Message']:
+        return cls._select_all(cls.c.mxid.in_(mxids), cls.c.mx_room == mx_room,
+                               cls.c.tg_space == tg_space)
+
     @classmethod
     def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
                        **values) -> None:

mautrix_telegram/db/portal.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Boolean, Text, func, sql
+from sqlalchemy import Column, BigInteger, String, Boolean, Text, func, sql
 
 from mautrix.types import RoomID, ContentURI
 from mautrix.util.db import Base
@@ -27,8 +27,8 @@ class Portal(Base):
     __tablename__ = "portal"
 
     # Telegram chat information
-    tgid: TelegramID = Column(Integer, primary_key=True)
+    tgid: TelegramID = Column(BigInteger, primary_key=True)
-    tg_receiver: TelegramID = Column(Integer, primary_key=True)
+    tg_receiver: TelegramID = Column(BigInteger, primary_key=True)
     peer_type: str = Column(String, nullable=False)
     megagroup: bool = Column(Boolean)
 

mautrix_telegram/db/puppet.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable
 
-from sqlalchemy import Column, Integer, String, Text, Boolean
+from sqlalchemy import Column, Integer, BigInteger, String, Text, Boolean
 from sqlalchemy.sql import expression, func
 
 from mautrix.types import UserID, SyncToken
@@ -27,13 +27,15 @@ from ..types import TelegramID
 class Puppet(Base):
     __tablename__ = "puppet"
 
-    id: TelegramID = Column(Integer, primary_key=True)
+    id: TelegramID = Column(BigInteger, primary_key=True)
     custom_mxid: UserID = Column(String, nullable=True)
     access_token: str = Column(String, nullable=True)
     next_batch: SyncToken = Column(String, nullable=True)
     base_url: str = Column(Text, nullable=True)
     displayname: str = Column(String, nullable=True)
-    displayname_source: TelegramID = Column(Integer, nullable=True)
+    displayname_source: TelegramID = Column(BigInteger, nullable=True)
+    displayname_contact: bool = Column(Boolean, nullable=False, server_default=expression.true())
+    displayname_quality: int = Column(Integer, nullable=False, server_default="0")
     username: str = Column(String, nullable=True)
     photo_id: str = Column(String, nullable=True)
     is_bot: bool = Column(Boolean, nullable=True)

mautrix_telegram/db/telegram_file.py

@@ -13,15 +13,17 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, cast, Dict, Any
+from typing import Optional, cast, Dict, Any, TYPE_CHECKING
 
 from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
                         TypeDecorator)
-from sqlalchemy.engine.result import RowProxy
 
 from mautrix.types import ContentURI, EncryptedFile
 from mautrix.util.db import Base
 
+if TYPE_CHECKING:
+    from sqlalchemy.engine.result import RowProxy
+
 
 class DBEncryptedFile(TypeDecorator):
     impl = Text
@@ -60,7 +62,7 @@ class TelegramFile(Base):
     thumbnail: Optional['TelegramFile'] = None
 
     @classmethod
-    def scan(cls, row: RowProxy) -> 'TelegramFile':
+    def scan(cls, row: 'RowProxy') -> 'TelegramFile':
         telegram_file = cast(TelegramFile, super().scan(row))
         if isinstance(telegram_file.thumbnail, str):
             telegram_file.thumbnail = cls.get(telegram_file.thumbnail)

mautrix_telegram/db/user.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Optional, Iterable, Tuple
 
-from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String, func
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, BigInteger, Integer, String, func
 
 from mautrix.types import UserID
 from mautrix.util.db import Base
@@ -27,7 +27,7 @@ class User(Base):
     __tablename__ = "user"
 
     mxid: UserID = Column(String, primary_key=True)
-    tgid: Optional[TelegramID] = Column(Integer, nullable=True, unique=True)
+    tgid: Optional[TelegramID] = Column(BigInteger, nullable=True, unique=True)
     tg_username: str = Column(String, nullable=True)
     tg_phone: str = Column(String, nullable=True)
     saved_contacts: int = Column(Integer, default=0, nullable=False)
@@ -91,10 +91,10 @@ class User(Base):
 class UserPortal(Base):
     __tablename__ = "user_portal"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE",
+    user: TelegramID = Column(BigInteger, ForeignKey("user.tgid", onupdate="CASCADE",
                                                   ondelete="CASCADE"), primary_key=True)
-    portal: TelegramID = Column(Integer, primary_key=True)
+    portal: TelegramID = Column(BigInteger, primary_key=True)
-    portal_receiver: TelegramID = Column(Integer, primary_key=True)
+    portal_receiver: TelegramID = Column(BigInteger, primary_key=True)
 
     __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
                                            ("portal.tgid", "portal.tg_receiver"),
@@ -104,5 +104,5 @@ class UserPortal(Base):
 class Contact(Base):
     __tablename__ = "contact"
 
-    user: TelegramID = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
+    user: TelegramID = Column(BigInteger, ForeignKey("user.tgid"), primary_key=True)
-    contact: TelegramID = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
+    contact: TelegramID = Column(BigInteger, ForeignKey("puppet.id"), primary_key=True)

mautrix_telegram/example-config.yaml

@@ -8,6 +8,12 @@ homeserver:
     # Only applies if address starts with https://
     verify_ssl: true
     asmux: false
+    # Number of retries for all HTTP requests if the homeserver isn't reachable.
+    http_retry_count: 4
+    # The URL to push real-time bridge status to.
+    # If set, the bridge will make POST requests to this URL whenever a user's Telegram connection state changes.
+    # The bridge will use the appservice as_token to authorize requests.
+    status_endpoint: null
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
@@ -194,8 +200,11 @@ bridge:
         example.com: foobar
     # Set to false to disable link previews in messages sent to Telegram.
     telegram_link_preview: true
+    # Whether or not the !tg join command should do a HTTP request
+    # to resolve redirects in invite links.
+    invite_link_resolve: false
     # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    # N.B. Inline images are not supported on all clients (e.g. Element iOS/Android).
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
@@ -204,6 +213,7 @@ bridge:
     # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
     # streaming from/to Matrix and using many connections for Telegram.
     # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
+    # This option uses internal Telethon implementation details and may break with minor updates.
     parallel_file_transfer: false
     # Whether or not created rooms should have federation enabled.
     # If false, created portal rooms will never be federated.
@@ -267,6 +277,15 @@ bridge:
     # This field will automatically be changed back to false after it,
     # except if the config file is not writable.
     resend_bridge_info: false
+    # When using double puppeting, should muted chats be muted in Matrix?
+    mute_bridging: false
+    # When using double puppeting, should pinned chats be moved to a specific tag in Matrix?
+    # The favorites tag is `m.favourite`.
+    pinned_tag: null
+    # Same as above for archived chats, the low priority tag is `m.lowpriority`.
+    archive_tag: null
+    # Whether or not mute status and tags should only be bridged when the portal room is created.
+    tag_only_on_create: true
     # Settings for backfilling messages from Telegram.
     backfill:
         # Whether or not the Telegram ghosts of logged in Matrix users should be

mautrix_telegram/formatter/from_telegram.py

@@ -79,7 +79,7 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
             try:
                 user = await source.client.get_entity(fwd_from.from_id)
                 if user:
-                    fwd_from_text = pu.Puppet.get_displayname(user, False)
+                    fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
                     fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
             except (ValueError, RPCError):
                 fwd_from_text = fwd_from_html = "unknown user"
@@ -87,7 +87,7 @@ async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventC
         from_id = (fwd_from.from_id.chat_id if isinstance(fwd_from.from_id, PeerChat)
                    else fwd_from.from_id.channel_id)
         portal = po.Portal.get_by_tgid(TelegramID(from_id))
-        if portal:
+        if portal and portal.title:
             fwd_from_text = portal.title
             if portal.alias:
                 fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"

mautrix_telegram/matrix.py

@@ -94,9 +94,7 @@ class MatrixHandler(BaseMatrixHandler):
                 except MatrixError:
                     pass
             portal.mxid = room_id
-            e2be_ok = None
+            e2be_ok = await portal.check_dm_encryption()
-            if self.config["bridge.encryption.default"] and self.e2ee:
-                e2be_ok = await portal.enable_dm_encryption()
             await portal.save()
             await inviter.register_portal(portal)
             if e2be_ok is True:
@@ -111,6 +109,7 @@ class MatrixHandler(BaseMatrixHandler):
                 if e2be_ok is False:
                     message += "\n\nWarning: Failed to enable end-to-bridge encryption"
                 await intent.send_notice(room_id, message)
+            await portal.update_bridge_info()
         else:
             await intent.join_room(room_id)
             await intent.send_notice(room_id, "This puppet will remain inactive until a "
@@ -283,13 +282,12 @@ class MatrixHandler(BaseMatrixHandler):
         portal = po.Portal.get_by_mxid(room_id)
         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if await sender.has_full_access(allow_bot=True) and portal:
-            events = new_events - old_events
+            if not new_events:
-            if len(events) > 0:
+                await portal.handle_matrix_unpin_all(sender, event_id)
-                # New event pinned, set that as pinned in Telegram.
+            else:
-                await portal.handle_matrix_pin(sender, EventID(events.pop()), event_id)
+                changes = {event_id: event_id in new_events
-            elif len(new_events) == 0:
+                           for event_id in new_events ^ old_events}
-                # All pinned events removed, remove pinned event in Telegram.
+                await portal.handle_matrix_pin(sender, changes, event_id)
-                await portal.handle_matrix_pin(sender, None, event_id)
 
     @staticmethod
     async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID,

mautrix_telegram/portal/base.py

@@ -15,22 +15,23 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, Set, Iterable, TYPE_CHECKING
 from abc import ABC, abstractmethod
+from datetime import datetime
 import asyncio
 import logging
 import json
 
 from telethon.tl.functions.messages import ExportChatInviteRequest
-from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, ChatInviteEmpty, InputChannel,
+from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, InputChannel,
                                InputPeerChannel, InputPeerChat, InputPeerUser, InputUser,
                                PeerChannel, PeerChat, PeerUser, TypeChat, TypeInputPeer, TypePeer,
                                TypeUser, TypeUserFull, User, UserFull, TypeInputChannel, Photo,
                                Document, TypePhotoSize, PhotoSize, InputPhotoFileLocation,
                                TypeChatParticipant, TypeChannelParticipant, PhotoEmpty, ChatPhoto,
-                               ChatPhotoEmpty)
+                               ChatPhotoEmpty, PhotoSizeProgressive, PhotoSizeEmpty)
 
 from mautrix.errors import MatrixRequestError, IntentError
 from mautrix.appservice import AppService, IntentAPI
-from mautrix.types import (RoomID, RoomAlias, UserID, EventID, EventType, MessageEventContent,
+from mautrix.types import (RoomID, RoomAlias, UserID, EventID, EventType,
                            PowerLevelStateEventContent, ContentURI)
 from mautrix.util.simple_template import SimpleTemplate
 from mautrix.util.simple_lock import SimpleLock
@@ -104,6 +105,7 @@ class BasePortal(MautrixBasePortal, ABC):
 
     dedup: PortalDedup
     send_lock: PortalSendLock
+    _pin_lock: asyncio.Lock
 
     _db_instance: DBPortal
     _main_intent: Optional[IntentAPI]
@@ -138,6 +140,7 @@ class BasePortal(MautrixBasePortal, ABC):
 
         self.dedup = PortalDedup(self)
         self.send_lock = PortalSendLock()
+        self._pin_lock = asyncio.Lock()
 
         if tgid:
             self.by_tgid[self.tgid_full] = self
@@ -181,6 +184,10 @@ class BasePortal(MautrixBasePortal, ABC):
         elif self.peer_type == "channel":
             return PeerChannel(channel_id=self.tgid)
 
+    @property
+    def is_direct(self) -> bool:
+        return self.peer_type == "user"
+
     @property
     def has_bot(self) -> bool:
         return (bool(self.bot)
@@ -215,7 +222,18 @@ class BasePortal(MautrixBasePortal, ABC):
         return config[f"bridge.{key}"]
 
     @staticmethod
-    def _get_largest_photo_size(photo: Union[Photo, Document]
+    def _photo_size_key(photo: TypePhotoSize) -> int:
+        if isinstance(photo, PhotoSize):
+            return photo.size
+        elif isinstance(photo, PhotoSizeProgressive):
+            return max(photo.sizes)
+        elif isinstance(photo, PhotoSizeEmpty):
+            return 0
+        else:
+            return len(photo.bytes)
+
+    @classmethod
+    def _get_largest_photo_size(cls, photo: Union[Photo, Document]
                                 ) -> Tuple[Optional[InputPhotoFileLocation],
                                            Optional[TypePhotoSize]]:
         if not photo or isinstance(photo, PhotoEmpty) or (isinstance(photo, Document)
@@ -223,9 +241,7 @@ class BasePortal(MautrixBasePortal, ABC):
             return None, None
 
         largest = max(photo.thumbs if isinstance(photo, Document) else photo.sizes,
-                      key=(lambda photo2: (len(photo2.bytes)
+                      key=cls._photo_size_key)
-                                           if not isinstance(photo2, PhotoSize)
-                                           else photo2.size)))
         return InputPhotoFileLocation(
             id=photo.id,
             access_hash=photo.access_hash,
@@ -264,14 +280,14 @@ class BasePortal(MautrixBasePortal, ABC):
                     return dialog.entity
             raise
 
-    async def get_invite_link(self, user: 'u.User') -> str:
+    async def get_invite_link(self, user: 'u.User', uses: Optional[int] = None,
+                              expire: Optional[datetime] = None) -> str:
         if self.peer_type == "user":
             raise ValueError("You can't invite users to private chats.")
         if self.username:
             return f"https://t.me/{self.username}"
-        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user)))
+        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user),
-        if isinstance(link, ChatInviteEmpty):
+                                                         expire_date=expire, usage_limit=uses))
-            raise ValueError("Failed to get invite link.")
         return link.link
 
     # endregion

mautrix_telegram/portal/deduplication.py

@@ -61,9 +61,9 @@ class PortalDedup:
         if isinstance(event, MessageService):
             hash_content = [event.date.timestamp(), event.from_id, event.action]
         else:
-            hash_content = [event.date.timestamp(), event.message]
+            hash_content = [event.date.timestamp(), event.message.strip()]
             if event.fwd_from:
-                hash_content += [event.fwd_from.from_id, event.fwd_from.channel_id]
+                hash_content += [event.fwd_from.from_id]
             elif isinstance(event, Message) and event.media:
                 try:
                     hash_content += {

mautrix_telegram/portal/matrix.py

@@ -22,11 +22,10 @@ import magic
 
 from telethon.tl.functions.messages import (EditChatPhotoRequest, EditChatTitleRequest,
                                             UpdatePinnedMessageRequest, SetTypingRequest,
-                                            EditChatAboutRequest)
+                                            EditChatAboutRequest, UnpinAllMessagesRequest)
 from telethon.tl.functions.channels import EditPhotoRequest, EditTitleRequest, JoinChannelRequest
-from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError,
+from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError, MessageIdInvalidError,
-                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError,
+                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, RPCError)
-                             RPCError)
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
                                InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
@@ -113,7 +112,19 @@ class PortalMatrix(BasePortal, ABC):
         space = self.tgid if self.peer_type == "channel" else user.tgid
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
-            return
+            message = DBMessage.find_last(self.mxid, space)
+            if not message:
+                self.log.debug(f"Dropping Matrix read receipt from {user.mxid}: "
+                               f"target message {event_id} not known and last message"
+                               " in chat not found")
+                return
+            else:
+                self.log.debug(f"Matrix read receipt target {event_id} not known, marking "
+                               f"messages up to most recent ({message.mxid}/{message.tgid}) "
+                               f"as read by {user.mxid}/{user.tgid}")
+        else:
+            self.log.debug("Handling Matrix read receipt: marking messages up to "
+                           f"{message.mxid}/{message.tgid} as read by {user.mxid}/{user.tgid}")
         await user.client.send_read_acknowledge(self.peer, max_id=message.tgid,
                                                 clear_mentions=True)
 
@@ -327,7 +338,7 @@ class PortalMatrix(BasePortal, ABC):
             self.log.exception("Failed to parse location")
             return None
         caption, entities = await formatter.matrix_to_telegram(client, text=content.body)
-        media = MessageMediaGeo(geo=GeoPoint(lat, long, access_hash=0))
+        media = MessageMediaGeo(geo=GeoPoint(lat=lat, long=long, access_hash=0))
 
         async with self.send_lock(sender_id):
             if await self._matrix_document_edit(client, content, space, caption, media, event_id):
@@ -412,23 +423,23 @@ class PortalMatrix(BasePortal, ABC):
         else:
             self.log.trace("Unhandled Matrix event: %s", content)
 
-    async def handle_matrix_pin(self, sender: 'u.User', pinned_message: Optional[EventID],
+    async def handle_matrix_unpin_all(self, sender: 'u.User', pin_event_id: EventID) -> None:
+        await sender.client(UnpinAllMessagesRequest(peer=self.peer))
+        await self._send_delivery_receipt(pin_event_id)
+
+    async def handle_matrix_pin(self, sender: 'u.User', changes: Dict[EventID, bool],
                                 pin_event_id: EventID) -> None:
-        if self.peer_type != "chat" and self.peer_type != "channel":
+        tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
-            return
+        ids = {msg.mxid: msg.tgid
-        try:
+               for msg in DBMessage.get_by_mxids(list(changes.keys()),
-            if not pinned_message:
+                                                 mx_room=self.mxid, tg_space=tg_space)}
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=0))
+        for event_id, pinned in changes.items():
-            else:
+            try:
-                tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
+                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=ids[event_id],
-                message = DBMessage.get_by_mxid(pinned_message, self.mxid, tg_space)
+                                                               unpin=not pinned))
-                if message is None:
+            except (ChatNotModifiedError, MessageIdInvalidError, KeyError):
-                    self.log.warning(f"Could not find pinned {pinned_message} in {self.mxid}")
+                pass
-                    return
+        await self._send_delivery_receipt(pin_event_id)
-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=message.tgid))
-            await self._send_delivery_receipt(pin_event_id)
-        except ChatNotModifiedError:
-            pass
 
     async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID,
                                      redaction_event_id: EventID) -> None:
@@ -436,12 +447,18 @@ class PortalMatrix(BasePortal, ABC):
         space = self.tgid if self.peer_type == "channel" else real_deleter.tgid
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
-            return
+            self.log.trace(f"Ignoring Matrix redaction of unknown event {event_id}")
-        if message.edit_index == 0:
+        elif message.redacted:
+            self.log.debug("Ignoring Matrix redaction of already redacted event "
+                           f"{message.mxid} in {message.mx_room}")
+        elif message.edit_index != 0:
+            message.edit(redacted=True)
+            self.log.debug("Ignoring Matrix redaction of edit event "
+                           f"{message.mxid} in {message.mx_room}")
+        else:
+            message.edit(redacted=True)
             await real_deleter.client.delete_messages(self.peer, [message.tgid])
             await self._send_delivery_receipt(redaction_event_id)
-        else:
-            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
 
     async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
                                            level: int) -> None:

mautrix_telegram/portal/metadata.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Optional, Iterable, Union, Dict, Any, TYPE_CHECKING
+from typing import List, Optional, Iterable, Union, Dict, Any, Tuple, TYPE_CHECKING
 from abc import ABC
 import asyncio
 
@@ -26,7 +26,8 @@ from telethon.tl.types import (
     Channel, ChatBannedRights, ChannelParticipantsRecent, ChannelParticipantsSearch, ChatPhoto,
     PhotoEmpty, InputChannel, InputUser, ChatPhotoEmpty, PeerUser, Photo, TypeChat, TypeInputPeer,
     TypeUser, User, InputPeerPhotoFileLocation, ChatParticipantAdmin, ChannelParticipantAdmin,
-    ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty)
+    ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty,
+    InputPeerUser)
 
 from mautrix.errors import MForbidden
 from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership,
@@ -58,21 +59,30 @@ class PortalMetadata(BasePortal, ABC):
 
     # region Matrix -> Telegram
 
-    async def _get_telegram_users_in_matrix_room(self) -> List[Union[InputUser, PeerUser]]:
+    async def get_telegram_users_in_matrix_room(self, source: 'u.User'
-        user_tgids = set()
+                                                ) -> Tuple[List[InputPeerUser], List[UserID]]:
+        user_tgids = {}
         user_mxids = await self.main_intent.get_room_members(self.mxid, (Membership.JOIN,
                                                                          Membership.INVITE))
-        for user_str in user_mxids:
+        for mxid in user_mxids:
-            user = UserID(user_str)
+            if mxid == self.az.bot_mxid:
-            if user == self.az.bot_mxid:
                 continue
-            mx_user = u.User.get_by_mxid(user, create=False)
+            mx_user = u.User.get_by_mxid(mxid, create=False)
             if mx_user and mx_user.tgid:
-                user_tgids.add(mx_user.tgid)
+                user_tgids[mx_user.tgid] = mxid
-            puppet_id = p.Puppet.get_id_from_mxid(user)
+            puppet_id = p.Puppet.get_id_from_mxid(mxid)
             if puppet_id:
-                user_tgids.add(puppet_id)
+                user_tgids[puppet_id] = mxid
-        return [PeerUser(user_id) for user_id in user_tgids]
+        input_users = []
+        errors = []
+        for tgid, mxid in user_tgids.items():
+            try:
+                input_users.append(await source.client.get_input_entity(tgid))
+            except ValueError as e:
+                source.log.debug(f"Failed to find the input entity for {tgid} ({mxid}) for "
+                                 f"creating a group: {e}")
+                errors.append(mxid)
+        return input_users, errors
 
     async def upgrade_telegram_chat(self, source: 'u.User') -> None:
         if self.peer_type != "chat":
@@ -116,13 +126,13 @@ class PortalMetadata(BasePortal, ABC):
         if await self._update_username(username):
             await self.save()
 
-    async def create_telegram_chat(self, source: 'u.User', supergroup: bool = False) -> None:
+    async def create_telegram_chat(self, source: 'u.User', invites: List[InputUser],
+                                   supergroup: bool = False) -> None:
         if not self.mxid:
             raise ValueError("Can't create Telegram chat for portal without Matrix room.")
         elif self.tgid:
             raise ValueError("Can't create Telegram chat for portal with existing Telegram chat.")
 
-        invites = await self._get_telegram_users_in_matrix_room()
         if len(invites) < 2:
             if self.bot is not None:
                 info, mxid = await self.bot.get_me()
@@ -160,6 +170,7 @@ class PortalMetadata(BasePortal, ABC):
             levels = self._get_base_power_levels(levels, entity)
             await self.main_intent.set_power_levels(self.mxid, levels)
         await self.handle_matrix_power_levels(source, levels.users, {}, None)
+        await self.update_bridge_info()
 
     async def invite_telegram(self, source: 'u.User',
                               puppet: Union[p.Puppet, 'AbstractUser']) -> None:
@@ -450,7 +461,7 @@ class PortalMetadata(BasePortal, ABC):
             levels.kick = overrides.get("kick", 50)
             levels.redact = overrides.get("redact", 50)
             levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
-            levels.events[EventType.ROOM_ENCRYPTION] = 99
+            levels.events[EventType.ROOM_ENCRYPTION] = 50 if self.matrix.e2ee else 99
             levels.events[EventType.ROOM_TOMBSTONE] = 99
             levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
             levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0

mautrix_telegram/portal/telegram.py

@@ -34,7 +34,8 @@ from telethon.tl.types import (
     MessageMediaPhoto, MessageMediaDice, MessageMediaGame, MessageMediaUnsupported, PeerUser,
     PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant, TypeDocumentAttribute,
     TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping, UpdateUserTyping,
-    MessageEntityPre, ChatPhotoEmpty)
+    MessageEntityPre, ChatPhotoEmpty, DocumentAttributeImageSize, DocumentAttributeAnimated,
+    UpdateChannelUserTyping, SendMessageTypingAction)
 
 from mautrix.appservice import IntentAPI
 from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType,
@@ -56,16 +57,18 @@ if TYPE_CHECKING:
 
 InviteList = Union[UserID, List[UserID]]
 TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
+UpdateTyping = Union[UpdateUserTyping, UpdateChatUserTyping, UpdateChannelUserTyping]
 DocAttrs = NamedTuple("DocAttrs", name=Optional[str], mime_type=Optional[str], is_sticker=bool,
-                      sticker_alt=Optional[str], width=int, height=int)
+                      sticker_alt=Optional[str], width=int, height=int, is_gif=bool)
 
 config: Optional['Config'] = None
 
 
 class PortalTelegram(BasePortal, ABC):
-    async def handle_telegram_typing(self, user: p.Puppet,
+    async def handle_telegram_typing(self, user: p.Puppet, update: UpdateTyping) -> None:
-                                     _: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+        is_typing = isinstance(update.action, SendMessageTypingAction)
-        await user.intent_for(self).set_typing(self.mxid, is_typing=True)
+        # Always use the default puppet here to avoid any problems with echoing
+        await user.default_mxid_intent.set_typing(self.mxid, is_typing=is_typing)
 
     def _get_external_url(self, evt: Message) -> Optional[str]:
         if self.peer_type == "channel" and self.username is not None:
@@ -109,8 +112,7 @@ class PortalTelegram(BasePortal, ABC):
             return await self._send_message(intent, content, timestamp=evt.date)
         info = ImageInfo(
             height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type,
-            size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize))
+            size=self._photo_size_key(largest_size))
-                  else largest_size.size))
         ext = sane_mimetypes.guess_extension(file.mime_type)
         name = f"disappearing_image{ext}" if media.ttl_seconds else f"image{ext}"
         await intent.set_typing(self.mxid, is_typing=False)
@@ -135,6 +137,7 @@ class PortalTelegram(BasePortal, ABC):
     @staticmethod
     def _parse_telegram_document_attributes(attributes: List[TypeDocumentAttribute]) -> DocAttrs:
         name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
+        is_gif = False
         for attr in attributes:
             if isinstance(attr, DocumentAttributeFilename):
                 name = name or attr.file_name
@@ -142,9 +145,13 @@ class PortalTelegram(BasePortal, ABC):
             elif isinstance(attr, DocumentAttributeSticker):
                 is_sticker = True
                 sticker_alt = attr.alt
+            elif isinstance(attr, DocumentAttributeAnimated):
+                is_gif = True
             elif isinstance(attr, DocumentAttributeVideo):
                 width, height = attr.w, attr.h
-        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height)
+            elif isinstance(attr, DocumentAttributeImageSize):
+                width, height = attr.w, attr.h
+        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height, is_gif)
 
     @staticmethod
     def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: DocAttrs,
@@ -185,7 +192,7 @@ class PortalTelegram(BasePortal, ABC):
                                                 width=file.thumbnail.width or thumb_size.w,
                                                 size=file.thumbnail.size)
         else:
-            # This is a hack for bad clients like Riot iOS that require a thumbnail
+            # This is a hack for bad clients like Element iOS that require a thumbnail
             if file.decryption_info:
                 info.thumbnail_file = file.decryption_info
             else:
@@ -226,9 +233,26 @@ class PortalTelegram(BasePortal, ABC):
         await intent.set_typing(self.mxid, is_typing=False)
 
         event_type = EventType.ROOM_MESSAGE
-        # Riot only supports images as stickers, so send animated webm stickers as m.video
+        # Elements only support images as stickers, so send animated webm stickers as m.video
         if attrs.is_sticker and file.mime_type.startswith("image/"):
             event_type = EventType.STICKER
+            # Tell clients to render the stickers as 256x256 if they're bigger
+            if info.width > 256 or info.height > 256:
+                if info.width > info.height:
+                    info.height = int(info.height / (info.width / 256))
+                    info.width = 256
+                else:
+                    info.width = int(info.width / (info.height / 256))
+                    info.height = 256
+            if info.thumbnail_info:
+                info.thumbnail_info.width = info.width
+                info.thumbnail_info.height = info.height
+        if attrs.is_gif:
+            info["fi.mau.telegram.gif"] = True
+            info["fi.mau.loop"] = True
+            info["fi.mau.autoplay"] = True
+            info["fi.mau.no_audio"] = True
+
         content = MediaMessageEventContent(
             body=name or "unnamed file", info=info, relates_to=relates_to,
             external_url=self._get_external_url(evt),
@@ -318,16 +342,63 @@ class PortalTelegram(BasePortal, ABC):
         await intent.set_typing(self.mxid, is_typing=False)
         return await self._send_message(intent, content, timestamp=evt.date)
 
+    @staticmethod
+    def _format_dice(roll: MessageMediaDice) -> str:
+        if roll.emoticon == "\U0001F3B0":
+            emojis = {
+                0: "\U0001F36B",  # "🍫",
+                1: "\U0001F352",  # "🍒",
+                2: "\U0001F34B",  # "🍋",
+                3: "7\ufe0f\u20e3"  # "7️⃣",
+            }
+            res = roll.value - 1
+            slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
+            return f"{slot1} {slot2} {slot3} ({roll.value})"
+        elif roll.emoticon == "\u26BD":
+            results = {
+                1: "miss",
+                2: "hit the woodwork",
+                3: "goal",  # seems to go in through the center
+                4: "goal",
+                5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
+            }
+        elif roll.emoticon == "\U0001F3B3":
+            results = {
+                1: "miss",
+                2: "1 pin down",
+                3: "3 pins down, split",
+                4: "4 pins down, split",
+                5: "5 pins down",
+                6: "strike 🎉",
+            }
+        # elif roll.emoticon == "\U0001F3C0":
+        #     results = {
+        #         2: "rolled off",
+        #         3: "stuck",
+        #     }
+        # elif roll.emoticon == "\U0001F3AF":
+        #     results = {
+        #         1: "bounced off",
+        #         2: "outer rim",
+        #
+        #         6: "bullseye",
+        #     }
+        else:
+            return str(roll.value)
+        return f"{results[roll.value]} ({roll.value})"
+
     async def handle_telegram_dice(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                    relates_to: RelatesTo) -> EventID:
         emoji_text = {
             "\U0001F3AF": " Dart throw",
             "\U0001F3B2": " Dice roll",
             "\U0001F3C0": " Basketball throw",
+            "\U0001F3B0": " Slot machine",
+            "\U0001F3B3": " Bowling",
             "\u26BD": " Football kick"
         }
         roll: MessageMediaDice = evt.media
-        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {roll.value}"
+        text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {self._format_dice(roll)}"
         content = TextMessageEventContent(msgtype=MessageType.TEXT, format=Format.HTML, body=text,
                                           formatted_body=f"<h4>{text}</h4>", relates_to=relates_to,
                                           external_url=self._get_external_url(evt))
@@ -575,6 +646,9 @@ class PortalTelegram(BasePortal, ABC):
                            "displayname, updating info...")
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
+            if not sender.displayname:
+                self.log.debug(f"Telegram user {sender.tgid} doesn't have a displayname even after"
+                               f" updating with data {entity!s}")
 
         allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
                          MessageMediaGame, MessageMediaDice, MessageMediaPoll,
@@ -694,13 +768,20 @@ class PortalTelegram(BasePortal, ABC):
             levels.users[puppet.mxid] = 50
         await self.main_intent.set_power_levels(self.mxid, levels)
 
-    async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
+    async def receive_telegram_pin_ids(self, msg_ids: List[TelegramID], receiver: TelegramID,
-        tg_space = receiver if self.peer_type != "channel" else self.tgid
+                                       remove: bool) -> None:
-        message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
+        async with self._pin_lock:
-        if message:
+            tg_space = receiver if self.peer_type != "channel" else self.tgid
-            await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
+            previously_pinned = await self.main_intent.get_pinned_messages(self.mxid)
-        else:
+            currently_pinned_dict = {event_id: True for event_id in previously_pinned}
-            await self.main_intent.set_pinned_messages(self.mxid, [])
+            for message in DBMessage.get_first_by_tgids(msg_ids, tg_space):
+                if remove:
+                    currently_pinned_dict.pop(message.mxid, None)
+                else:
+                    currently_pinned_dict[message.mxid] = True
+            currently_pinned = list(currently_pinned_dict.keys())
+            if currently_pinned != previously_pinned:
+                await self.main_intent.set_pinned_messages(self.mxid, currently_pinned)
 
     async def set_telegram_admins_enabled(self, enabled: bool) -> None:
         level = 50 if enabled else 10

mautrix_telegram/puppet.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Any, Dict, Iterable, Optional, Union, TYPE_CHECKING
+from typing import Awaitable, Any, Dict, Iterable, Optional, Union, Tuple, TYPE_CHECKING
 from difflib import SequenceMatcher
 import unicodedata
 import asyncio
@@ -24,10 +24,11 @@ from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser,
 from yarl import URL
 
 from mautrix.appservice import AppService, IntentAPI
-from mautrix.errors import MatrixRequestError
+from mautrix.errors import MatrixRequestError, MatrixError
 from mautrix.bridge import BasePuppet
-from mautrix.types import UserID, SyncToken, RoomID
+from mautrix.types import UserID, SyncToken, RoomID, ContentURI
 from mautrix.util.simple_template import SimpleTemplate
+from mautrix.util.logging import TraceLogger
 
 from .types import TelegramID
 from .db import Puppet as DBPuppet
@@ -43,7 +44,7 @@ config: Optional['Config'] = None
 
 
 class Puppet(BasePuppet):
-    log: logging.Logger = logging.getLogger("mau.puppet")
+    log: TraceLogger = logging.getLogger("mau.puppet")
     az: AppService
     mx: 'MatrixHandler'
     loop: asyncio.AbstractEventLoop
@@ -64,6 +65,8 @@ class Puppet(BasePuppet):
     username: Optional[str]
     displayname: Optional[str]
     displayname_source: Optional[TelegramID]
+    displayname_contact: bool
+    displayname_quality: int
     photo_id: Optional[str]
     is_bot: bool
     is_registered: bool
@@ -85,6 +88,8 @@ class Puppet(BasePuppet):
                  username: Optional[str] = None,
                  displayname: Optional[str] = None,
                  displayname_source: Optional[TelegramID] = None,
+                 displayname_contact: bool = True,
+                 displayname_quality: int = 0,
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
@@ -100,6 +105,8 @@ class Puppet(BasePuppet):
         self.username = username
         self.displayname = displayname
         self.displayname_source = displayname_source
+        self.displayname_contact = displayname_contact
+        self.displayname_quality = displayname_quality
         self.photo_id = photo_id
         self.is_bot = is_bot
         self.is_registered = is_registered
@@ -164,8 +171,10 @@ class Puppet(BasePuppet):
         return dict(access_token=self.access_token, next_batch=self._next_batch,
                     custom_mxid=self.custom_mxid, username=self.username, is_bot=self.is_bot,
                     displayname=self.displayname, displayname_source=self.displayname_source,
-                    photo_id=self.photo_id, matrix_registered=self.is_registered,
+                    displayname_contact=self.displayname_contact,
-                    disable_updates=self.disable_updates, base_url=self.base_url)
+                    displayname_quality=self.displayname_quality, photo_id=self.photo_id,
+                    matrix_registered=self.is_registered, disable_updates=self.disable_updates,
+                    base_url=str(self.base_url) if self.base_url else None)
 
     def new_db_instance(self) -> DBPuppet:
         return DBPuppet(id=self.id, **self._fields)
@@ -177,9 +186,10 @@ class Puppet(BasePuppet):
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
                       db_puppet.next_batch, db_puppet.base_url, db_puppet.username,
-                      db_puppet.displayname, db_puppet.displayname_source, db_puppet.photo_id,
+                      db_puppet.displayname, db_puppet.displayname_source,
-                      db_puppet.is_bot, db_puppet.matrix_registered, db_puppet.disable_updates,
+                      db_puppet.displayname_contact, db_puppet.displayname_quality,
-                      db_instance=db_puppet)
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
     # endregion
     # region Info updating
@@ -199,11 +209,13 @@ class Puppet(BasePuppet):
         whitespace = ("\t\n\r\v\f \u00a0\u034f\u180e\u2063\u202f\u205f\u2800\u3000\u3164\ufeff"
                       "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b"
                       "\u200c\u200d\u200e\u200f\ufe0f")
-        name = "".join(c for c in name.strip(whitespace) if unicodedata.category(c) != 'Cf')
+        allowed_other_format = ("\u200d", "\u200c")
+        name = "".join(c for c in name.strip(whitespace) if unicodedata.category(c) != 'Cf'
+                                                         or c in allowed_other_format)
         return name
 
     @classmethod
-    def get_displayname(cls, info: User, enable_format: bool = True) -> str:
+    def get_displayname(cls, info: User, enable_format: bool = True) -> Tuple[str, int]:
         fn = cls._filter_name(info.first_name)
         ln = cls._filter_name(info.last_name)
         data = {
@@ -216,19 +228,21 @@ class Puppet(BasePuppet):
         }
         preferences = config["bridge.displayname_preference"]
         name = None
+        quality = 99
         for preference in preferences:
             name = data[preference]
             if name:
                 break
+            quality -= 1
 
         if isinstance(info, User) and info.deleted:
             name = f"Deleted account {info.id}"
+            quality = 99
         elif not name:
             name = str(info.id)
+            quality = 0
 
-        if not enable_format:
+        return (cls.displayname_template.format_full(name) if enable_format else name), quality
-            return name
-        return cls.displayname_template.format_full(name)
 
     async def try_update_info(self, source: 'AbstractUser', info: User) -> None:
         try:
@@ -264,27 +278,40 @@ class Puppet(BasePuppet):
             allow_because = "user is the primary source"
         elif not isinstance(info, UpdateUserName) and not info.contact:
             allow_because = "user is not a contact"
-        elif self.displayname_source is None:
+        elif not self.displayname_source:
             allow_because = "no primary source set"
+        elif not self.displayname:
+            allow_because = "user has no name"
         else:
             return False
 
         if isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))
+        if not info.contact:
+            self.displayname_contact = False
+        elif not self.displayname_contact:
+            if not self.displayname:
+                self.displayname_contact = True
+            else:
+                return False
 
-        displayname = self.get_displayname(info)
+        displayname, quality = self.get_displayname(info)
-        if displayname != self.displayname:
+        if displayname != self.displayname and quality >= self.displayname_quality:
+            allow_because = f"{allow_because} and quality {quality} >= {self.displayname_quality}"
             self.log.debug(f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
                            f"because {allow_because}) from {self.displayname} to {displayname}")
+            self.log.trace("Displayname source data: %s", info)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            self.displayname_quality = quality
             try:
                 await self.default_mxid_intent.set_displayname(
                     displayname[:config["bridge.displayname_max_length"]])
-            except MatrixRequestError:
+            except MatrixError:
                 self.log.exception("Failed to set displayname")
                 self.displayname = ""
                 self.displayname_source = None
+                self.displayname_quality = 0
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
@@ -309,8 +336,8 @@ class Puppet(BasePuppet):
             if not photo_id:
                 self.photo_id = ""
                 try:
-                    await self.default_mxid_intent.set_avatar_url("")
+                    await self.default_mxid_intent.set_avatar_url(ContentURI(""))
-                except MatrixRequestError:
+                except MatrixError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
                 return True
@@ -326,13 +353,13 @@ class Puppet(BasePuppet):
                 self.photo_id = photo_id
                 try:
                     await self.default_mxid_intent.set_avatar_url(file.mxc)
-                except MatrixRequestError:
+                except MatrixError:
                     self.log.exception("Failed to set avatar")
                     self.photo_id = ""
                 return True
         return False
 
-    def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
+    async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
         portal: p.Portal = p.Portal.get_by_mxid(room_id)
         return portal and not portal.backfill_lock.locked and portal.peer_type != "user"
 

mautrix_telegram/user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,27 +15,28 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import (Awaitable, Dict, List, Iterable, NamedTuple, Optional, Tuple, Any, cast,
                     TYPE_CHECKING)
-from collections import defaultdict
+from datetime import datetime, timezone
 import logging
 import asyncio
 
-from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
+from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage,
                                UpdateShortChatMessage, UpdateShortMessage, User as TLUser, Chat,
-                               ChatForbidden)
+                               ChatForbidden, UpdateFolderPeers, UpdatePinnedDialogs,
+                               UpdateNotifySettings, NotifyPeer)
 from telethon.tl.custom import Dialog
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.account import UpdateStatusRequest
 
 from mautrix.client import Client
-from mautrix.errors import MatrixRequestError
+from mautrix.errors import MatrixRequestError, MNotFound
-from mautrix.types import UserID, RoomID
+from mautrix.types import UserID, RoomID, PushRuleScope, PushRuleKind, PushActionType, RoomTagInfo
-from mautrix.bridge import BaseUser
+from mautrix.bridge import BaseUser, BridgeState
 from mautrix.util.logging import TraceLogger
 from mautrix.util.opt_prometheus import Gauge
 
 from .types import TelegramID
-from .db import User as DBUser, Portal as DBPortal
+from .db import User as DBUser, Portal as DBPortal, Message as DBMessage
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
@@ -50,6 +51,11 @@ SearchResult = NamedTuple('SearchResult', puppet='pu.Puppet', similarity=int)
 METRIC_LOGGED_IN = Gauge('bridge_logged_in', 'Users logged into bridge')
 METRIC_CONNECTED = Gauge('bridge_connected', 'Users connected to Telegram')
 
+BridgeState.human_readable_errors.update({
+    "tg-not-connected": "Your Telegram connection failed",
+    "logged-out": "You're not logged into Telegram",
+})
+
 
 class User(AbstractUser, BaseUser):
     log: TraceLogger = logging.getLogger("mau.user")
@@ -72,8 +78,9 @@ class User(AbstractUser, BaseUser):
                  saved_contacts: int = 0, is_bot: bool = False,
                  db_portals: Optional[Iterable[Tuple[TelegramID, TelegramID]]] = None,
                  db_instance: Optional[DBUser] = None) -> None:
-        super().__init__()
+        AbstractUser.__init__(self)
         self.mxid = mxid
+        BaseUser.__init__(self)
         self.tgid = tgid
         self.is_bot = is_bot
         self.username = username
@@ -85,12 +92,8 @@ class User(AbstractUser, BaseUser):
         self.db_portals = db_portals or []
         self._db_instance = db_instance
         self._ensure_started_lock = asyncio.Lock()
-        self.dm_update_lock = asyncio.Lock()
-        self._metric_value = defaultdict(lambda: False)
         self._track_connection_task = None
 
-        self.command_status = None
-
         (self.relaybot_whitelisted,
          self.whitelisted,
          self.puppet_whitelisted,
@@ -102,8 +105,6 @@ class User(AbstractUser, BaseUser):
         if tgid:
             self.by_tgid[tgid] = self
 
-        self.log = self.log.getChild(self.mxid)
-
     @property
     def name(self) -> str:
         return self.mxid
@@ -217,6 +218,21 @@ class User(AbstractUser, BaseUser):
             connected = bool(self.client._sender._transport_connected
                              if self.client and self.client._sender else False)
             self._track_metric(METRIC_CONNECTED, connected)
+            await self.push_bridge_state(ok=connected, ttl=3600 if connected else 240,
+                                         error="tg-not-connected" if not connected else None)
+
+    async def fill_bridge_state(self, state: BridgeState) -> None:
+        await super().fill_bridge_state(state)
+        state.remote_id = str(self.tgid)
+        state.remote_name = self.human_tg_id
+
+    async def get_bridge_state(self) -> BridgeState:
+        if not self.client:
+            return BridgeState(ok=False, error="logged-out")
+        elif not self.client._sender or not self.client._sender._transport_connected:
+            return BridgeState(ok=False, error="tg-not-connected")
+        else:
+            return BridgeState(ok=True)
 
     async def stop(self) -> None:
         await super().stop()
@@ -224,6 +240,7 @@ class User(AbstractUser, BaseUser):
             self._track_connection_task.cancel()
             self._track_connection_task = None
         self._track_metric(METRIC_CONNECTED, False)
+        await self.push_bridge_state(ok=False, error="tg-not-connected")
 
     async def post_login(self, info: TLUser = None, first_login: bool = False) -> None:
         if config["metrics.enabled"] and not self._track_connection_task:
@@ -328,6 +345,7 @@ class User(AbstractUser, BaseUser):
         self.delete()
         await self.stop()
         self._track_metric(METRIC_LOGGED_IN, False)
+        await self.push_bridge_state(ok=False, error="logged-out")
         return True
 
     def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
@@ -376,6 +394,101 @@ class User(AbstractUser, BaseUser):
             if portal.mxid
         }
 
+    async def _tag_room(self, puppet: pu.Puppet, portal: po.Portal, tag: str, active: bool
+                        ) -> None:
+        if not tag or not portal or not portal.mxid:
+            return
+        tag_info = await puppet.intent.get_room_tag(portal.mxid, tag)
+        if active and tag_info is None:
+            tag_info = RoomTagInfo(order=0.5)
+            tag_info[self.bridge.real_user_content_key] = True
+            await puppet.intent.set_room_tag(portal.mxid, tag, tag_info)
+        elif not active and tag_info and tag_info.get(self.bridge.real_user_content_key, False):
+            await puppet.intent.remove_room_tag(portal.mxid, tag)
+
+    @staticmethod
+    async def _mute_room(puppet: pu.Puppet, portal: po.Portal, mute_until: datetime) -> None:
+        if not config["bridge.mute_bridging"] or not portal or not portal.mxid:
+            return
+        now = datetime.utcnow().replace(tzinfo=timezone.utc)
+        if mute_until is not None and mute_until > now:
+            await puppet.intent.set_push_rule(PushRuleScope.GLOBAL, PushRuleKind.ROOM, portal.mxid,
+                                              actions=[PushActionType.DONT_NOTIFY])
+        else:
+            try:
+                await puppet.intent.remove_push_rule(PushRuleScope.GLOBAL, PushRuleKind.ROOM,
+                                                     portal.mxid)
+            except MNotFound:
+                pass
+
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        for peer in update.folder_peers:
+            portal = po.Portal.get_by_entity(peer.peer, receiver_id=self.tgid, create=False)
+            await self._tag_room(puppet, portal, config["bridge.archive_tag"],
+                                 peer.folder_id == 1)
+
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        # TODO bridge unpinning properly
+        for pinned in update.order:
+            portal = po.Portal.get_by_entity(pinned.peer, receiver_id=self.tgid, create=False)
+            await self._tag_room(puppet, portal, config["bridge.pinned_tag"], True)
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        if config["bridge.tag_only_on_create"]:
+            return
+        elif not isinstance(update.peer, NotifyPeer):
+            # TODO handle global notification setting changes?
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        portal = po.Portal.get_by_entity(update.peer.peer, receiver_id=self.tgid, create=False)
+        await self._mute_room(puppet, portal, update.notify_settings.mute_until)
+
+    async def _sync_dialog(self, portal: po.Portal, dialog: Dialog, should_create: bool,
+                           puppet: Optional[pu.Puppet]) -> None:
+        was_created = False
+        if portal.mxid:
+            try:
+                await portal.backfill(self, last_id=dialog.message.id)
+            except Exception:
+                self.log.exception(f"Error while backfilling {portal.tgid_log}")
+            try:
+                await portal.update_matrix_room(self, dialog.entity)
+            except Exception:
+                self.log.exception(f"Error while updating {portal.tgid_log}")
+        elif should_create:
+            try:
+                await portal.create_matrix_room(self, dialog.entity, invites=[self.mxid])
+                was_created = True
+            except Exception:
+                self.log.exception(f"Error while creating {portal.tgid_log}")
+        if portal.mxid and puppet and puppet.is_real_user:
+            tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+            if dialog.unread_count == 0:
+                # This is usually more reliable than finding a specific message
+                # e.g. if the last read message is a service message that isn't in the message db
+                last_read = DBMessage.find_last(portal.mxid, tg_space)
+            else:
+                last_read = DBMessage.get_one_by_tgid(portal.tgid, tg_space,
+                                                      dialog.dialog.read_inbox_max_id)
+            if last_read:
+                await puppet.intent.mark_read(last_read.mx_room, last_read.mxid)
+            if was_created or not config["bridge.tag_only_on_create"]:
+                await self._mute_room(puppet, portal, dialog.dialog.notify_settings.mute_until)
+                await self._tag_room(puppet, portal, config["bridge.pinned_tag"], dialog.pinned)
+                await self._tag_room(puppet, portal, config["bridge.archive_tag"], dialog.archived)
+
     async def sync_dialogs(self) -> None:
         if self.is_bot:
             return
@@ -385,6 +498,7 @@ class User(AbstractUser, BaseUser):
         index = 0
         self.log.debug(f"Syncing dialogs (update_limit={update_limit}, "
                        f"create_limit={create_limit})")
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
         dialog: Dialog
         async for dialog in self.client.iter_dialogs(limit=update_limit, ignore_migrated=True,
                                                      archived=False):
@@ -400,17 +514,9 @@ class User(AbstractUser, BaseUser):
                 continue
             portal = po.Portal.get_by_entity(entity, receiver_id=self.tgid)
             self.portals[portal.tgid_full] = portal
-            if portal.mxid:
+            coro = self._sync_dialog(portal=portal, dialog=dialog, puppet=puppet,
-                update_task = portal.update_matrix_room(self, entity)
+                                     should_create=not create_limit or index < create_limit)
-                backfill_task = portal.backfill(self, last_id=dialog.message.id)
+            creators.append(self.loop.create_task(coro))
-                creators.append(self._catch(f"updating {portal.tgid_log}",
-                                            self.loop.create_task(update_task)))
-                creators.append(self._catch(f"backfilling {portal.tgid_log}",
-                                            self.loop.create_task(backfill_task)))
-            elif not create_limit or index < create_limit:
-                create_task = portal.create_matrix_room(self, entity, invites=[self.mxid])
-                creators.append(self._catch(f"creating {portal.tgid_log}",
-                                            self.loop.create_task(create_task)))
             index += 1
         await self.save(portals=True)
         await asyncio.gather(*creators)

mautrix_telegram/util/file_transfer.py

@@ -30,7 +30,6 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio
                              SecurityError, FileIdInvalidError)
 
 from mautrix.appservice import IntentAPI
-from mautrix.util.network_retry import call_with_net_retry
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
@@ -145,8 +144,7 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if encrypt:
         file, decryption_info = encrypt_attachment(file)
         upload_mime_type = "application/octet-stream"
-    content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
+    content_uri = await intent.upload_media(file, upload_mime_type)
-                                            _action="upload media")
     if decryption_info:
         decryption_info.url = content_uri
 
@@ -222,9 +220,9 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
 
         image_converted = False
         # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
-        is_tgs = (mime_type == "application/gzip" or (mime_type == "application/octet-stream"
+        is_tgs = (mime_type == "application/gzip"
-                                                      and magic.from_buffer(file).startswith(
+                  or (mime_type == "application/octet-stream"
-                "gzip")))
+                      and magic.from_buffer(file).startswith("gzip")))
         if is_sticker and tgs_convert and is_tgs:
             converted_anim = await convert_tgs_to(file, tgs_convert["target"],
                                                   **tgs_convert["args"])
@@ -234,21 +232,12 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
             image_converted = mime_type != "application/gzip"
             thumbnail = None
 
-        if mime_type == "image/webp":
-            new_mime_type, file, width, height = convert_image(
-                file, source_mime="image/webp", target_type="png",
-                thumbnail_to=(256, 256) if is_sticker else None)
-            image_converted = new_mime_type != mime_type
-            mime_type = new_mime_type
-            thumbnail = None
-
         decryption_info = None
         upload_mime_type = mime_type
         if encrypt and encrypt_attachment:
             file, decryption_info = encrypt_attachment(file)
             upload_mime_type = "application/octet-stream"
-        content_uri = await call_with_net_retry(intent.upload_media, file, upload_mime_type,
+        content_uri = await intent.upload_media(file, upload_mime_type)
-                                                _action="upload media")
         if decryption_info:
             decryption_info.url = content_uri
 

mautrix_telegram/util/parallel_file_transfer.py

@@ -27,8 +27,10 @@ from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLoc
                                InputPhotoFileLocation, InputPeerPhotoFileLocation, TypeInputFile,
                                InputFileBig, InputFile)
 from telethon.tl.functions.auth import ExportAuthorizationRequest, ImportAuthorizationRequest
+from telethon.tl.functions import InvokeWithLayerRequest
 from telethon.tl.functions.upload import (GetFileRequest, SaveFilePartRequest,
                                           SaveBigFilePartRequest)
+from telethon.tl.alltlobjects import LAYER
 from telethon.network import MTProtoSender
 from telethon.crypto import AuthKey
 from telethon import utils, helpers
@@ -193,9 +195,9 @@ class ParallelTransferrer:
         if not self.auth_key:
             log.debug(f"Exporting auth to DC {self.dc_id}")
             auth = await self.client(ExportAuthorizationRequest(self.dc_id))
-            req = self.client._init_with(ImportAuthorizationRequest(
+            self.client._init_request.query = ImportAuthorizationRequest(id=auth.id,
-                id=auth.id, bytes=auth.bytes
+                                                                         bytes=auth.bytes)
-            ))
+            req = InvokeWithLayerRequest(LAYER, self.client._init_request)
             await sender.send(req)
             self.auth_key = sender.auth_key
         return sender

optional-requirements.txt

@@ -7,24 +7,21 @@ cchardet
 aiodns
 brotli
 
-#/webp_convert
-pillow>=4,<8
-
 #/qr_login
-pillow>=4,<8
+pillow>=4,<9
 qrcode>=6,<7
 
 #/hq_thumbnails
 moviepy>=1,<2
 
 #/metrics
-prometheus_client>=0.6,<0.9
+prometheus_client>=0.6,<0.12
 
 #/postgres
 psycopg2-binary>=2,<3
 
 #/e2be
-asyncpg>=0.20,<0.22
+asyncpg>=0.20,<0.24
 python-olm>=3,<4
 pycryptodome>=3,<4
 unpaddedbase64>=1,<2

requirements.txt

@@ -1,10 +1,10 @@
-SQLAlchemy>=1.2,<2
+SQLAlchemy>=1.2,<1.4
 alembic>=1,<2
-ruamel.yaml>=0.15.35,<0.17
+ruamel.yaml>=0.15.35,<0.18
 python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.8.3,<0.9
+mautrix>=0.9.3,<0.10
-telethon>=1.17,<1.18
+telethon>=1.20,<1.22
 telethon-session-sqlalchemy>=0.2.14,<0.3

setup.py

@@ -49,7 +49,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.6",
+    python_requires="~=3.7",
 
     setup_requires=["pytest-runner"],
     tests_require=["pytest", "pytest-asyncio", "pytest-mock"],