Hacky fix for null m.relates_to's and bump version to 0.5.2

.gitignore

@@ -5,13 +5,8 @@ env/
 pip-selfcheck.json
 *.pyc
 __pycache__
-build
-dist
-*.egg-info
-.eggs
 
 config.yaml
 registration.yaml
 *.log*
 *.db
-*.bak

.gitlab-ci.yml

@@ -1,38 +0,0 @@
-image: docker:stable
-
-stages:
-- build
-- push
-
-default:
-  before_script:
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-
-build:
-  stage: build
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-
-push latest:
-  stage: push
-  only:
-  - master
-  variables:
-    GIT_STRATEGY: none
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
-  - docker push $CI_REGISTRY_IMAGE:latest
-
-push tag:
-  stage: push
-  variables:
-    GIT_STRATEGY: none
-  except:
-  - master
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/alpine:3.10
+FROM docker.io/alpine:3.9
 
 ENV UID=1337 \
     GID=1337 \
@@ -10,11 +10,13 @@ RUN apk add --no-cache \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
+      py3-lxml \
       py3-magic \
       py3-sqlalchemy \
       py3-markdown \
       py3-psycopg2 \
-      py3-ruamel.yaml \
+      # Not yet in stable repos:
+      #py3-ruamel \
       # Indirect dependencies
       #commonmark
         py3-future \
@@ -32,7 +34,6 @@ RUN apk add --no-cache \
         py3-rsa \
       # Other dependencies
       python3-dev \
-      libffi-dev \
       build-base \
       ffmpeg \
       ca-certificates \

ROADMAP.md

@@ -3,7 +3,6 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
-  * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
   * [x] Typing notifications

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -1,27 +0,0 @@
-"""Add disable_updates field for puppets
-
-Revision ID: 17574c57f3f8
-Revises: a9119be92164
-Create Date: 2019-05-15 00:24:46.967529
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '17574c57f3f8'
-down_revision = 'a9119be92164'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
-                                      server_default=sa.sql.expression.false()))
-
-
-def downgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column("disable_updates")

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -17,8 +17,7 @@ depends_on = None
 
 
 def upgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
+    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
 
 
 def downgrade():

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -16,8 +16,7 @@ depends_on = None
 
 
 def upgrade():
-    with op.batch_alter_table("portal") as batch_op:
-        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
+    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
 
 
 def downgrade():

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -57,8 +57,7 @@ class Puppet(Base):
 
 
 def upgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+    op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False,
                                       server_default=sa.sql.expression.false()))
     op.create_table("mx_room_state",
                     sa.Column("room_id", sa.String(), nullable=False),

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -1,48 +0,0 @@
-"""Add edit index to messages
-
-Revision ID: 9e9c89b0b877
-Revises: 17574c57f3f8
-Create Date: 2019-05-29 15:28:23.128377
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '9e9c89b0b877'
-down_revision = '17574c57f3f8'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table('_message_temp',
-                    sa.Column('mxid', sa.String),
-                    sa.Column('mx_room', sa.String),
-                    sa.Column('tgid', sa.Integer),
-                    sa.Column('tg_space', sa.Integer),
-                    sa.Column('edit_index', sa.Integer),
-                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
-                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
-    c = op.get_bind()
-    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
-              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
-              "FROM message")
-    c.execute("DROP TABLE message")
-    c.execute("ALTER TABLE _message_temp RENAME TO message")
-
-
-def downgrade():
-    op.create_table('_message_temp',
-                    sa.Column('mxid', sa.String),
-                    sa.Column('mx_room', sa.String),
-                    sa.Column('tgid', sa.Integer),
-                    sa.Column('tg_space', sa.Integer),
-                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
-                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
-    c = op.get_bind()
-    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
-              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
-              "FROM message")
-    c.execute("DROP TABLE message")
-    c.execute("ALTER TABLE _message_temp RENAME TO message")

example-config.yaml

@@ -60,19 +60,10 @@ appservice:
     bot_displayname: Telegram bridge bot
     bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
 
-    # Community ID for bridged users (changes registration file) and rooms.
-    # Must be created manually.
-    community_id: false
-
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
 
-# Prometheus telemetry config. Requires prometheus-client to be installed.
-metrics:
-    enabled: false
-    listen_port: 8000
-
 # Bridge config
 bridge:
     # Localpart template of MXIDs for Telegram users.
@@ -135,11 +126,15 @@ bridge:
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
+    # Show message editing as a reply to the original message.
+    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
+    edits_as_replies: true
+    # Highlight changed/added parts in edits. Requires lxml.
+    highlight_edits: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
     # Currently only works for private chats and normal groups.
-    # WARNING: This feature seems to be broken in the telegram library.
     catch_up: false
     # Whether or not to use /sync to get presence, read receipts and typing notifications when using
     # your own Matrix account as the Matrix puppet for your Telegram account.
@@ -151,8 +146,6 @@ bridge:
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
-    # Maximum size of Telegram documents in megabytes to bridge.
-    max_document_size: 100
 
     # Whether to bridge Telegram bot messages as m.notices or m.texts.
     bot_messages_as_notices: true
@@ -258,40 +251,6 @@ telegram:
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
-
-    # Telethon connection options.
-    connection:
-        # The timeout in seconds to be used when connecting.
-        timeout: 120
-        # How many times the reconnection should retry, either on the initial connection or when
-        # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
-        # this is not recommended, since the program can get stuck in an infinite loop.
-        retries: 5
-        # The delay in seconds to sleep between automatic reconnections.
-        retry_delay: 1
-        # The threshold below which the library should automatically sleep on flood wait errors
-        # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
-        # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
-        # the error instead. Values larger than a day (86400) will be changed to a day.
-        flood_sleep_threshold: 60
-        # How many times a request should be retried. Request are retried when Telegram is having
-        # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
-        # there's a migrate error. May take a negative or null value for infinite retries, but this
-        # is not recommended, since some requests can always trigger a call fail (such as searching
-        # for messages).
-        request_retries: 5
-
-    # Device info sent to Telegram.
-    device_info:
-        # "auto" = OS name+version.
-        device_model: auto
-        # "auto" = Telethon version.
-        system_version: auto
-        # "auto" = mautrix-telegram version.
-        app_version: auto
-        lang_code: en
-        system_lang_code: en
-
     # Custom server to connect to.
     server:
         # Set to true to use these server settings. If false, will automatically
@@ -303,7 +262,6 @@ telegram:
         ip: 149.154.167.40
         # The port to connect to. 443 may not work, 80 is better and both are equally secure.
         port: 80
-
     # Telethon proxy configuration.
     # You must install PySocks from pip for proxies to work.
     proxy:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.6.1"
+__version__ = "0.5.2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -22,7 +22,6 @@ import logging.config
 import sys
 import copy
 import signal
-import os
 
 import sqlalchemy as sql
 
@@ -44,11 +43,6 @@ from .sqlstatestore import SQLStateStore
 from .user import User, init as init_user
 from . import __version__
 
-try:
-    import prometheus_client as prometheus
-except ImportError:
-    prometheus = None
-
 parser = argparse.ArgumentParser(
     description="A Matrix-Telegram puppeting bridge.",
     prog="python -m mautrix-telegram")
@@ -63,7 +57,7 @@ parser.add_argument("-r", "--registration", type=str, default="registration.yaml
                     metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()
 
-config = Config(args.config, args.registration, args.base_config, os.environ)
+config = Config(args.config, args.registration, args.base_config)
 config.load()
 config.update()
 
@@ -120,12 +114,6 @@ if config["appservice.provisioning.enabled"]:
 
 context.mx = MatrixHandler(context)
 
-if config["metrics.enabled"]:
-    if prometheus:
-        prometheus.start_http_server(config["metrics.listen_port"])
-    else:
-        log.warn("Metrics are enabled in the config, but prometheus-async is not installed.")
-
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
     start_ts = time()
     init_db(db_engine)

mautrix_telegram/abstract_user.py

@@ -14,12 +14,11 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional, List, Union, Dict, TYPE_CHECKING
+from typing import Tuple, Optional, List, Union, TYPE_CHECKING
 from abc import ABC, abstractmethod
 import asyncio
 import logging
 import platform
-import time
 
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
@@ -51,14 +50,6 @@ UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChann
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
-try:
-    from prometheus_client import Histogram
-
-    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
-                            ["update_type"])
-except ImportError:
-    Histogram = None
-    UPDATE_TIME = None
 
 class AbstractUser(ABC):
     session_container = None  # type: AlchemySessionContainer
@@ -104,43 +95,27 @@ class AbstractUser(ABC):
 
     def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
-
+        device = f"{platform.system()} {platform.release()}"
+        sysversion = MautrixTelegramClient.__version__
         self.session = self.session_container.new_session(self.name)
         if config["telegram.server.enabled"]:
             self.session.set_dc(config["telegram.server.dc"],
                                 config["telegram.server.ip"],
                                 config["telegram.server.port"])
-
         if self.is_relaybot:
             base_logger = logging.getLogger("telethon.relaybot")
         else:
             base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
-
-        device = config["telegram.device_info.device_model"]
-        sysversion = config["telegram.device_info.system_version"]
-        appversion = config["telegram.device_info.app_version"]
-
-        self.client = MautrixTelegramClient(
-            session=self.session,
-
-            api_id=config["telegram.api_id"],
-            api_hash=config["telegram.api_hash"],
-
-            app_version=__version__ if appversion == "auto" else appversion,
-            system_version=MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion,
-            device_model=f"{platform.system()} {platform.release()}" if device == "auto" else device,
-
-            timeout=config["telegram.connection.timeout"],
-            connection_retries=config["telegram.connection.retries"],
-            retry_delay=config["telegram.connection.retry_delay"],
-            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
-            request_retries=config["telegram.connection.request_retries"],
-
-            proxy=self._proxy_settings,
-
-            loop=self.loop,
-            base_logger=base_logger
-        )
+        self.client = MautrixTelegramClient(session=self.session,
+                                            api_id=config["telegram.api_id"],
+                                            api_hash=config["telegram.api_hash"],
+                                            loop=self.loop,
+                                            app_version=__version__,
+                                            system_version=sysversion,
+                                            device_model=device,
+                                            timeout=120,
+                                            base_logger=base_logger,
+                                            proxy=self._proxy_settings)
         self.client.add_event_handler(self._update_catch)
 
     @abstractmethod
@@ -160,14 +135,11 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     async def _update_catch(self, update: TypeUpdate) -> None:
-        start_time = time.time()
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception("Failed to handle Telegram update")
-        if UPDATE_TIME:
-            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
 
     async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
         if self.is_bot:
@@ -213,8 +185,6 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
-        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
-                              loop=self.loop)
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -264,7 +234,7 @@ class AbstractUser(ABC):
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
+        message = DBMessage.get_by_tgid(TelegramID(update.max_id), self.tgid)
         if not message:
             return
 
@@ -291,16 +261,6 @@ class AbstractUser(ABC):
         sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
-    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
-                                     ) -> None:
-        try:
-            users = (entity for entity in entities.values() if isinstance(entity, User))
-            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
-            await asyncio.gather(*[puppet.update_info(self, info)
-                                   for puppet, info in puppets if puppet])
-        except Exception:
-            self.log.exception("Failed to handle entity updates")
-
     async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
         # TODO duplication not checked
         puppet = pu.Puppet.get(TelegramID(update.user_id))
@@ -309,7 +269,7 @@ class AbstractUser(ABC):
             if await puppet.update_displayname(self, update):
                 puppet.save()
         elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo):
+            if await puppet.update_avatar(self, update.photo.photo_big):
                 puppet.save()
         else:
             self.log.warning("Unexpected other user info update: %s", update)
@@ -349,8 +309,7 @@ class AbstractUser(ABC):
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(message: DBMessage) -> None:
-        portal = po.Portal.get_by_mxid(message.mx_room)
+    async def _try_redact(portal: po.Portal, message: DBMessage) -> None:
         if not portal:
             return
         try:
@@ -362,26 +321,30 @@ class AbstractUser(ABC):
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        for message_id in update.messages:
-            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
-            for message in messages:
-                message.delete()
-                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
-                if number_left == 0:
-                    portal = po.Portal.get_by_mxid(message.mx_room)
-                    await self._try_redact(message)
+        for message in update.messages:
+            message = DBMessage.get_by_tgid(TelegramID(message), self.tgid)
+            if not message:
+                continue
+            message.delete()
+            number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+            if number_left == 0:
+                portal = po.Portal.get_by_mxid(message.mx_room)
+                await self._try_redact(portal, message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        channel_id = TelegramID(update.channel_id)
+        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        if not portal:
+            return
 
-        for message_id in update.messages:
-            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
-            for message in messages:
-                message.delete()
-                await self._try_redact(message)
+        for message in update.messages:
+            message = DBMessage.get_by_tgid(TelegramID(message), portal.tgid)
+            if not message:
+                continue
+            message.delete()
+            await self._try_redact(portal, message)
 
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
@@ -407,7 +370,10 @@ class AbstractUser(ABC):
 
         user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            return await portal.handle_telegram_edit(self, sender, update)
+            if config["bridge.edits_as_replies"]:
+                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
+                return await portal.handle_telegram_edit(self, sender, update)
+            return
 
         self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
         return await portal.handle_telegram_message(self, sender, update)

mautrix_telegram/bot.py

@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
+from typing import Awaitable, Callable, Dict, List, Optional, Pattern, TYPE_CHECKING
 import logging
 import re
 
@@ -23,7 +23,7 @@ from telethon.tl.types import (
     ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
     ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
     MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
-    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
@@ -60,14 +60,6 @@ class Bot(AbstractUser):
         self.tg_whitelist = []  # type: List[int]
         self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
                                        or False)  # type: bool
-        self._me_info = None  # type: Optional[User]
-        self._me_mxid = None  # type: Optional[MatrixUserID]
-
-    async def get_me(self, use_cache: bool = True) -> Tuple[User, MatrixUserID]:
-        if not use_cache or not self._me_mxid:
-            self._me_info = await self.client.get_me()
-            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
-        return self._me_info, self._me_mxid
 
     async def init_permissions(self) -> None:
         whitelist = config["bridge.relaybot.whitelist"] or []

mautrix_telegram/commands/portal/bridge.py

@@ -36,10 +36,6 @@ async def bridge(evt: CommandEvent) -> Dict:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** "
                                "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
-    force_use_bot = False
-    if evt.args[0] == "--usebot" and evt.sender.is_admin:
-        force_use_bot = True
-        evt.args = evt.args[1:]
     room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
     that_this = "This" if room_id == evt.room_id else "That"
 
@@ -84,7 +80,6 @@ async def bridge(evt: CommandEvent) -> Dict:
             "bridge_to_mxid": room_id,
             "tgid": portal.tgid,
             "peer_type": portal.peer_type,
-            "force_use_bot": force_use_bot,
         }
         return await evt.reply(f"{has_portal_message}"
                                "However, you have the permissions to unbridge that room.\n\n"
@@ -98,7 +93,6 @@ async def bridge(evt: CommandEvent) -> Dict:
         "bridge_to_mxid": room_id,
         "tgid": portal.tgid,
         "peer_type": portal.peer_type,
-        "force_use_bot": force_use_bot,
     }
     return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
                            "chat to this room, use `$cmdprefix+sp continue`")
@@ -155,7 +149,7 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
                                "`$cmdprefix+sp cancel` to cancel.")
 
     evt.sender.command_status = None
-    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+    is_logged_in = await evt.sender.is_logged_in()
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)

mautrix_telegram/commands/portal/config.py

@@ -77,6 +77,7 @@ def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
 def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
     stream = StringIO()
     yaml.dump({
+        "edits_as_replies": evt.config["bridge.edits_as_replies"],
         "bridge_notices": {
             "default": evt.config["bridge.bridge_notices.default"],
             "exceptions": evt.config["bridge.bridge_notices.exceptions"],

mautrix_telegram/commands/telegram/account.py

@@ -17,10 +17,10 @@
 from typing import Dict, Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
+                             HashInvalidError, AuthKeyError)
 from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
-                                           ResetAuthorizationRequest, UpdateProfileRequest)
+                                           ResetAuthorizationRequest)
 
 from .. import command_handler, CommandEvent, SECTION_AUTH
 
@@ -53,25 +53,6 @@ async def username(evt: CommandEvent) -> Optional[Dict]:
         await evt.reply(f"Username changed to {evt.sender.username}")
 
 
-@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
-                 help_text="Change your Telegram displayname.")
-async def displayname(evt: CommandEvent) -> Optional[Dict]:
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
-    if evt.sender.is_bot:
-        return await evt.reply("Bots can't set their own displayname.")
-
-    first_name, last_name = ((evt.args[0], "")
-                             if len(evt.args) == 1
-                             else (" ".join(evt.args[:-1]), evt.args[-1]))
-    try:
-        await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
-    except FirstNameInvalidError:
-        return await evt.reply("Invalid first name")
-    await evt.sender.update_info()
-    await evt.reply("Displayname updated")
-
-
 def _format_session(sess: Authorization) -> str:
     return (f"**{sess.app_name} {sess.app_version}**  \n"
             f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
@@ -106,7 +87,9 @@ async def session(evt: CommandEvent) -> Optional[Dict]:
         try:
             session_hash = int(evt.args[1])
         except ValueError:
-            return await evt.reply("Hash must be an integer")
+            return await evt.reply("Hash must be a positive integer")
+        if session_hash <= 0:
+            return await evt.reply("Hash must be a positive integer")
         try:
             ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
         except HashInvalidError:

mautrix_telegram/commands/telegram/auth.py

@@ -33,8 +33,8 @@ from ...util import format_duration, ignore_coro
                  help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> Optional[Dict]:
     me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
+    human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
     if me:
-        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
         return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")
@@ -46,9 +46,11 @@ async def ping(evt: CommandEvent) -> Optional[Dict]:
 async def ping_bot(evt: CommandEvent) -> Optional[Dict]:
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
-    info, mxid = await evt.tgbot.get_me(use_cache=False)
+    bot_info = await evt.tgbot.client.get_me()
+    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
+    displayname = bot_info.first_name
     return await evt.reply("Telegram message relay bot is active: "
-                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
+                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
                            "To use the bot, simply invite it to a portal room.")
 
 
@@ -124,20 +126,41 @@ async def login(evt: CommandEvent) -> Optional[Dict]:
     if evt.config["appservice.public.enabled"]:
         prefix = evt.config["appservice.public.external"]
         url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
+        if allow_matrix_login:
+            if override_sender:
+                return await evt.reply(
+                    "This bridge instance allows you to log in inside or outside of Matrix, but "
+                    "logging in as another user is only possible via the web interface.\n\n"
+                    f"Please visit [the login page]({url}) to log in as "
+                    f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).\n\n")
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your phone number or bot "
+                "auth token here.\n"
+                "If you would like to log in outside of Matrix, please visit [the login page]"
+                f"({url}).\n\n"
+                "Logging in outside of Matrix is recommended if you have two-factor authentication "
+                "enabled, because in-Matrix login would save your password in the message history."
+                f"\n\n{nb}")
         if override_sender:
-            return await evt.reply(f"[Click here to log in]({url}) as "
-                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
-        elif allow_matrix_login:
-            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
-                                   f" number (or bot auth token) here to log in.\n\n{nb}")
-        return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
+            return await evt.reply(
+                "This bridge instance does not allow logging in inside Matrix, and logging in as "
+                "another user inside Matrix isn't possible anyway.\n\n"
+                f"Please visit [the login page]({url}) to log in as "
+                f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+        return await evt.reply(
+            "This bridge instance does not allow logging in inside Matrix.\n\n"
+            f"Please visit [the login page]({url}) to log in.\n\n"
+            f"{nb}")
     elif allow_matrix_login:
         if override_sender:
             return await evt.reply(
                 "This bridge instance does not allow you to log in outside of Matrix. "
                 "Logging in as another user inside Matrix is not currently possible.")
-        return await evt.reply("Please send your phone number (or bot auth token) here to start "
-                               f"the login process.\n\n{nb}")
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your phone number or bot auth token here to start the login process.\n\n"
+            f"{nb}")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 

mautrix_telegram/commands/telegram/misc.py

@@ -191,7 +191,7 @@ async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
         raise MessageIDError(f"Invalid {type_name} ID (format)") from e
 
     if peer_type == PEER_TYPE_CHAT:
-        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
+        orig_msg = DBMessage.get_by_tgid(msg_id, space)
         if not orig_msg:
             raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
         new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
@@ -237,11 +237,11 @@ async def play(evt: CommandEvent) -> Optional[Dict]:
 
 
 @command_handler(help_section=SECTION_MISC,
-                 help_args="<_poll ID_> <_choice number_>",
+                 help_args="<_poll ID_> <_choice ID_>",
                  help_text="Vote in a Telegram poll.")
 async def vote(evt: CommandEvent) -> Optional[Dict]:
-    if len(evt.args) < 1:
-        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
+    if len(evt.args) < 2:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice ID>`")
     elif not await evt.sender.is_logged_in():
         return await evt.reply("You must be logged in with a real account to vote in polls.")
     elif evt.sender.is_bot:
@@ -255,25 +255,7 @@ async def vote(evt: CommandEvent) -> Optional[Dict]:
     if not isinstance(msg.media, MessageMediaPoll):
         return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
 
-    options = []
-    for option in evt.args[1:]:
-        try:
-            if len(option) > 10:
-                raise ValueError("option index too long")
-            option_index = int(option) - 1
-        except ValueError:
-            option_index = None
-        if option_index is None:
-            return await evt.reply(f"Invalid option number \"{option}\"",
-                                   render_markdown=False, allow_html=False)
-        elif option_index < 0:
-            return await evt.reply(f"Invalid option number {option}. "
-                                   f"Option numbers must be positive.")
-        elif option_index >= len(msg.media.poll.answers):
-            return await evt.reply(f"Invalid option number {option}. "
-                                   f"The poll only has {len(msg.media.poll.answers)} options.")
-        options.append(msg.media.poll.answers[option_index].option)
-    options = [msg.media.poll.answers[int(option) - 1].option
+    options = [base64.b64decode(option + (3 - (len(option) + 3) % 4) * "=")
                for option in evt.args[1:]]
     try:
         resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))

mautrix_telegram/config.py

@@ -103,20 +103,12 @@ class DictWithRecursion:
 
 
 class Config(DictWithRecursion):
-    def __init__(self, path: str, registration_path: str, base_path: str,
-                 overrides: Dict[str, Any] = None) -> None:
+    def __init__(self, path: str, registration_path: str, base_path: str) -> None:
         super().__init__()
         self.path = path  # type: str
         self.registration_path = registration_path  # type: str
         self.base_path = base_path  # type: str
         self._registration = None  # type: Optional[Dict]
-        self._overrides = overrides or {}  # type: Dict[str, Any]
-
-    def __getitem__(self, key: str) -> Any:
-        try:
-            return self._overrides[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
-        except KeyError:
-            return super().__getitem__(key)
 
     def load(self) -> None:
         with open(self.path, 'r') as stream:
@@ -189,14 +181,9 @@ class Config(DictWithRecursion):
         copy("appservice.bot_displayname")
         copy("appservice.bot_avatar")
 
-        copy("appservice.community_id")
-
         copy("appservice.as_token")
         copy("appservice.hs_token")
 
-        copy("metrics.enabled")
-        copy("metrics.listen_port")
-
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
@@ -212,13 +199,14 @@ class Config(DictWithRecursion):
         copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
         copy("bridge.plaintext_highlights")
+        copy("bridge.edits_as_replies")
+        copy("bridge.highlight_edits")
         copy("bridge.public_portals")
         copy("bridge.catch_up")
         copy("bridge.sync_with_custom_puppets")
         copy("bridge.telegram_link_preview")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
-        copy("bridge.max_document_size")
 
         copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):
@@ -269,24 +257,10 @@ class Config(DictWithRecursion):
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
-
-        copy("telegram.connection.timeout")
-        copy("telegram.connection.retries")
-        copy("telegram.connection.retry_delay")
-        copy("telegram.connection.flood_sleep_threshold")
-        copy("telegram.connection.request_retries")
-
-        copy("telegram.device_info.device_model")
-        copy("telegram.device_info.system_version")
-        copy("telegram.device_info.app_version")
-        copy("telegram.device_info.lang_code")
-        copy("telegram.device_info.system_lang_code")
-
         copy("telegram.server.enabled")
         copy("telegram.server.dc")
         copy("telegram.server.ip")
         copy("telegram.server.port")
-
         copy("telegram.proxy.type")
         copy("telegram.proxy.address")
         copy("telegram.proxy.port")
@@ -354,6 +328,3 @@ class Config(DictWithRecursion):
             "sender_localpart": self["appservice.bot_username"],
             "rate_limited": False
         }
-        if self["appservice.community_id"]:
-            self._registration["namespaces"]["users"][0]["group_id"] \
-                = self["appservice.community_id"]

mautrix_telegram/db/message.py

@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, select
 from sqlalchemy.engine.result import RowProxy
 from typing import Optional, List
 
@@ -29,44 +29,25 @@ class Message(Base):
     mx_room = Column(String)  # type: MatrixRoomID
     tgid = Column(Integer, primary_key=True)  # type: TelegramID
     tg_space = Column(Integer, primary_key=True)  # type: TelegramID
-    edit_index = Column(Integer, primary_key=True)  # type: int
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
         try:
-            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
-            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
-                       edit_index=edit_index)
+            mxid, mx_room, tgid, tg_space = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space)
         except StopIteration:
             return None
 
     @staticmethod
     def _all(rows: RowProxy) -> List['Message']:
-        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
-                        edit_index=row[4])
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3])
                 for row in rows]
 
     @classmethod
-    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
-        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
-                                                                 cls.c.tg_space == tg_space))))
-
-    @classmethod
-    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
-                        ) -> Optional['Message']:
-        query = cls.t.select()
-        if edit_index < 0:
-            query = (query
-                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
-                     .order_by(desc(cls.c.edit_index))
-                     .limit(1)
-                     .offset(-edit_index - 1))
-        else:
-            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
-                                     cls.c.edit_index == edit_index))
-        return cls._one_or_none(cls.db.execute(query))
+    def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']:
+        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
 
     @classmethod
     def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
@@ -86,12 +67,10 @@ class Message(Base):
                                             cls.c.tg_space == tg_space))
 
     @classmethod
-    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
-                       **values) -> None:
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
         with cls.db.begin() as conn:
             conn.execute(cls.t.update()
-                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
-                                     cls.c.edit_index == s_edit_index))
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
                          .values(**values))
 
     @classmethod
@@ -103,11 +82,9 @@ class Message(Base):
 
     @property
     def _edit_identity(self):
-        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
-                    self.c.edit_index == self.edit_index)
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)
 
     def insert(self) -> None:
         with self.db.begin() as conn:
             conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
-                                                tgid=self.tgid, tg_space=self.tg_space,
-                                                edit_index=self.edit_index))
+                                                tgid=self.tgid, tg_space=self.tg_space))

mautrix_telegram/db/puppet.py

@@ -35,16 +35,15 @@ class Puppet(Base):
     photo_id = Column(String, nullable=True)
     is_bot = Column(Boolean, nullable=True)
     matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
-    disable_updates = Column(Boolean, nullable=False, server_default=expression.false())
 
     @classmethod
     def scan(cls, row) -> Optional['Puppet']:
         (id, custom_mxid, access_token, displayname, displayname_source, username, photo_id,
-         is_bot, matrix_registered, disable_updates) = row
+         is_bot, matrix_registered) = row
         return cls(id=id, custom_mxid=custom_mxid, access_token=access_token,
                    displayname=displayname, displayname_source=displayname_source,
                    username=username, photo_id=photo_id, is_bot=is_bot,
-                   matrix_registered=matrix_registered, disable_updates=disable_updates)
+                   matrix_registered=matrix_registered)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Puppet']:
@@ -85,4 +84,4 @@ class Puppet(Base):
                 id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
                 displayname=self.displayname, displayname_source=self.displayname_source,
                 username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
-                matrix_registered=self.matrix_registered, disable_updates=self.disable_updates))
+                matrix_registered=self.matrix_registered))

mautrix_telegram/db/user.py

@@ -106,8 +106,8 @@ class User(Base):
 
     def delete(self) -> None:
         super().delete()
-        self.portals = []
-        self.contacts = []
+        self.portals = None
+        self.contacts = None
 
 
 class UserPortal(Base):

mautrix_telegram/formatter/__init__.py

@@ -1,8 +1,9 @@
 from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
                           init_mx)
-from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
+from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
 from .. import context as c
 
 
 def init(context: c.Context) -> None:
     init_mx(context)
+    init_tg(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -87,28 +87,25 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
 
 def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
                              room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
-    relates_to = content.get("m.relates_to", None) or {}
-    if not relates_to:
-        return None
-    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
-             else relates_to.get("m.in_reply_to", None) or {})
-    if not reply:
-        return None
-    room_id = room_id or reply.get("room_id", None)
-    event_id = reply.get("event_id", None)
-    if not event_id:
-        return
-
     try:
-        if content["format"] == "org.matrix.custom.html":
-            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
+        reply = content.get("m.relates_to", {}).get("m.in_reply_to", {})
+        if not reply:
+            return None
+        room_id = room_id or reply["room_id"]
+        event_id = reply["event_id"]
+
+        try:
+            if content["format"] == "org.matrix.custom.html":
+                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
+        except KeyError:
+            pass
+        content["body"] = trim_reply_fallback_text(content["body"])
+
+        message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+        if message:
+            return message.tgid
     except KeyError:
         pass
-    content["body"] = trim_reply_fallback_text(content["body"])
-
-    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
-    if message:
-        return message.tgid
     return None
 
 

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -29,10 +29,6 @@ class HTMLNode(list):
 
 
 class NodeifyingParser(HTMLParser):
-    # From https://www.w3.org/TR/html5/syntax.html#writing-html-documents-elements
-    void_tags = ("area", "base", "br", "col", "command", "embed", "hr", "img", "input", "link",
-                 "meta", "param", "source", "track", "wbr")
-
     def __init__(self):
         super().__init__()
         self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
@@ -40,11 +36,7 @@ class NodeifyingParser(HTMLParser):
     def handle_starttag(self, tag, attrs):
         node = HTMLNode(tag, attrs)
         self.stack[-1].append(node)
-        if tag not in self.void_tags:
-            self.stack.append(node)
-
-    def handle_startendtag(self, tag, attrs):
-        self.stack[-1].append(HTMLNode(tag, attrs))
+        self.stack.append(node)
 
     def handle_endtag(self, tag):
         if tag == self.stack[-1].tag:

mautrix_telegram/formatter/from_matrix/parser.py

@@ -18,15 +18,15 @@ from typing import List, Tuple, Pattern
 import re
 
 from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
-                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
-                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
+                               MessageEntityMentionName as MentionName, MessageEntityEmail as Email,
+                               MessageEntityUrl as URL, MessageEntityTextUrl as TextURL,
                                MessageEntityBold as Bold, MessageEntityItalic as Italic,
                                MessageEntityCode as Code, MessageEntityPre as Pre,
-                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
-                               MessageEntityBlockquote as Blockquote, TypeMessageEntity)
+                               TypeMessageEntity)
 
 from ... import user as u, puppet as pu, portal as po
 from ...types import MatrixUserID
+from ..util import html_to_unicode
 from .telegram_message import TelegramMessage, Entity, offset_length_multiply
 
 from .html_reader import HTMLNode, read_html
@@ -101,6 +101,13 @@ class MatrixParser:
             children.append(child)
         return TelegramMessage.join(children, "\n")
 
+    @classmethod
+    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")
+
     @classmethod
     def header_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         children = cls.node_to_tmessages(node, ctx)
@@ -115,14 +122,15 @@ class MatrixParser:
             msg.format(Bold)
         elif node.tag in ("i", "em"):
             msg.format(Italic)
-        elif node.tag in ("s", "strike", "del"):
-            msg.format(Strike)
-        elif node.tag in ("u", "ins"):
-            msg.format(Underline)
-        elif node == "blockquote":
-            msg.format(Blockquote)
         elif node.tag == "command":
             msg.format(Command)
+        elif node.tag in ("s", "strike", "del"):
+            msg.text = html_to_unicode(msg.text, "\u0336")
+        elif node.tag in ("u", "ins"):
+            msg.text = html_to_unicode(msg.text, "\u0332")
+
+        if node.tag in ("s", "strike", "del", "u", "ins"):
+            msg.entities = Entity.adjust(msg.entities, offset_length_multiply(2))
 
         return msg
 
@@ -161,17 +169,10 @@ class MatrixParser:
                 if msg.text == href
                 else msg.format(TextURL, url=href))
 
-    @classmethod
-    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        msg = cls.tag_aware_parse_node(node, ctx)
-        children = msg.trim().split("\n")
-        children = [child.prepend("> ") for child in children]
-        return TelegramMessage.join(children, "\n")
-
     @classmethod
     def node_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        if node.tag == "mx-reply":
-            return TelegramMessage("")
+        if node.tag == "blockquote":
+            return cls.blockquote_to_tmessage(node, ctx)
         elif node.tag == "ol":
             return cls.list_to_tmessage(node, ctx)
         elif node.tag == "ul":
@@ -182,11 +183,6 @@ class MatrixParser:
             return TelegramMessage("\n")
         elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
             return cls.basic_format_to_tmessage(node, ctx)
-        elif node.tag == "blockquote":
-            # Telegram already has blockquote entities in the protocol schema, but it strips them
-            # server-side and none of the official clients support them.
-            # TODO once Telegram changes that, use the above if block for blockquotes too.
-            return cls.blockquote_to_tmessage(node, ctx)
         elif node.tag == "a":
             return cls.link_to_tstring(node, ctx)
         elif node.tag == "p":

mautrix_telegram/formatter/from_telegram.py

@@ -24,8 +24,7 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, M
                                MessageEntityItalic, MessageEntityCode, MessageEntityPre,
                                MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
                                MessageEntityPhone, TypeMessageEntity, Message, PeerChannel,
-                               MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
-                               MessageEntityUnderline, PeerUser)
+                               MessageFwdHeader, PeerUser)
 
 from mautrix_appservice import MatrixRequestError
 from mautrix_appservice.intent_api import IntentAPI
@@ -34,12 +33,19 @@ from .. import user as u, puppet as pu, portal as po
 from ..types import TelegramID
 from ..db import Message as DBMessage
 from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text)
+                   trim_reply_fallback_text, unicode_to_html)
 
 if TYPE_CHECKING:
     from ..abstract_user import AbstractUser
+    from ..context import Context
+
+try:
+    from lxml.html.diff import htmldiff
+except ImportError:
+    htmldiff = None  # type: ignore
 
 log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
+should_highlight_edits = False  # type: bool
 
 
 def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
@@ -47,16 +53,13 @@ def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
         space = (evt.to_id.channel_id
                  if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
+        msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
         if msg:
             return {
                 "m.in_reply_to": {
                     "event_id": msg.mxid,
                     "room_id": msg.mx_room,
-                },
-                "rel_type": "m.reference",
-                "event_id": msg.mxid,
-                "room_id": msg.mx_room,
+                }
             }
     return {}
 
@@ -70,30 +73,27 @@ async def _add_forward_header(source, text: str, html: Optional[str],
         user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
-            fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
-                             f"{escape(fwd_from_text)}</a>")
+            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
 
         if not fwd_from_text:
             puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
-                fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                                 f"{escape(fwd_from_text)}</a>")
+                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
 
         if not fwd_from_text:
             user = await source.client.get_entity(PeerUser(fwd_from.from_id))
             if user:
                 fwd_from_text = pu.Puppet.get_displayname(user, False)
-                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+                fwd_from_html = f"<b>{fwd_from_text}</b>"
     else:
         portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
         if portal:
             fwd_from_text = portal.title
             if portal.alias:
-                fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
-                                 f"{escape(fwd_from_text)}</a>")
+                fwd_from_html = f"<a href='https://matrix.to/#/{portal.alias}'>{fwd_from_text}</a>"
             else:
-                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+                fwd_from_html = f"<b>{fwd_from_text}</b>"
         else:
             channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
             if channel:
@@ -114,19 +114,32 @@ async def _add_forward_header(source, text: str, html: Optional[str],
     return text, html
 
 
+def highlight_edits(new_html: str, old_html: str) -> str:
+    # Don't include `Edit:` text in diff.
+    if old_html.startswith("<u>Edit:</u> "):
+        old_html = old_html[len("<u>Edit:</u> "):]
+
+    # Generate diff with lxml
+    new_html = htmldiff(old_html, new_html)
+
+    # Replace <ins> with <u> since Riot doesn't allow <ins>
+    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
+    # Remove <del>s since we just want to hide deletions.
+    new_html = re.sub("<del>.+?</del>", "", new_html)
+    return new_html
+
+
 async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
-                            relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]:
+                            relates_to: Dict, main_intent: IntentAPI, is_edit: bool
+                            ) -> Tuple[str, str]:
     space = (evt.to_id.channel_id
              if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
+    msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
     if not msg:
         return text, html
 
-    relates_to["rel_type"] = "m.reference"
-    relates_to["event_id"] = msg.mxid
-    relates_to["room_id"] = msg.mx_room
     relates_to["m.in_reply_to"] = {
         "event_id": msg.mxid,
         "room_id": msg.mx_room,
@@ -145,14 +158,22 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
         puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
         r_displayname = puppet.displayname if puppet else r_sender
-        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{escape(r_displayname)}</a>"
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
+
+        if is_edit and should_highlight_edits:
+            html = highlight_edits(html or escape(text), r_html_body)
     except (ValueError, KeyError, MatrixRequestError):
         r_sender_link = "unknown user"
         r_displayname = "unknown user"
         r_text_body = "Failed to fetch message"
         r_html_body = "<em>Failed to fetch message</em>"
 
-    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>In reply to</a>"
+    if is_edit:
+        html = f"<u>Edit:</u> {html or escape(text)}"
+        text = f"Edit: {text}"
+
+    r_keyword = "In reply to" if not is_edit else "Edit to"
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
     html = (
         f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
         + (html or escape(text)))
@@ -169,8 +190,8 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
 async def telegram_to_matrix(evt: Message, source: "AbstractUser",
                              main_intent: Optional[IntentAPI] = None,
-                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
-                             override_text: str = None,
+                             is_edit: bool = False, prefix_text: Optional[str] = None,
+                             prefix_html: Optional[str] = None, override_text: str = None,
                              override_entities: List[TypeMessageEntity] = None,
                              no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
     text = add_surrogates(override_text or evt.message)
@@ -187,7 +208,8 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
         text, html = await _add_forward_header(source, text, html, evt.fwd_from)
 
     if evt.reply_to_msg_id and not no_reply_fallback:
-        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent)
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
+                                             is_edit)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
         if not html:
@@ -195,6 +217,9 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
         text += f"\n- {evt.post_author}"
         html += f"<br/><i>- <u>{evt.post_author}</u></i>"
 
+    html = unicode_to_html(text, html, "\u0336", "del")
+    html = unicode_to_html(text, html, "\u0332", "u")
+
     if html:
         html = html.replace("\n", "<br/>")
 
@@ -212,43 +237,29 @@ def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEnti
     return "[failed conversion in _telegram_entities_to_matrix]"
 
 
-def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity],
-                                 offset: int = 0, length: int = None) -> str:
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
     if not entities:
-        return escape(text)
-    if length is None:
-        length = len(text)
+        return text
     html = []
     last_offset = 0
-    for i, entity in enumerate(entities):
-        if entity.offset > offset + length:
-            break
-        relative_offset = entity.offset - offset
-        if relative_offset > last_offset:
-            html.append(escape(text[last_offset:relative_offset]))
-        elif relative_offset < last_offset:
+    for entity in entities:
+        if entity.offset > last_offset:
+            html.append(escape(text[last_offset:entity.offset]))
+        elif entity.offset < last_offset:
             continue
 
         skip_entity = False
-        entity_text = _telegram_entities_to_matrix(
-            text=text[relative_offset:relative_offset + entity.length],
-            entities=entities[i + 1:], offset=entity.offset, length=entity.length)
+        entity_text = escape(text[entity.offset:entity.offset + entity.length])
         entity_type = type(entity)
 
         if entity_type == MessageEntityBold:
             html.append(f"<strong>{entity_text}</strong>")
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
-        elif entity_type == MessageEntityUnderline:
-            html.append(f"<u>{entity_text}</u>")
-        elif entity_type == MessageEntityStrike:
-            html.append(f"<del>{entity_text}</del>")
-        elif entity_type == MessageEntityBlockquote:
-            html.append(f"<blockquote>{entity_text}</blockquote>")
         elif entity_type == MessageEntityCode:
-            html.append(f"<pre><code>{entity_text}</code></pre>"
-                        if "\n" in entity_text
-                        else f"<code>{entity_text}</code>")
+            html.append(("<pre><code>{entity_text}</code></pre>"
+                         if "\n" in entity_text
+                         else "<code>{entity_text}</code>").format(entity_text=entity_text))
         elif entity_type == MessageEntityPre:
             skip_entity = _parse_pre(html, entity_text, entity.language)
         elif entity_type == MessageEntityMention:
@@ -266,8 +277,8 @@ def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity],
             html.append(f"<font color='blue'>{entity_text}</font>")
         else:
             skip_entity = True
-        last_offset = relative_offset + (0 if skip_entity else entity.length)
-    html.append(escape(text[last_offset:]))
+        last_offset = entity.offset + (0 if skip_entity else entity.length)
+    html.append(text[last_offset:])
 
     return "".join(html)
 
@@ -329,9 +340,14 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
 
         portal = po.Portal.find_by_username(group)
         if portal:
-            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
+            message = DBMessage.get_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 
     html.append(f"<a href='{url}'>{entity_text}</a>")
     return False
+
+
+def init_tg(context: "Context") -> None:
+    global should_highlight_edits
+    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/formatter/util.py

@@ -20,6 +20,38 @@ import struct
 import re
 
 
+def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
+    if ctrl not in text:
+        return html
+    if not html:
+        html = escape(text)
+    tag_start = f"<{tag}>"
+    tag_end = f"</{tag}>"
+    characters = html.split(ctrl)
+    html = ""
+    in_tag = False
+    for char in characters:
+        if not in_tag:
+            if len(char) > 1:
+                html += char[0:-1]
+                char = char[-1]
+            html += tag_start
+            in_tag = True
+            html += char
+        else:
+            if len(char) > 1:
+                html += tag_end
+                in_tag = False
+            html += char
+    if in_tag:
+        html += tag_end
+    return html
+
+
+def html_to_unicode(text: str, ctrl: str) -> str:
+    return ctrl.join(text) + ctrl
+
+
 # add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
 # Licensed under the MIT license.
 # https://github.com/LonamiWebs/Telethon/blob/7cce7aa3e4c6c7019a55530391b1761d33e5a04e/telethon/helpers.py

mautrix_telegram/matrix.py

@@ -17,7 +17,6 @@
 from typing import Dict, List, Match, Optional, Set, Tuple, TYPE_CHECKING
 import logging
 import asyncio
-import time
 import re
 
 from mautrix_appservice import MatrixRequestError, IntentError
@@ -28,15 +27,6 @@ from . import user as u, portal as po, puppet as pu, commands as com
 if TYPE_CHECKING:
     from .context import Context
 
-try:
-    from prometheus_client import Histogram
-
-    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events",
-                           ["event_type"])
-except ImportError:
-    Histogram = None
-    EVENT_TIME = None
-
 
 class MatrixHandler:
     log = logging.getLogger("mau.mx")  # type: logging.Logger
@@ -136,31 +126,12 @@ class MatrixHandler:
 
         if not inviter.whitelisted:
             await self.az.intent.send_notice(
-                room_id,
-                text="You are not whitelisted to use this bridge.\n\n"
+                room_id, text="",
+                html="You are not whitelisted to use this bridge.<br/><br/>"
                      "If you are the owner of this bridge, see the "
-                     "`bridge.permissions` section in your config file.",
-                html="<p>You are not whitelisted to use this bridge.</p>"
-                     "<p>If you are the owner of this bridge, see the "
-                     "<code>bridge.permissions</code> section in your config file.</p>")
+                     "<code>bridge.permissions</code> section in your config file.")
             await self.az.intent.leave_room(room_id)
 
-        try:
-            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
-        except MatrixRequestError:
-            is_management = False
-        cmd_prefix = self.commands.command_prefix
-        text = html = "Hello, I'm a Telegram bridge bot. "
-        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
-            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
-            html += (f"Use <code>{cmd_prefix} help</code> for help"
-                     f" or <code>{cmd_prefix} login</code> to log in.")
-            pass
-        else:
-            text += f"Use `{cmd_prefix} help` for help."
-            html += f"Use <code>{cmd_prefix} help</code> for help."
-        await self.az.intent.send_notice(room_id, text=text, html=html)
-
     async def handle_invite(self, room_id: MatrixRoomID, user_id: MatrixUserID,
                             inviter_mxid: MatrixUserID) -> None:
         self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
@@ -242,7 +213,7 @@ class MatrixHandler:
         prefix = self.config["bridge.command_prefix"]
         is_command = text.startswith(prefix)
         if is_command:
-            text = text[len(prefix) + 1:].lstrip()
+            text = text[len(prefix) + 1:]
         return is_command, text
 
     async def handle_message(self, room: MatrixRoomID, sender_id: MatrixUserID, message: Dict,
@@ -399,79 +370,63 @@ class MatrixHandler:
         return (sender == self.az.bot_mxid
                 or pu.Puppet.get_id_from_mxid(sender) is not None)
 
-    async def try_handle_ephemeral_event(self, evt: MatrixEvent) -> None:
+    async def try_handle_event(self, evt: MatrixEvent) -> None:
         try:
-            await self.handle_ephemeral_event(evt)
+            await self.handle_event(evt)
         except Exception:
             self.log.exception("Error handling manually received Matrix event")
 
-    async def handle_ephemeral_event(self, evt: MatrixEvent) -> None:
-        evt_type = evt.get("type", "m.unknown")  # type: str
-        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
-        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
-        content = evt.get("content", {})  # type: Dict
-        if evt_type == "m.receipt":
-            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
-        elif evt_type == "m.presence":
-            await self.handle_presence(sender, content.get("presence", "offline"))
-        elif evt_type == "m.typing":
-            await self.handle_typing(room_id, content.get("user_ids", []))
-
     async def handle_event(self, evt: MatrixEvent) -> None:
         if self.filter_matrix_event(evt):
             return
-        start_time = time.time()
         self.log.debug("Received event: %s", evt)
         evt_type = evt.get("type", "m.unknown")  # type: str
         room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
         event_id = evt.get("event_id", None)  # type: Optional[MatrixEventID]
         sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
-        state_key = evt.get("state_key", None)
         content = evt.get("content", {})  # type: Dict
-        if state_key is not None:
-            if evt_type == "m.room.member":
-                prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
-                membership = content.get("membership", "")  # type: str
-                prev_membership = prev_content.get("membership", "leave")  # type: str
-                if membership == prev_membership:
-                    match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
-                    mxid = match.group(0)  # type: str
-                    displayname = content.get("displayname", None) or mxid  # type: str
-                    prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
-                    if displayname != prev_displayname:
-                        await self.handle_name_change(room_id, state_key, displayname,
-                                                      prev_displayname, event_id)
-                elif membership == "invite":
-                    await self.handle_invite(room_id, state_key, sender)
-                elif prev_membership == "join" and membership == "leave":
-                    await self.handle_part(room_id, state_key, sender, event_id)
-                elif membership == "join":
-                    await self.handle_join(room_id, state_key, event_id)
-            elif evt_type == "m.room.power_levels":
-                prev_content = evt.get("unsigned", {}).get("prev_content", {})
-                await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
-            elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
-                await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
-            elif evt_type == "m.room.pinned_events":
-                new_events = set(evt["content"]["pinned"])
-                try:
-                    old_events = set(evt["unsigned"]["prev_content"]["pinned"])
-                except KeyError:
-                    old_events = set()
-                await self.handle_room_pin(room_id, sender, new_events, old_events)
-            elif evt_type == "m.room.tombstone":
-                await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
-            else:
-                return
-        else:
-            if evt_type in ("m.room.message", "m.sticker"):
-                if evt_type != "m.room.message":
-                    content["msgtype"] = evt_type
-                await self.handle_message(room_id, sender, content, event_id)
-            elif evt_type == "m.room.redaction":
-                await self.handle_redaction(room_id, sender, evt["redacts"])
-            else:
-                return
-
-        if EVENT_TIME:
-            EVENT_TIME.labels(event_type=evt_type).observe(time.time() - start_time)
+        if evt_type == "m.room.member":
+            state_key = evt["state_key"]  # type: MatrixUserID
+            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
+            membership = content.get("membership", "")  # type: str
+            prev_membership = prev_content.get("membership", "leave")  # type: str
+            if membership == prev_membership:
+                match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
+                mxid = match.group(0)  # type: str
+                displayname = content.get("displayname", None) or mxid  # type: str
+                prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
+                if displayname != prev_displayname:
+                    await self.handle_name_change(room_id, state_key, displayname,
+                                                  prev_displayname, event_id)
+            elif membership == "invite":
+                await self.handle_invite(room_id, state_key, sender)
+            elif prev_membership == "join" and membership == "leave":
+                await self.handle_part(room_id, state_key, sender, event_id)
+            elif membership == "join":
+                await self.handle_join(room_id, state_key, event_id)
+        elif evt_type in ("m.room.message", "m.sticker"):
+            if evt_type != "m.room.message":
+                content["msgtype"] = evt_type
+            await self.handle_message(room_id, sender, content, event_id)
+        elif evt_type == "m.room.redaction":
+            await self.handle_redaction(room_id, sender, evt["redacts"])
+        elif evt_type == "m.room.power_levels":
+            prev_content = evt.get("unsigned", {}).get("prev_content", {})
+            await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
+        elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
+            await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
+        elif evt_type == "m.room.pinned_events":
+            new_events = set(evt["content"]["pinned"])
+            try:
+                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+            except KeyError:
+                old_events = set()
+            await self.handle_room_pin(room_id, sender, new_events, old_events)
+        elif evt_type == "m.room.tombstone":
+            await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
+        elif evt_type == "m.receipt":
+            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
+        elif evt_type == "m.presence":
+            await self.handle_presence(sender, content.get("presence", "offline"))
+        elif evt_type == "m.typing":
+            await self.handle_typing(room_id, content.get("user_ids", []))

mautrix_telegram/portal.py

@@ -43,33 +43,31 @@ from telethon.tl.functions.channels import (
     UpdateUsernameRequest)
 from telethon.tl.functions.messages import ReadHistoryRequest as ReadMessageHistoryRequest
 from telethon.tl.functions.channels import ReadHistoryRequest as ReadChannelHistoryRequest
-from telethon.errors import (ChatAdminRequiredError, ChatNotModifiedError, PhotoExtInvalidError,
-                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError)
+from telethon.errors import ChatAdminRequiredError, ChatNotModifiedError
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
-    Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin, Document,
-    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat,
-    ChatFull, ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll,
-    DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker, PhotoEmpty,
-    DocumentAttributeVideo, GeoPoint, InputChannel, InputChatUploadedPhoto, InputPhotoFileLocation,
-    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf, MessageMediaPoll,
-    MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate, ChatPhotoEmpty,
+    Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin,
+    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat, ChatFull,
+    ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll, PollAnswer,
+    DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker,
+    DocumentAttributeVideo, FileLocation, GeoPoint, InputChannel, InputChatUploadedPhoto,
+    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf,
+    MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate,
     MessageActionChatDeletePhoto, MessageActionChatDeleteUser, MessageActionChatEditPhoto,
     MessageActionChatEditTitle, MessageActionChatJoinedByLink, MessageActionChatMigrateTo,
     MessageActionPinMessage, MessageActionGameScore, MessageMediaContact, MessageMediaDocument,
-    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame,
+    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, MessageMediaPoll,
     PeerChannel, PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction,
     SendMessageTypingAction, TypeChannelParticipant, TypeChat, TypeChatParticipant,
     TypeDocumentAttribute, TypeInputPeer, TypeMessageAction, TypeMessageEntity, TypePeer,
     TypePhotoSize, TypeUpdates, TypeUser, PhotoSize, TypeUserFull, UpdateChatUserTyping,
-    UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping, User, UserFull, MessageEntityPre,
-    InputMediaUploadedDocument, InputPeerPhotoFileLocation)
+    UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping, User, UserFull, MessageEntityPre)
 from mautrix_appservice import MatrixRequestError, IntentError, AppService, IntentAPI
 
 from .types import MatrixEventID, MatrixRoomID, MatrixUserID, TelegramID
 from .context import Context
 from .db import Portal as DBPortal, Message as DBMessage, TelegramFile as DBTelegramFile
-from .util import ignore_coro, sane_mimetypes
+from .util import ignore_coro
 from . import puppet as p, user as u, formatter, util
 
 if TYPE_CHECKING:
@@ -78,6 +76,8 @@ if TYPE_CHECKING:
     from .config import Config
     from .tgclient import MautrixTelegramClient
 
+mimetypes.init()
+
 config = None  # type: Config
 
 TypeMessage = Union[Message, MessageService]
@@ -346,10 +346,7 @@ class Portal:
                 await self.invite_to_matrix(invites or [])
             return self.mxid
         async with self._room_create_lock:
-            try:
-                return await self._create_matrix_room(user, entity, invites)
-            except Exception:
-                self.log.exception("Fatal error creating Matrix room")
+            return await self._create_matrix_room(user, entity, invites)
 
     async def _create_matrix_room(self, user: 'AbstractUser', entity: TypeChat, invites: InviteList
                                   ) -> Optional[MatrixRoomID]:
@@ -400,11 +397,6 @@ class Portal:
             "type": "m.room.power_levels",
             "content": power_levels,
         }]
-        if config["appservice.community_id"]:
-            initial_state.append({
-                "type": "m.room.related_groups",
-                "content": {"groups": [config["appservice.community_id"]]},
-            })
 
         room_id = await self.main_intent.create_room(alias=alias, is_public=public,
                                                      is_direct=direct, invitees=invites or [],
@@ -581,7 +573,7 @@ class Portal:
         changed = await self.update_title(entity.title) or changed
 
         if isinstance(entity.photo, ChatPhoto):
-            changed = await self.update_avatar(user, entity.photo) or changed
+            changed = await self.update_avatar(user, entity.photo.photo_big) or changed
 
         if changed:
             self.save()
@@ -622,23 +614,12 @@ class Portal:
         return False
 
     @staticmethod
-    def _get_largest_photo_size(photo: Union[Photo, Document]
-                                ) -> Tuple[Optional[InputPhotoFileLocation],
-                                           Optional[TypePhotoSize]]:
+    def _get_largest_photo_size(photo: Union[Photo, List[TypePhotoSize]]
+                                ) -> Optional[TypePhotoSize]:
         if not photo:
-            return None, None
-        if isinstance(photo, Document) and not photo.thumbs:
-            return None, None
-        largest = max(photo.sizes if isinstance(photo, Photo) else photo.thumbs,
-                      key=(lambda photo2: (len(photo2.bytes)
-                                           if not isinstance(photo2, PhotoSize)
-                                           else photo2.size)))
-        return InputPhotoFileLocation(
-            id=photo.id,
-            access_hash=photo.access_hash,
-            file_reference=photo.file_reference,
-            thumb_size=largest.type,
-        ), largest
+            return None
+        return max(photo.sizes if isinstance(photo, Photo) else photo, key=(lambda photo2: (
+            len(photo2.bytes) if not isinstance(photo2, PhotoSize) else photo2.size)))
 
     async def remove_avatar(self, _: 'AbstractUser', save: bool = False) -> None:
         await self.main_intent.set_room_avatar(self.mxid, None)
@@ -646,33 +627,11 @@ class Portal:
         if save:
             self.save()
 
-    async def update_avatar(self, user: 'AbstractUser',
-                            photo: Union[ChatPhoto, ChatPhotoEmpty, Photo, PhotoEmpty],
+    async def update_avatar(self, user: 'AbstractUser', photo: FileLocation,
                             save: bool = False) -> bool:
-        if isinstance(photo, ChatPhoto):
-            loc = InputPeerPhotoFileLocation(
-                peer=await self.get_input_entity(user),
-                local_id=photo.photo_big.local_id,
-                volume_id=photo.photo_big.volume_id,
-                big=True
-            )
-            photo_id = f"{loc.volume_id}-{loc.local_id}"
-        elif isinstance(photo, Photo):
-            loc, largest = self._get_largest_photo_size(photo)
-            photo_id = f"{largest.location.volume_id}-{largest.location.local_id}"
-        elif isinstance(photo, (ChatPhotoEmpty, PhotoEmpty)):
-            photo_id = ""
-            loc = None
-        else:
-            raise ValueError(f"Unknown photo type {type(photo)}")
+        photo_id = f"{photo.volume_id}-{photo.local_id}"
         if self.photo_id != photo_id:
-            if not photo_id:
-                await self.main_intent.set_room_avatar(self.mxid, "")
-                self.photo_id = ""
-                if save:
-                    self.save()
-                return True
-            file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
+            file = await util.transfer_file_to_matrix(user.client, self.main_intent, photo)
             if file:
                 await self.main_intent.set_room_avatar(self.mxid, file.mxc)
                 self.photo_id = photo_id
@@ -791,9 +750,14 @@ class Portal:
                 return body + current_extension
         except (ValueError, KeyError):
             pass
+        ext_override = {
+            "image/jpeg": ".jpg"
+        }
         if mime:
-            return f"matrix_upload{sane_mimetypes.guess_extension(mime)}"
-        return ""
+            ext = ext_override.get(mime, mimetypes.guess_extension(mime))
+            return f"matrix_upload{ext}"
+        else:
+            return ""
 
     def get_config(self, key: str) -> Any:
         local = util.recursive_get(self.local_config, key)
@@ -935,7 +899,7 @@ class Portal:
                                 ) -> None:
         if "formatted_body" not in message:
             message["format"] = "org.matrix.custom.html"
-            message["formatted_body"] = escape_html(message.get("body", "")).replace("\n", "<br/>")
+            message["formatted_body"] = escape_html(message.get("body", ""))
         body = message["formatted_body"]
 
         tpl = (self.get_config(f"message_formats.[{msgtype}]")
@@ -952,14 +916,9 @@ class Portal:
         msgtype = message.get("msgtype", "m.text")
         if msgtype == "m.emote":
             await self._apply_msg_format(sender, msgtype, message)
-            if "m.new_content" in message:
-                await self._apply_msg_format(sender, msgtype, message["m.new_content"])
-                message["m.new_content"]["msgtype"] = "m.text"
             message["msgtype"] = "m.text"
         elif use_relaybot:
             await self._apply_msg_format(sender, msgtype, message)
-            if "m.new_content" in message:
-                await self._apply_msg_format(sender, msgtype, message["m.new_content"])
 
     @staticmethod
     def _matrix_event_to_entities(event: Dict[str, Any]
@@ -991,26 +950,15 @@ class Portal:
             return None
 
     async def _handle_matrix_text(self, sender_id: TelegramID, event_id: MatrixEventID,
-                                  space: TelegramID, client: 'MautrixTelegramClient',
-                                  message: Dict, reply_to: TelegramID) -> None:
+                                  space: TelegramID, client: 'MautrixTelegramClient', message: Dict,
+                                  reply_to: TelegramID) -> None:
         lock = self.require_send_lock(sender_id)
         async with lock:
             lp = self.get_config("telegram_link_preview")
-            relates_to = message.get("m.relates_to", None) or {}
-            if relates_to.get("rel_type", None) == "m.replace":
-                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
-                if orig_msg and "m.new_content" in message:
-                    message = message["m.new_content"]
-                    formatter.matrix_reply_to_telegram(message, space, room_id=self.mxid)
-                    response = await client.edit_message(self.peer, orig_msg.tgid, message,
-                                                         parse_mode=self._matrix_event_to_entities,
-                                                         link_preview=lp)
-                    self._add_telegram_message_to_db(event_id, space, -1, response)
-                    return
             response = await client.send_message(self.peer, message, reply_to=reply_to,
                                                  parse_mode=self._matrix_event_to_entities,
                                                  link_preview=lp)
-            self._add_telegram_message_to_db(event_id, space, 0, response)
+            self._add_telegram_message_to_db(event_id, space, response)
 
     async def _handle_matrix_file(self, msgtype: str, sender_id: TelegramID,
                                   event_id: MatrixEventID, space: TelegramID,
@@ -1045,23 +993,9 @@ class Portal:
             max_image_size=config["bridge.image_as_file_size"] * 1000 ** 2)
         lock = self.require_send_lock(sender_id)
         async with lock:
-            relates_to = message.get("m.relates_to", None) or {}
-            if relates_to.get("rel_type", None) == "m.replace":
-                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
-                if orig_msg:
-                    response = await client.edit_message(self.peer, orig_msg.tgid,
-                                                         caption, file=media)
-                    self._add_telegram_message_to_db(event_id, space, -1, response)
-                    return
-            try:
-                response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption)
-            except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
-                media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
-                                                   attributes=attributes)
-                response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                                   caption=caption)
-            self._add_telegram_message_to_db(event_id, space, 0, response)
+            response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                               caption=caption)
+            self._add_telegram_message_to_db(event_id, space, response)
 
     async def _handle_matrix_location(self, sender_id: TelegramID, event_id: MatrixEventID,
                                       space: TelegramID, client: 'MautrixTelegramClient',
@@ -1077,31 +1011,19 @@ class Portal:
 
         lock = self.require_send_lock(sender_id)
         async with lock:
-            relates_to = message.get("m.relates_to", None) or {}
-            if relates_to.get("rel_type", None) == "m.replace":
-                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
-                if orig_msg:
-                    response = await client.edit_message(self.peer, orig_msg.tgid,
-                                                         caption, file=media)
-                    self._add_telegram_message_to_db(event_id, space, -1, response)
-                    return
             response = await client.send_media(self.peer, media, reply_to=reply_to,
                                                caption=caption, entities=entities)
-            self._add_telegram_message_to_db(event_id, space, 0, response)
+            self._add_telegram_message_to_db(event_id, space, response)
 
     def _add_telegram_message_to_db(self, event_id: MatrixEventID, space: TelegramID,
-                                    edit_index: int, response: TypeMessage) -> None:
+                                    response: TypeMessage) -> None:
         self.log.debug("Handled Matrix message: %s", response)
-        self.is_duplicate(response, (event_id, space), force_hash=edit_index != 0)
-        if edit_index < 0:
-            prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1)
-            edit_index = prev_edit.edit_index + 1
+        self.is_duplicate(response, (event_id, space))
         DBMessage(
             tgid=TelegramID(response.id),
             tg_space=space,
             mx_room=self.mxid,
-            mxid=event_id,
-            edit_index=edit_index).insert()
+            mxid=event_id).insert()
 
     async def handle_matrix_message(self, sender: 'u.User', message: Dict[str, Any],
                                     event_id: MatrixEventID) -> None:
@@ -1165,10 +1087,7 @@ class Portal:
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
             return
-        if message.edit_index == 0:
-            await real_deleter.client.delete_messages(self.peer, [message.tgid])
-        else:
-            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
+        await real_deleter.client.delete_messages(self.peer, [message.tgid])
 
     async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
                                            level: int) -> None:
@@ -1232,7 +1151,7 @@ class Portal:
 
         file = await self.main_intent.download_file(url)
         mime = magic.from_buffer(file, mime=True)
-        ext = sane_mimetypes.guess_extension(mime)
+        ext = mimetypes.guess_extension(mime)
         uploaded = await sender.client.upload_file(file, file_name=f"avatar{ext}", use_cache=False)
         photo = InputChatUploadedPhoto(file=uploaded)
 
@@ -1247,8 +1166,8 @@ class Portal:
                                and isinstance(update.message, MessageService)
                                and isinstance(update.message.action, MessageActionChatEditPhoto))
             if is_photo_update:
-                loc, size = self._get_largest_photo_size(update.message.action.photo)
-                self.photo_id = f"{size.location.volume_id}-{size.location.local_id}"
+                loc = self._get_largest_photo_size(update.message.action.photo).location
+                self.photo_id = f"{loc.volume_id}-{loc.local_id}"
                 self.save()
                 break
 
@@ -1339,13 +1258,8 @@ class Portal:
 
         invites = await self._get_telegram_users_in_matrix_room()
         if len(invites) < 2:
-            if self.bot is not None:
-                info, mxid = await self.bot.get_me()
-                raise ValueError("Not enough Telegram users to create a chat. "
-                                 "Invite more Telegram ghost users to the room, such as the "
-                                 f"relaybot ([{info.first_name}](https://matrix.to/#/{mxid})).")
             raise ValueError("Not enough Telegram users to create a chat. "
-                             "Invite more Telegram ghost users to the room.")
+                             "Invite more Telegram ghost users to the room, such as the relaybot.")
         if self.peer_type == "chat":
             response = await source.client(CreateChatRequest(title=self.title, users=invites))
             entity = response.chats[0]
@@ -1397,14 +1311,12 @@ class Portal:
     def get_external_url(self, evt: Message) -> Optional[str]:
         if self.peer_type == "channel" and self.username is not None:
             return f"https://t.me/{self.username}/{evt.id}"
-        elif self.peer_type != "user":
-            return f"https://t.me/c/{self.tgid}/{evt.id}"
         return None
 
     async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                     relates_to: Dict = None) -> Optional[Dict]:
-        loc, largest_size = self._get_largest_photo_size(evt.media.photo)
-        file = await util.transfer_file_to_matrix(source.client, intent, loc)
+        largest_size = self._get_largest_photo_size(evt.media.photo)
+        file = await util.transfer_file_to_matrix(source.client, intent, largest_size.location)
         if not file:
             return None
         if self.get_config("inline_images") and (evt.message
@@ -1425,7 +1337,10 @@ class Portal:
             "orientation": 0,
             "mimetype": file.mime_type,
         }
-        name = f"image{sane_mimetypes.guess_extension(file.mime_type)}"
+        ext_override = {
+            "image/jpeg": ".jpg"
+        }
+        name = "image" + ext_override.get(file.mime_type, mimetypes.guess_extension(file.mime_type))
         await intent.set_typing(self.mxid, is_typing=False)
         result = await intent.send_image(self.mxid, file.mxc, info=info, text=name,
                                          relates_to=relates_to, timestamp=evt.date,
@@ -1460,7 +1375,7 @@ class Portal:
 
     @staticmethod
     def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: Dict,
-                                      thumb_size: TypePhotoSize) -> Tuple[Dict, str]:
+                                      thumb: TypePhotoSize) -> Tuple[Dict, str]:
         document = evt.media.document
         name = evt.message or attrs["name"]
         if attrs["is_sticker"]:
@@ -1471,11 +1386,7 @@ class Portal:
                 except ValueError:
                     name = alt
 
-        generic_types = ("text/plain", "application/octet-stream")
-        if file.mime_type in generic_types and document.mime_type not in generic_types:
-            mime_type = document.mime_type or file.mime_type
-        else:
-            mime_type = file.mime_type or document.mime_type
+        mime_type = document.mime_type or file.mime_type
         info = {
             "size": file.size,
             "mimetype": mime_type,
@@ -1492,8 +1403,8 @@ class Portal:
             info["thumbnail_url"] = file.thumbnail.mxc
             info["thumbnail_info"] = {
                 "mimetype": file.thumbnail.mime_type,
-                "h": file.thumbnail.height or thumb_size.h,
-                "w": file.thumbnail.width or thumb_size.w,
+                "h": file.thumbnail.height or thumb.h,
+                "w": file.thumbnail.width or thumb.w,
                 "size": file.thumbnail.size,
             }
 
@@ -1502,25 +1413,18 @@ class Portal:
     async def handle_telegram_document(self, source: 'AbstractUser', intent: IntentAPI,
                                        evt: Message, relates_to: dict = None) -> Optional[Dict]:
         document = evt.media.document
-
         attrs = self._parse_telegram_document_attributes(document.attributes)
 
-        if document.size > config["bridge.max_document_size"] * 1000 ** 2:
-            name = attrs["name"] or ""
-            caption = f"\n{evt.message}" if evt.message else ""
-            return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
-
-        thumb_loc, thumb_size = self._get_largest_photo_size(document)
-        if thumb_size and not isinstance(thumb_size, (PhotoSize, PhotoCachedSize)):
-            self.log.debug(f"Unsupported thumbnail type {type(thumb_size)}")
-            thumb_loc = None
-            thumb_size = None
-        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
+        thumb = self._get_largest_photo_size(document.thumbs)
+        if thumb and not isinstance(thumb, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb)}")
+            thumb = None
+        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb,
                                                   is_sticker=attrs["is_sticker"])
         if not file:
             return None
 
-        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb_size)
+        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb)
 
         await intent.set_typing(self.mxid, is_typing=False)
 
@@ -1587,7 +1491,7 @@ class Portal:
                                       external_url=self.get_external_url(evt))
 
     async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
-                                          evt: Message, relates_to: dict = None) -> dict:
+                                          evt: Message, _: dict = None) -> dict:
         override_text = ("This message is not supported on your version of Mautrix-Telegram. "
                          "Please check https://github.com/tulir/mautrix-telegram or ask your "
                          "bridge administrator about possible updates.")
@@ -1608,24 +1512,20 @@ class Portal:
         poll = evt.media.poll  # type: Poll
         poll_id = self._encode_msgid(source, evt)
 
-        _n = 0
+        def enc(answer: PollAnswer) -> str:
+            return base64.b64encode(answer.option).decode("utf-8").rstrip("=")
 
-        def n() -> int:
-            nonlocal _n
-            _n += 1
-            return _n
-
-        text = (f"Poll: {poll.question}\n"
-                + "\n".join(f"{n()}. {answer.text}" for answer in poll.answers) +
+        text = (f"Poll ID {poll_id}: {poll.question}\n"
+                + "\n".join(f"* {enc(answer)}: {answer.text}" for answer in poll.answers) +
                 "\n"
-                f"Vote with !tg vote {poll_id} <choice number>")
+                f"Vote with !tg vote <poll ID> <choice ID>")
 
-        html = (f"<strong>Poll</strong>: {poll.question}<br/>\n"
-                f"<ol>"
-                + "\n".join(f"<li>{answer.text}</li>"
+        html = (f"<strong>Poll</strong> ID <code>{poll_id}</code>: {poll.question}<br/>\n"
+                f"<ul>"
+                + "\n".join(f"<li><code>{enc(answer)}</code>: {answer.text}</li>"
                             for answer in poll.answers) +
-                "</ol>\n"
-                f"Vote with <code>!tg vote {poll_id} &lt;choice number&gt;</code>")
+                "</ul>\n"
+                f"Vote with <code>!tg vote &lt;poll ID&gt; &lt;choice ID&gt;</code>")
         await intent.set_typing(self.mxid, is_typing=False)
         return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
                                       msgtype="m.text", timestamp=evt.date,
@@ -1660,8 +1560,7 @@ class Portal:
         play_id = self._encode_msgid(source, evt)
         command = f"!tg play {play_id}"
         override_text = f"Run {command} in your bridge management room to play {game.title}"
-        override_entities = [
-            MessageEntityPre(offset=len("Run "), length=len(command), language="")]
+        override_entities = [MessageEntityPre(offset=len("Run "), length=len(command), language="")]
         text, html, relates_to = await formatter.telegram_to_matrix(
             evt, source, self.main_intent,
             override_text=override_text, override_entities=override_entities)
@@ -1679,6 +1578,9 @@ class Portal:
                                    evt: Message) -> None:
         if not self.mxid:
             return
+        elif not self.get_config("edits_as_replies"):
+            self.log.debug("Edits as replies disabled, ignoring edit event...")
+            return
         elif hasattr(evt, "media") and isinstance(evt.media, (MessageMediaGame,)):
             self.log.debug("Ignoring game message edit event")
             return
@@ -1696,52 +1598,28 @@ class Portal:
         if duplicate_found:
             mxid, other_tg_space = duplicate_found
             if tg_space != other_tg_space:
-                prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1)
-                if not prev_edit_msg:
-                    return
-                DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
-                          edit_index=prev_edit_msg.edit_index + 1).insert()
+                DBMessage.update_by_tgid(TelegramID(evt.id), tg_space,
+                                         mxid=mxid,
+                                         mx_room=self.mxid)
             return
 
-        text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent,
-                                                           no_reply_fallback=True)
-        editing_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
-        if not editing_msg:
-            self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) "
-                          "in database.")
-            return
-
-        msgtype = ("m.notice"
-                   if sender and sender.is_bot and self.get_config("bot_messages_as_notices")
-                   else "m.text")
-        content = {
-            "body": f"Edit: {text}",
-            "msgtype": msgtype,
-            "format": "org.matrix.custom.html",
-            "formatted_body": (f"<a href='https://matrix.to/#/{editing_msg.mx_room}/"
-                               f"{editing_msg.mxid}'>Edit</a>: "
-                               f"{html or escape_html(text)}"),
-            "external_url": self.get_external_url(evt),
-            "m.new_content": {
-                "body": text,
-                "msgtype": "m.text",
-                **({"format": "org.matrix.custom.html",
-                    "formatted_body": html} if html else {}),
-            },
-            "m.relates_to": {
-                "rel_type": "m.replace",
-                "event_id": editing_msg.mxid,
-            },
-        }
-
+        evt.reply_to_msg_id = evt.id
+        text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                                    is_edit=True)
         intent = sender.intent if sender else self.main_intent
         await intent.set_typing(self.mxid, is_typing=False)
-        response = await intent.send_message(self.mxid, content)
+        response = await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
+                                          external_url=self.get_external_url(evt))
+
         mxid = response["event_id"]
 
-        prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
-        DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
-                  edit_index=prev_edit_msg.edit_index + 1).insert()
+        msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
+        if not msg:
+            self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) "
+                          "in database.")
+            # Oh crap
+            return
+        msg.update(mxid=mxid, mx_room=self.mxid)
         DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
 
     async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
@@ -1765,11 +1643,11 @@ class Portal:
                            f"as it was already handled (in space {other_tg_space})")
             if tg_space != other_tg_space:
                 DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                          tg_space=tg_space, edit_index=0).insert()
+                          tg_space=tg_space).insert()
             return
 
         if self.dedup_pre_db_check and self.peer_type == "channel":
-            msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
+            msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
             if msg:
                 self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
                                f"handled into {msg.mxid}. This duplicate was catched in the db "
@@ -1778,13 +1656,13 @@ class Portal:
                 return
 
         if sender and not sender.displayname:
-            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a "
+            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a"
                            "displayname, updating info...")
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
 
-        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
-                         MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
+        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo, MessageMediaGame,
+                         MessageMediaPoll, MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
         intent = sender.intent if sender else self.main_intent
@@ -1824,7 +1702,7 @@ class Portal:
         self.log.debug("Handled Telegram message: %s", evt)
         try:
             DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                      tg_space=tg_space, edit_index=0).insert()
+                      tg_space=tg_space).insert()
             DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
         except IntegrityError as e:
             self.log.exception(f"{e.__class__.__name__} while saving message mapping. "
@@ -1857,7 +1735,8 @@ class Portal:
         if isinstance(action, MessageActionChatEditTitle):
             await self.update_title(action.title, save=True)
         elif isinstance(action, MessageActionChatEditPhoto):
-            await self.update_avatar(source, action.photo, save=True)
+            largest_size = self._get_largest_photo_size(action.photo)
+            await self.update_avatar(source, largest_size.location, save=True)
         elif isinstance(action, MessageActionChatDeletePhoto):
             await self.remove_avatar(source, save=True)
         elif isinstance(action, MessageActionChatAddUser):
@@ -1902,7 +1781,7 @@ class Portal:
         self._temp_pinned_message_id = None
         self._temp_pinned_message_sender = None
 
-        message = DBMessage.get_one_by_tgid(msg_id, self._temp_pinned_message_id_space)
+        message = DBMessage.get_by_tgid(msg_id, self._temp_pinned_message_id_space)
         if message:
             await intent.set_pinned_messages(self.mxid, [message.mxid])
         else:
@@ -2024,7 +1903,7 @@ class Portal:
             existing.delete()
         except KeyError:
             pass
-        self.db_instance.update(tgid=new_id, tg_receiver=new_id, peer_type=self.peer_type)
+        self.db_instance.update(tgid=new_id, tg_receiver=new_id)
         old_id = self.tgid
         self.tgid = new_id
         self.tg_receiver = new_id

mautrix_telegram/puppet.py

@@ -22,8 +22,7 @@ import asyncio
 import logging
 import re
 
-from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
-                               InputPeerPhotoFileLocation, UserProfilePhotoEmpty)
+from telethon.tl.types import UserProfilePhoto, User, FileLocation, UpdateUserName, PeerUser
 from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
 from .types import MatrixUserID, TelegramID
@@ -62,7 +61,6 @@ class Puppet:
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
-                 disable_updates: bool = False,
                  db_instance: Optional[DBPuppet] = None) -> None:
         self.id = id  # type: TelegramID
         self.access_token = access_token  # type: Optional[str]
@@ -75,7 +73,6 @@ class Puppet:
         self.photo_id = photo_id  # type: Optional[str]
         self.is_bot = is_bot  # type: bool
         self.is_registered = is_registered  # type: bool
-        self.disable_updates = disable_updates  # type: bool
         self._db_instance = db_instance  # type: Optional[DBPuppet]
 
         self.default_mxid_intent = self.az.intent.user(self.default_mxid)
@@ -114,9 +111,6 @@ class Puppet:
         match = regex.match(self.displayname)
         return match.group(1) or self.displayname
 
-    def get_input_entity(self, user: 'AbstractUser') -> Awaitable[TypeInputPeer]:
-        return user.client.get_input_entity(PeerUser(user_id=self.tgid))
-
     # region Custom puppet management
     def _fresh_intent(self) -> IntentAPI:
         return (self.az.intent.user(self.custom_mxid, self.access_token)
@@ -221,13 +215,13 @@ class Puppet:
         return new_events
 
     def handle_sync(self, presence: List, ephemeral: Dict) -> None:
-        presence_events = [self.mx.try_handle_ephemeral_event(event) for event in presence]
+        presence_events = [self.mx.try_handle_event(event) for event in presence]
 
         for room_id, events in ephemeral.items():
             for event in events:
                 event["room_id"] = room_id
 
-        ephemeral_events = [self.mx.try_handle_ephemeral_event(event)
+        ephemeral_events = [self.mx.try_handle_event(event)
                             for events in ephemeral.values()
                             for event in self.filter_events(events)]
 
@@ -287,26 +281,23 @@ class Puppet:
         return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
                         username=self.username, displayname=self.displayname,
                         displayname_source=self.displayname_source, photo_id=self.photo_id,
-                        is_bot=self.is_bot, matrix_registered=self.is_registered,
-                        disable_updates=self.disable_updates)
+                        is_bot=self.is_bot, matrix_registered=self.is_registered)
 
     @classmethod
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
                       db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
                       db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
-                      db_puppet.disable_updates, db_instance=db_puppet)
+                      db_instance=db_puppet)
 
     def save(self) -> None:
         self.db_instance.update(access_token=self.access_token, custom_mxid=self.custom_mxid,
                                 username=self.username, displayname=self.displayname,
                                 displayname_source=self.displayname_source, photo_id=self.photo_id,
-                                is_bot=self.is_bot, matrix_registered=self.is_registered,
-                                disable_updates=self.disable_updates)
+                                is_bot=self.is_bot, matrix_registered=self.is_registered)
 
     # endregion
     # region Info updating
-
     def similarity(self, query: str) -> int:
         username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                                if self.username else 0)
@@ -343,8 +334,6 @@ class Puppet:
             displayname=name)
 
     async def update_info(self, source: 'AbstractUser', info: User) -> None:
-        if self.disable_updates:
-            return
         changed = False
         if self.username != info.username:
             self.username = info.username
@@ -352,7 +341,7 @@ class Puppet:
 
         changed = await self.update_displayname(source, info) or changed
         if isinstance(info.photo, UserProfilePhoto):
-            changed = await self.update_avatar(source, info.photo) or changed
+            changed = await self.update_avatar(source, info.photo.photo_big) or changed
 
         self.is_bot = info.bot
 
@@ -361,68 +350,33 @@ class Puppet:
 
     async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
                                  ) -> bool:
-        if self.disable_updates:
+        ignore_source = (not source.is_relaybot
+                         and self.displayname_source is not None
+                         and self.displayname_source != source.tgid)
+        if ignore_source:
             return False
-        allow_source = (source.is_relaybot
-                        or self.displayname_source == source.tgid
-                        # No displayname source, so just trust anything
-                        or self.displayname_source is None
-                        # No phone -> not in contact list -> can't set custom name
-                        or (isinstance(info, User) and info.phone is None))
-        if not allow_source:
-            return False
-        elif isinstance(info, UpdateUserName):
+        if isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
+            await self.default_mxid_intent.set_display_name(displayname)
             self.displayname = displayname
             self.displayname_source = source.tgid
-            try:
-                await self.default_mxid_intent.set_display_name(displayname[:100])
-            except MatrixRequestError:
-                self.log.exception("Failed to set displayname")
-                self.displayname = ""
-                self.displayname_source = None
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
             return True
         return False
 
-    async def update_avatar(self, source: 'AbstractUser',
-                            photo: Union[UserProfilePhoto, UserProfilePhotoEmpty]) -> bool:
-        if self.disable_updates:
-            return False
-
-        if isinstance(photo, UserProfilePhotoEmpty):
-            photo_id = ""
-        else:
-            photo_id = str(photo.photo_id)
+    async def update_avatar(self, source: 'AbstractUser', photo: FileLocation) -> bool:
+        photo_id = f"{photo.volume_id}-{photo.local_id}"
         if self.photo_id != photo_id:
-            if not photo_id:
-                self.photo_id = ""
-                try:
-                    await self.default_mxid_intent.set_avatar("")
-                except MatrixRequestError:
-                    self.log.exception("Failed to set avatar")
-                    self.photo_id = ""
-                return True
-
-            loc = InputPeerPhotoFileLocation(
-                peer=await self.get_input_entity(source),
-                local_id=photo.photo_big.local_id,
-                volume_id=photo.photo_big.volume_id,
-                big=True
-            )
-            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
+            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent,
+                                                      photo)
             if file:
+                await self.default_mxid_intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
-                try:
-                    await self.default_mxid_intent.set_avatar(file.mxc)
-                except MatrixRequestError:
-                    self.log.exception("Failed to set avatar")
-                    self.photo_id = ""
                 return True
         return False
 

mautrix_telegram/user.py

@@ -161,12 +161,6 @@ class User(AbstractUser):
     # endregion
     # region Telegram connection management
 
-    async def try_ensure_started(self) -> None:
-        try:
-            await self.ensure_started()
-        except Exception:
-            self.log.exception("Exception in ensure_started")
-
     def ensure_started(self, even_if_no_session=False) -> Awaitable['User']:
         return super().ensure_started(even_if_no_session)
 
@@ -406,9 +400,9 @@ class User(AbstractUser):
     # endregion
 
 
-def init(context: 'Context') -> List[Awaitable[None]]:
+def init(context: 'Context') -> List[Awaitable['User']]:
     global config
     config = context.config
 
     users = [User.from_db(user) for user in DBUser.all()]
-    return [user.try_ensure_started() for user in users if user.tgid]
+    return [user.ensure_started() for user in users if user.tgid]

mautrix_telegram/util/__init__.py

@@ -3,5 +3,6 @@ from .format_duration import format_duration
 from .signed_token import sign_token, verify_token
 from .recursive_dict import recursive_del, recursive_set, recursive_get
 
+
 def ignore_coro(coro):
     pass

mautrix_telegram/util/file_transfer.py

@@ -23,16 +23,14 @@ import asyncio
 import magic
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
 
-from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLocation,
-                               TypePhotoSize, PhotoSize, PhotoCachedSize, InputPhotoFileLocation,
-                               InputPeerPhotoFileLocation)
+from telethon.tl.types import (Document, FileLocation, InputFileLocation, InputDocumentFileLocation,
+                               TypePhotoSize, PhotoSize, PhotoCachedSize)
 from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
-                             SecurityError, FileIdInvalidError)
+                             SecurityError)
 from mautrix_appservice import IntentAPI
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
-from ..util import sane_mimetypes
 
 try:
     from PIL import Image
@@ -49,8 +47,7 @@ except ImportError:
 
 log = logging.getLogger("mau.util")  # type: logging.Logger
 
-TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
-                     InputFileLocation, InputPhotoFileLocation]
+TypeLocation = Union[Document, InputDocumentFileLocation, FileLocation, InputFileLocation]
 
 
 def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
@@ -102,9 +99,9 @@ def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str =
 
 
 def _location_to_id(location: TypeLocation) -> str:
-    if isinstance(location, (Document, InputDocumentFileLocation, InputPhotoFileLocation)):
+    if isinstance(location, (Document, InputDocumentFileLocation)):
         return f"{location.id}-{location.access_hash}"
-    elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
+    elif isinstance(location, (FileLocation, InputFileLocation)):
         return f"{location.volume_id}-{location.local_id}"
 
 
@@ -122,7 +119,7 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if db_file:
         return db_file
 
-    video_ext = sane_mimetypes.guess_extension(mime)
+    video_ext = mimetypes.guess_extension(mime)
     if VideoFileClip and video_ext:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
@@ -184,7 +181,7 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
 
     try:
         file = await client.download_file(location)
-    except (LocationInvalidError, FileIdInvalidError):
+    except LocationInvalidError:
         return None
     except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
         log.exception(f"{e.__class__.__name__} while downloading a file.")

mautrix_telegram/util/sane_mimetypes.py

@@ -1,38 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import mimetypes
-
-mimetypes.init()
-
-sanity_overrides = {
-    "image/jpeg": ".jpeg",
-    "image/tiff": ".tiff",
-    "text/plain": ".txt",
-    "text/html": ".html",
-    "audio/mpeg": ".mp3",
-    "audio/ogg": ".ogg",
-    "application/xml": ".xml",
-    "application/octet-stream": "",
-    "application/x-msdos-program": ".exe",
-}
-
-
-def guess_extension(mime: str) -> str:
-    try:
-        return sanity_overrides[mime]
-    except KeyError:
-        return mimetypes.guess_extension(mime)

mautrix_telegram/web/common/auth_api.py

@@ -56,7 +56,7 @@ class AuthAPI(abc.ABC):
                                               error="You have already logged in with your Matrix "
                                                     "account.", errcode="already-logged-in")
 
-        resp = await puppet.switch_mxid(token.strip(), user.mxid)
+        resp = await puppet.switch_mxid(token, user.mxid)
         if resp == PuppetError.OnlyLoginSelf:
             return self.get_mx_login_response(status=403, errcode="only-login-self",
                                               error="You can only log in as your own Matrix user.")
@@ -72,15 +72,10 @@ class AuthAPI(abc.ABC):
                                           errcode="not-yet-implemented")
 
     async def post_login_phone(self, user: User, phone: str) -> web.Response:
-        if not phone or not phone.strip():
-            return self.get_login_response(mxid=user.mxid, state="request", status=400,
-                                           errcode="phone_number_invalid",
-                                           error="Phone number not given.")
         try:
-            await user.client.sign_in(phone.strip())
+            await user.client.sign_in(phone or "+123")
             return self.get_login_response(mxid=user.mxid, state="code", status=200,
-                                           message="Code requested successfully. Check your SMS "
-                                                   "or Telegram client and enter the code below.")
+                                           message="Code requested successfully.")
         except PhoneNumberInvalidError:
             return self.get_login_response(mxid=user.mxid, state="request", status=400,
                                            errcode="phone_number_invalid",
@@ -122,9 +117,10 @@ class AuthAPI(abc.ABC):
         if user.command_status and user.command_status["action"] == "Login":
             user.command_status = None
 
+
     async def post_login_token(self, user: User, token: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(bot_token=token.strip())
+            user_info = await user.client.sign_in(bot_token=token)
             await self.postprocess_login(user, user_info)
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
                                            username=user_info.username, phone=None,
@@ -178,7 +174,7 @@ class AuthAPI(abc.ABC):
 
     async def post_login_password(self, user: User, password: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(password=password.strip())
+            user_info = await user.client.sign_in(password=password)
             await self.postprocess_login(user, user_info)
             human_tg_id = f"@{user_info.username}" if user_info.username else f"+{user_info.phone}"
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,

mautrix_telegram/web/provisioning/__init__.py

@@ -365,7 +365,7 @@ class ProvisioningAPI(AuthAPI):
 
     async def bridge_info(self, request: web.Request) -> web.Response:
         return web.json_response({
-            "relaybot_username": self.context.bot.username if self.context.bot is not None else None,
+            "relaybot_username": self.context.bot.username,
         }, status=200)
 
     @staticmethod

mautrix_telegram/web/public/__init__.py

@@ -87,8 +87,7 @@ class PublicBridgeWebsite(AuthAPI):
         return self.get_login_response(mxid=user.mxid, human_tg_id=user.human_tg_id)
 
     async def get_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None),
-                                 endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
         user = User.get_by_mxid(mxid, create=False) if mxid else None
@@ -125,8 +124,7 @@ class PublicBridgeWebsite(AuthAPI):
                                                       error=error, message=message, mxid=mxid))
 
     async def post_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None),
-                                 endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
 
@@ -169,13 +167,7 @@ class PublicBridgeWebsite(AuthAPI):
         elif "bot_token" in data:
             return await self.post_login_token(user, data["bot_token"])
         elif "code" in data:
-            try:
-                code = int(data["code"].strip())
-            except ValueError:
-                return self.get_login_response(mxid=user.mxid, state="code", status=400,
-                                               errcode="phone_code_invalid",
-                                               error="Phone code must be a number.")
-            resp = await self.post_login_code(user, code,
+            resp = await self.post_login_code(user, data["code"],
                                               password_in_data="password" in data)
             if resp or "password" not in data:
                 return resp

mautrix_telegram/web/public/login.html.mako

@@ -95,8 +95,8 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 				% if state == "request":
 					<label for="value">Phone number</label>
 					<input type="tel" id="value" name="phone" placeholder="Enter phone number"/>
-					<button type="submit">Start</button>
-					<button class="button-clear float-right" type="button" onclick="switchToBotLogin()">
+					<button type="submit">Request code</button>
+					<button class="button-clear" type="button" onclick="switchToBotLogin()">
 						Use bot token
 					</button>
 				% elif state == "bot_token":

optional-requirements.txt

@@ -1,4 +1,4 @@
+lxml
 cryptg
 Pillow
 moviepy
-prometheus-client

setup.py

@@ -3,10 +3,10 @@ import glob
 import mautrix_telegram
 
 extras = {
-    "fast_crypto": ["cryptg>=0.1,<0.3"],
-    "webp_convert": ["Pillow>=4.3.0,<7"],
+    "highlight_edits": ["lxml>=4.1.1,<5"],
+    "fast_crypto": ["cryptg>=0.1,<0.2"],
+    "webp_convert": ["Pillow>=4.3.0,<6"],
     "hq_thumbnails": ["moviepy>=1.0,<2.0"],
-    "metrics": ["prometheus-client>=0.6.0,<0.8.0"],
 }
 extras["all"] = list({dep for deps in extras.values() for dep in deps})
 
@@ -31,15 +31,15 @@ setuptools.setup(
 
     install_requires=[
         "aiohttp>=3.0.1,<4",
-        "mautrix-appservice>=0.3.11,<0.4.0",
+        "mautrix-appservice>=0.3.10.dev1,<0.4.0",
         "SQLAlchemy>=1.2.3,<2",
         "alembic>=1.0.0,<2",
         "commonmark>=0.8.1,<1",
         "ruamel.yaml>=0.15.35,<0.16",
         "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
-        "telethon>=1.9,<1.10",
-        "telethon-session-sqlalchemy>=0.2.14,<0.3",
+        "telethon>=1.5.5,<1.7",
+        "telethon-session-sqlalchemy>=0.2.12,<0.3",
     ],
     extras_require=extras,