Add native Matrix edit support

Warning: may break everything and/or edit your cat

Dockerfile

@@ -10,7 +10,6 @@ RUN apk add --no-cache \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
-      py3-lxml \
       py3-magic \
       py3-sqlalchemy \
       py3-markdown \

ROADMAP.md

@@ -3,6 +3,7 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
   * [x] Typing notifications

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,50 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM portal")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

example-config.yaml

@@ -126,11 +126,6 @@ bridge:
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: true
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.

mautrix_telegram/abstract_user.py

@@ -252,7 +252,7 @@ class AbstractUser(ABC):
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.get_by_tgid(TelegramID(update.max_id), self.tgid)
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
         if not message:
             return
 
@@ -336,7 +336,8 @@ class AbstractUser(ABC):
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal: po.Portal, message: DBMessage) -> None:
+    async def _try_redact(message: DBMessage) -> None:
+        portal = po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
@@ -348,30 +349,26 @@ class AbstractUser(ABC):
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), self.tgid)
-            if not message:
-                continue
-            message.delete()
-            number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
-            if number_left == 0:
-                portal = po.Portal.get_by_mxid(message.mx_room)
-                await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
+            for message in messages:
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    portal = po.Portal.get_by_mxid(message.mx_room)
+                    await self._try_redact(message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
-        if not portal:
-            return
+        channel_id = TelegramID(update.channel_id)
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), portal.tgid)
-            if not message:
-                continue
-            message.delete()
-            await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
+            for message in messages:
+                message.delete()
+                await self._try_redact(message)
 
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
@@ -397,10 +394,7 @@ class AbstractUser(ABC):
 
         user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            if config["bridge.edits_as_replies"]:
-                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-                return await portal.handle_telegram_edit(self, sender, update)
-            return
+            return await portal.handle_telegram_edit(self, sender, update)
 
         self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
         return await portal.handle_telegram_message(self, sender, update)

mautrix_telegram/commands/portal/config.py

@@ -77,7 +77,6 @@ def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
 def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
     stream = StringIO()
     yaml.dump({
-        "edits_as_replies": evt.config["bridge.edits_as_replies"],
         "bridge_notices": {
             "default": evt.config["bridge.bridge_notices.default"],
             "exceptions": evt.config["bridge.bridge_notices.exceptions"],

mautrix_telegram/commands/telegram/auth.py

@@ -33,8 +33,8 @@ from ...util import format_duration, ignore_coro
                  help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> Optional[Dict]:
     me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
-    human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
     if me:
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
         return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")

mautrix_telegram/commands/telegram/misc.py

@@ -191,7 +191,7 @@ async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
         raise MessageIDError(f"Invalid {type_name} ID (format)") from e
 
     if peer_type == PEER_TYPE_CHAT:
-        orig_msg = DBMessage.get_by_tgid(msg_id, space)
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
         if not orig_msg:
             raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
         new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)

mautrix_telegram/config.py

@@ -199,8 +199,6 @@ class Config(DictWithRecursion):
         copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
         copy("bridge.plaintext_highlights")
-        copy("bridge.edits_as_replies")
-        copy("bridge.highlight_edits")
         copy("bridge.public_portals")
         copy("bridge.catch_up")
         copy("bridge.sync_with_custom_puppets")

mautrix_telegram/db/message.py

@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, select
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
 from sqlalchemy.engine.result import RowProxy
 from typing import Optional, List
 
@@ -29,25 +29,44 @@ class Message(Base):
     mx_room = Column(String)  # type: MatrixRoomID
     tgid = Column(Integer, primary_key=True)  # type: TelegramID
     tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+    edit_index = Column(Integer, primary_key=True)  # type: int
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
         try:
-            mxid, mx_room, tgid, tg_space = next(rows)
-            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space)
+            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
+                       edit_index=edit_index)
         except StopIteration:
             return None
 
     @staticmethod
     def _all(rows: RowProxy) -> List['Message']:
-        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3])
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
+                        edit_index=row[4])
                 for row in rows]
 
     @classmethod
-    def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']:
-        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
+        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
+                                                                 cls.c.tg_space == tg_space))))
+
+    @classmethod
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
+        query = cls.t.select()
+        if edit_index < 0:
+            query = (query
+                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                     .order_by(desc(cls.c.edit_index))
+                     .limit(1)
+                     .offset(-edit_index - 1))
+        else:
+            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                     cls.c.edit_index == edit_index))
+        return cls._one_or_none(cls.db.execute(query))
 
     @classmethod
     def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
@@ -67,10 +86,12 @@ class Message(Base):
                                             cls.c.tg_space == tg_space))
 
     @classmethod
-    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
         with cls.db.begin() as conn:
             conn.execute(cls.t.update()
-                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
                          .values(**values))
 
     @classmethod
@@ -82,9 +103,11 @@ class Message(Base):
 
     @property
     def _edit_identity(self):
-        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
+                    self.c.edit_index == self.edit_index)
 
     def insert(self) -> None:
         with self.db.begin() as conn:
             conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
-                                                tgid=self.tgid, tg_space=self.tg_space))
+                                                tgid=self.tgid, tg_space=self.tg_space,
+                                                edit_index=self.edit_index))

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,8 @@
 from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
                           init_mx)
-from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 
 
 def init(context: c.Context) -> None:
     init_mx(context)
-    init_tg(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -87,25 +87,28 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
 
 def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
                              room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
+    relates_to = content.get("m.relates_to", None) or {}
+    if not relates_to:
+        return None
+    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
+             else relates_to.get("m.in_reply_to", None) or {})
+    if not reply:
+        return None
+    room_id = room_id or reply.get("room_id", None)
+    event_id = reply.get("event_id", None)
+    if not event_id:
+        return
+
     try:
-        reply = (content.get("m.relates_to", None) or {}).get("m.in_reply_to", {})
-        if not reply:
-            return None
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
-        if message:
-            return message.tgid
+        if content["format"] == "org.matrix.custom.html":
+            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
     except KeyError:
         pass
+    content["body"] = trim_reply_fallback_text(content["body"])
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
     return None
 
 

mautrix_telegram/formatter/from_telegram.py

@@ -37,15 +37,8 @@ from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
 
 if TYPE_CHECKING:
     from ..abstract_user import AbstractUser
-    from ..context import Context
-
-try:
-    from lxml.html.diff import htmldiff
-except ImportError:
-    htmldiff = None  # type: ignore
 
 log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
-should_highlight_edits = False  # type: bool
 
 
 def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
@@ -53,13 +46,16 @@ def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
         space = (evt.to_id.channel_id
                  if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+        msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
         if msg:
             return {
                 "m.in_reply_to": {
                     "event_id": msg.mxid,
                     "room_id": msg.mx_room,
-                }
+                },
+                "rel_type": "m.reference",
+                "event_id": msg.mxid,
+                "room_id": msg.mx_room,
             }
     return {}
 
@@ -114,32 +110,19 @@ async def _add_forward_header(source, text: str, html: Optional[str],
     return text, html
 
 
-def highlight_edits(new_html: str, old_html: str) -> str:
-    # Don't include `Edit:` text in diff.
-    if old_html.startswith("<u>Edit:</u> "):
-        old_html = old_html[len("<u>Edit:</u> "):]
-
-    # Generate diff with lxml
-    new_html = htmldiff(old_html, new_html)
-
-    # Replace <ins> with <u> since Riot doesn't allow <ins>
-    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
-    # Remove <del>s since we just want to hide deletions.
-    new_html = re.sub("<del>.+?</del>", "", new_html)
-    return new_html
-
-
 async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
-                            relates_to: Dict, main_intent: IntentAPI, is_edit: bool
-                            ) -> Tuple[str, str]:
+                            relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]:
     space = (evt.to_id.channel_id
              if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+    msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
     if not msg:
         return text, html
 
+    relates_to["rel_type"] = "m.reference"
+    relates_to["event_id"] = msg.mxid
+    relates_to["room_id"] = msg.mx_room
     relates_to["m.in_reply_to"] = {
         "event_id": msg.mxid,
         "room_id": msg.mx_room,
@@ -159,21 +142,13 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
         puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
         r_displayname = puppet.displayname if puppet else r_sender
         r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
-
-        if is_edit and should_highlight_edits:
-            html = highlight_edits(html or escape(text), r_html_body)
     except (ValueError, KeyError, MatrixRequestError):
         r_sender_link = "unknown user"
         r_displayname = "unknown user"
         r_text_body = "Failed to fetch message"
         r_html_body = "<em>Failed to fetch message</em>"
 
-    if is_edit:
-        html = f"<u>Edit:</u> {html or escape(text)}"
-        text = f"Edit: {text}"
-
-    r_keyword = "In reply to" if not is_edit else "Edit to"
-    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>In reply to</a>"
     html = (
         f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
         + (html or escape(text)))
@@ -190,8 +165,8 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
 async def telegram_to_matrix(evt: Message, source: "AbstractUser",
                              main_intent: Optional[IntentAPI] = None,
-                             is_edit: bool = False, prefix_text: Optional[str] = None,
-                             prefix_html: Optional[str] = None, override_text: str = None,
+                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
+                             override_text: str = None,
                              override_entities: List[TypeMessageEntity] = None,
                              no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
     text = add_surrogates(override_text or evt.message)
@@ -208,8 +183,7 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
         text, html = await _add_forward_header(source, text, html, evt.fwd_from)
 
     if evt.reply_to_msg_id and not no_reply_fallback:
-        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
-                                             is_edit)
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
         if not html:
@@ -340,14 +314,9 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
 
         portal = po.Portal.find_by_username(group)
         if portal:
-            message = DBMessage.get_by_tgid(TelegramID(msgid), portal.tgid)
+            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 
     html.append(f"<a href='{url}'>{entity_text}</a>")
     return False
-
-
-def init_tg(context: "Context") -> None:
-    global should_highlight_edits
-    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/portal.py

@@ -47,16 +47,16 @@ from telethon.errors import ChatAdminRequiredError, ChatNotModifiedError
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
     Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin,
-    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat, ChatFull,
-    ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll, PollAnswer,
+    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat,
+    ChatFull, ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll,
     DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker,
     DocumentAttributeVideo, FileLocation, GeoPoint, InputChannel, InputChatUploadedPhoto,
-    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf,
+    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf, MessageMediaPoll,
     MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate,
     MessageActionChatDeletePhoto, MessageActionChatDeleteUser, MessageActionChatEditPhoto,
     MessageActionChatEditTitle, MessageActionChatJoinedByLink, MessageActionChatMigrateTo,
     MessageActionPinMessage, MessageActionGameScore, MessageMediaContact, MessageMediaDocument,
-    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, MessageMediaPoll,
+    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame,
     PeerChannel, PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction,
     SendMessageTypingAction, TypeChannelParticipant, TypeChat, TypeChatParticipant,
     TypeDocumentAttribute, TypeInputPeer, TypeMessageAction, TypeMessageEntity, TypePeer,
@@ -950,15 +950,25 @@ class Portal:
             return None
 
     async def _handle_matrix_text(self, sender_id: TelegramID, event_id: MatrixEventID,
-                                  space: TelegramID, client: 'MautrixTelegramClient', message: Dict,
-                                  reply_to: TelegramID) -> None:
+                                  space: TelegramID, client: 'MautrixTelegramClient',
+                                  message: Dict, reply_to: TelegramID) -> None:
         lock = self.require_send_lock(sender_id)
         async with lock:
             lp = self.get_config("telegram_link_preview")
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         message.get("m.new_content", message),
+                                                         parse_mode=self._matrix_event_to_entities,
+                                                         link_preview=lp)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
             response = await client.send_message(self.peer, message, reply_to=reply_to,
                                                  parse_mode=self._matrix_event_to_entities,
                                                  link_preview=lp)
-            self._add_telegram_message_to_db(event_id, space, response)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     async def _handle_matrix_file(self, msgtype: str, sender_id: TelegramID,
                                   event_id: MatrixEventID, space: TelegramID,
@@ -993,9 +1003,17 @@ class Portal:
             max_image_size=config["bridge.image_as_file_size"] * 1000 ** 2)
         lock = self.require_send_lock(sender_id)
         async with lock:
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         caption, file=media)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
             response = await client.send_media(self.peer, media, reply_to=reply_to,
                                                caption=caption)
-            self._add_telegram_message_to_db(event_id, space, response)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     async def _handle_matrix_location(self, sender_id: TelegramID, event_id: MatrixEventID,
                                       space: TelegramID, client: 'MautrixTelegramClient',
@@ -1011,19 +1029,31 @@ class Portal:
 
         lock = self.require_send_lock(sender_id)
         async with lock:
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         caption, file=media)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
             response = await client.send_media(self.peer, media, reply_to=reply_to,
                                                caption=caption, entities=entities)
-            self._add_telegram_message_to_db(event_id, space, response)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     def _add_telegram_message_to_db(self, event_id: MatrixEventID, space: TelegramID,
-                                    response: TypeMessage) -> None:
+                                    edit_index: int, response: TypeMessage) -> None:
         self.log.debug("Handled Matrix message: %s", response)
         self.is_duplicate(response, (event_id, space))
+        if edit_index < 0:
+            prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1)
+            edit_index = prev_edit.edit_index + 1
         DBMessage(
             tgid=TelegramID(response.id),
             tg_space=space,
             mx_room=self.mxid,
-            mxid=event_id).insert()
+            mxid=event_id,
+            edit_index=edit_index).insert()
 
     async def handle_matrix_message(self, sender: 'u.User', message: Dict[str, Any],
                                     event_id: MatrixEventID) -> None:
@@ -1087,7 +1117,10 @@ class Portal:
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
             return
-        await real_deleter.client.delete_messages(self.peer, [message.tgid])
+        if message.edit_index == 0:
+            await real_deleter.client.delete_messages(self.peer, [message.tgid])
+        else:
+            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
 
     async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
                                            level: int) -> None:
@@ -1342,7 +1375,8 @@ class Portal:
         ext_override = {
             "image/jpeg": ".jpg"
         }
-        name = "image" + ext_override.get(file.mime_type, mimetypes.guess_extension(file.mime_type))
+        name = "image" + ext_override.get(file.mime_type,
+                                          mimetypes.guess_extension(file.mime_type))
         await intent.set_typing(self.mxid, is_typing=False)
         result = await intent.send_image(self.mxid, file.mxc, info=info, text=name,
                                          relates_to=relates_to, timestamp=evt.date,
@@ -1570,7 +1604,8 @@ class Portal:
         play_id = self._encode_msgid(source, evt)
         command = f"!tg play {play_id}"
         override_text = f"Run {command} in your bridge management room to play {game.title}"
-        override_entities = [MessageEntityPre(offset=len("Run "), length=len(command), language="")]
+        override_entities = [
+            MessageEntityPre(offset=len("Run "), length=len(command), language="")]
         text, html, relates_to = await formatter.telegram_to_matrix(
             evt, source, self.main_intent,
             override_text=override_text, override_entities=override_entities)
@@ -1588,9 +1623,6 @@ class Portal:
                                    evt: Message) -> None:
         if not self.mxid:
             return
-        elif not self.get_config("edits_as_replies"):
-            self.log.debug("Edits as replies disabled, ignoring edit event...")
-            return
         elif hasattr(evt, "media") and isinstance(evt.media, (MessageMediaGame,)):
             self.log.debug("Ignoring game message edit event")
             return
@@ -1608,28 +1640,50 @@ class Portal:
         if duplicate_found:
             mxid, other_tg_space = duplicate_found
             if tg_space != other_tg_space:
-                DBMessage.update_by_tgid(TelegramID(evt.id), tg_space,
-                                         mxid=mxid,
-                                         mx_room=self.mxid)
+                prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1)
+                if not prev_edit_msg:
+                    return
+                DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
+                          edit_index=prev_edit_msg.edit_index + 1).insert()
             return
 
-        evt.reply_to_msg_id = evt.id
-        text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent,
-                                                                    is_edit=True)
+        text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                           no_reply_fallback=True)
+        editing_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
+
+        content = {
+            "body": f"Edit: {text}",
+            "msgtype": "m.text",
+            "format": "org.matrix.custom.html",
+            "formatted_body": (f"<a href='https://matrix.to/#/{editing_msg.mx_room}/"
+                               f"{editing_msg.mxid}'>Edit</a>: "
+                               f"{html or escape_html(text)}"),
+            "external_url": self.get_external_url(evt),
+            "m.new_content": {
+                "body": text,
+                "msgtype": "m.text",
+                **({"format": "org.matrix.custom.html",
+                    "formatted_body": html} if html else {}),
+            },
+            "m.relates_to": {
+                "rel_type": "m.replace",
+                "event_id": editing_msg.mxid,
+            },
+        }
+
         intent = sender.intent if sender else self.main_intent
         await intent.set_typing(self.mxid, is_typing=False)
-        response = await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
-                                          external_url=self.get_external_url(evt))
-
+        response = await intent.send_message(self.mxid, content)
         mxid = response["event_id"]
 
-        msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
-        if not msg:
+        prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1)
+        if not prev_edit_msg:
             self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) "
                           "in database.")
             # Oh crap
             return
-        msg.update(mxid=mxid, mx_room=self.mxid)
+        DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
+                  edit_index=prev_edit_msg.edit_index + 1).insert()
         DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
 
     async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
@@ -1653,11 +1707,11 @@ class Portal:
                            f"as it was already handled (in space {other_tg_space})")
             if tg_space != other_tg_space:
                 DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                          tg_space=tg_space).insert()
+                          tg_space=tg_space, edit_index=0).insert()
             return
 
         if self.dedup_pre_db_check and self.peer_type == "channel":
-            msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
+            msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
             if msg:
                 self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
                                f"handled into {msg.mxid}. This duplicate was catched in the db "
@@ -1671,8 +1725,8 @@ class Portal:
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
 
-        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo, MessageMediaGame,
-                         MessageMediaPoll, MessageMediaUnsupported)
+        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
+                         MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
         intent = sender.intent if sender else self.main_intent
@@ -1712,7 +1766,7 @@ class Portal:
         self.log.debug("Handled Telegram message: %s", evt)
         try:
             DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                      tg_space=tg_space).insert()
+                      tg_space=tg_space, edit_index=0).insert()
             DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
         except IntegrityError as e:
             self.log.exception(f"{e.__class__.__name__} while saving message mapping. "
@@ -1791,7 +1845,7 @@ class Portal:
         self._temp_pinned_message_id = None
         self._temp_pinned_message_sender = None
 
-        message = DBMessage.get_by_tgid(msg_id, self._temp_pinned_message_id_space)
+        message = DBMessage.get_one_by_tgid(msg_id, self._temp_pinned_message_id_space)
         if message:
             await intent.set_pinned_messages(self.mxid, [message.mxid])
         else:

optional-requirements.txt

@@ -1,4 +1,3 @@
-lxml
 cryptg
 Pillow
 moviepy

setup.py

@@ -3,7 +3,6 @@ import glob
 import mautrix_telegram
 
 extras = {
-    "highlight_edits": ["lxml>=4.1.1,<5"],
     "fast_crypto": ["cryptg>=0.1,<0.2"],
     "webp_convert": ["Pillow>=4.3.0,<6"],
     "hq_thumbnails": ["moviepy>=1.0,<2.0"],