diff --git a/Dockerfile b/Dockerfile index a1a71156..7b1a7914 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,6 @@ RUN apk add --no-cache \ py3-virtualenv \ py3-pillow \ py3-aiohttp \ - py3-lxml \ py3-magic \ py3-sqlalchemy \ py3-markdown \ diff --git a/ROADMAP.md b/ROADMAP.md index 1fcd5f26..bb32ece0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -3,6 +3,7 @@ * Matrix → Telegram * [x] Message content (text, formatting, files, etc..) * [x] Message redactions + * [x] Message edits * [ ] ‡ Message history * [x] Presence * [x] Typing notifications diff --git a/alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py b/alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py new file mode 100644 index 00000000..87ca53c4 --- /dev/null +++ b/alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py @@ -0,0 +1,50 @@ +"""Add edit index to messages + +Revision ID: 9e9c89b0b877 +Revises: 17574c57f3f8 +Create Date: 2019-05-29 15:28:23.128377 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '9e9c89b0b877' +down_revision = '17574c57f3f8' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('_message_temp', + sa.Column('mxid', sa.String), + sa.Column('mx_room', sa.String), + sa.Column('tgid', sa.Integer), + sa.Column('tg_space', sa.Integer), + sa.Column('edit_index', sa.Integer), + sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'), + sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room")) + c = op.get_bind() + c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) " + "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 " + "FROM message") + c.execute("DROP TABLE message") + c.execute("ALTER TABLE _message_temp RENAME TO message") + + + +def downgrade(): + op.create_table('_message_temp', + sa.Column('mxid', sa.String), + sa.Column('mx_room', sa.String), + sa.Column('tgid', sa.Integer), + sa.Column('tg_space', sa.Integer), + sa.PrimaryKeyConstraint('tgid', 'tg_space'), + sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room")) + c = op.get_bind() + c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) " + "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space " + "FROM portal") + c.execute("DROP TABLE message") + c.execute("ALTER TABLE _message_temp RENAME TO message") diff --git a/example-config.yaml b/example-config.yaml index 86f64372..bb46d803 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -126,11 +126,6 @@ bridge: # Only enable this if your displayname_template has some static part that the bridge can use to # reliably identify what is a plaintext highlight. plaintext_highlights: false - # Show message editing as a reply to the original message. - # If this is false, message edits are not shown at all, as Matrix does not support editing yet. - edits_as_replies: true - # Highlight changed/added parts in edits. Requires lxml. - highlight_edits: false # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix. public_portals: true # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down. diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index 6244999e..43701cc8 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -252,7 +252,7 @@ class AbstractUser(ABC): return # We check that these are user read receipts, so tg_space is always the user ID. - message = DBMessage.get_by_tgid(TelegramID(update.max_id), self.tgid) + message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1) if not message: return @@ -336,7 +336,8 @@ class AbstractUser(ABC): return update, sender, portal @staticmethod - async def _try_redact(portal: po.Portal, message: DBMessage) -> None: + async def _try_redact(message: DBMessage) -> None: + portal = po.Portal.get_by_mxid(message.mx_room) if not portal: return try: @@ -348,30 +349,26 @@ class AbstractUser(ABC): if len(update.messages) > MAX_DELETIONS: return - for message in update.messages: - message = DBMessage.get_by_tgid(TelegramID(message), self.tgid) - if not message: - continue - message.delete() - number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room) - if number_left == 0: - portal = po.Portal.get_by_mxid(message.mx_room) - await self._try_redact(portal, message) + for message_id in update.messages: + messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid) + for message in messages: + message.delete() + number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room) + if number_left == 0: + portal = po.Portal.get_by_mxid(message.mx_room) + await self._try_redact(message) async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None: if len(update.messages) > MAX_DELETIONS: return - portal = po.Portal.get_by_tgid(TelegramID(update.channel_id)) - if not portal: - return + channel_id = TelegramID(update.channel_id) - for message in update.messages: - message = DBMessage.get_by_tgid(TelegramID(message), portal.tgid) - if not message: - continue - message.delete() - await self._try_redact(portal, message) + for message_id in update.messages: + messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id) + for message in messages: + message.delete() + await self._try_redact(message) async def update_message(self, original_update: UpdateMessage) -> None: update, sender, portal = self.get_message_details(original_update) @@ -397,10 +394,7 @@ class AbstractUser(ABC): user = sender.tgid if sender else "admin" if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)): - if config["bridge.edits_as_replies"]: - self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user) - return await portal.handle_telegram_edit(self, sender, update) - return + return await portal.handle_telegram_edit(self, sender, update) self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user) return await portal.handle_telegram_message(self, sender, update) diff --git a/mautrix_telegram/commands/portal/config.py b/mautrix_telegram/commands/portal/config.py index 0cacfbcc..95fcfabb 100644 --- a/mautrix_telegram/commands/portal/config.py +++ b/mautrix_telegram/commands/portal/config.py @@ -77,7 +77,6 @@ def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]: def config_defaults(evt: CommandEvent) -> Awaitable[Dict]: stream = StringIO() yaml.dump({ - "edits_as_replies": evt.config["bridge.edits_as_replies"], "bridge_notices": { "default": evt.config["bridge.bridge_notices.default"], "exceptions": evt.config["bridge.bridge_notices.exceptions"], diff --git a/mautrix_telegram/commands/telegram/auth.py b/mautrix_telegram/commands/telegram/auth.py index 5f56c4ca..efe34763 100644 --- a/mautrix_telegram/commands/telegram/auth.py +++ b/mautrix_telegram/commands/telegram/auth.py @@ -33,8 +33,8 @@ from ...util import format_duration, ignore_coro help_text="Check if you're logged into Telegram.") async def ping(evt: CommandEvent) -> Optional[Dict]: me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None - human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}" if me: + human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}" return await evt.reply(f"You're logged in as {human_tg_id}") else: return await evt.reply("You're not logged in.") diff --git a/mautrix_telegram/commands/telegram/misc.py b/mautrix_telegram/commands/telegram/misc.py index 0d3c8fbb..5555bd1c 100644 --- a/mautrix_telegram/commands/telegram/misc.py +++ b/mautrix_telegram/commands/telegram/misc.py @@ -191,7 +191,7 @@ async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str raise MessageIDError(f"Invalid {type_name} ID (format)") from e if peer_type == PEER_TYPE_CHAT: - orig_msg = DBMessage.get_by_tgid(msg_id, space) + orig_msg = DBMessage.get_one_by_tgid(msg_id, space) if not orig_msg: raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)") new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid) diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 4cd27018..ef9b983b 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -199,8 +199,6 @@ class Config(DictWithRecursion): copy("bridge.sync_matrix_state") copy("bridge.allow_matrix_login") copy("bridge.plaintext_highlights") - copy("bridge.edits_as_replies") - copy("bridge.highlight_edits") copy("bridge.public_portals") copy("bridge.catch_up") copy("bridge.sync_with_custom_puppets") diff --git a/mautrix_telegram/db/message.py b/mautrix_telegram/db/message.py index a704b197..327ba508 100644 --- a/mautrix_telegram/db/message.py +++ b/mautrix_telegram/db/message.py @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, select +from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select from sqlalchemy.engine.result import RowProxy from typing import Optional, List @@ -29,25 +29,44 @@ class Message(Base): mx_room = Column(String) # type: MatrixRoomID tgid = Column(Integer, primary_key=True) # type: TelegramID tg_space = Column(Integer, primary_key=True) # type: TelegramID + edit_index = Column(Integer, primary_key=True) # type: int __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),) @classmethod def _one_or_none(cls, rows: RowProxy) -> Optional['Message']: try: - mxid, mx_room, tgid, tg_space = next(rows) - return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space) + mxid, mx_room, tgid, tg_space, edit_index = next(rows) + return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space, + edit_index=edit_index) except StopIteration: return None @staticmethod def _all(rows: RowProxy) -> List['Message']: - return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3]) + return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3], + edit_index=row[4]) for row in rows] @classmethod - def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']: - return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space)) + def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']: + return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid, + cls.c.tg_space == tg_space)))) + + @classmethod + def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0 + ) -> Optional['Message']: + query = cls.t.select() + if edit_index < 0: + query = (query + .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space)) + .order_by(desc(cls.c.edit_index)) + .limit(1) + .offset(-edit_index - 1)) + else: + query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space, + cls.c.edit_index == edit_index)) + return cls._one_or_none(cls.db.execute(query)) @classmethod def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int: @@ -67,10 +86,12 @@ class Message(Base): cls.c.tg_space == tg_space)) @classmethod - def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None: + def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int, + **values) -> None: with cls.db.begin() as conn: conn.execute(cls.t.update() - .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space)) + .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space, + cls.c.edit_index == s_edit_index)) .values(**values)) @classmethod @@ -82,9 +103,11 @@ class Message(Base): @property def _edit_identity(self): - return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space) + return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space, + self.c.edit_index == self.edit_index) def insert(self) -> None: with self.db.begin() as conn: conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room, - tgid=self.tgid, tg_space=self.tg_space)) + tgid=self.tgid, tg_space=self.tg_space, + edit_index=self.edit_index)) diff --git a/mautrix_telegram/formatter/__init__.py b/mautrix_telegram/formatter/__init__.py index d78fc36e..ed3066eb 100644 --- a/mautrix_telegram/formatter/__init__.py +++ b/mautrix_telegram/formatter/__init__.py @@ -1,9 +1,8 @@ from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram, init_mx) -from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg) +from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix from .. import context as c def init(context: c.Context) -> None: init_mx(context) - init_tg(context) diff --git a/mautrix_telegram/formatter/from_matrix/__init__.py b/mautrix_telegram/formatter/from_matrix/__init__.py index 2adc76ac..cfcb8ba2 100644 --- a/mautrix_telegram/formatter/from_matrix/__init__.py +++ b/mautrix_telegram/formatter/from_matrix/__init__.py @@ -87,25 +87,28 @@ def matrix_to_telegram(html: str) -> ParsedMessage: def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID, room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]: + relates_to = content.get("m.relates_to", None) or {} + if not relates_to: + return None + reply = (relates_to if relates_to.get("rel_type", None) == "m.reference" + else relates_to.get("m.in_reply_to", None) or {}) + if not reply: + return None + room_id = room_id or reply.get("room_id", None) + event_id = reply.get("event_id", None) + if not event_id: + return + try: - reply = (content.get("m.relates_to", None) or {}).get("m.in_reply_to", {}) - if not reply: - return None - room_id = room_id or reply["room_id"] - event_id = reply["event_id"] - - try: - if content["format"] == "org.matrix.custom.html": - content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"]) - except KeyError: - pass - content["body"] = trim_reply_fallback_text(content["body"]) - - message = DBMessage.get_by_mxid(event_id, room_id, tg_space) - if message: - return message.tgid + if content["format"] == "org.matrix.custom.html": + content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"]) except KeyError: pass + content["body"] = trim_reply_fallback_text(content["body"]) + + message = DBMessage.get_by_mxid(event_id, room_id, tg_space) + if message: + return message.tgid return None diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 2afe7a6e..ddb93ac2 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -37,15 +37,8 @@ from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, if TYPE_CHECKING: from ..abstract_user import AbstractUser - from ..context import Context - -try: - from lxml.html.diff import htmldiff -except ImportError: - htmldiff = None # type: ignore log = logging.getLogger("mau.fmt.tg") # type: logging.Logger -should_highlight_edits = False # type: bool def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict: @@ -53,13 +46,16 @@ def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict: space = (evt.to_id.channel_id if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel) else source.tgid) - msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space) + msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space) if msg: return { "m.in_reply_to": { "event_id": msg.mxid, "room_id": msg.mx_room, - } + }, + "rel_type": "m.reference", + "event_id": msg.mxid, + "room_id": msg.mx_room, } return {} @@ -114,32 +110,19 @@ async def _add_forward_header(source, text: str, html: Optional[str], return text, html -def highlight_edits(new_html: str, old_html: str) -> str: - # Don't include `Edit:` text in diff. - if old_html.startswith("Edit: "): - old_html = old_html[len("Edit: "):] - - # Generate diff with lxml - new_html = htmldiff(old_html, new_html) - - # Replace with since Riot doesn't allow - new_html = new_html.replace("", "").replace("", "") - # Remove s since we just want to hide deletions. - new_html = re.sub(".+?", "", new_html) - return new_html - - async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message, - relates_to: Dict, main_intent: IntentAPI, is_edit: bool - ) -> Tuple[str, str]: + relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]: space = (evt.to_id.channel_id if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel) else source.tgid) - msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space) + msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space) if not msg: return text, html + relates_to["rel_type"] = "m.reference" + relates_to["event_id"] = msg.mxid + relates_to["room_id"] = msg.mx_room relates_to["m.in_reply_to"] = { "event_id": msg.mxid, "room_id": msg.mx_room, @@ -159,21 +142,13 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M puppet = pu.Puppet.get_by_mxid(r_sender, create=False) r_displayname = puppet.displayname if puppet else r_sender r_sender_link = f"{r_displayname}" - - if is_edit and should_highlight_edits: - html = highlight_edits(html or escape(text), r_html_body) except (ValueError, KeyError, MatrixRequestError): r_sender_link = "unknown user" r_displayname = "unknown user" r_text_body = "Failed to fetch message" r_html_body = "Failed to fetch message" - if is_edit: - html = f"Edit: {html or escape(text)}" - text = f"Edit: {text}" - - r_keyword = "In reply to" if not is_edit else "Edit to" - r_msg_link = f"{r_keyword}" + r_msg_link = f"In reply to" html = ( f"
{r_msg_link} {r_sender_link}\n{r_html_body}
" + (html or escape(text))) @@ -190,8 +165,8 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M async def telegram_to_matrix(evt: Message, source: "AbstractUser", main_intent: Optional[IntentAPI] = None, - is_edit: bool = False, prefix_text: Optional[str] = None, - prefix_html: Optional[str] = None, override_text: str = None, + prefix_text: Optional[str] = None, prefix_html: Optional[str] = None, + override_text: str = None, override_entities: List[TypeMessageEntity] = None, no_reply_fallback: bool = False) -> Tuple[str, str, Dict]: text = add_surrogates(override_text or evt.message) @@ -208,8 +183,7 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser", text, html = await _add_forward_header(source, text, html, evt.fwd_from) if evt.reply_to_msg_id and not no_reply_fallback: - text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent, - is_edit) + text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent) if isinstance(evt, Message) and evt.post and evt.post_author: if not html: @@ -340,14 +314,9 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool: portal = po.Portal.find_by_username(group) if portal: - message = DBMessage.get_by_tgid(TelegramID(msgid), portal.tgid) + message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid) if message: url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}" html.append(f"{entity_text}") return False - - -def init_tg(context: "Context") -> None: - global should_highlight_edits - should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"] diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 0590c5e6..df86d751 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -47,16 +47,16 @@ from telethon.errors import ChatAdminRequiredError, ChatNotModifiedError from telethon.tl.patched import Message, MessageService from telethon.tl.types import ( Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin, - ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat, ChatFull, - ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll, PollAnswer, + ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat, + ChatFull, ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll, DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker, DocumentAttributeVideo, FileLocation, GeoPoint, InputChannel, InputChatUploadedPhoto, - InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf, + InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf, MessageMediaPoll, MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate, MessageActionChatDeletePhoto, MessageActionChatDeleteUser, MessageActionChatEditPhoto, MessageActionChatEditTitle, MessageActionChatJoinedByLink, MessageActionChatMigrateTo, MessageActionPinMessage, MessageActionGameScore, MessageMediaContact, MessageMediaDocument, - MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, MessageMediaPoll, + MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, PeerChannel, PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction, SendMessageTypingAction, TypeChannelParticipant, TypeChat, TypeChatParticipant, TypeDocumentAttribute, TypeInputPeer, TypeMessageAction, TypeMessageEntity, TypePeer, @@ -950,15 +950,25 @@ class Portal: return None async def _handle_matrix_text(self, sender_id: TelegramID, event_id: MatrixEventID, - space: TelegramID, client: 'MautrixTelegramClient', message: Dict, - reply_to: TelegramID) -> None: + space: TelegramID, client: 'MautrixTelegramClient', + message: Dict, reply_to: TelegramID) -> None: lock = self.require_send_lock(sender_id) async with lock: lp = self.get_config("telegram_link_preview") + relates_to = message.get("m.relates_to", None) or {} + if relates_to.get("rel_type", None) == "m.replace": + orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space) + if orig_msg: + response = await client.edit_message(self.peer, orig_msg.tgid, + message.get("m.new_content", message), + parse_mode=self._matrix_event_to_entities, + link_preview=lp) + self._add_telegram_message_to_db(event_id, space, -1, response) + return response = await client.send_message(self.peer, message, reply_to=reply_to, parse_mode=self._matrix_event_to_entities, link_preview=lp) - self._add_telegram_message_to_db(event_id, space, response) + self._add_telegram_message_to_db(event_id, space, 0, response) async def _handle_matrix_file(self, msgtype: str, sender_id: TelegramID, event_id: MatrixEventID, space: TelegramID, @@ -993,9 +1003,17 @@ class Portal: max_image_size=config["bridge.image_as_file_size"] * 1000 ** 2) lock = self.require_send_lock(sender_id) async with lock: + relates_to = message.get("m.relates_to", None) or {} + if relates_to.get("rel_type", None) == "m.replace": + orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space) + if orig_msg: + response = await client.edit_message(self.peer, orig_msg.tgid, + caption, file=media) + self._add_telegram_message_to_db(event_id, space, -1, response) + return response = await client.send_media(self.peer, media, reply_to=reply_to, caption=caption) - self._add_telegram_message_to_db(event_id, space, response) + self._add_telegram_message_to_db(event_id, space, 0, response) async def _handle_matrix_location(self, sender_id: TelegramID, event_id: MatrixEventID, space: TelegramID, client: 'MautrixTelegramClient', @@ -1011,19 +1029,31 @@ class Portal: lock = self.require_send_lock(sender_id) async with lock: + relates_to = message.get("m.relates_to", None) or {} + if relates_to.get("rel_type", None) == "m.replace": + orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space) + if orig_msg: + response = await client.edit_message(self.peer, orig_msg.tgid, + caption, file=media) + self._add_telegram_message_to_db(event_id, space, -1, response) + return response = await client.send_media(self.peer, media, reply_to=reply_to, caption=caption, entities=entities) - self._add_telegram_message_to_db(event_id, space, response) + self._add_telegram_message_to_db(event_id, space, 0, response) def _add_telegram_message_to_db(self, event_id: MatrixEventID, space: TelegramID, - response: TypeMessage) -> None: + edit_index: int, response: TypeMessage) -> None: self.log.debug("Handled Matrix message: %s", response) self.is_duplicate(response, (event_id, space)) + if edit_index < 0: + prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1) + edit_index = prev_edit.edit_index + 1 DBMessage( tgid=TelegramID(response.id), tg_space=space, mx_room=self.mxid, - mxid=event_id).insert() + mxid=event_id, + edit_index=edit_index).insert() async def handle_matrix_message(self, sender: 'u.User', message: Dict[str, Any], event_id: MatrixEventID) -> None: @@ -1087,7 +1117,10 @@ class Portal: message = DBMessage.get_by_mxid(event_id, self.mxid, space) if not message: return - await real_deleter.client.delete_messages(self.peer, [message.tgid]) + if message.edit_index == 0: + await real_deleter.client.delete_messages(self.peer, [message.tgid]) + else: + self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}") async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID, level: int) -> None: @@ -1342,7 +1375,8 @@ class Portal: ext_override = { "image/jpeg": ".jpg" } - name = "image" + ext_override.get(file.mime_type, mimetypes.guess_extension(file.mime_type)) + name = "image" + ext_override.get(file.mime_type, + mimetypes.guess_extension(file.mime_type)) await intent.set_typing(self.mxid, is_typing=False) result = await intent.send_image(self.mxid, file.mxc, info=info, text=name, relates_to=relates_to, timestamp=evt.date, @@ -1570,7 +1604,8 @@ class Portal: play_id = self._encode_msgid(source, evt) command = f"!tg play {play_id}" override_text = f"Run {command} in your bridge management room to play {game.title}" - override_entities = [MessageEntityPre(offset=len("Run "), length=len(command), language="")] + override_entities = [ + MessageEntityPre(offset=len("Run "), length=len(command), language="")] text, html, relates_to = await formatter.telegram_to_matrix( evt, source, self.main_intent, override_text=override_text, override_entities=override_entities) @@ -1588,9 +1623,6 @@ class Portal: evt: Message) -> None: if not self.mxid: return - elif not self.get_config("edits_as_replies"): - self.log.debug("Edits as replies disabled, ignoring edit event...") - return elif hasattr(evt, "media") and isinstance(evt.media, (MessageMediaGame,)): self.log.debug("Ignoring game message edit event") return @@ -1608,28 +1640,50 @@ class Portal: if duplicate_found: mxid, other_tg_space = duplicate_found if tg_space != other_tg_space: - DBMessage.update_by_tgid(TelegramID(evt.id), tg_space, - mxid=mxid, - mx_room=self.mxid) + prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) + if not prev_edit_msg: + return + DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id, + edit_index=prev_edit_msg.edit_index + 1).insert() return - evt.reply_to_msg_id = evt.id - text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent, - is_edit=True) + text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent, + no_reply_fallback=True) + editing_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space) + + content = { + "body": f"Edit: {text}", + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "formatted_body": (f"Edit: " + f"{html or escape_html(text)}"), + "external_url": self.get_external_url(evt), + "m.new_content": { + "body": text, + "msgtype": "m.text", + **({"format": "org.matrix.custom.html", + "formatted_body": html} if html else {}), + }, + "m.relates_to": { + "rel_type": "m.replace", + "event_id": editing_msg.mxid, + }, + } + intent = sender.intent if sender else self.main_intent await intent.set_typing(self.mxid, is_typing=False) - response = await intent.send_text(self.mxid, text, html=html, relates_to=relates_to, - external_url=self.get_external_url(evt)) - + response = await intent.send_message(self.mxid, content) mxid = response["event_id"] - msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space) - if not msg: + prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) + if not prev_edit_msg: self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) " "in database.") # Oh crap return - msg.update(mxid=mxid, mx_room=self.mxid) + DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id, + edit_index=prev_edit_msg.edit_index + 1).insert() DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid) async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet, @@ -1653,11 +1707,11 @@ class Portal: f"as it was already handled (in space {other_tg_space})") if tg_space != other_tg_space: DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid, - tg_space=tg_space).insert() + tg_space=tg_space, edit_index=0).insert() return if self.dedup_pre_db_check and self.peer_type == "channel": - msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space) + msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space) if msg: self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already" f"handled into {msg.mxid}. This duplicate was catched in the db " @@ -1671,8 +1725,8 @@ class Portal: entity = await source.client.get_entity(PeerUser(sender.tgid)) await sender.update_info(source, entity) - allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo, MessageMediaGame, - MessageMediaPoll, MessageMediaUnsupported) + allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo, + MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported) media = evt.media if hasattr(evt, "media") and isinstance(evt.media, allowed_media) else None intent = sender.intent if sender else self.main_intent @@ -1712,7 +1766,7 @@ class Portal: self.log.debug("Handled Telegram message: %s", evt) try: DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid, - tg_space=tg_space).insert() + tg_space=tg_space, edit_index=0).insert() DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid) except IntegrityError as e: self.log.exception(f"{e.__class__.__name__} while saving message mapping. " @@ -1791,7 +1845,7 @@ class Portal: self._temp_pinned_message_id = None self._temp_pinned_message_sender = None - message = DBMessage.get_by_tgid(msg_id, self._temp_pinned_message_id_space) + message = DBMessage.get_one_by_tgid(msg_id, self._temp_pinned_message_id_space) if message: await intent.set_pinned_messages(self.mxid, [message.mxid]) else: diff --git a/optional-requirements.txt b/optional-requirements.txt index a4400877..7cc6ceea 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,4 +1,3 @@ -lxml cryptg Pillow moviepy diff --git a/setup.py b/setup.py index 0104b638..54a8db9d 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,6 @@ import glob import mautrix_telegram extras = { - "highlight_edits": ["lxml>=4.1.1,<5"], "fast_crypto": ["cryptg>=0.1,<0.2"], "webp_convert": ["Pillow>=4.3.0,<6"], "hq_thumbnails": ["moviepy>=1.0,<2.0"],