From 26b8efb1e68b2d9125becaf4b504bf89725345b1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 4 Mar 2018 20:59:45 +0200 Subject: [PATCH 01/41] Send thumbnail and size info with Telegram -> Matrix videos --- mautrix_telegram/portal.py | 24 ++++++++++++++++++++---- mautrix_telegram/util/file_transfer.py | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 6111be8e..88258062 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -839,25 +839,41 @@ class Portal: return await intent.send_image(self.mxid, file.mxc, info=info, text=name, relates_to=relates_to) - async def handle_telegram_document(self, source, intent, evt, relates_to=None): + async def handle_telegram_document(self, source, intent, evt: Message, relates_to=None): document = evt.media.document file = await util.transfer_file_to_matrix(self.db, source.client, intent, document) if not file: return None name = evt.message + width, height = 0, 0 for attr in document.attributes: - if not name and isinstance(attr, DocumentAttributeFilename): - name = attr.file_name + if isinstance(attr, DocumentAttributeFilename): + name = name or attr.file_name if not file.was_converted: (mime_from_name, _) = mimetypes.guess_type(name) file.mime_type = mime_from_name or file.mime_type elif isinstance(attr, DocumentAttributeSticker): name = f"Sticker for {attr.alt}" + elif isinstance(attr, DocumentAttributeVideo): + width, height = attr.w, attr.h mime_type = document.mime_type or file.mime_type info = { "size": document.size, "mimetype": mime_type, } + if document.thumb: + thumbnail = await util.transfer_file_to_matrix(self.db, source.client, intent, + document.thumb.location) + info["thumbnail_info"] = { + "mimetype": thumbnail.mime_type, + "h": document.thumb.h, + "w": document.thumb.w, + "size": len(document.thumb.bytes) + } + info["thumbnail_url"] = thumbnail.mxc + if height and width: + info["h"] = height + info["w"] = width type = "m.file" if mime_type.startswith("video/"): type = "m.video" @@ -985,7 +1001,7 @@ class Portal: if not response: return - + self.log.debug("Handled Telegram message: %s", evt) mxid = response["event_id"] DBMessage.query \ .filter(DBMessage.mx_room == self.mxid, diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py index 8c56113d..044b982f 100644 --- a/mautrix_telegram/util/file_transfer.py +++ b/mautrix_telegram/util/file_transfer.py @@ -23,7 +23,7 @@ from PIL import Image from sqlalchemy.exc import IntegrityError from telethon_aio.tl.types import (Document, FileLocation, InputFileLocation, - InputDocumentFileLocation) + InputDocumentFileLocation, PhotoCachedSize) from telethon_aio.errors import LocationInvalidError from ..db import TelegramFile as DBTelegramFile From e98acf39ae6d54afc3c1dd0292bc2e4cdbbad6fd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 4 Mar 2018 21:12:24 +0200 Subject: [PATCH 02/41] Fix messages with URL previews not being bridged --- mautrix_telegram/portal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 88258062..4f9d5f7b 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -979,7 +979,8 @@ class Portal: DBMessage(tgid=evt.id, mx_room=self.mxid, mxid=mxid, tg_space=tg_space)) self.db.commit() return - media = evt.media if hasattr(evt, "media") else None + allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo) + media = evt.media if hasattr(evt, "media") and isinstance(media, allowed_media) else None intent = sender.intent if sender else self.main_intent if not media and evt.message: response = await self.handle_telegram_text(source, intent, evt) From e8c031283949125942091b85acb6af13427901ae Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 4 Mar 2018 21:22:19 +0200 Subject: [PATCH 03/41] Fix messages again --- mautrix_telegram/portal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 4f9d5f7b..48176407 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -980,7 +980,8 @@ class Portal: self.db.commit() return allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo) - media = evt.media if hasattr(evt, "media") and isinstance(media, allowed_media) else None + media = evt.media if hasattr(evt, "media") and isinstance(evt.media, + allowed_media) else None intent = sender.intent if sender else self.main_intent if not media and evt.message: response = await self.handle_telegram_text(source, intent, evt) From a8322992cce5c94c72fd27cc650db0d8b20fe8d1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 4 Mar 2018 23:17:12 +0200 Subject: [PATCH 04/41] Escape HTML tags in quoted text of non-native replies --- mautrix_telegram/formatter/from_telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 82b38c10..34d81e7f 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -93,7 +93,7 @@ async def _add_reply_header(source, text, html, evt, relates_to, content = event["content"] body = (content["formatted_body"] if "formatted_body" in content - else content["body"]) + else escape(content["body"])) sender = event['sender'] puppet = pu.Puppet.get_by_mxid(sender, create=False) reply_displayname = puppet.displayname if puppet else sender From ed8c9337725a02fb7349ff3f278c9f8458466c34 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 5 Mar 2018 15:04:18 +0200 Subject: [PATCH 05/41] Fix possible web login bug --- mautrix_telegram/public/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix_telegram/public/__init__.py b/mautrix_telegram/public/__init__.py index 3f57e369..45b91c03 100644 --- a/mautrix_telegram/public/__init__.py +++ b/mautrix_telegram/public/__init__.py @@ -144,7 +144,7 @@ class PublicBridgeWebsite: if "mxid" not in data: return self.render_login(error="Please enter your Matrix ID.", status=400) - user = await User.get_by_mxid(data["mxid"]).ensure_started() + user = await User.get_by_mxid(data["mxid"]).ensure_started(even_if_no_session=True) if not user.whitelisted: return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403) elif user.logged_in: From e2ba4780952de8937e370a4a6443536810ba497a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Mar 2018 00:27:35 +0200 Subject: [PATCH 06/41] Fix highlighting Telegram users without usernames --- mautrix_telegram/formatter/from_matrix.py | 10 ++++------ mautrix_telegram/portal.py | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index c74ac1ae..72d2edfe 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -53,17 +53,15 @@ class MatrixParser(HTMLParser): mention = self.mention_regex.match(url) if mention: mxid = mention.group(1) - user = (pu.Puppet.get_by_mxid(mxid, create=False) + user = (pu.Puppet.get_by_mxid(mxid) or u.User.get_by_mxid(mxid, create=False)) if not user: return None, None if user.username: - entity_type = MessageEntityMention - url = f"@{user.username}" + return MessageEntityMention, f"@{user.username}" else: - entity_type = MessageEntityMentionName - args["user_id"] = user.tgid - return entity_type, url + args["user_id"] = InputUser(user.tgid, 0) + return InputMessageEntityMentionName, user.displayname or None room = self.room_regex.match(url) if room: diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 48176407..bbd13273 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -580,13 +580,22 @@ class Portal: message["body"] = f"<{sender.displayname}> {message['body']}" return type - def _handle_matrix_text(self, client, message, reply_to): - if "format" in message and message["format"] == "org.matrix.custom.html": + async def _handle_matrix_text(self, client, message, reply_to): + is_formatted = ("format" in message + and message["format"] == "org.matrix.custom.html" + and "formatted_body" in message) + if is_formatted: message, entities = formatter.matrix_to_telegram(message["formatted_body"]) - return client.send_message(self.peer, message, entities=entities, reply_to=reply_to) + + # TODO remove this crap + for entity in entities: + if isinstance(entity, InputMessageEntityMentionName): + entity.user_id = await client.get_input_entity(entity.user_id.user_id) + + return await client.send_message(self.peer, message, entities=entities, reply_to=reply_to) else: message = formatter.matrix_text_to_telegram(message["body"]) - return client.send_message(self.peer, message, reply_to=reply_to) + return await client.send_message(self.peer, message, reply_to=reply_to) async def _handle_matrix_file(self, client, message, reply_to): file = await self.main_intent.download_file(message["url"]) From fe00145d1c136358a87fe47d13462d997015ed86 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Mar 2018 15:13:13 +0200 Subject: [PATCH 07/41] Fix bridging documents without thumbnails --- mautrix_telegram/portal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index bbd13273..25e642e1 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -870,7 +870,7 @@ class Portal: "size": document.size, "mimetype": mime_type, } - if document.thumb: + if document.thumb and not isinstance(document.thumb, PhotoSizeEmpty): thumbnail = await util.transfer_file_to_matrix(self.db, source.client, intent, document.thumb.location) info["thumbnail_info"] = { From 97957a5731f13924475ba91482964c2cc74fc641 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Mar 2018 21:14:37 +0200 Subject: [PATCH 08/41] Use native reply fallback format. Fixes #86 --- example-config.yaml | 6 -- mautrix_telegram/formatter/from_matrix.py | 10 ++- mautrix_telegram/formatter/from_telegram.py | 76 ++++++++++++--------- mautrix_telegram/formatter/util.py | 21 +++++- mautrix_telegram/portal.py | 24 +++---- 5 files changed, 80 insertions(+), 57 deletions(-) diff --git a/example-config.yaml b/example-config.yaml index 2534728d..9a601c8f 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -70,12 +70,6 @@ bridge: - username - phone number - # Whether or not to use native Matrix replies. At the time of writing, only riot-web supports - # replies and the format of them is subject to change. - native_replies: true - # If native replies are disabled, should the custom replies contain a link to the message being - # replied to? - link_in_reply: false # Show message editing as a reply to the original message. # If this is false, message edits are not shown at all, as Matrix does not support editing yet. edits_as_replies: false diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index 72d2edfe..be9b88f8 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -25,7 +25,7 @@ from telethon_aio.tl.types import * from .. import user as u, puppet as pu, portal as po from ..db import Message as DBMessage -from .util import add_surrogates, remove_surrogates +from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, trim_reply_fallback_text) log = logging.getLogger("mau.fmt.mx") @@ -232,6 +232,14 @@ def matrix_reply_to_telegram(content, tg_space, room_id=None): reply = content["m.relates_to"]["m.in_reply_to"] room_id = room_id or reply["room_id"] event_id = reply["event_id"] + + try: + if content["format"] == "org.custom.matrix.html": + content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"]) + except KeyError: + pass + content["body"] = trim_reply_fallback_text(content["body"]) + message = DBMessage.query.filter(DBMessage.mxid == event_id, DBMessage.tg_space == tg_space, DBMessage.mx_room == room_id).one_or_none() diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 34d81e7f..276f68c3 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -22,7 +22,8 @@ from mautrix_appservice import MatrixRequestError from .. import user as u, puppet as pu, portal as po from ..db import Message as DBMessage -from .util import add_surrogates, remove_surrogates +from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, + trim_reply_fallback_text) log = logging.getLogger("mau.fmt.tg") @@ -67,8 +68,7 @@ async def _add_forward_header(source, text, html, fwd_from_id): return text, html -async def _add_reply_header(source, text, html, evt, relates_to, - native_replies, message_link_in_reply, main_intent, reply_text): +async def _add_reply_header(source, text, html, evt, relates_to, main_intent, is_edit): space = (evt.to_id.channel_id if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel) else source.tgid) @@ -77,42 +77,50 @@ async def _add_reply_header(source, text, html, evt, relates_to, if not msg: return text, html - if native_replies: - relates_to["m.in_reply_to"] = { - "event_id": msg.mxid, - "room_id": msg.mx_room, - } - if reply_text == "Edit": - html = f"Edit: {html or escape(text)}" - text = f"Edit: {text}" - return text, html + relates_to["m.in_reply_to"] = { + "event_id": msg.mxid, + "room_id": msg.mx_room, + } + if is_edit: + html = f"Edit: {html or escape(text)}" + text = f"Edit: {text}" - reply_displayname = "unknown user" try: event = await main_intent.get_event(msg.mx_room, msg.mxid) + content = event["content"] - body = (content["formatted_body"] - if "formatted_body" in content - else escape(content["body"])) - sender = event['sender'] - puppet = pu.Puppet.get_by_mxid(sender, create=False) - reply_displayname = puppet.displayname if puppet else sender - reply_to_user = f"{reply_displayname}" - reply_to_msg = (("{reply_text}") - if message_link_in_reply else "Reply") - quote = f"{reply_to_msg} to {reply_to_user}
{body}
" + r_sender = event["sender"] + + r_text_body = trim_reply_fallback_text(content["body"]) + r_html_body = trim_reply_fallback_html(content["formatted_body"] + if "formatted_body" in content + else escape(content["body"])) + + puppet = pu.Puppet.get_by_mxid(r_sender, create=False) + r_displayname = puppet.displayname if puppet else r_sender + r_sender_link = f"{r_displayname}" except (ValueError, KeyError, MatrixRequestError): - quote = f"{reply_text} to unknown user (Failed to fetch message):
" - if not html: - html = escape(text) - html = quote + html - text = f"{reply_text} to {reply_displayname}:\n{text}" - return text, html + r_sender_link = "unknown user" + # r_sender = "unknown user" + r_text_body = "Failed to fetch message" + r_html_body = "Failed to fetch message" + + r_keyword = "In reply to" if not is_edit else "Edit to" + r_msg_link = f"{r_keyword}" + html = (f"
{r_msg_link} {r_sender_link} {r_html_body}
" + + (html or escape(text))) + + lines = r_text_body.strip().split("\n") + text_with_quote = f"> <{r_displayname}> {lines.pop(0)}" + for line in lines: + if line: + text_with_quote += f"\n> {line}" + text_with_quote += "\n\n" + text_with_quote += text + return text_with_quote, html -async def telegram_to_matrix(evt, source, native_replies=False, message_link_in_reply=False, - main_intent=None, reply_text="Reply"): +async def telegram_to_matrix(evt, source, main_intent=None, is_edit=False): text = add_surrogates(evt.message) html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None relates_to = {} @@ -121,8 +129,8 @@ async def telegram_to_matrix(evt, source, native_replies=False, message_link_in_ text, html = await _add_forward_header(source, text, html, evt.fwd_from.from_id) if evt.reply_to_msg_id: - text, html = await _add_reply_header(source, text, html, evt, relates_to, native_replies, - message_link_in_reply, main_intent, reply_text) + text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent, + is_edit) if isinstance(evt, Message) and evt.post and evt.post_author: if not html: diff --git a/mautrix_telegram/formatter/util.py b/mautrix_telegram/formatter/util.py index ff35519d..ec51844c 100644 --- a/mautrix_telegram/formatter/util.py +++ b/mautrix_telegram/formatter/util.py @@ -1,8 +1,9 @@ -# Unicode surrogate handling -# From https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py import struct +import re +# Unicode surrogate handling from +# https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py def add_surrogates(text): if text is None: return None @@ -14,3 +15,19 @@ def remove_surrogates(text): if text is None: return None return text.encode("utf-16", "surrogatepass").decode("utf-16") + + +def trim_reply_fallback_text(text): + if not text.startswith("> ") or "\n" not in text: + return text + lines = text.split("\n") + while len(lines) > 0 and lines[0].startswith("> "): + lines.pop(0) + return "\n".join(lines) + + +HTML_REPLY_FALLBACK_REGEX = re.compile(r"^
[\s\S]+?
") + + +def trim_reply_fallback_html(html): + return HTML_REPLY_FALLBACK_REGEX.sub("", html) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 25e642e1..4b737f6e 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -33,6 +33,7 @@ from mautrix_appservice import MatrixRequestError, IntentError from .db import Portal as DBPortal, Message as DBMessage from . import puppet as p, user as u, formatter, util +from .formatter.util import trim_reply_fallback_html, trim_reply_fallback_text mimetypes.init() @@ -575,9 +576,10 @@ class Portal: message["msgtype"] = "m.text" elif not sender.logged_in: if "formatted_body" in message: - message["formatted_body"] = (f"<{sender.displayname}> " - f"{message['formatted_body']}") - message["body"] = f"<{sender.displayname}> {message['body']}" + html = message["formatted_body"] + message["formatted_body"] = f"<{sender.displayname}> {html}" + text = message["body"] + message["body"] = f"<{sender.displayname}> {text}" return type async def _handle_matrix_text(self, client, message, reply_to): @@ -592,7 +594,8 @@ class Portal: if isinstance(entity, InputMessageEntityMentionName): entity.user_id = await client.get_input_entity(entity.user_id.user_id) - return await client.send_message(self.peer, message, entities=entities, reply_to=reply_to) + return await client.send_message(self.peer, message, entities=entities, + reply_to=reply_to) else: message = formatter.matrix_text_to_telegram(message["body"]) return await client.send_message(self.peer, message, reply_to=reply_to) @@ -922,11 +925,7 @@ class Portal: async def handle_telegram_text(self, source, intent, evt): self.log.debug(f"Sending {evt.message} to {self.mxid} by {intent.mxid}") - text, html, relates_to = await formatter.telegram_to_matrix( - evt, source, - config["bridge.native_replies"], - config["bridge.link_in_reply"], - self.main_intent) + text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent) await intent.set_typing(self.mxid, is_typing=False) return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to) @@ -950,11 +949,8 @@ class Portal: return evt.reply_to_msg_id = evt.id - text, html, relates_to = await formatter.telegram_to_matrix( - evt, source, - config["bridge.native_replies"], - config["bridge.link_in_reply"], - self.main_intent, reply_text="Edit") + text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent, + is_edit=True) intent = sender.intent if sender else self.main_intent await intent.set_typing(self.mxid, is_typing=False) response = await intent.send_text(self.mxid, text, html=html, relates_to=relates_to) From 3aff450bae728e2aeee9f7ea027c1d688fed393b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Mar 2018 22:43:31 +0200 Subject: [PATCH 09/41] Fix error with large thumbnails --- mautrix_telegram/portal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 4b737f6e..1de1f907 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -880,7 +880,9 @@ class Portal: "mimetype": thumbnail.mime_type, "h": document.thumb.h, "w": document.thumb.w, - "size": len(document.thumb.bytes) + "size": (len(document.thumb.bytes) + if isinstance(document.thumb, PhotoCachedSize) + else document.thumb.size) } info["thumbnail_url"] = thumbnail.mxc if height and width: From 13dddb4c10fd2468177251ed60e7d2eaa0102cad Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 13:00:13 +0200 Subject: [PATCH 10/41] Override alias if it already exists --- mautrix_appservice/intent_api.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mautrix_appservice/intent_api.py b/mautrix_appservice/intent_api.py index fa148b2e..5a7495a5 100644 --- a/mautrix_appservice/intent_api.py +++ b/mautrix_appservice/intent_api.py @@ -277,11 +277,16 @@ class IntentAPI: content["info"] = info return self.send_state_event(room_id, "m.room.avatar", content) - async def add_room_alias(self, room_id, localpart): + async def add_room_alias(self, room_id, localpart, override=True): await self.ensure_registered() content = {"room_id": room_id} alias = f"#{localpart}:{self.client.domain}" - return await self.client.request("PUT", f"/directory/room/{quote(alias)}", content) + try: + return await self.client.request("PUT", f"/directory/room/{quote(alias)}", content) + except MatrixRequestError as e: + if override and e.code == 409: + await self.remove_room_alias(localpart) + return await self.client.request("PUT", f"/directory/room/{quote(alias)}", content) async def remove_room_alias(self, localpart): await self.ensure_registered() From a6f26c16fc4f59bfa07f2445ac06dd24e43690cf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 14:03:38 +0200 Subject: [PATCH 11/41] Add strikethrough/underline <-> unicode converter to formatter --- mautrix_telegram/formatter/from_matrix.py | 13 +++++++-- mautrix_telegram/formatter/from_telegram.py | 5 +++- mautrix_telegram/formatter/util.py | 31 +++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index be9b88f8..842ee3ba 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -25,7 +25,8 @@ from telethon_aio.tl.types import * from .. import user as u, puppet as pu, portal as po from ..db import Message as DBMessage -from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, trim_reply_fallback_text) +from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, + trim_reply_fallback_text, html_to_unicode) log = logging.getLogger("mau.fmt.mx") @@ -35,7 +36,7 @@ class MatrixParser(HTMLParser): room_regex = re.compile("https://matrix.to/#/(#.+:.+)") block_tags = ("br", "p", "pre", "blockquote", "ol", "ul", "li", - "h1", "h2", "h3", "h4", "h5", "h6" + "h1", "h2", "h3", "h4", "h5", "h6", "div", "hr", "table") def __init__(self): @@ -159,6 +160,14 @@ class MatrixParser(HTMLParser): text = url elif previous_tag == "command": text = f"/{text}" + + # Strikethrough + if "del" in self._open_tags: + text = html_to_unicode(text, "\u0336") + # Underline + if "u" in self._open_tags: + text = html_to_unicode(text, "\u0332") + list_entry_handled_once = False # In order to maintain order of things like blockquotes in lists or lists in blockquotes, # we can't just have ifs/elses and we need to actually loop through the open tags in order. diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 276f68c3..7ed974e4 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -23,7 +23,7 @@ from mautrix_appservice import MatrixRequestError from .. import user as u, puppet as pu, portal as po from ..db import Message as DBMessage from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, - trim_reply_fallback_text) + trim_reply_fallback_text, unicode_to_html) log = logging.getLogger("mau.fmt.tg") @@ -138,6 +138,9 @@ async def telegram_to_matrix(evt, source, main_intent=None, is_edit=False): text += f"\n- {evt.post_author}" html += f"
- {evt.post_author}" + html = unicode_to_html(text, html, "\u0336", "del") + html = unicode_to_html(text, html, "\u0332", "u") + if html: html = html.replace("\n", "
") diff --git a/mautrix_telegram/formatter/util.py b/mautrix_telegram/formatter/util.py index ec51844c..86c263fd 100644 --- a/mautrix_telegram/formatter/util.py +++ b/mautrix_telegram/formatter/util.py @@ -1,3 +1,4 @@ +from html import escape import struct import re @@ -31,3 +32,33 @@ HTML_REPLY_FALLBACK_REGEX = re.compile(r"^
[\s\S]+?" + tag_end = f"" + characters = html.split(ctrl) + html = "" + in_del = False + for char in characters: + if not in_del: + if len(char) > 1: + html += char[0:-1] + char = char[-1] + html += tag_start + in_del = True + html += char + else: + if len(char) > 1: + html += tag_end + in_del = False + html += char + return html + + +def html_to_unicode(text, ctrl): + return ctrl.join(text) + ctrl From 8bbd1f7db14b4eb4522b79d000cea9c326c4b039 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 14:12:37 +0200 Subject: [PATCH 12/41] Fix duplicate unicode formatting when mixing strikethrough and underline --- mautrix_telegram/formatter/from_matrix.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index 842ee3ba..b5dee1d6 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -161,11 +161,12 @@ class MatrixParser(HTMLParser): elif previous_tag == "command": text = f"/{text}" - # Strikethrough - if "del" in self._open_tags: + strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags + if strikethrough and underline: + text = html_to_unicode(text, "\u0336\u0332") + elif strikethrough: text = html_to_unicode(text, "\u0336") - # Underline - if "u" in self._open_tags: + elif underline: text = html_to_unicode(text, "\u0332") list_entry_handled_once = False From 7b4b7509f32a8675d3a0ba2ded7a210adbf19c91 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 14:50:36 +0200 Subject: [PATCH 13/41] Minor improvements to unicode->html formatter --- mautrix_telegram/formatter/util.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mautrix_telegram/formatter/util.py b/mautrix_telegram/formatter/util.py index 86c263fd..54502fbc 100644 --- a/mautrix_telegram/formatter/util.py +++ b/mautrix_telegram/formatter/util.py @@ -35,7 +35,7 @@ def trim_reply_fallback_html(html): def unicode_to_html(text, html, ctrl, tag): - if "\u0336" not in text and "\u0332" not in text: + if ctrl not in text: return html if not html: html = escape(text) @@ -43,20 +43,22 @@ def unicode_to_html(text, html, ctrl, tag): tag_end = f"" characters = html.split(ctrl) html = "" - in_del = False + in_tag = False for char in characters: - if not in_del: + if not in_tag: if len(char) > 1: html += char[0:-1] char = char[-1] html += tag_start - in_del = True + in_tag = True html += char else: if len(char) > 1: html += tag_end - in_del = False + in_tag = False html += char + if in_tag: + html += tag_end return html From 0e6940eea55cc6aa31189dd6b8a085ac9ee0213b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 15:37:19 +0200 Subject: [PATCH 14/41] Add bridge command to !help (ref #87) --- mautrix_telegram/commands/meta.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mautrix_telegram/commands/meta.py b/mautrix_telegram/commands/meta.py index 5d2f38df..16505a4e 100644 --- a/mautrix_telegram/commands/meta.py +++ b/mautrix_telegram/commands/meta.py @@ -73,6 +73,9 @@ def help(evt): Only works for group chats; to delete a private chat portal, simply leave the room. **unbridge** - Remove puppets from the current portal room and forget the portal. +**bridge** [_id_] - Bridge the current Matrix room to the Telegram chat with the given + ID. The ID must be the prefixed version that you get with the `/id` + command of the Telegram-side bot. **group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash (`-`) as the name. **clean-rooms** - Clean up unused portal/management rooms. From 7004da92689be2a48e60f787f5e8292bac8a4bc3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 19:11:23 +0200 Subject: [PATCH 15/41] Handle SQL InvalidRequestErrors when transferring files --- mautrix_telegram/util/file_transfer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py index 044b982f..0a2b47dd 100644 --- a/mautrix_telegram/util/file_transfer.py +++ b/mautrix_telegram/util/file_transfer.py @@ -20,7 +20,7 @@ import logging import magic from PIL import Image -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, InvalidRequestError from telethon_aio.tl.types import (Document, FileLocation, InputFileLocation, InputDocumentFileLocation, PhotoCachedSize) @@ -73,9 +73,9 @@ async def transfer_file_to_matrix(db, client, intent, location): try: db.add(db_file) db.commit() - except IntegrityError: + except (IntegrityError, InvalidRequestError) as e: db.rollback() - log.exception("Integrity error while saving transferred file data. " + log.exception(f"{e.__class__.__name__} while saving transferred file data. " "This was probably caused by two simultaneous transfers of the same file, " "and should not cause any problems.") From 30768d0a0674e3196d10d04a598238b2033df523 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 21:16:09 +0200 Subject: [PATCH 16/41] Add option to use inline images for better captions. Fixes #83 --- example-config.yaml | 5 +++- mautrix_telegram/config.py | 52 ++++++++++++++++++++++++++++++-------- mautrix_telegram/portal.py | 10 ++++++++ 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/example-config.yaml b/example-config.yaml index 9a601c8f..f1b72855 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -83,6 +83,9 @@ bridge: allow_matrix_login: true # Whether or not to allow creating portals from Telegram. authless_relaybot_portals: true + # Use inline images instead of m.image to make rich captions possible. + # N.B. Inline images are not supported on all clients (e.g. Riot iOS). + inline_images: false # The prefix for commands. Only required in non-management rooms. command_prefix: "!tg" @@ -112,4 +115,4 @@ telegram: # The version of the config. The bridge will read this and automatically update the config if # the schema has changed. For the latest version, check the example config. -version: 1 +version: 2 diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 7bab5563..b7bcf8bc 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -16,6 +16,8 @@ # along with this program. If not, see . from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap +from ruamel.yaml.tokens import CommentToken +from ruamel.yaml.error import CommentMark import random import string @@ -42,6 +44,9 @@ class DictWithRecursion: def __getitem__(self, key): return self.get(key, None) + def __contains__(self, key): + return self[key] is not None + def _recursive_set(self, data, key, value): if '.' in key: key, next_key = key.split('.', 1) @@ -71,6 +76,7 @@ class DictWithRecursion: return try: del data[key] + del data.ca.items[key] except KeyError: pass @@ -80,6 +86,7 @@ class DictWithRecursion: return try: del self._data[key] + del self._data.ca.items[key] except KeyError: pass @@ -93,10 +100,19 @@ class DictWithRecursion: except ValueError: path = None entry = self[path] if path else self._data - c = self._data.ca.items.setdefault(key, [None, [], None, None]) + c = entry.ca.items.setdefault(key, [None, [], None, None]) c[1] = [] entry.yaml_set_comment_before_after_key(key=key, before=message, indent=indent) + def comment_newline(self, key): + try: + path, key = key.rsplit(".", 1) + except ValueError: + path = None + entry = self[path] if path else self._data + c = entry.ca.items.setdefault(key, [None, [], None, None]) + c[2] = CommentToken("\n\n", CommentMark(0), None) + class Config(DictWithRecursion): def __init__(self, path, registration_path): @@ -131,10 +147,10 @@ class Config(DictWithRecursion): del self["bridge.whitelist"] del self["bridge.admins"] - self["bridge.authless_relaybot_portals"] = self.get("bridge.authless_relaybot_portals", - True) - self.comment("bridge.authless_relaybot_portals", - "Whether or not to allow creating portals from Telegram.") + if "bridge.authless_relaybot_portals" not in self: + self["bridge.authless_relaybot_portals"] = True + self.comment("bridge.authless_relaybot_portals", + "Whether or not to allow creating portals from Telegram.") self.comment("bridge.permissions", "\n".join(( "", @@ -156,13 +172,29 @@ class Config(DictWithRecursion): "\nThe version of the config. The bridge will read this and automatically " "update the config if\nthe schema has changed. For the latest version, " "check the example config.") + return self["version"] + + def update_1_2(self): + del self["bridge.link_in_reply"] + del self["bridge.native_replies"] + if "bridge.inline_images" not in self: + self["bridge.inline_images"] = False + self.comment("bridge.inline_images", + "Use inline images instead of m.image to make rich captions possible.\n" + "N.B. Inline images are not supported on all clients (e.g. Riot iOS).") + self.comment_newline("bridge.inline_images") + self["version"] = 2 + return self["version"] def check_updates(self): - if self.get("version", 0) == 0: - self.update_0_1() - else: - return - self.save() + version = self.get("version", 0) + new_version = version + if version < 1: + new_version = self.update_0_1() + if version < 2: + new_version = self.update_1_2() + if new_version != version: + self.save() def _get_permissions(self, key): level = self["bridge.permissions"].get(key, "") diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 1de1f907..01055ab6 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -16,6 +16,7 @@ # along with this program. If not, see . from collections import deque from datetime import datetime +from html import escape import asyncio import random import mimetypes @@ -838,6 +839,15 @@ class Portal: largest_size.location) if not file: return None + if config["bridge.inline_images"] and evt.message: + text, html, relates_to = await formatter.telegram_to_matrix(evt, source, + self.main_intent) + await intent.set_typing(self.mxid, is_typing=False) + print(self.main_intent.client.get_download_url(file.mxc)) + inline_img = f"Inline Telegram photo
\n" + html = inline_img + (html or escape(text)) + text = f"Inline image: {text}" + return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to) info = { "h": largest_size.h, "w": largest_size.w, From 6ff89d1fe4aaa37a59086f503d20e790d971424e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 21:18:44 +0200 Subject: [PATCH 17/41] Add option to disable homeserver SSL verification --- example-config.yaml | 1 + mautrix_appservice/appservice.py | 8 ++++++-- mautrix_telegram/__main__.py | 3 ++- mautrix_telegram/config.py | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/example-config.yaml b/example-config.yaml index f1b72855..f2a96e96 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -2,6 +2,7 @@ homeserver: address: https://matrix.org domain: matrix.org + verify_ssl: true # Application service host/registration related details # Changing these values requires regeneration of the registration. diff --git a/mautrix_appservice/appservice.py b/mautrix_appservice/appservice.py index bb340607..bd775df4 100644 --- a/mautrix_appservice/appservice.py +++ b/mautrix_appservice/appservice.py @@ -28,9 +28,10 @@ from .state_store import StateStore class AppService: def __init__(self, server, domain, as_token, hs_token, bot_localpart, loop=None, log=None, - query_user=None, query_alias=None): + verify_ssl=True, query_user=None, query_alias=None): self.server = server self.domain = domain + self.verify_ssl = verify_ssl self.as_token = as_token self.hs_token = hs_token self.bot_mxid = f"@{bot_localpart}:{domain}" @@ -80,7 +81,10 @@ class AppService: @contextmanager def run(self, host="127.0.0.1", port=8080): - self._http_session = aiohttp.ClientSession(loop=self.loop) + connector = None + if self.server.startswith("https://") and not self.verify_ssl: + connector = aiohttp.TCPConnector(verify_ssl=False) + self._http_session = aiohttp.ClientSession(loop=self.loop, connector=connector) self._intent = HTTPAPI(base_url=self.server, domain=self.domain, bot_mxid=self.bot_mxid, token=self.as_token, log=self.log, state_store=self.state_store, client_session=self._http_session).bot_intent() diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index 2c5ce1ff..6023a7c8 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -85,7 +85,8 @@ loop = asyncio.get_event_loop() appserv = AppService(config["homeserver.address"], config["homeserver.domain"], config["appservice.as_token"], config["appservice.hs_token"], - config["appservice.bot_username"], log="mau.as", loop=loop) + config["appservice.bot_username"], log="mau.as", loop=loop, + verify_ssl=config["homeserver.verify_ssl"]) context = Context(appserv, db_session, config, loop, None, None, telethon_session_container) diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index b7bcf8bc..070f04ec 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -183,6 +183,8 @@ class Config(DictWithRecursion): "Use inline images instead of m.image to make rich captions possible.\n" "N.B. Inline images are not supported on all clients (e.g. Riot iOS).") self.comment_newline("bridge.inline_images") + if "homeserver.verify_ssl" not in self: + self["homeserver.verify_ssl"] = True self["version"] = 2 return self["version"] From 2c68bd7378205eefeae42d36c943bad924f3d405 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 21:25:49 +0200 Subject: [PATCH 18/41] Update config updater --- mautrix_telegram/config.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 070f04ec..a00a6ae0 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -151,6 +151,12 @@ class Config(DictWithRecursion): self["bridge.authless_relaybot_portals"] = True self.comment("bridge.authless_relaybot_portals", "Whether or not to allow creating portals from Telegram.") + if "bridge.max_telegram_delete" not in self: + self["bridge.max_telegram_delete"] = 10 + self.comment("bridge.max_telegram_delete", + "The maximum number of simultaneous Telegram deletions to handle.\n" + "A large number of simultaneous redactions could put strain on your " + "homeserver.") self.comment("bridge.permissions", "\n".join(( "", @@ -177,12 +183,21 @@ class Config(DictWithRecursion): def update_1_2(self): del self["bridge.link_in_reply"] del self["bridge.native_replies"] + if "bridge.bridge_notices" not in self: + self["bridge.bridge_notices"] = False + self.comment("bridge.bridge_notices", + "Whether or not Matrix bot messages (type m.notice) should be bridged.") + if "bridge.allow_matrix_login" not in self: + self["bridge.allow_matrix_login"] = True + self.comment("bridge.allow_matrix_login", + "Allow logging in within Matrix. If false, the only way to log in is " + "using the out-of-Matrix login website (see appservice.public config " + "section)") if "bridge.inline_images" not in self: self["bridge.inline_images"] = False self.comment("bridge.inline_images", "Use inline images instead of m.image to make rich captions possible.\n" "N.B. Inline images are not supported on all clients (e.g. Riot iOS).") - self.comment_newline("bridge.inline_images") if "homeserver.verify_ssl" not in self: self["homeserver.verify_ssl"] = True self["version"] = 2 From cf9a1f3afbd7cebaf19fe284b5353297819b163d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 21:31:27 +0200 Subject: [PATCH 19/41] Add appservice.public to config in v2 update --- mautrix_telegram/config.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index a00a6ae0..d3ad67d3 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -198,6 +198,21 @@ class Config(DictWithRecursion): self.comment("bridge.inline_images", "Use inline images instead of m.image to make rich captions possible.\n" "N.B. Inline images are not supported on all clients (e.g. Riot iOS).") + if "appservice.public" not in self: + self["appservice.public.enabled"] = False + self["appservice.public.prefix"] = "/public" + self["appservice.public.external"] = "https://example.com/public" + self.comment("appservice.public", + "Public part of web server for out-of-Matrix interaction with the " + "bridge.\nUsed for things like login if the user wants to make sure the " + "2FA password isn't stored in the HS database.") + self.comment("appservice.public.enabled", + "Whether or not the public-facing endpoints should be enabled.") + self.comment("appservice.public.prefix", + "The prefix to use in the public-facing endpoints.") + self.comment("appservice.public.external", + "The base URL where the public-facing endpoints are available. The " + "prefix is not added\nimplicitly.") if "homeserver.verify_ssl" not in self: self["homeserver.verify_ssl"] = True self["version"] = 2 From 8d1de218a1b6d845ca7cfa0a1c3267a2d047ba8c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 22:05:53 +0200 Subject: [PATCH 20/41] Implement registering (untested), fix auth stuff and possibly break stuff. Fixes #44 --- mautrix_telegram/commands/auth.py | 88 +++++++++++++++++++++------- mautrix_telegram/commands/handler.py | 3 +- mautrix_telegram/portal.py | 1 - mautrix_telegram/public/__init__.py | 7 ++- 4 files changed, 74 insertions(+), 25 deletions(-) diff --git a/mautrix_telegram/commands/auth.py b/mautrix_telegram/commands/auth.py index 60ce69d4..2250b6f3 100644 --- a/mautrix_telegram/commands/auth.py +++ b/mautrix_telegram/commands/auth.py @@ -51,6 +51,50 @@ def register(evt): return evt.reply("Not yet implemented.") +@command_handler(needs_auth=False, management_only=True) +async def register(evt): + if evt.sender.logged_in: + return await evt.reply("You are already logged in.") + elif len(evt.args) < 1: + return await evt.reply("**Usage:** `$cmdprefix+sp register `") + + phone_number = evt.args[0] + full_name = evt.args[1:].split(" ", 1) + if len(full_name) == 1: + full_name.append("") + await request_code(evt, phone_number, { + "next": enter_code_register, + "action": "Register", + "full_name": full_name, + }) + + +async def enter_code_register(evt): + if len(evt.args) == 0: + return await evt.reply("**Usage:** `$cmdprefix+sp `") + try: + await evt.sender.ensure_started(even_if_no_session=True) + first_name, last_name = evt.sender.command_status["full_name"] + user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name) + asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop) + evt.sender.command_status = None + return await evt.reply(f"Successfully registered to Telegram.") + except PhoneNumberOccupiedError: + return await evt.reply("That phone number has already been registered. " + "You can log in with `$cmdprefix+sp login`.") + except FirstNameInvalidError: + return await evt.reply("Invalid name. Please set a Matrix displayname before registering.") + except PhoneCodeExpiredError: + return await evt.reply( + "Phone code expired. Try again with `$cmdprefix+sp register `.") + except PhoneCodeInvalidError: + return await evt.reply("Invalid phone code.") + except Exception: + evt.log.exception("Error sending phone code") + return await evt.reply("Unhandled exception while sending code. " + "Check console for more details.") + + @command_handler(needs_auth=False, management_only=True) async def login(evt): if evt.sender.logged_in: @@ -80,22 +124,12 @@ async def login(evt): return await evt.reply("This bridge instance has been configured to not allow logging in.") -@command_handler(needs_auth=False) -async def enter_phone(evt): - if len(evt.args) == 0: - return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone `") - elif not evt.config.get("bridge.allow_matrix_login", True): - return await evt.reply("This bridge instance does not allow in-Matrix login. " - "Please use `$cmdprefix+sp login` to get login instructions") - - phone_number = evt.args[0] +async def request_code(evt, phone_number, next_status): + ok = False try: await evt.sender.ensure_started(even_if_no_session=True) await evt.sender.client.sign_in(phone_number) - evt.sender.command_status = { - "next": enter_code, - "action": "Login", - } + ok = True return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.") except PhoneNumberAppSignupForbiddenError: return await evt.reply( @@ -109,17 +143,31 @@ async def enter_phone(evt): "Your phone number has been temporarily blocked for flooding. " f"Please wait for {format_duration(e.seconds)} before trying again.") except PhoneNumberBannedError: - return await evt.reply("Your phone number has been banned from Telegram.") + return await evt.reply("Your phone number has been banned from Telegram.") except PhoneNumberUnoccupiedError: - return await evt.reply("That phone number has not been registered. " - "Please register with `$cmdprefix+sp register `.") + return await evt.reply("That phone number has not been registered. " + "Please register with `$cmdprefix+sp register `.") except Exception: evt.log.exception("Error requesting phone code") return await evt.reply("Unhandled exception while requesting code. " "Check console for more details.") finally: - if evt.sender.command_status["next"] == enter_phone: - evt.sender.command_status = None + evt.sender.command_status = next_status if ok else None + + +@command_handler(needs_auth=False) +async def enter_phone(evt): + if len(evt.args) == 0: + return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone `") + elif not evt.config.get("bridge.allow_matrix_login", True): + return await evt.reply("This bridge instance does not allow in-Matrix login. " + "Please use `$cmdprefix+sp login` to get login instructions") + + phone_number = evt.args[0] + await request_code(evt, phone_number, { + "next": enter_code, + "action": "Login", + }) @command_handler(needs_auth=False) @@ -136,8 +184,7 @@ async def enter_code(evt): evt.sender.command_status = None return await evt.reply(f"Successfully logged in as @{user.username}") except PhoneCodeExpiredError: - return await evt.reply( - "Phone code expired. Try again with `$cmdprefix+sp login `.") + return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.") except PhoneCodeInvalidError: return await evt.reply("Invalid phone code.") except SessionPasswordNeededError: @@ -160,7 +207,6 @@ async def enter_password(evt): elif not evt.config.get("bridge.allow_matrix_login", True): return await evt.reply("This bridge instance does not allow in-Matrix login. " "Please use `$cmdprefix+sp login` to get login instructions") - try: await evt.sender.ensure_started(even_if_no_session=True) user = await evt.sender.client.sign_in(password=evt.args[0]) diff --git a/mautrix_telegram/commands/handler.py b/mautrix_telegram/commands/handler.py index fbbbfadc..1309d058 100644 --- a/mautrix_telegram/commands/handler.py +++ b/mautrix_telegram/commands/handler.py @@ -81,12 +81,13 @@ class CommandHandler: async def handle(self, room, sender, command, args, is_management, is_portal): evt = CommandEvent(self, room, sender, command, args, is_management, is_portal) + orig_command = command command = command.lower() try: command = command_handlers[command] except KeyError: if sender.command_status and "next" in sender.command_status: - args.insert(0, command) + args.insert(0, orig_command) evt.command = "" command = sender.command_status["next"] else: diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 01055ab6..72397a12 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -843,7 +843,6 @@ class Portal: text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent) await intent.set_typing(self.mxid, is_typing=False) - print(self.main_intent.client.get_download_url(file.mxc)) inline_img = f"Inline Telegram photo
\n" html = inline_img + (html or escape(text)) text = f"Inline image: {text}" diff --git a/mautrix_telegram/public/__init__.py b/mautrix_telegram/public/__init__.py index 45b91c03..4924d759 100644 --- a/mautrix_telegram/public/__init__.py +++ b/mautrix_telegram/public/__init__.py @@ -46,7 +46,9 @@ class PublicBridgeWebsite: user = (User.get_by_mxid(request.rel_url.query["mxid"], create=False) if "mxid" in request.rel_url.query else None) if not user: - return self.render_login(mxid=request.rel_url.query["mxid"], state="request") + return self.render_login( + mxid=request.rel_url.query["mxid"] if "mxid" in request.rel_url.query else None, + state="request") elif not user.whitelisted: return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403) await user.ensure_started() @@ -153,7 +155,8 @@ class PublicBridgeWebsite: if "phone" in data: return await self.post_login_phone(user, data["phone"]) elif "code" in data: - resp = await self.post_login_code(user, data["code"], password_in_data="password" in data) + resp = await self.post_login_code(user, data["code"], + password_in_data="password" in data) if resp or "password" not in data: return resp elif "password" not in data: From 96d3ca106a205dca7b6eab02e50a1eae631edf1f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 7 Mar 2018 23:28:36 +0200 Subject: [PATCH 21/41] Fix Matrix -> Telegram code block bridging --- mautrix_telegram/formatter/from_matrix.py | 8 ++++---- mautrix_telegram/portal.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index b5dee1d6..d005c05e 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -143,7 +143,7 @@ class MatrixParser(HTMLParser): return indent def _newline(self, allow_multi=False): - if self._line_is_new or allow_multi: + if self._line_is_new and not allow_multi: return self.text += "\n" self._line_is_new = True @@ -210,13 +210,13 @@ class MatrixParser(HTMLParser): except IndexError: pass - if tag in self.block_tags: - self._newline(allow_multi=tag == "br") - entity = self._building_entities.pop(tag, None) if entity: self.entities.append(entity) + if tag in self.block_tags: + self._newline(allow_multi=tag == "br") + command_regex = re.compile("(\s|^)!([A-Za-z0-9@]+)") diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 72397a12..7603704b 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -633,6 +633,7 @@ class Portal: else: self.log.debug("Unhandled Matrix event: %s", message) return + self.log.debug("Handled Matrix message: %s", response) self.is_duplicate(response, (event_id, space)) self.db.add(DBMessage( tgid=response.id, From ee8531143f6e537143d0854b25e1ab2ea58f1db9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 11:42:53 +0200 Subject: [PATCH 22/41] Fix small typo --- mautrix_telegram/formatter/from_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index d005c05e..b5b3576f 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -244,7 +244,7 @@ def matrix_reply_to_telegram(content, tg_space, room_id=None): event_id = reply["event_id"] try: - if content["format"] == "org.custom.matrix.html": + if content["format"] == "org.matrix.custom.html": content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"]) except KeyError: pass From 3eefbc4e34ae2cdba5b9e5d910c36e320db4189c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 16:53:20 +0200 Subject: [PATCH 23/41] Update README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 57704bd0..c053f520 100644 --- a/README.md +++ b/README.md @@ -8,5 +8,4 @@ A Matrix-Telegram hybrid puppeting/relaybot bridge. ## Discussion Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net) -A Telegram chat bridged to the Matrix room will be created once the bridge supports using a bot -for unauthenticated users. +A Telegram chat bridged to the Matrix room might be created at some point. From 150321a4d7448d10f2037837e3b3adb29076e8f2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 18:01:58 +0200 Subject: [PATCH 24/41] Fix replies/forwards to/of images --- mautrix_telegram/formatter/from_telegram.py | 8 +++++++- mautrix_telegram/portal.py | 13 ++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 7ed974e4..6dfbb1fe 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -120,11 +120,17 @@ async def _add_reply_header(source, text, html, evt, relates_to, main_intent, is return text_with_quote, html -async def telegram_to_matrix(evt, source, main_intent=None, is_edit=False): +async def telegram_to_matrix(evt, source, main_intent=None, is_edit=False, prefix_text=None, + prefix_html=None): text = add_surrogates(evt.message) html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None relates_to = {} + if prefix_html: + html = prefix_html + (html or escape(text)) + if prefix_text: + text = prefix_text + text + if evt.fwd_from: text, html = await _add_forward_header(source, text, html, evt.fwd_from.from_id) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 7603704b..b6bd56e0 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -834,19 +834,18 @@ class Portal: if self.mxid: await user.intent.set_typing(self.mxid, is_typing=True) - async def handle_telegram_photo(self, source, intent, evt, relates_to=None): + async def handle_telegram_photo(self, source: u.User, intent, evt: Message, relates_to=None): largest_size = self._get_largest_photo_size(evt.media.photo) file = await util.transfer_file_to_matrix(self.db, source.client, intent, largest_size.location) if not file: return None - if config["bridge.inline_images"] and evt.message: - text, html, relates_to = await formatter.telegram_to_matrix(evt, source, - self.main_intent) + if config["bridge.inline_images"] and (evt.message or evt.fwd_from or evt.reply_to_msg_id): + text, html, relates_to = await formatter.telegram_to_matrix( + evt, source, self.main_intent, + prefix_html=f"Inline Telegram photo
\n", + prefix_text="Inline image: ") await intent.set_typing(self.mxid, is_typing=False) - inline_img = f"Inline Telegram photo
\n" - html = inline_img + (html or escape(text)) - text = f"Inline image: {text}" return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to) info = { "h": largest_size.h, From 61d9d6890a6d68a2fe55aeca91d117071745f538 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 18:39:27 +0200 Subject: [PATCH 25/41] Bridge plaintext mentions of Telegram puppets into Telegram mentions --- mautrix_telegram/__main__.py | 2 + mautrix_telegram/formatter/__init__.py | 10 ++++- mautrix_telegram/formatter/from_matrix.py | 46 ++++++++++++++++++++- mautrix_telegram/formatter/from_telegram.py | 4 ++ mautrix_telegram/portal.py | 7 +--- mautrix_telegram/puppet.py | 15 +++++++ 6 files changed, 76 insertions(+), 8 deletions(-) diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index 6023a7c8..0936e514 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -35,6 +35,7 @@ from .user import init as init_user, User from .bot import init as init_bot from .portal import init as init_portal from .puppet import init as init_puppet +from .formatter import init as init_formatter from .public import PublicBridgeWebsite from .context import Context @@ -99,6 +100,7 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st init_abstract_user(context) context.bot = init_bot(context) context.mx = MatrixHandler(context) + init_formatter(context) init_portal(context) init_puppet(context) startup_actions = init_user(context) + [start, context.mx.init_as_bot()] diff --git a/mautrix_telegram/formatter/__init__.py b/mautrix_telegram/formatter/__init__.py index 0428e723..6252455a 100644 --- a/mautrix_telegram/formatter/__init__.py +++ b/mautrix_telegram/formatter/__init__.py @@ -1,2 +1,8 @@ -from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram -from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix +from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram, + init_mx) +from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg) + + +def init(context): + init_mx(context) + init_tg(context) diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index b5b3576f..971e7a97 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -219,11 +219,46 @@ class MatrixParser(HTMLParser): command_regex = re.compile("(\s|^)!([A-Za-z0-9@]+)") +plain_mention_regex = None def matrix_text_to_telegram(text): text = command_regex.sub(r"\1/\2", text) - return text + entities, pmr_replacer = plain_mention_to_text() + text = plain_mention_regex.sub(pmr_replacer, text) + return text, entities + + +def plain_mention_to_text(): + entities = [] + + def replacer(match): + puppet = pu.Puppet.find_by_displayname(match.group(2)) + if puppet: + offset = match.start() + length = match.end() - offset + if puppet.username: + entity = MessageEntityMention(offset, length) + text = f"@{puppet.username}" + else: + entity = InputMessageEntityMentionName(offset, length, + user_id=InputUser(puppet.tgid, 0)) + text = puppet.displayname + entities.append(entity) + return text + return "".join(match.groups()) + + return entities, replacer + + +def plain_mention_to_html(match): + puppet = pu.Puppet.find_by_displayname(match.group(2)) + if puppet: + return (f"{match.group(1)}" + f"" + f"{puppet.displayname}" + "") + return "".join(match.groups()) def matrix_to_telegram(html): @@ -231,6 +266,7 @@ def matrix_to_telegram(html): parser = MatrixParser() html = html.replace("\n", "") html = command_regex.sub(r"\1\2", html) + html = plain_mention_regex.sub(plain_mention_to_html, html) parser.feed(add_surrogates(html)) return remove_surrogates(parser.text.strip()), parser.entities except Exception: @@ -258,3 +294,11 @@ def matrix_reply_to_telegram(content, tg_space, room_id=None): except KeyError: pass return None + + +def init_mx(context): + global plain_mention_regex + config = context.config + dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)") + dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+") + plain_mention_regex = re.compile(f"(\s|^)({dn_template})") diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 6dfbb1fe..7d4be35e 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -254,3 +254,7 @@ def _parse_url(html, entity_text, url): url = "http://" + url html.append(f"{entity_text}") return False + + +def init_tg(context): + pass diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index b6bd56e0..94153ff8 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -594,12 +594,9 @@ class Portal: for entity in entities: if isinstance(entity, InputMessageEntityMentionName): entity.user_id = await client.get_input_entity(entity.user_id.user_id) - - return await client.send_message(self.peer, message, entities=entities, - reply_to=reply_to) else: - message = formatter.matrix_text_to_telegram(message["body"]) - return await client.send_message(self.peer, message, reply_to=reply_to) + message, entities = formatter.matrix_text_to_telegram(message["body"]) + return await client.send_message(self.peer, message, reply_to=reply_to) async def _handle_matrix_file(self, client, message, reply_to): file = await self.main_intent.download_file(message["url"]) diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index 4340498a..782ad2a5 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -191,6 +191,21 @@ class Puppet: return None + @classmethod + def find_by_displayname(cls, displayname): + if not displayname: + return None + + for _, puppet in cls.cache.items(): + if puppet.displayname and puppet.displayname == displayname: + return puppet + + puppet = DBPuppet.query.filter(DBPuppet.displayname == displayname).one_or_none() + if puppet: + return cls.from_db(puppet) + + return None + def init(context): global config From b3082da99973d4a8d51f7334e9cc55409a3bfbee Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 19:44:53 +0200 Subject: [PATCH 26/41] Add option to underline edited part of message in edits. Fixes #61 --- example-config.yaml | 2 ++ mautrix_telegram/formatter/from_telegram.py | 34 ++++++++++++++++++--- requirements/optional.txt | 1 + setup.py | 3 ++ 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 requirements/optional.txt diff --git a/example-config.yaml b/example-config.yaml index f2a96e96..59ffaa3b 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -74,6 +74,8 @@ bridge: # Show message editing as a reply to the original message. # If this is false, message edits are not shown at all, as Matrix does not support editing yet. edits_as_replies: false + # Highlight changed/added parts in edits. Requires lxml. + highlight_edits: false # Whether or not Matrix bot messages (type m.notice) should be bridged. bridge_notices: false # The maximum number of simultaneous Telegram deletions to handle. diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 7d4be35e..576f51be 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -15,7 +15,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . from html import escape +try: + from lxml.html.diff import htmldiff +except ImportError: + htmldiff = None import logging +import re from telethon_aio.tl.types import * from mautrix_appservice import MatrixRequestError @@ -26,6 +31,7 @@ from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, trim_reply_fallback_text, unicode_to_html) log = logging.getLogger("mau.fmt.tg") +should_highlight_edits = False def telegram_reply_to_matrix(evt, source): @@ -68,6 +74,21 @@ async def _add_forward_header(source, text, html, fwd_from_id): return text, html +def highlight_edits(new_html, old_html): + # Don't include `Edit:` text in diff. + if old_html.startswith("Edit: "): + old_html = old_html[len("Edit: "):] + + # Generate diff with lxml + new_html = htmldiff(old_html, new_html) + + # Replace with since Riot doesn't allow + new_html = new_html.replace("", "").replace("", "") + # Remove s since we just want to hide deletions. + new_html = re.sub(".+?", "", new_html) + return new_html + + async def _add_reply_header(source, text, html, evt, relates_to, main_intent, is_edit): space = (evt.to_id.channel_id if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel) @@ -81,9 +102,6 @@ async def _add_reply_header(source, text, html, evt, relates_to, main_intent, is "event_id": msg.mxid, "room_id": msg.mx_room, } - if is_edit: - html = f"Edit: {html or escape(text)}" - text = f"Edit: {text}" try: event = await main_intent.get_event(msg.mx_room, msg.mxid) @@ -99,12 +117,19 @@ async def _add_reply_header(source, text, html, evt, relates_to, main_intent, is puppet = pu.Puppet.get_by_mxid(r_sender, create=False) r_displayname = puppet.displayname if puppet else r_sender r_sender_link = f"{r_displayname}" + + if is_edit and should_highlight_edits: + html = highlight_edits(html or escape(text), r_html_body) except (ValueError, KeyError, MatrixRequestError): r_sender_link = "unknown user" # r_sender = "unknown user" r_text_body = "Failed to fetch message" r_html_body = "Failed to fetch message" + if is_edit: + html = f"Edit: {html or escape(text)}" + text = f"Edit: {text}" + r_keyword = "In reply to" if not is_edit else "Edit to" r_msg_link = f"{r_keyword}" html = (f"
{r_msg_link} {r_sender_link} {r_html_body}
" @@ -257,4 +282,5 @@ def _parse_url(html, entity_text, url): def init_tg(context): - pass + global should_highlight_edits + should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"] diff --git a/requirements/optional.txt b/requirements/optional.txt new file mode 100644 index 00000000..ab90481d --- /dev/null +++ b/requirements/optional.txt @@ -0,0 +1 @@ +lxml diff --git a/setup.py b/setup.py index dbc732ed..eabecf3d 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,9 @@ setuptools.setup( dependency_links=[ "https://github.com/tulir/telethon-asyncio/tarball/9b389cfb4b6d3876e9661c23507f17e96897e4b0#egg=telethon-aio-git-0.18.0+1" ], + extras_require={ + "highlight_edits": ["lxml>=4.1.1,<5"], + }, classifiers=[ "Development Status :: 4 Beta", From f6e3903b45ce06cd54e52075fce87bba81931abc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 19:58:49 +0200 Subject: [PATCH 27/41] Add command to force set a Matrix power level without affecting Telegram. Fixes #60 --- mautrix_telegram/commands/meta.py | 7 ++++--- mautrix_telegram/commands/portal.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/mautrix_telegram/commands/meta.py b/mautrix_telegram/commands/meta.py index 16505a4e..ca66fdd3 100644 --- a/mautrix_telegram/commands/meta.py +++ b/mautrix_telegram/commands/meta.py @@ -57,7 +57,8 @@ def help(evt): #### Miscellaneous things **search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users. **sync** [`chats`|`contacts`|`me`] - Synchronize your chat portals, contacts and/or own info. -**ping-bot** - Get info of the message relay Telegram bot. +**ping-bot** - Get info of the message relay Telegram bot. +**set-pl** <_level_> - Set a temporary power level without affecting Telegram. #### Initiating chats **pm** <_identifier_> - Open a private chat with the given Telegram user. The identifier is either @@ -72,10 +73,10 @@ def help(evt): **delete-portal** - Remove all users from the current portal room and forget the portal. Only works for group chats; to delete a private chat portal, simply leave the room. -**unbridge** - Remove puppets from the current portal room and forget the portal. +**unbridge** - Remove puppets from the current portal room and forget the portal. **bridge** [_id_] - Bridge the current Matrix room to the Telegram chat with the given ID. The ID must be the prefixed version that you get with the `/id` - command of the Telegram-side bot. + command of the Telegram-side bot. **group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash (`-`) as the name. **clean-rooms** - Clean up unused portal/management rooms. diff --git a/mautrix_telegram/commands/portal.py b/mautrix_telegram/commands/portal.py index 666baa21..1ca17878 100644 --- a/mautrix_telegram/commands/portal.py +++ b/mautrix_telegram/commands/portal.py @@ -23,6 +23,23 @@ from .. import portal as po from . import command_handler, CommandEvent +@command_handler(needs_admin=True, needs_auth=False, name="set-pl") +async def set_power_level(evt: CommandEvent): + try: + level = int(evt.args[0]) + except KeyError: + return await evt.reply("**Usage:** `$cmdprefix+sp set-power `") + except ValueError: + return await evt.reply("The level must be an integer.") + levels = await evt.az.intent.get_power_levels(evt.room_id) + levels["users"][evt.sender.mxid] = level + try: + await evt.az.intent.set_power_levels(evt.room_id, levels) + except MatrixRequestError: + evt.log.exception("Failed to set power level.") + return await evt.reply("Failed to set power level.") + + @command_handler() async def invite_link(evt: CommandEvent): portal = po.Portal.get_by_mxid(evt.room_id) From 9709768b17f5d94a346b4a63a6b3a228b3d909ee Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 20:01:48 +0200 Subject: [PATCH 28/41] Add mxid parameter to set-pl --- mautrix_telegram/commands/meta.py | 2 +- mautrix_telegram/commands/portal.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mautrix_telegram/commands/meta.py b/mautrix_telegram/commands/meta.py index ca66fdd3..1f07d3f1 100644 --- a/mautrix_telegram/commands/meta.py +++ b/mautrix_telegram/commands/meta.py @@ -58,7 +58,7 @@ def help(evt): **search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users. **sync** [`chats`|`contacts`|`me`] - Synchronize your chat portals, contacts and/or own info. **ping-bot** - Get info of the message relay Telegram bot. -**set-pl** <_level_> - Set a temporary power level without affecting Telegram. +**set-pl** <_level_> [_mxid_] - Set a temporary power level without affecting Telegram. #### Initiating chats **pm** <_identifier_> - Open a private chat with the given Telegram user. The identifier is either diff --git a/mautrix_telegram/commands/portal.py b/mautrix_telegram/commands/portal.py index 1ca17878..aa5850b8 100644 --- a/mautrix_telegram/commands/portal.py +++ b/mautrix_telegram/commands/portal.py @@ -28,11 +28,12 @@ async def set_power_level(evt: CommandEvent): try: level = int(evt.args[0]) except KeyError: - return await evt.reply("**Usage:** `$cmdprefix+sp set-power `") + return await evt.reply("**Usage:** `$cmdprefix+sp set-power [mxid]`") except ValueError: return await evt.reply("The level must be an integer.") levels = await evt.az.intent.get_power_levels(evt.room_id) - levels["users"][evt.sender.mxid] = level + mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid + levels["users"][mxid] = level try: await evt.az.intent.set_power_levels(evt.room_id, levels) except MatrixRequestError: From 8bfb41673588840499d6acb9c6eaf9faf46ec73c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 20:33:13 +0200 Subject: [PATCH 29/41] Add config option for plaintext highlight bridging --- example-config.yaml | 4 ++++ mautrix_telegram/formatter/from_matrix.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/example-config.yaml b/example-config.yaml index 59ffaa3b..562c44e3 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -89,6 +89,10 @@ bridge: # Use inline images instead of m.image to make rich captions possible. # N.B. Inline images are not supported on all clients (e.g. Riot iOS). inline_images: false + # Whether or not to bridge plaintext highlights. + # Only enable this if your displayname_template has some static part that the bridge can use to + # reliably identify what is a plaintext highlight. + plaintext_highlights: false # The prefix for commands. Only required in non-management rooms. command_prefix: "!tg" diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index 971e7a97..9091d2fc 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -29,6 +29,7 @@ from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, trim_reply_fallback_text, html_to_unicode) log = logging.getLogger("mau.fmt.mx") +should_bridge_plaintext_highlights = False class MatrixParser(HTMLParser): @@ -224,8 +225,11 @@ plain_mention_regex = None def matrix_text_to_telegram(text): text = command_regex.sub(r"\1/\2", text) - entities, pmr_replacer = plain_mention_to_text() - text = plain_mention_regex.sub(pmr_replacer, text) + if should_bridge_plaintext_highlights: + entities, pmr_replacer = plain_mention_to_text() + text = plain_mention_regex.sub(pmr_replacer, text) + else: + entities = [] return text, entities @@ -266,7 +270,8 @@ def matrix_to_telegram(html): parser = MatrixParser() html = html.replace("\n", "") html = command_regex.sub(r"\1\2", html) - html = plain_mention_regex.sub(plain_mention_to_html, html) + if should_bridge_plaintext_highlights: + html = plain_mention_regex.sub(plain_mention_to_html, html) parser.feed(add_surrogates(html)) return remove_surrogates(parser.text.strip()), parser.entities except Exception: @@ -297,8 +302,9 @@ def matrix_reply_to_telegram(content, tg_space, room_id=None): def init_mx(context): - global plain_mention_regex + global plain_mention_regex, should_bridge_plaintext_highlights config = context.config dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)") dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+") plain_mention_regex = re.compile(f"(\s|^)({dn_template})") + should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False From 517c7d8b70eb0be45d41b4330d796803caf7d0ce Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 23:16:58 +0200 Subject: [PATCH 30/41] Move mautrix-appservice to separate repo. Fixes #37 --- mautrix_appservice/__init__.py | 5 - mautrix_appservice/appservice.py | 183 --------- mautrix_appservice/errors.py | 38 -- mautrix_appservice/intent_api.py | 592 ------------------------------ mautrix_appservice/state_store.py | 155 -------- requirements/base.txt | 2 +- requirements/optional.txt | 1 + setup.py | 5 +- 8 files changed, 5 insertions(+), 976 deletions(-) delete mode 100644 mautrix_appservice/__init__.py delete mode 100644 mautrix_appservice/appservice.py delete mode 100644 mautrix_appservice/errors.py delete mode 100644 mautrix_appservice/intent_api.py delete mode 100644 mautrix_appservice/state_store.py diff --git a/mautrix_appservice/__init__.py b/mautrix_appservice/__init__.py deleted file mode 100644 index 7a5ef73c..00000000 --- a/mautrix_appservice/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .appservice import AppService -from .errors import MatrixError, MatrixRequestError, IntentError - -__version__ = "0.1.0" -__author__ = "Tulir Asokan " diff --git a/mautrix_appservice/appservice.py b/mautrix_appservice/appservice.py deleted file mode 100644 index bd775df4..00000000 --- a/mautrix_appservice/appservice.py +++ /dev/null @@ -1,183 +0,0 @@ -# -*- coding: future_fstrings -*- -# matrix-appservice-python - A Matrix Application Service framework written in Python. -# Copyright (C) 2018 Tulir Asokan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Partly based on github.com/Cadair/python-appservice-framework (MIT license) -from contextlib import contextmanager -from aiohttp import web -import aiohttp -import asyncio -import logging - -from .intent_api import HTTPAPI -from .state_store import StateStore - - -class AppService: - def __init__(self, server, domain, as_token, hs_token, bot_localpart, loop=None, log=None, - verify_ssl=True, query_user=None, query_alias=None): - self.server = server - self.domain = domain - self.verify_ssl = verify_ssl - self.as_token = as_token - self.hs_token = hs_token - self.bot_mxid = f"@{bot_localpart}:{domain}" - self.state_store = StateStore(autosave_file="mx-state.json") - self.state_store.load("mx-state.json") - - self.transactions = [] - - self._http_session = None - self._intent = None - - self.loop = loop or asyncio.get_event_loop() - self.log = (logging.getLogger(log) if isinstance(log, str) - else log or logging.getLogger("mautrix_appservice")) - - async def default_query_handler(_): - return None - - self.query_user = query_user or default_query_handler - self.query_alias = query_alias or default_query_handler - - self.event_handlers = [] - - self.app = web.Application(loop=self.loop) - self.app.router.add_route("PUT", "/transactions/{transaction_id}", - self._http_handle_transaction) - self.app.router.add_route("GET", "/rooms/{alias}", self._http_query_alias) - self.app.router.add_route("GET", "/users/{user_id}", self._http_query_user) - - self.matrix_event_handler(self.update_state_store) - - @property - def http_session(self): - if self._http_session is None: - raise AttributeError("the http_session attribute can only be used " - "from within the `AppService.run` context manager") - else: - return self._http_session - - @property - def intent(self): - if self._intent is None: - raise AttributeError("the intent attribute can only be used from " - "within the `AppService.run` context manager") - else: - return self._intent - - @contextmanager - def run(self, host="127.0.0.1", port=8080): - connector = None - if self.server.startswith("https://") and not self.verify_ssl: - connector = aiohttp.TCPConnector(verify_ssl=False) - self._http_session = aiohttp.ClientSession(loop=self.loop, connector=connector) - self._intent = HTTPAPI(base_url=self.server, domain=self.domain, bot_mxid=self.bot_mxid, - token=self.as_token, log=self.log, state_store=self.state_store, - client_session=self._http_session).bot_intent() - - yield self.loop.create_server(self.app.make_handler(), host, port) - - self._intent = None - self._http_session.close() - self._http_session = None - - def _check_token(self, request): - try: - token = request.rel_url.query["access_token"] - except KeyError: - return False - - if token != self.hs_token: - return False - - return True - - async def _http_query_user(self, request): - if not self._check_token(request): - return web.Response(status=401) - - user_id = request.match_info["userId"] - - try: - response = await self.query_user(user_id) - except Exception: - self.log.exception("Exception in user query handler") - return web.Response(status=500) - - if not response: - return web.Response(status=404) - return web.json_response(response) - - async def _http_query_alias(self, request): - if not self._check_token(request): - return web.Response(status=401) - - alias = request.match_info["alias"] - - try: - response = await self.query_alias(alias) - except Exception: - self.log.exception("Exception in alias query handler") - return web.Response(status=500) - - if not response: - return web.Response(status=404) - return web.json_response(response) - - async def _http_handle_transaction(self, request): - if not self._check_token(request): - return web.Response(status=401) - - transaction_id = request.match_info["transaction_id"] - if transaction_id in self.transactions: - return web.Response(status=200) - - json = await request.json() - - try: - events = json["events"] - except KeyError: - return web.Response(status=400) - - for event in events: - self.handle_matrix_event(event) - - self.transactions.append(transaction_id) - - return web.json_response({}) - - async def update_state_store(self, event): - event_type = event["type"] - if event_type == "m.room.power_levels": - self.state_store.set_power_levels(event["room_id"], event["content"]) - elif event_type == "m.room.member": - self.state_store.set_membership(event["room_id"], event["state_key"], - event["content"]["membership"]) - - def handle_matrix_event(self, event): - async def try_handle(handler): - try: - await handler(event) - except Exception: - self.log.exception("Exception in Matrix event handler") - - for handler in self.event_handlers: - asyncio.ensure_future(try_handle(handler), loop=self.loop) - - def matrix_event_handler(self, func): - self.event_handlers.append(func) - return func diff --git a/mautrix_appservice/errors.py b/mautrix_appservice/errors.py deleted file mode 100644 index 02c4f87b..00000000 --- a/mautrix_appservice/errors.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: future_fstrings -*- -# mautrix-telegram - A Matrix-Telegram puppeting bridge -# Copyright (C) 2018 Tulir Asokan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class MatrixError(Exception): - """A generic Matrix error. Specific errors will subclass this.""" - pass - - -class IntentError(MatrixError): - def __init__(self, message, source): - super().__init__(message) - self.source = source - - -class MatrixRequestError(MatrixError): - """ The home server returned an error response. """ - - def __init__(self, code=0, text="", errcode=None, message=None): - super().__init__(f"{code}: {text}") - self.code = code - self.text = text - self.errcode = errcode - self.message = message diff --git a/mautrix_appservice/intent_api.py b/mautrix_appservice/intent_api.py deleted file mode 100644 index 5a7495a5..00000000 --- a/mautrix_appservice/intent_api.py +++ /dev/null @@ -1,592 +0,0 @@ -# -*- coding: future_fstrings -*- -# mautrix-telegram - A Matrix-Telegram puppeting bridge -# Copyright (C) 2018 Tulir Asokan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from urllib.parse import quote -from time import time -from json.decoder import JSONDecodeError -from aiohttp.client_exceptions import ContentTypeError -import re -import json -import magic -import asyncio - -from .errors import MatrixError, MatrixRequestError, IntentError - - -class HTTPAPI: - def __init__(self, base_url, domain=None, bot_mxid=None, token=None, identity=None, log=None, - state_store=None, client_session=None, child=False): - self.base_url = base_url - self.token = token - self.identity = identity - self.validate_cert = True - self.session = client_session - - self.domain = domain - self.bot_mxid = bot_mxid - self._bot_intent = None - self.state_store = state_store - - if child: - self.log = log - else: - self.intent_log = log.getChild("intent") - self.log = log.getChild("api") - self.txn_id = 0 - self.children = {} - - def user(self, user): - try: - return self.children[user] - except KeyError: - child = ChildHTTPAPI(user, self) - self.children[user] = child - return child - - def bot_intent(self): - if self._bot_intent: - return self._bot_intent - return IntentAPI(self.bot_mxid, self, state_store=self.state_store, log=self.intent_log) - - def intent(self, user): - return IntentAPI(user, self.user(user), self.bot_intent(), self.state_store, - self.intent_log) - - async def _send(self, method, endpoint, content, query_params, headers): - while True: - query_params["access_token"] = self.token - request = self.session.request(method, endpoint, params=query_params, - data=content, headers=headers) - async with request as response: - if response.status < 200 or response.status >= 300: - errcode = message = None - try: - response_data = await response.json() - errcode = response_data["errcode"] - message = response_data["error"] - except (JSONDecodeError, ContentTypeError, KeyError): - pass - raise MatrixRequestError(code=response.status, text=await response.text(), - errcode=errcode, message=message) - - if response.status == 429: - await asyncio.sleep(response.json()["retry_after_ms"] / 1000) - else: - return await response.json() - - def _log_request(self, method, path, content, query_params): - log_content = content if not isinstance(content, bytes) else f"<{len(content)} bytes>" - log_content = log_content or "(No content)" - query_identity = query_params["user_id"] if "user_id" in query_params else "No identity" - self.log.debug("%s %s %s as user %s", method, path, log_content, query_identity) - - def request(self, method, path, content=None, query_params=None, headers=None, - api_path="/_matrix/client/r0"): - content = content or {} - query_params = query_params or {} - headers = headers or {} - - method = method.upper() - if method not in ["GET", "PUT", "DELETE", "POST"]: - raise MatrixError("Unsupported HTTP method: %s" % method) - - if "Content-Type" not in headers: - headers["Content-Type"] = "application/json" - if headers["Content-Type"] == "application/json": - content = json.dumps(content) - - if self.identity: - query_params["user_id"] = self.identity - - self._log_request(method, path, content, query_params) - - endpoint = self.base_url + api_path + path - return self._send(method, endpoint, content, query_params, headers or {}) - - def get_download_url(self, mxcurl): - if mxcurl.startswith('mxc://'): - return f"{self.base_url}/_matrix/media/r0/download/{mxcurl[6:]}" - else: - raise ValueError("MXC URL did not begin with 'mxc://'") - - async def get_display_name(self, user_id): - content = await self.request("GET", f"/profile/{user_id}/displayname") - return content.get('displayname', None) - - async def get_avatar_url(self, user_id): - content = await self.request("GET", f"/profile/{user_id}/avatar_url") - return content.get('avatar_url', None) - - async def get_room_id(self, room_alias): - content = await self.request("GET", f"/directory/room/{quote(room_alias)}") - return content.get("room_id", None) - - def set_typing(self, room_id, is_typing=True, timeout=5000, user=None): - content = { - "typing": is_typing - } - if is_typing: - content["timeout"] = timeout - user = user or self.identity - return self.request("PUT", f"/rooms/{room_id}/typing/{user}", content) - - -class ChildHTTPAPI(HTTPAPI): - def __init__(self, user, parent): - super().__init__(parent.base_url, parent.domain, parent.bot_mxid, parent.token, user, - parent.log, parent.state_store, parent.session, child=True) - self.parent = parent - - @property - def txn_id(self): - return self.parent.txn_id - - @txn_id.setter - def txn_id(self, value): - self.parent.txn_id = value - - -class IntentAPI: - mxid_regex = re.compile("@(.+):(.+)") - - def __init__(self, mxid, client, bot=None, state_store=None, log=None): - self.client = client - self.bot = bot - self.mxid = mxid - self.log = log - - results = self.mxid_regex.match(mxid) - if not results: - raise ValueError("invalid MXID") - self.localpart = results.group(1) - - self.state_store = state_store - - def user(self, user): - if not self.bot: - return self.client.intent(user) - else: - self.log.warning("Called IntentAPI#user() of child intent object.") - return self.bot.client.intent(user) - - # region User actions - - async def get_joined_rooms(self): - await self.ensure_registered() - response = await self.client.request("GET", "/joined_rooms") - return response["joined_rooms"] - - async def set_display_name(self, name): - await self.ensure_registered() - content = {"displayname": name} - return await self.client.request("PUT", f"/profile/{self.mxid}/displayname", content) - - async def set_presence(self, status="online", ignore_cache=False): - await self.ensure_registered() - if not ignore_cache and self.state_store.has_presence(self.mxid, status): - return - content = { - "presence": status - } - resp = await self.client.request("PUT", f"/presence/{self.mxid}/status", content) - self.state_store.set_presence(self.mxid, status) - return resp - - async def set_avatar(self, url): - await self.ensure_registered() - content = {"avatar_url": url} - return await self.client.request("PUT", f"/profile/{self.mxid}/avatar_url", content) - - async def upload_file(self, data, mime_type=None): - await self.ensure_registered() - mime_type = mime_type or magic.from_buffer(data, mime=True) - return await self.client.request("POST", "", content=data, - headers={"Content-Type": mime_type}, - api_path="/_matrix/media/r0/upload") - - async def download_file(self, url): - await self.ensure_registered() - url = self.client.get_download_url(url) - async with self.client.session.get(url) as response: - return await response.read() - - # endregion - # region Room actions - - async def create_room(self, alias=None, is_public=False, name=None, topic=None, - is_direct=False, invitees=None, initial_state=None, - guests_can_join=False): - await self.ensure_registered() - content = { - "visibility": "private", - "is_direct": is_direct, - "preset": "public_chat" if is_public else "private_chat", - "guests_can_join": guests_can_join, - } - if alias: - content["room_alias_name"] = alias - if invitees: - content["invite"] = invitees - if name: - content["name"] = name - if topic: - content["topic"] = topic - if initial_state: - content["initial_state"] = initial_state - - return await self.client.request("POST", "/createRoom", content) - - def _invite_direct(self, room_id, user_id): - content = {"user_id": user_id} - return self.client.request("POST", "/rooms/" + room_id + "/invite", content) - - async def invite(self, room_id, user_id, check_cache=False): - await self.ensure_joined(room_id) - try: - ok_states = {"invite", "join"} - do_invite = (not check_cache - or self.state_store.get_membership(room_id, user_id) not in ok_states) - if do_invite: - response = await self._invite_direct(room_id, user_id) - self.state_store.invited(room_id, user_id) - return response - except MatrixRequestError as e: - if e.errcode != "M_FORBIDDEN": - raise IntentError(f"Failed to invite {user_id} to {room_id}", e) - if "is already in the room" in e.message: - self.state_store.joined(room_id, user_id) - - def set_room_avatar(self, room_id, avatar_url, info=None): - content = { - "url": avatar_url, - } - if info: - content["info"] = info - return self.send_state_event(room_id, "m.room.avatar", content) - - async def add_room_alias(self, room_id, localpart, override=True): - await self.ensure_registered() - content = {"room_id": room_id} - alias = f"#{localpart}:{self.client.domain}" - try: - return await self.client.request("PUT", f"/directory/room/{quote(alias)}", content) - except MatrixRequestError as e: - if override and e.code == 409: - await self.remove_room_alias(localpart) - return await self.client.request("PUT", f"/directory/room/{quote(alias)}", content) - - async def remove_room_alias(self, localpart): - await self.ensure_registered() - alias = f"#{localpart}:{self.client.domain}" - return await self.client.request("DELETE", f"/directory/room/{quote(alias)}") - - def set_room_name(self, room_id, name): - body = {"name": name} - return self.send_state_event(room_id, "m.room.name", body) - - async def get_power_levels(self, room_id, ignore_cache=False): - await self.ensure_joined(room_id) - if not ignore_cache: - try: - return self.state_store.get_power_levels(room_id) - except KeyError: - pass - levels = await self.client.request("GET", - f"/rooms/{quote(room_id)}/state/m.room.power_levels") - self.state_store.set_power_levels(room_id, levels) - return levels - - async def set_power_levels(self, room_id, content): - if "events" not in content: - content["events"] = {} - response = await self.send_state_event(room_id, "m.room.power_levels", content) - self.state_store.set_power_levels(room_id, content) - return response - - async def get_pinned_messages(self, room_id): - await self.ensure_joined(room_id) - response = await self.client.request("GET", f"/rooms/{room_id}/state/m.room.pinned_events") - return response["content"]["pinned"] - - def set_pinned_messages(self, room_id, events): - return self.send_state_event(room_id, "m.room.pinned_events", { - "pinned": events - }) - - async def pin_message(self, room_id, event_id): - events = await self.get_pinned_messages(room_id) - if event_id not in events: - events.append(event_id) - await self.set_pinned_messages(room_id, events) - - async def unpin_message(self, room_id, event_id): - events = await self.get_pinned_messages(room_id) - if event_id in events: - events.remove(event_id) - await self.set_pinned_messages(room_id, events) - - async def set_join_rule(self, room_id, join_rule): - if join_rule not in ("public", "knock", "invite", "private"): - raise ValueError(f"Invalid join rule \"{join_rule}\"") - await self.send_state_event(room_id, "m.room.join_rules", { - "join_rule": join_rule, - }) - - async def get_event(self, room_id, event_id): - await self.ensure_joined(room_id) - return await self.client.request("GET", f"/rooms/{room_id}/event/{event_id}") - - async def set_typing(self, room_id, is_typing=True, timeout=5000, ignore_cache=False): - await self.ensure_joined(room_id) - if not ignore_cache and is_typing == self.state_store.is_typing(room_id, self.mxid): - return - content = { - "typing": is_typing - } - if is_typing: - content["timeout"] = timeout - resp = await self.client.request("PUT", f"/rooms/{room_id}/typing/{self.mxid}", content) - self.state_store.set_typing(room_id, self.mxid, is_typing, timeout) - return resp - - async def mark_read(self, room_id, event_id): - await self.ensure_joined(room_id) - return await self.client.request("POST", f"/rooms/{room_id}/receipt/m.read/{event_id}", - content={}) - - def send_notice(self, room_id, text, html=None, relates_to=None): - return self.send_text(room_id, text, html, "m.notice", relates_to) - - def send_emote(self, room_id, text, html=None, relates_to=None): - return self.send_text(room_id, text, html, "m.emote", relates_to) - - def send_image(self, room_id, url, info=None, text=None, relates_to=None): - return self.send_file(room_id, url, info or {}, text, "m.image", relates_to) - - def send_file(self, room_id, url, info=None, text=None, file_type="m.file", relates_to=None): - return self.send_message(room_id, { - "msgtype": file_type, - "url": url, - "body": text or "Uploaded file", - "info": info or {}, - "m.relates_to": relates_to or None, - }) - - def send_text(self, room_id, text, html=None, msgtype="m.text", relates_to=None): - if html: - if not text: - text = html - return self.send_message(room_id, { - "body": text, - "msgtype": msgtype, - "format": "org.matrix.custom.html", - "formatted_body": html or text, - "m.relates_to": relates_to or None, - }) - else: - return self.send_message(room_id, { - "body": text, - "msgtype": msgtype, - "m.relates_to": relates_to or None, - }) - - def send_message(self, room_id, body): - return self.send_event(room_id, "m.room.message", body) - - async def error_and_leave(self, room_id, text, html=None): - await self.ensure_joined(room_id) - await self.send_notice(room_id, text, html=html) - await self.leave_room(room_id) - - def kick(self, room_id, user_id, message): - return self.set_membership(room_id, user_id, "leave", message) - - def get_membership(self, room_id, user_id): - return self.get_state_event(room_id, "m.room.member", state_key=user_id) - - def set_membership(self, room_id, user_id, membership, reason="", profile=None): - body = { - "membership": membership, - "reason": reason - } - profile = profile or {} - if "displayname" in profile: - body["displayname"] = profile["displayname"] - if "avatar_url" in profile: - body["avatar_url"] = profile["avatar_url"] - - return self.send_state_event(room_id, "m.room.member", body, state_key=user_id) - - def redact(self, room_id, event_id, reason=None, txn_id=None): - txn_id = txn_id or str(self.client.txn_id) + str(int(time() * 1000)) - self.client.txn_id += 1 - content = {} - if reason: - content["reason"] = reason - return self.client.request("PUT", - f"/rooms/{quote(room_id)}/redact/{quote(event_id)}/{txn_id}", - content) - - @staticmethod - def _get_event_url(room_id, event_type, txn_id): - if not room_id: - raise ValueError("Room ID not given") - elif not event_type: - raise ValueError("Event type not given") - elif not txn_id: - raise ValueError("Transaction ID not given") - return f"/rooms/{quote(room_id)}/send/{quote(event_type)}/{quote(txn_id)}" - - async def send_event(self, room_id, event_type, content, txn_id=None): - if not room_id: - raise ValueError("Room ID not given") - elif not event_type: - raise ValueError("Event type not given") - await self.ensure_joined(room_id) - await self._ensure_has_power_level_for(room_id, event_type) - - txn_id = txn_id or str(self.client.txn_id) + str(int(time() * 1000)) - self.client.txn_id += 1 - - url = self._get_event_url(room_id, event_type, txn_id) - - return await self.client.request("PUT", url, content) - - @staticmethod - def _get_state_url(room_id, event_type, state_key=""): - if not room_id: - raise ValueError("Room ID not given") - elif not event_type: - raise ValueError("Event type not given") - url = f"/rooms/{quote(room_id)}/state/{quote(event_type)}" - if state_key: - url += f"/{quote(state_key)}" - return url - - async def send_state_event(self, room_id, event_type, content, state_key=""): - if not room_id: - raise ValueError("Room ID not given") - elif not event_type: - raise ValueError("Event type not given") - await self.ensure_joined(room_id) - await self._ensure_has_power_level_for(room_id, event_type, is_state_event=True) - url = self._get_state_url(room_id, event_type, state_key) - return await self.client.request("PUT", url, content) - - async def get_state_event(self, room_id, event_type, state_key=""): - if not room_id: - raise ValueError("Room ID not given") - elif not event_type: - raise ValueError("Event type not given") - await self.ensure_joined(room_id) - url = self._get_state_url(room_id, event_type, state_key) - return await self.client.request("GET", url) - - def join_room(self, room_id): - if not room_id: - raise ValueError("Room ID not given") - return self.ensure_joined(room_id, ignore_cache=True) - - def _join_room_direct(self, room): - if not room: - raise ValueError("Room ID not given") - return self.client.request("POST", f"/join/{quote(room)}") - - def leave_room(self, room_id): - if not room_id: - raise ValueError("Room ID not given") - try: - self.state_store.left(room_id, self.mxid) - return self.client.request("POST", f"/rooms/{quote(room_id)}/leave") - except MatrixRequestError as e: - if "not in room" not in e.message: - raise - - def get_room_memberships(self, room_id): - if not room_id: - raise ValueError("Room ID not given") - return self.client.request("GET", f"/rooms/{quote(room_id)}/members") - - async def get_room_members(self, room_id, allowed_memberships=("join",)): - memberships = await self.get_room_memberships(room_id) - return [membership["state_key"] for membership in memberships["chunk"] if - membership["content"]["membership"] in allowed_memberships] - - async def get_room_state(self, room_id): - await self.ensure_joined(room_id) - state = await self.client.request("GET", f"/rooms/{quote(room_id)}/state") - # TODO update values based on state? - return state - - # endregion - # region Ensure functions - - async def ensure_joined(self, room_id, ignore_cache=False): - if not room_id: - raise ValueError("Room ID not given") - if not ignore_cache and self.state_store.is_joined(room_id, self.mxid): - return - await self.ensure_registered() - try: - await self._join_room_direct(room_id) - self.state_store.joined(room_id, self.mxid) - except MatrixRequestError as e: - if e.errcode != "M_FORBIDDEN" or not self.bot: - raise IntentError(f"Failed to join room {room_id} as {self.mxid}", e) - try: - await self.bot.invite(room_id, self.mxid) - await self._join_room_direct(room_id) - self.state_store.joined(room_id, self.mxid) - except MatrixRequestError as e2: - raise IntentError(f"Failed to join room {room_id} as {self.mxid}", e2) - - def _register(self): - content = {"username": self.localpart} - query_params = {"kind": "user"} - return self.client.request("POST", "/register", content, query_params) - - async def ensure_registered(self): - if self.state_store.is_registered(self.mxid): - return - try: - await self._register() - except MatrixRequestError as e: - if e.errcode != "M_USER_IN_USE": - self.log.exception(f"Failed to register {self.mxid}!") - # raise IntentError(f"Failed to register {self.mxid}", e) - return - self.state_store.registered(self.mxid) - - async def _ensure_has_power_level_for(self, room_id, event_type, is_state_event=False): - if not room_id: - raise ValueError("Room ID not given") - elif not event_type: - raise ValueError("Event type not given") - - if not self.state_store.has_power_levels(room_id): - await self.get_power_levels(room_id) - if self.state_store.has_power_level(room_id, self.mxid, event_type, - is_state_event=is_state_event): - return - elif not self.bot: - self.log.warning( - f"Power level of {self.mxid} is not enough for {event_type} in {room_id}") - # raise IntentError(f"Power level of {self.mxid} is not enough" - # f"for {event_type} in {room_id}") - return - # TODO implement - - # endregion diff --git a/mautrix_appservice/state_store.py b/mautrix_appservice/state_store.py deleted file mode 100644 index f376426e..00000000 --- a/mautrix_appservice/state_store.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: future_fstrings -*- -# matrix-appservice-python - A Matrix Application Service framework written in Python. -# Copyright (C) 2018 Tulir Asokan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import json -import time - - -class StateStore: - def __init__(self, autosave_file=None): - self.autosave_file = autosave_file - - # Persistent storage - self.registrations = set() - self.memberships = {} - self.power_levels = {} - - # Non-persistent storage - self.presence = {} - self.typing = {} - - def save(self, file): - if isinstance(file, str): - output = open(file, "w") - else: - output = file - - json.dump({ - "registrations": list(self.registrations), - "memberships": self.memberships, - "power_levels": self.power_levels, - }, output) - - if isinstance(file, str): - output.close() - - def load(self, file): - if isinstance(file, str): - try: - input_source = open(file, "r") - except FileNotFoundError: - return - else: - input_source = file - - data = json.load(input_source) - if "registrations" in data: - self.registrations = set(data["registrations"]) - if "memberships" in data: - self.memberships = data["memberships"] - if "power_levels" in data: - self.power_levels = data["power_levels"] - - if isinstance(file, str): - input_source.close() - - def _autosave(self): - if self.autosave_file: - self.save(self.autosave_file) - - def set_presence(self, user, presence): - self.presence[user] = presence - - def has_presence(self, user, presence): - try: - return self.presence[user] == presence - except KeyError: - return False - - def set_typing(self, room_id, user, is_typing, timeout=0): - if is_typing: - ts = int(round(time.time() * 1000)) - self.typing[(room_id, user)] = ts + timeout - else: - del self.typing[(room_id, user)] - - def is_typing(self, room_id, user): - ts = int(round(time.time() * 1000)) - try: - return self.typing[(room_id, user)] > ts - except KeyError: - return False - - def is_registered(self, user): - return user in self.registrations - - def registered(self, user): - self.registrations.add(user) - self._autosave() - - def get_membership(self, room, user): - return self.memberships.get(room, {}).get(user, "left") - - def is_joined(self, room, user): - return self.get_membership(room, user) == "join" - - def set_membership(self, room, user, membership): - if room not in self.memberships: - self.memberships[room] = {} - self.memberships[room][user] = membership - self._autosave() - - def joined(self, room, user): - return self.set_membership(room, user, "join") - - def invited(self, room, user): - return self.set_membership(room, user, "invite") - - def left(self, room, user): - return self.set_membership(room, user, "left") - - def has_power_levels(self, room): - return room in self.power_levels - - def get_power_levels(self, room): - return self.power_levels[room] - - def has_power_level(self, room, user, event, is_state_event=False, default=None): - room_levels = self.power_levels.get(room, {}) - default_required = default or (room_levels.get("state_default", 50) if is_state_event - else room_levels.get("events_default", 0)) - required = room_levels.get("events", {}).get(event, default_required) - has = room_levels.get("users", {}).get(user, room_levels.get("users_default", 0)) - return has >= required - - def set_power_level(self, room, user, level): - if room not in self.power_levels: - self.power_levels[room] = { - "users": {}, - "events": {}, - } - elif "users" not in self.power_levels[room]: - self.power_levels[room]["users"] = {} - self.power_levels[room]["users"][user] = level - self._autosave() - - def set_power_levels(self, room, content): - if "events" not in content: - content["events"] = {} - if "users" not in content: - content["users"] = {} - self.power_levels[room] = content - self._autosave() diff --git a/requirements/base.txt b/requirements/base.txt index e16861af..b7b8cc90 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,5 @@ aiohttp +mautrix-appservice ruamel.yaml python-magic SQLAlchemy @@ -6,4 +7,3 @@ alembic Markdown Pillow future-fstrings -cryptg diff --git a/requirements/optional.txt b/requirements/optional.txt index ab90481d..10087cb8 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -1 +1,2 @@ lxml +cryptg diff --git a/setup.py b/setup.py index eabecf3d..a9292e9e 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ setuptools.setup( install_requires=[ "aiohttp>=3.0.1,<4", + "mautrix-telegram>=0.1,<0.2", "SQLAlchemy>=1.2.3,<2", "alembic>=0.9.8,<0.10", "Markdown>=2.6.11,<3", @@ -25,7 +26,6 @@ setuptools.setup( "Pillow>=5.0.0,<6", "future-fstrings>=0.4.2", "python-magic>=0.4.15,<0.5", - "cryptg>=0.1,<0.2", "telethon-aio>=0.18,<0.19" if sys.version_info >= (3, 6) else "telethon-aio-git", ], dependency_links=[ @@ -33,10 +33,11 @@ setuptools.setup( ], extras_require={ "highlight_edits": ["lxml>=4.1.1,<5"], + "fast_crypto": ["cryptg>=0.1,<0.2"], }, classifiers=[ - "Development Status :: 4 Beta", + "Development Status :: 4 - Beta", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", "Topic :: Communications :: Chat", "Programming Language :: Python", From a469e6ed10a0c52500d6ef7c7a1659bba7774f5b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 8 Mar 2018 23:47:48 +0200 Subject: [PATCH 31/41] Switch to AGPLv3 --- LICENSE | 147 +++++++++----------- mautrix_telegram/__main__.py | 8 +- mautrix_telegram/abstract_user.py | 8 +- mautrix_telegram/bot.py | 8 +- mautrix_telegram/commands/auth.py | 8 +- mautrix_telegram/commands/clean_rooms.py | 8 +- mautrix_telegram/commands/handler.py | 8 +- mautrix_telegram/commands/meta.py | 8 +- mautrix_telegram/commands/portal.py | 8 +- mautrix_telegram/commands/telegram.py | 8 +- mautrix_telegram/config.py | 8 +- mautrix_telegram/context.py | 8 +- mautrix_telegram/db.py | 8 +- mautrix_telegram/formatter/from_matrix.py | 8 +- mautrix_telegram/formatter/from_telegram.py | 8 +- mautrix_telegram/formatter/util.py | 19 ++- mautrix_telegram/matrix.py | 8 +- mautrix_telegram/portal.py | 9 +- mautrix_telegram/public/__init__.py | 8 +- mautrix_telegram/public/login.css | 17 +++ mautrix_telegram/public/login.html.mako | 17 +++ mautrix_telegram/puppet.py | 8 +- mautrix_telegram/tgclient.py | 8 +- mautrix_telegram/user.py | 8 +- mautrix_telegram/util/file_transfer.py | 8 +- mautrix_telegram/util/format_duration.py | 8 +- 26 files changed, 207 insertions(+), 170 deletions(-) diff --git a/LICENSE b/LICENSE index 94a9ed02..be3f7b28 100644 --- a/LICENSE +++ b/LICENSE @@ -1,23 +1,21 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 - Copyright (C) 2007 Free Software Foundation, Inc. + Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble - The GNU General Public License is a free, copyleft license for -software and other kinds of works. + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to +our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. +software for all its users. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you @@ -26,44 +24,34 @@ them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. The precise terms and conditions for copying, distribution and modification follow. @@ -72,7 +60,7 @@ modification follow. 0. Definitions. - "This License" refers to version 3 of the GNU General Public License. + "This License" refers to version 3 of the GNU Affero General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. @@ -549,35 +537,45 @@ to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. - 13. Use with the GNU Affero General Public License. + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single +under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General +Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published +GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's +versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. @@ -635,40 +633,29 @@ the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by + it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. + GNU Affero General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program. If not, see . + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index 0936e514..dc9355b1 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . import argparse import sys import logging diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index 0fe33d2c..04554a32 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . import platform import os diff --git a/mautrix_telegram/bot.py b/mautrix_telegram/bot.py index 4947da1b..9e1051cd 100644 --- a/mautrix_telegram/bot.py +++ b/mautrix_telegram/bot.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . import logging import re diff --git a/mautrix_telegram/commands/auth.py b/mautrix_telegram/commands/auth.py index 2250b6f3..0e4fabab 100644 --- a/mautrix_telegram/commands/auth.py +++ b/mautrix_telegram/commands/auth.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . import asyncio from telethon_aio.errors import * diff --git a/mautrix_telegram/commands/clean_rooms.py b/mautrix_telegram/commands/clean_rooms.py index 5c515645..71ef2a91 100644 --- a/mautrix_telegram/commands/clean_rooms.py +++ b/mautrix_telegram/commands/clean_rooms.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from mautrix_appservice import MatrixRequestError from . import command_handler diff --git a/mautrix_telegram/commands/handler.py b/mautrix_telegram/commands/handler.py index 1309d058..77fbe9a6 100644 --- a/mautrix_telegram/commands/handler.py +++ b/mautrix_telegram/commands/handler.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . import markdown import logging diff --git a/mautrix_telegram/commands/meta.py b/mautrix_telegram/commands/meta.py index 1f07d3f1..f8d1fe30 100644 --- a/mautrix_telegram/commands/meta.py +++ b/mautrix_telegram/commands/meta.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from . import command_handler diff --git a/mautrix_telegram/commands/portal.py b/mautrix_telegram/commands/portal.py index aa5850b8..b0df61c9 100644 --- a/mautrix_telegram/commands/portal.py +++ b/mautrix_telegram/commands/portal.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . import asyncio from telethon_aio.errors import * diff --git a/mautrix_telegram/commands/telegram.py b/mautrix_telegram/commands/telegram.py index 7be428e0..34e81c0a 100644 --- a/mautrix_telegram/commands/telegram.py +++ b/mautrix_telegram/commands/telegram.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from telethon_aio.errors import * from telethon_aio.tl.types import User as TLUser from telethon_aio.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index d3ad67d3..0d414542 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap from ruamel.yaml.tokens import CommentToken diff --git a/mautrix_telegram/context.py b/mautrix_telegram/context.py index 5b7fb8cf..530f1eed 100644 --- a/mautrix_telegram/context.py +++ b/mautrix_telegram/context.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . class Context: diff --git a/mautrix_telegram/db.py b/mautrix_telegram/db.py index be849b78..ba82001b 100644 --- a/mautrix_telegram/db.py +++ b/mautrix_telegram/db.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer, BigInteger, String, Boolean) from sqlalchemy.orm import relationship diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index 9091d2fc..a6a976bb 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from html import unescape from html.parser import HTMLParser from collections import deque diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 576f51be..5ed2865d 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from html import escape try: from lxml.html.diff import htmldiff diff --git a/mautrix_telegram/formatter/util.py b/mautrix_telegram/formatter/util.py index 54502fbc..a9806d53 100644 --- a/mautrix_telegram/formatter/util.py +++ b/mautrix_telegram/formatter/util.py @@ -1,9 +1,26 @@ +# -*- coding: future_fstrings -*- +# mautrix-telegram - A Matrix-Telegram puppeting bridge +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from html import escape import struct import re -# Unicode surrogate handling from +# add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon. +# Licensed under the MIT license. # https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py def add_surrogates(text): if text is None: diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index fb7911d0..42c079b8 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . import logging from mautrix_appservice import MatrixRequestError diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 94153ff8..8a40ae0f 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -3,20 +3,19 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from collections import deque from datetime import datetime -from html import escape import asyncio import random import mimetypes diff --git a/mautrix_telegram/public/__init__.py b/mautrix_telegram/public/__init__.py index 4924d759..8a4d9e8d 100644 --- a/mautrix_telegram/public/__init__.py +++ b/mautrix_telegram/public/__init__.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from aiohttp import web from mako.template import Template import asyncio diff --git a/mautrix_telegram/public/login.css b/mautrix_telegram/public/login.css index 95d461b8..c7ade95b 100644 --- a/mautrix_telegram/public/login.css +++ b/mautrix_telegram/public/login.css @@ -1,3 +1,20 @@ +/* + * mautrix-telegram - A Matrix-Telegram puppeting bridge + * Copyright (C) 2018 Tulir Asokan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ form > div { display: none; } diff --git a/mautrix_telegram/public/login.html.mako b/mautrix_telegram/public/login.html.mako index 4da8d7d5..a63e30e3 100644 --- a/mautrix_telegram/public/login.html.mako +++ b/mautrix_telegram/public/login.html.mako @@ -1,3 +1,20 @@ + diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index 782ad2a5..de00a107 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from difflib import SequenceMatcher import re import logging diff --git a/mautrix_telegram/tgclient.py b/mautrix_telegram/tgclient.py index d70511a7..86bbd711 100644 --- a/mautrix_telegram/tgclient.py +++ b/mautrix_telegram/tgclient.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from io import BytesIO from telethon_aio import TelegramClient diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index 26f1d79e..2af7d46a 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . import logging import asyncio import re diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py index 0a2b47dd..b64f6346 100644 --- a/mautrix_telegram/util/file_transfer.py +++ b/mautrix_telegram/util/file_transfer.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . from io import BytesIO import time import logging diff --git a/mautrix_telegram/util/format_duration.py b/mautrix_telegram/util/format_duration.py index ffbac714..c873e9e5 100644 --- a/mautrix_telegram/util/format_duration.py +++ b/mautrix_telegram/util/format_duration.py @@ -3,17 +3,17 @@ # Copyright (C) 2018 Tulir Asokan # # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by +# it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# GNU Affero General Public License for more details. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . def format_duration(seconds): From b1c85d5cda805605de3def712604bfa6cb6c571d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Mar 2018 16:54:17 +0200 Subject: [PATCH 32/41] Add moviepy as optional dep for HQ thumbnails, make Pillow optional [db updated] --- ...c972368e50_add_metadata_to_telegramfile.py | 35 +++++++ mautrix_telegram/db.py | 9 +- mautrix_telegram/portal.py | 26 +++-- mautrix_telegram/util/file_transfer.py | 97 +++++++++++++++++-- requirements/base.txt | 1 - requirements/optional.txt | 2 + setup.py | 14 ++- 7 files changed, 153 insertions(+), 31 deletions(-) create mode 100644 alembic/versions/cfc972368e50_add_metadata_to_telegramfile.py diff --git a/alembic/versions/cfc972368e50_add_metadata_to_telegramfile.py b/alembic/versions/cfc972368e50_add_metadata_to_telegramfile.py new file mode 100644 index 00000000..205cbf3a --- /dev/null +++ b/alembic/versions/cfc972368e50_add_metadata_to_telegramfile.py @@ -0,0 +1,35 @@ +"""Add metadata to TelegramFile + +Revision ID: cfc972368e50 +Revises: 501dad2868bc +Create Date: 2018-03-09 16:07:01.236712 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'cfc972368e50' +down_revision = '501dad2868bc' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("telegram_file") as batch_op: + batch_op.add_column(sa.Column('size', sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column('width', sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column('height', sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column('thumbnail', sa.String(), nullable=True)) + batch_op.create_foreign_key(constraint_name="fk_file_thumbnail", + referent_table="telegram_file", + local_cols=['thumbnail'], + remote_cols=['id']) + + +def downgrade(): + with op.batch_alter_table("telegram_file") as batch_op: + batch_op.drop_column('size') + batch_op.drop_column('width') + batch_op.drop_column('height') + batch_op.drop_column('thumbnail') diff --git a/mautrix_telegram/db.py b/mautrix_telegram/db.py index ba82001b..a057e15f 100644 --- a/mautrix_telegram/db.py +++ b/mautrix_telegram/db.py @@ -81,8 +81,8 @@ class Contact(Base): query = None __tablename__ = "contact" - user = Column("user", Integer, ForeignKey("user.tgid"), primary_key=True) - contact = Column("contact", Integer, ForeignKey("puppet.id"), primary_key=True) + user = Column(Integer, ForeignKey("user.tgid"), primary_key=True) + contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True) class Puppet(Base): @@ -112,6 +112,11 @@ class TelegramFile(Base): mime_type = Column(String) was_converted = Column(Boolean) timestamp = Column(BigInteger) + size = Column(Integer, nullable=True) + width = Column(Integer, nullable=True) + height = Column(Integer, nullable=True) + thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True) + thumbnail = relationship("TelegramFile", uselist=False) def init(db_session): diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 8a40ae0f..a60da924 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -33,7 +33,6 @@ from mautrix_appservice import MatrixRequestError, IntentError from .db import Portal as DBPortal, Message as DBMessage from . import puppet as p, user as u, formatter, util -from .formatter.util import trim_reply_fallback_html, trim_reply_fallback_text mimetypes.init() @@ -858,11 +857,12 @@ class Portal: async def handle_telegram_document(self, source, intent, evt: Message, relates_to=None): document = evt.media.document - file = await util.transfer_file_to_matrix(self.db, source.client, intent, document) + file = await util.transfer_file_to_matrix(self.db, source.client, intent, document, + document.thumb) if not file: return None name = evt.message - width, height = 0, 0 + width, height = file.width, file.height for attr in document.attributes: if isinstance(attr, DocumentAttributeFilename): name = name or attr.file_name @@ -871,25 +871,21 @@ class Portal: file.mime_type = mime_from_name or file.mime_type elif isinstance(attr, DocumentAttributeSticker): name = f"Sticker for {attr.alt}" - elif isinstance(attr, DocumentAttributeVideo): + elif isinstance(attr, DocumentAttributeVideo) and (not width or not height): width, height = attr.w, attr.h mime_type = document.mime_type or file.mime_type info = { - "size": document.size, + "size": file.size, "mimetype": mime_type, } - if document.thumb and not isinstance(document.thumb, PhotoSizeEmpty): - thumbnail = await util.transfer_file_to_matrix(self.db, source.client, intent, - document.thumb.location) + if file.thumbnail: + info["thumbnail_url"] = file.thumbnail.mxc info["thumbnail_info"] = { - "mimetype": thumbnail.mime_type, - "h": document.thumb.h, - "w": document.thumb.w, - "size": (len(document.thumb.bytes) - if isinstance(document.thumb, PhotoCachedSize) - else document.thumb.size) + "mimetype": file.thumbnail.mime_type, + "h": file.thumbnail.height or document.thumb.h, + "w": file.thumbnail.width or document.thumb.w, + "size": file.thumbnail.size, } - info["thumbnail_url"] = thumbnail.mxc if height and width: info["h"] = height info["w"] = width diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py index b64f6346..d8542247 100644 --- a/mautrix_telegram/util/file_transfer.py +++ b/mautrix_telegram/util/file_transfer.py @@ -19,11 +19,22 @@ import time import logging import magic -from PIL import Image from sqlalchemy.exc import IntegrityError, InvalidRequestError +try: + from PIL import Image +except ImportError: + Image = None +try: + from moviepy.editor import VideoFileClip + import random + import string + import os + import mimetypes +except ImportError: + VideoFileClip = random = string = os = mimetypes = None from telethon_aio.tl.types import (Document, FileLocation, InputFileLocation, - InputDocumentFileLocation, PhotoCachedSize) + InputDocumentFileLocation, PhotoSize, PhotoCachedSize) from telethon_aio.errors import LocationInvalidError from ..db import TelegramFile as DBTelegramFile @@ -32,24 +43,86 @@ log = logging.getLogger("mau.util") def _convert_webp(file, to="png"): + if not Image: + return "image/webp", file try: image = Image.open(BytesIO(file)).convert("RGBA") new_file = BytesIO() image.save(new_file, to) - return f"image/{to}", new_file.getvalue() + w, h = image.size + return f"image/{to}", new_file.getvalue(), w, h except Exception: log.exception(f"Failed to convert webp to {to}") return "image/webp", file -async def transfer_file_to_matrix(db, client, intent, location): +def _temp_file_name(ext): + return ("/tmp/mxtg-video-" + + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + + ext) + + +def _read_video_thumbnail(data, video_ext="mp4", frame_ext="png", max_size=(1024, 720)): + # We don't have any way to read the video from memory, so save it to disk. + temp_file = _temp_file_name(video_ext) + with open(temp_file, "wb") as file: + file.write(data) + + # Read temp file and get frame + clip = VideoFileClip(temp_file) + frame = clip.get_frame(0) + + # Convert to png and save to BytesIO + image = Image.fromarray(frame).convert("RGBA") + thumbnail_file = BytesIO() + if max_size: + image.thumbnail(max_size, Image.ANTIALIAS) + image.save(thumbnail_file, frame_ext) + + os.remove(temp_file) + + w, h = image.size + return thumbnail_file.getvalue(), w, h + + +def _location_to_id(location): if isinstance(location, (Document, InputDocumentFileLocation)): - id = f"{location.id}-{location.version}" + return f"{location.id}-{location.version}" elif isinstance(location, (FileLocation, InputFileLocation)): - id = f"{location.volume_id}-{location.local_id}" + return f"{location.volume_id}-{location.local_id}" else: return None + +async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mime): + if not Image or not VideoFileClip: + return None + + id = _location_to_id(thumbnail_loc) + if not id: + return None + + video_ext = mimetypes.guess_extension(mime) + if VideoFileClip and video_ext: + file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png") + mime_type = "image/png" + else: + file = await client.download_file_bytes(thumbnail_loc) + width, height = None, None + mime_type = magic.from_buffer(file, mime=True) + + uploaded = await intent.upload_file(file, mime_type) + + return DBTelegramFile(id=id, mxc=uploaded["content_uri"], mime_type=mime_type, + was_converted=False, timestamp=int(time.time()), size=len(file), + width=width, height=height) + + +async def transfer_file_to_matrix(db, client, intent, location, thumbnail=None): + id = _location_to_id(location) + if not id: + return None + db_file = DBTelegramFile.query.get(id) if db_file: return db_file @@ -58,18 +131,26 @@ async def transfer_file_to_matrix(db, client, intent, location): file = await client.download_file_bytes(location) except LocationInvalidError: return None + width, height = None, None mime_type = magic.from_buffer(file, mime=True) image_converted = False if mime_type == "image/webp": - mime_type, file = _convert_webp(file, to="png") + mime_type, file, width, height = _convert_webp(file, to="png") image_converted = True uploaded = await intent.upload_file(file, mime_type) db_file = DBTelegramFile(id=id, mxc=uploaded["content_uri"], mime_type=mime_type, was_converted=image_converted, - timestamp=int(time.time())) + timestamp=int(time.time()), size=len(file), + width=width, height=height) + if thumbnail: + if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)): + thumbnail = thumbnail.location + db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file, + mime_type) + try: db.add(db_file) db.commit() diff --git a/requirements/base.txt b/requirements/base.txt index b7b8cc90..6e955265 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,5 +5,4 @@ python-magic SQLAlchemy alembic Markdown -Pillow future-fstrings diff --git a/requirements/optional.txt b/requirements/optional.txt index 10087cb8..a4400877 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -1,2 +1,4 @@ lxml cryptg +Pillow +moviepy diff --git a/setup.py b/setup.py index a9292e9e..5166748e 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,14 @@ import sys import glob import mautrix_telegram +extras = { + "highlight_edits": ["lxml>=4.1.1,<5"], + "fast_crypto": ["cryptg>=0.1,<0.2"], + "webp_convert": ["Pillow>=5.0.0,<6"], + "hq_thumbnails": ["moviepy>=0.2,<0.3"], +} +extras["all"] = [deps[0] for deps in extras.values()] + setuptools.setup( name="mautrix-telegram", version=mautrix_telegram.__version__, @@ -23,7 +31,6 @@ setuptools.setup( "alembic>=0.9.8,<0.10", "Markdown>=2.6.11,<3", "ruamel.yaml>=0.15.35,<0.16", - "Pillow>=5.0.0,<6", "future-fstrings>=0.4.2", "python-magic>=0.4.15,<0.5", "telethon-aio>=0.18,<0.19" if sys.version_info >= (3, 6) else "telethon-aio-git", @@ -31,10 +38,7 @@ setuptools.setup( dependency_links=[ "https://github.com/tulir/telethon-asyncio/tarball/9b389cfb4b6d3876e9661c23507f17e96897e4b0#egg=telethon-aio-git-0.18.0+1" ], - extras_require={ - "highlight_edits": ["lxml>=4.1.1,<5"], - "fast_crypto": ["cryptg>=0.1,<0.2"], - }, + extras_require=extras, classifiers=[ "Development Status :: 4 - Beta", From 85a4982ad95f32214cdef785990f197f70870bda Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Mar 2018 17:49:26 +0200 Subject: [PATCH 33/41] Update roadmap and remove unnecessary newline --- ROADMAP.md | 19 +------------------ mautrix_telegram/portal.py | 2 +- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index e0584dd7..e6a0a5d7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -45,7 +45,7 @@ * [x] Video messages * [x] Documents * [x] Message deletions - * [ ] Message edits (not yet supported in Matrix) + * [x] Message edits * [x] Avatars * [x] Presence * [x] Typing notifications @@ -74,23 +74,6 @@ * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room * [x] Option to use bot to relay messages for unauthenticated Matrix users * [ ] Option to use own Matrix account for messages sent from other Telegram clients -* [Commands](https://github.com/tulir/mautrix-telegram/wiki/Management-commands) - * [x] Logging in and out (`login` + code entering) - * [x] Logging out - * [ ] Registering (`register`) - * [x] Searching for users (`search`) - * [x] Starting private chats (`pm`) - * [x] Joining chats with invite links (`join`) - * [x] Creating a Telegram chat for an existing Matrix room (`create`) - * [x] Upgrading the chat of a portal room into a supergroup (`upgrade`) - * [x] Change username of supergroup/channel (`group-name`) - * [x] Getting the Telegram invite link to a Matrix room (`invite-link`) - * [ ] Bridging existing Matrix rooms to existing Telegram chats (`bridge`) - * [ ] Unbridging Matrix rooms from Telegram chats (`unbridge`) - * Bridge administration - * [x] Clean up and forget a portal room (`delete-portal`) - * [x] Find and clean up old portal rooms (`clean-rooms`) - * [ ] Setting Matrix-only power levels (`powerlevel`) † Information not automatically sent from source, i.e. implementation may not be possible ‡ Maybe, i.e. this feature may or may not be implemented at some point diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index a60da924..0c5ed860 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -838,7 +838,7 @@ class Portal: if config["bridge.inline_images"] and (evt.message or evt.fwd_from or evt.reply_to_msg_id): text, html, relates_to = await formatter.telegram_to_matrix( evt, source, self.main_intent, - prefix_html=f"Inline Telegram photo
\n", + prefix_html=f"Inline Telegram photo
", prefix_text="Inline image: ") await intent.set_typing(self.mxid, is_typing=False) return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to) From c98e822e6d0b843634a84315c1b2b6e94e5c7908 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Mar 2018 21:31:32 +0200 Subject: [PATCH 34/41] Add some extra checks before generating thumbnail --- mautrix_telegram/util/file_transfer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py index d8542247..530b78be 100644 --- a/mautrix_telegram/util/file_transfer.py +++ b/mautrix_telegram/util/file_transfer.py @@ -104,7 +104,10 @@ async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mim video_ext = mimetypes.guess_extension(mime) if VideoFileClip and video_ext: - file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png") + try: + file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png") + except OSError: + return None mime_type = "image/png" else: file = await client.download_file_bytes(thumbnail_loc) @@ -137,6 +140,7 @@ async def transfer_file_to_matrix(db, client, intent, location, thumbnail=None): image_converted = False if mime_type == "image/webp": mime_type, file, width, height = _convert_webp(file, to="png") + thumbnail = None image_converted = True uploaded = await intent.upload_file(file, mime_type) @@ -145,7 +149,7 @@ async def transfer_file_to_matrix(db, client, intent, location, thumbnail=None): mime_type=mime_type, was_converted=image_converted, timestamp=int(time.time()), size=len(file), width=width, height=height) - if thumbnail: + if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"): if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)): thumbnail = thumbnail.location db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file, From 77c57eb64b50b35eefad41ed1a7c5820843d34bc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 9 Mar 2018 23:42:33 +0200 Subject: [PATCH 35/41] Handle FlushError in transfer_file_to_matrix --- mautrix_telegram/util/file_transfer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py index 530b78be..a1424987 100644 --- a/mautrix_telegram/util/file_transfer.py +++ b/mautrix_telegram/util/file_transfer.py @@ -20,6 +20,7 @@ import logging import magic from sqlalchemy.exc import IntegrityError, InvalidRequestError +from sqlalchemy.orm.exc import FlushError try: from PIL import Image except ImportError: @@ -158,6 +159,10 @@ async def transfer_file_to_matrix(db, client, intent, location, thumbnail=None): try: db.add(db_file) db.commit() + except FlushError as e: + log.exception(f"{e.__class__.__name__} while saving transferred file data. " + "This was probably caused by two simultaneous transfers of the same file, " + "and should not cause any problems.") except (IntegrityError, InvalidRequestError) as e: db.rollback() log.exception(f"{e.__class__.__name__} while saving transferred file data. " From 2d63c5b3cebbf7effc4ad49870f2bd0d1ffcd57d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 10 Mar 2018 09:39:53 +0200 Subject: [PATCH 36/41] Fix and refactor Matrix->Telegram formatter --- mautrix_telegram/formatter/from_matrix.py | 115 +++++++++++--------- mautrix_telegram/formatter/from_telegram.py | 2 +- mautrix_telegram/portal.py | 2 +- 3 files changed, 68 insertions(+), 51 deletions(-) diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index a6a976bb..ffbc6878 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -151,17 +151,22 @@ class MatrixParser(HTMLParser): for entity in self._building_entities.values(): entity.length += 1 - def handle_data(self, text): - text = unescape(text) + def _handle_special_previous_tags(self, text): + if "pre" not in self._open_tags and "code" not in self._open_tags: + text = text.replace("\n", "") + else: + text = text.strip() + previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else "" - extra_offset = 0 if previous_tag == "a": url = self._open_tags_meta[0] if url: text = url elif previous_tag == "command": text = f"/{text}" + return text + def _html_to_unicode(self, text): strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags if strikethrough and underline: text = html_to_unicode(text, "\u0336\u0332") @@ -169,7 +174,10 @@ class MatrixParser(HTMLParser): text = html_to_unicode(text, "\u0336") elif underline: text = html_to_unicode(text, "\u0332") + return text + def _handle_tags_for_data(self, text): + extra_offset = 0 list_entry_handled_once = False # In order to maintain order of things like blockquotes in lists or lists in blockquotes, # we can't just have ifs/elses and we need to actually loop through the open tags in order. @@ -197,10 +205,19 @@ class MatrixParser(HTMLParser): text = indent + prefix + text self._list_entry_is_new = False list_entry_handled_once = True + return text, extra_offset + + def _extend_entities_in_construction(self, text, extra_offset): for tag, entity in self._building_entities.items(): entity.length += len(text) - extra_offset entity.offset += extra_offset + def handle_data(self, text): + text = unescape(text) + text = self._handle_special_previous_tags(text) + text = self._html_to_unicode(text) + text, extra_offset = self._handle_tags_for_data(text) + self._extend_entities_in_construction(text, extra_offset) self._line_is_new = False self.text += text @@ -223,6 +240,52 @@ command_regex = re.compile("(\s|^)!([A-Za-z0-9@]+)") plain_mention_regex = None +def plain_mention_to_html(match): + puppet = pu.Puppet.find_by_displayname(match.group(2)) + if puppet: + return (f"{match.group(1)}" + f"" + f"{puppet.displayname}" + "") + return "".join(match.groups()) + + +def matrix_to_telegram(html): + try: + parser = MatrixParser() + html = command_regex.sub(r"\1\2", html) + if should_bridge_plaintext_highlights: + html = plain_mention_regex.sub(plain_mention_to_html, html) + parser.feed(add_surrogates(html)) + print([str(e) for e in parser.entities]) + return remove_surrogates(parser.text.strip()), parser.entities + except Exception: + log.exception("Failed to convert Matrix format:\nhtml=%s", html) + + +def matrix_reply_to_telegram(content, tg_space, room_id=None): + try: + reply = content["m.relates_to"]["m.in_reply_to"] + room_id = room_id or reply["room_id"] + event_id = reply["event_id"] + + try: + if content["format"] == "org.matrix.custom.html": + content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"]) + except KeyError: + pass + content["body"] = trim_reply_fallback_text(content["body"]) + + message = DBMessage.query.filter(DBMessage.mxid == event_id, + DBMessage.tg_space == tg_space, + DBMessage.mx_room == room_id).one_or_none() + if message: + return message.tgid + except KeyError: + pass + return None + + def matrix_text_to_telegram(text): text = command_regex.sub(r"\1/\2", text) if should_bridge_plaintext_highlights: @@ -255,52 +318,6 @@ def plain_mention_to_text(): return entities, replacer -def plain_mention_to_html(match): - puppet = pu.Puppet.find_by_displayname(match.group(2)) - if puppet: - return (f"{match.group(1)}" - f"" - f"{puppet.displayname}" - "") - return "".join(match.groups()) - - -def matrix_to_telegram(html): - try: - parser = MatrixParser() - html = html.replace("\n", "") - html = command_regex.sub(r"\1\2", html) - if should_bridge_plaintext_highlights: - html = plain_mention_regex.sub(plain_mention_to_html, html) - parser.feed(add_surrogates(html)) - return remove_surrogates(parser.text.strip()), parser.entities - except Exception: - log.exception("Failed to convert Matrix format:\nhtml=%s", html) - - -def matrix_reply_to_telegram(content, tg_space, room_id=None): - try: - reply = content["m.relates_to"]["m.in_reply_to"] - room_id = room_id or reply["room_id"] - event_id = reply["event_id"] - - try: - if content["format"] == "org.matrix.custom.html": - content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"]) - except KeyError: - pass - content["body"] = trim_reply_fallback_text(content["body"]) - - message = DBMessage.query.filter(DBMessage.mxid == event_id, - DBMessage.tg_space == tg_space, - DBMessage.mx_room == room_id).one_or_none() - if message: - return message.tgid - except KeyError: - pass - return None - - def init_mx(context): global plain_mention_regex, should_bridge_plaintext_highlights config = context.config diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 5ed2865d..7a2bf442 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -132,7 +132,7 @@ async def _add_reply_header(source, text, html, evt, relates_to, main_intent, is r_keyword = "In reply to" if not is_edit else "Edit to" r_msg_link = f"{r_keyword}" - html = (f"
{r_msg_link} {r_sender_link} {r_html_body}
" + html = (f"
{r_msg_link} {r_sender_link}\n{r_html_body}
" + (html or escape(text))) lines = r_text_body.strip().split("\n") diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 0c5ed860..0c248de0 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -594,7 +594,7 @@ class Portal: entity.user_id = await client.get_input_entity(entity.user_id.user_id) else: message, entities = formatter.matrix_text_to_telegram(message["body"]) - return await client.send_message(self.peer, message, reply_to=reply_to) + return await client.send_message(self.peer, message, entities=entities, reply_to=reply_to) async def _handle_matrix_file(self, client, message, reply_to): file = await self.main_intent.download_file(message["url"]) From ae88aa0553655d94ef39cbf549e3a3fd7cc4ddbd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 10 Mar 2018 10:23:50 +0200 Subject: [PATCH 37/41] Add type hints to formatter --- mautrix_telegram/formatter/__init__.py | 3 +- mautrix_telegram/formatter/from_matrix.py | 40 ++++++++++++--------- mautrix_telegram/formatter/from_telegram.py | 39 ++++++++++++-------- mautrix_telegram/formatter/util.py | 17 ++++----- 4 files changed, 60 insertions(+), 39 deletions(-) diff --git a/mautrix_telegram/formatter/__init__.py b/mautrix_telegram/formatter/__init__.py index 6252455a..7cb102f7 100644 --- a/mautrix_telegram/formatter/__init__.py +++ b/mautrix_telegram/formatter/__init__.py @@ -1,8 +1,9 @@ from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram, init_mx) from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg) +from ..context import Context -def init(context): +def init(context: Context): init_mx(context) init_tg(context) diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index ffbc6878..aadc7b26 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -17,12 +17,18 @@ from html import unescape from html.parser import HTMLParser from collections import deque +from typing import Optional, List, Tuple, Type, Callable, Dict import math import re import logging -from telethon_aio.tl.types import * +from telethon_aio.tl.types import (TypeMessageEntity, MessageEntityMention, + InputMessageEntityMentionName, MessageEntityEmail, + MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold, + MessageEntityItalic, MessageEntityCode, MessageEntityPre, + MessageEntityBotCommand, InputUser) +from ..context import Context from .. import user as u, puppet as pu, portal as po from ..db import Message as DBMessage from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, @@ -51,7 +57,8 @@ class MatrixParser(HTMLParser): self._line_is_new = True self._list_entry_is_new = False - def _parse_url(self, url, args): + def _parse_url(self, url: str, args: Dict[str, str] + ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]: mention = self.mention_regex.match(url) if mention: mxid = mention.group(1) @@ -80,7 +87,7 @@ class MatrixParser(HTMLParser): args["url"] = url return MessageEntityTextUrl, None - def handle_starttag(self, tag, attrs): + def handle_starttag(self, tag: str, attrs: List[Tuple[str, str]]): self._open_tags.appendleft(tag) self._open_tags_meta.appendleft(0) @@ -127,7 +134,7 @@ class MatrixParser(HTMLParser): self._building_entities[tag] = entity_type(offset=offset, length=0, **args) @property - def _list_indent(self): + def _list_indent(self) -> int: indent = 0 first_skipped = False for index, tag in enumerate(self._open_tags): @@ -143,7 +150,7 @@ class MatrixParser(HTMLParser): indent += 3 return indent - def _newline(self, allow_multi=False): + def _newline(self, allow_multi: bool = False): if self._line_is_new and not allow_multi: return self.text += "\n" @@ -151,7 +158,7 @@ class MatrixParser(HTMLParser): for entity in self._building_entities.values(): entity.length += 1 - def _handle_special_previous_tags(self, text): + def _handle_special_previous_tags(self, text: str) -> str: if "pre" not in self._open_tags and "code" not in self._open_tags: text = text.replace("\n", "") else: @@ -166,7 +173,7 @@ class MatrixParser(HTMLParser): text = f"/{text}" return text - def _html_to_unicode(self, text): + def _html_to_unicode(self, text: str) -> str: strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags if strikethrough and underline: text = html_to_unicode(text, "\u0336\u0332") @@ -176,7 +183,7 @@ class MatrixParser(HTMLParser): text = html_to_unicode(text, "\u0332") return text - def _handle_tags_for_data(self, text): + def _handle_tags_for_data(self, text: str) -> Tuple[str, int]: extra_offset = 0 list_entry_handled_once = False # In order to maintain order of things like blockquotes in lists or lists in blockquotes, @@ -207,12 +214,12 @@ class MatrixParser(HTMLParser): list_entry_handled_once = True return text, extra_offset - def _extend_entities_in_construction(self, text, extra_offset): + def _extend_entities_in_construction(self, text: str, extra_offset: int): for tag, entity in self._building_entities.items(): entity.length += len(text) - extra_offset entity.offset += extra_offset - def handle_data(self, text): + def handle_data(self, text: str): text = unescape(text) text = self._handle_special_previous_tags(text) text = self._html_to_unicode(text) @@ -221,7 +228,7 @@ class MatrixParser(HTMLParser): self._line_is_new = False self.text += text - def handle_endtag(self, tag): + def handle_endtag(self, tag: str): try: self._open_tags.popleft() self._open_tags_meta.popleft() @@ -250,7 +257,7 @@ def plain_mention_to_html(match): return "".join(match.groups()) -def matrix_to_telegram(html): +def matrix_to_telegram(html: str) -> Tuple[str, List[TypeMessageEntity]]: try: parser = MatrixParser() html = command_regex.sub(r"\1\2", html) @@ -263,7 +270,8 @@ def matrix_to_telegram(html): log.exception("Failed to convert Matrix format:\nhtml=%s", html) -def matrix_reply_to_telegram(content, tg_space, room_id=None): +def matrix_reply_to_telegram(content: dict, tg_space: int, room_id: Optional[str] = None + ) -> Optional[int]: try: reply = content["m.relates_to"]["m.in_reply_to"] room_id = room_id or reply["room_id"] @@ -286,7 +294,7 @@ def matrix_reply_to_telegram(content, tg_space, room_id=None): return None -def matrix_text_to_telegram(text): +def matrix_text_to_telegram(text: str) -> Tuple[str, List[TypeMessageEntity]]: text = command_regex.sub(r"\1/\2", text) if should_bridge_plaintext_highlights: entities, pmr_replacer = plain_mention_to_text() @@ -296,7 +304,7 @@ def matrix_text_to_telegram(text): return text, entities -def plain_mention_to_text(): +def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]: entities = [] def replacer(match): @@ -318,7 +326,7 @@ def plain_mention_to_text(): return entities, replacer -def init_mx(context): +def init_mx(context: Context): global plain_mention_regex, should_bridge_plaintext_highlights config = context.config dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)") diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 7a2bf442..2ec2d8f9 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from html import escape +from typing import Optional, List, Tuple + try: from lxml.html.diff import htmldiff except ImportError: @@ -22,10 +24,16 @@ except ImportError: import logging import re -from telethon_aio.tl.types import * +from telethon_aio.tl.types import (MessageEntityMention, MessageEntityMentionName, + MessageEntityEmail, MessageEntityUrl, MessageEntityTextUrl, + MessageEntityBold, MessageEntityItalic, MessageEntityCode, + MessageEntityPre, MessageEntityBotCommand, Message, PeerChannel, + MessageEntityHashtag, TypeMessageEntity) from mautrix_appservice import MatrixRequestError +from mautrix_appservice.intent_api import IntentAPI from .. import user as u, puppet as pu, portal as po +from ..context import Context from ..db import Message as DBMessage from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, trim_reply_fallback_text, unicode_to_html) @@ -34,7 +42,7 @@ log = logging.getLogger("mau.fmt.tg") should_highlight_edits = False -def telegram_reply_to_matrix(evt, source): +def telegram_reply_to_matrix(evt: Message, source: u.User) -> dict: if evt.reply_to_msg_id: space = (evt.to_id.channel_id if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel) @@ -50,7 +58,8 @@ def telegram_reply_to_matrix(evt, source): return {} -async def _add_forward_header(source, text, html, fwd_from_id): +async def _add_forward_header(source, text: str, html: Optional[str], + fwd_from_id: Optional[int]) -> Tuple[str, str]: if not html: html = escape(text) user = u.User.get_by_tgid(fwd_from_id) @@ -74,7 +83,7 @@ async def _add_forward_header(source, text, html, fwd_from_id): return text, html -def highlight_edits(new_html, old_html): +def highlight_edits(new_html: str, old_html: str) -> str: # Don't include `Edit:` text in diff. if old_html.startswith("Edit: "): old_html = old_html[len("Edit: "):] @@ -89,7 +98,8 @@ def highlight_edits(new_html, old_html): return new_html -async def _add_reply_header(source, text, html, evt, relates_to, main_intent, is_edit): +async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, relates_to: dict, + main_intent: IntentAPI, is_edit: bool) -> Tuple[str, str]: space = (evt.to_id.channel_id if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel) else source.tgid) @@ -145,8 +155,9 @@ async def _add_reply_header(source, text, html, evt, relates_to, main_intent, is return text_with_quote, html -async def telegram_to_matrix(evt, source, main_intent=None, is_edit=False, prefix_text=None, - prefix_html=None): +async def telegram_to_matrix(evt: Message, source: u.User, main_intent: Optional[IntentAPI] = None, + is_edit: bool = False, prefix_text: Optional[str] = None, + prefix_html: Optional[str] = None) -> Tuple[str, str, dict]: text = add_surrogates(evt.message) html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None relates_to = {} @@ -178,7 +189,7 @@ async def telegram_to_matrix(evt, source, main_intent=None, is_edit=False, prefi return remove_surrogates(text), remove_surrogates(html), relates_to -def _telegram_entities_to_matrix_catch(text, entities): +def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEntity]) -> str: try: return _telegram_entities_to_matrix(text, entities) except Exception: @@ -188,7 +199,7 @@ def _telegram_entities_to_matrix_catch(text, entities): text, entities) -def _telegram_entities_to_matrix(text, entities): +def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str: if not entities: return text html = [] @@ -232,7 +243,7 @@ def _telegram_entities_to_matrix(text, entities): return "".join(html) -def _parse_pre(html, entity_text, language): +def _parse_pre(html: List[str], entity_text: str, language: str) -> bool: if language: html.append("
"
                     f"{entity_text}"
@@ -242,7 +253,7 @@ def _parse_pre(html, entity_text, language):
     return False
 
 
-def _parse_mention(html, entity_text):
+def _parse_mention(html: List[str], entity_text: str) -> bool:
     username = entity_text[1:]
 
     user = u.User.find_by_username(username) or pu.Puppet.find_by_username(username)
@@ -259,7 +270,7 @@ def _parse_mention(html, entity_text):
     return False
 
 
-def _parse_name_mention(html, entity_text, user_id):
+def _parse_name_mention(html: List[str], entity_text: str, user_id: int) -> bool:
     user = u.User.get_by_tgid(user_id)
     if user:
         mxid = user.mxid
@@ -273,7 +284,7 @@ def _parse_name_mention(html, entity_text, user_id):
     return False
 
 
-def _parse_url(html, entity_text, url):
+def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
     url = escape(url) if url else entity_text
     if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
         url = "http://" + url
@@ -281,6 +292,6 @@ def _parse_url(html, entity_text, url):
     return False
 
 
-def init_tg(context):
+def init_tg(context: Context):
     global should_highlight_edits
     should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]
diff --git a/mautrix_telegram/formatter/util.py b/mautrix_telegram/formatter/util.py
index a9806d53..f10146b4 100644
--- a/mautrix_telegram/formatter/util.py
+++ b/mautrix_telegram/formatter/util.py
@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see .
 from html import escape
+from typing import Optional
 import struct
 import re
 
@@ -22,20 +23,20 @@ import re
 # add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
 # Licensed under the MIT license.
 # https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py
-def add_surrogates(text):
+def add_surrogates(text: Optional[str]) -> Optional[str]:
     if text is None:
         return None
     return "".join("".join(chr(y) for y in struct.unpack(" Optional[str]:
     if text is None:
         return None
     return text.encode("utf-16", "surrogatepass").decode("utf-16")
 
 
-def trim_reply_fallback_text(text):
+def trim_reply_fallback_text(text: str) -> str:
     if not text.startswith("> ") or "\n" not in text:
         return text
     lines = text.split("\n")
@@ -44,14 +45,14 @@ def trim_reply_fallback_text(text):
     return "\n".join(lines)
 
 
-HTML_REPLY_FALLBACK_REGEX = re.compile(r"^
[\s\S]+?
")
+html_reply_fallback_regex = re.compile(r"^
[\s\S]+?
")
 
 
-def trim_reply_fallback_html(html):
-    return HTML_REPLY_FALLBACK_REGEX.sub("", html)
+def trim_reply_fallback_html(html: str) -> str:
+    return html_reply_fallback_regex.sub("", html)
 
 
-def unicode_to_html(text, html, ctrl, tag):
+def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
     if ctrl not in text:
         return html
     if not html:
@@ -79,5 +80,5 @@ def unicode_to_html(text, html, ctrl, tag):
     return html
 
 
-def html_to_unicode(text, ctrl):
+def html_to_unicode(text: str, ctrl: str) -> str:
     return ctrl.join(text) + ctrl

From 7f52238fbbb9900a6831e417b60c4f9719f29008 Mon Sep 17 00:00:00 2001
From: Tulir Asokan 
Date: Sat, 10 Mar 2018 14:36:44 +0200
Subject: [PATCH 38/41] Add Telegram bot command access whitelist. Fixes #80

---
 example-config.yaml        | 13 +++++-
 mautrix_telegram/bot.py    | 84 ++++++++++++++++++++++++++++++--------
 mautrix_telegram/config.py | 29 +++++++++++++
 3 files changed, 106 insertions(+), 20 deletions(-)

diff --git a/example-config.yaml b/example-config.yaml
index 562c44e3..08688e6b 100644
--- a/example-config.yaml
+++ b/example-config.yaml
@@ -84,8 +84,6 @@ bridge:
     # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
     # login website (see appservice.public config section)
     allow_matrix_login: true
-    # Whether or not to allow creating portals from Telegram.
-    authless_relaybot_portals: true
     # Use inline images instead of m.image to make rich captions possible.
     # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
     inline_images: false
@@ -112,6 +110,17 @@ bridge:
         "public.example.com": "full"
         "@admin:example.com": "admin"
 
+    # Options related to the message relay Telegram bot.
+    relaybot:
+        # Whether or not to allow creating portals from Telegram.
+        authless_portals: true
+        # Whether or not to allow Telegram group admins to use the bot commands.
+        whitelist_group_admins: true
+        # List of usernames/user IDs who are also allowed to use the bot commands.
+        whitelist:
+        - myusername
+        - 12345678
+
 # Telegram config
 telegram:
     # Get your own API keys at https://my.telegram.org/apps
diff --git a/mautrix_telegram/bot.py b/mautrix_telegram/bot.py
index 9e1051cd..b1d9e2ca 100644
--- a/mautrix_telegram/bot.py
+++ b/mautrix_telegram/bot.py
@@ -14,12 +14,13 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see .
+from typing import Awaitable, Callable
 import logging
 import re
 
 from telethon_aio.tl.types import *
-from telethon_aio.tl.functions.messages import GetChatsRequest
-from telethon_aio.tl.functions.channels import GetChannelsRequest
+from telethon_aio.tl.functions.messages import GetChatsRequest, GetFullChatRequest
+from telethon_aio.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon_aio.errors import ChannelInvalidError, ChannelPrivateError
 
 from .abstract_user import AbstractUser
@@ -29,16 +30,33 @@ from . import puppet as pu, portal as po, user as u
 config = None
 
 
+ReplyFunc = Callable[[str], Awaitable[Message]]
+
+
 class Bot(AbstractUser):
     log = logging.getLogger("mau.bot")
     mxid_regex = re.compile("@.+:.+")
 
-    def __init__(self, token):
+    def __init__(self, token: str):
         super().__init__()
         self.token = token
         self.whitelisted = True
         self.username = None
         self.chats = {chat.id: chat.type for chat in BotChat.query.all()}
+        self.tg_whitelist = []
+        self.whitelist_group_admins = config["bridge.relaybot.whitelist_group_admins"] or False
+
+    async def init_permissions(self):
+        whitelist = config["bridge.relaybot.whitelist"] or []
+        for id in whitelist:
+            if isinstance(id, str):
+                entity = await self.client.get_input_entity(id)
+                if isinstance(entity, InputUser):
+                    id = entity.user_id
+                else:
+                    id = None
+            if isinstance(id, int):
+                self.tg_whitelist.append(id)
 
     async def start(self):
         await super().start()
@@ -48,6 +66,7 @@ class Bot(AbstractUser):
         return self
 
     async def post_login(self):
+        await self.init_permissions()
         info = await self.client.get_me()
         self.tgid = info.id
         self.username = info.username
@@ -68,19 +87,19 @@ class Bot(AbstractUser):
             except (ChannelPrivateError, ChannelInvalidError):
                 self.remove_chat(id.channel_id)
 
-    def register_portal(self, portal):
+    def register_portal(self, portal: po.Portal):
         self.add_chat(portal.tgid, portal.peer_type)
 
-    def unregister_portal(self, portal):
+    def unregister_portal(self, portal: po.Portal):
         self.remove_chat(portal.tgid)
 
-    def add_chat(self, id, type):
+    def add_chat(self, id: int, type: str):
         if id not in self.chats:
             self.chats[id] = type
             self.db.add(BotChat(id=id, type=type))
             self.db.commit()
 
-    def remove_chat(self, id):
+    def remove_chat(self, id: int):
         try:
             del self.chats[id]
         except KeyError:
@@ -88,8 +107,34 @@ class Bot(AbstractUser):
         self.db.delete(BotChat.query.get(id))
         self.db.commit()
 
-    async def handle_command_portal(self, portal, reply):
-        if not config["bridge.authless_relaybot_portals"]:
+    async def _can_use_commands(self, chat, tgid):
+        if tgid in self.tg_whitelist:
+            return True
+
+        user = u.User.get_by_tgid(tgid)
+        if user and user.is_admin:
+            self.tg_whitelist.append(user.tgid)
+            return True
+
+        if self.whitelist_group_admins:
+            if isinstance(chat, PeerChannel):
+                p = await self.client(GetParticipantRequest(chat, tgid))
+                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
+            elif isinstance(chat, PeerChat):
+                chat = await self.client(GetFullChatRequest(chat.chat_id))
+                participants = chat.full_chat.participants.participants
+                for p in participants:
+                    if p.user_id == tgid:
+                        return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+
+    async def check_can_use_commands(self, event: Message, reply: ReplyFunc):
+        if not await self._can_use_commands(event.to_id, event.from_id):
+            await reply("You do not have the permission to use that command.")
+            return False
+        return True
+
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc):
+        if not config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
         await portal.create_matrix_room(self)
@@ -101,7 +146,7 @@ class Bot(AbstractUser):
                 return await reply(
                     "Portal is not public. Use `/invite ` to get an invite.")
 
-    async def handle_command_invite(self, portal, reply, mxid):
+    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc, mxid: str):
         if len(mxid) == 0:
             return await reply("Usage: `/invite `")
         elif not portal.mxid:
@@ -120,14 +165,14 @@ class Bot(AbstractUser):
             await portal.main_intent.invite(portal.mxid, user.mxid)
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
-    def handle_command_id(self, message, reply):
+    def handle_command_id(self, message: Message, reply: ReplyFunc):
         # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
         # chat is a normal group or a supergroup/channel when using the ID.
         if isinstance(message.to_id, PeerChannel):
             return reply(f"-100{message.to_id.channel_id}")
         return reply(str(-message.to_id.chat_id))
 
-    def match_command(self, text, command):
+    def match_command(self, text: str, command: str) -> bool:
         text = text.lower()
         command = f"/{command.lower()}"
         command_targeted = f"{command}@{self.username.lower()}"
@@ -142,7 +187,7 @@ class Bot(AbstractUser):
 
         return False
 
-    async def handle_command(self, message):
+    async def handle_command(self, message: Message):
         def reply(reply_text):
             return self.client.send_message(message.to_id, reply_text, markdown=True,
                                             reply_to=message.id)
@@ -155,15 +200,19 @@ class Bot(AbstractUser):
         portal = po.Portal.get_by_entity(message.to_id)
 
         if self.match_command(text, "portal"):
+            if not await self.check_can_use_commands(message, reply):
+                return
             await self.handle_command_portal(portal, reply)
         elif self.match_command(text, "invite"):
+            if not await self.check_can_use_commands(message, reply):
+                return
             try:
                 mxid = text[text.index(" ") + 1:]
             except ValueError:
                 mxid = ""
             await self.handle_command_invite(portal, reply, mxid=mxid)
 
-    def handle_service_message(self, message):
+    def handle_service_message(self, message: MessageService):
         to_id = message.to_id
         if isinstance(to_id, PeerChannel):
             to_id = to_id.channel_id
@@ -180,10 +229,9 @@ class Bot(AbstractUser):
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
             self.remove_chat(to_id)
 
-    async def update(self, update):
+    async def update(self, update: TypeUpdate):
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
             return
-
         if isinstance(update.message, MessageService):
             return self.handle_service_message(update.message)
 
@@ -193,11 +241,11 @@ class Bot(AbstractUser):
         if is_command:
             return await self.handle_command(update.message)
 
-    def is_in_chat(self, peer_id):
+    def is_in_chat(self, peer_id) -> bool:
         return peer_id in self.chats
 
     @property
-    def name(self):
+    def name(self) -> str:
         return "bot"
 
 
diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py
index 0d414542..cfc00357 100644
--- a/mautrix_telegram/config.py
+++ b/mautrix_telegram/config.py
@@ -218,6 +218,33 @@ class Config(DictWithRecursion):
         self["version"] = 2
         return self["version"]
 
+    def update_2_3(self):
+        if "bridge.plaintext_highlights" not in self:
+            self["bridge.plaintext_highlights"] = False
+            self.comment("bridge.plaintext_highlights",
+                         "Whether or not to bridge plaintext highlights.\n"
+                         "Only enable this if your displayname_template has some static part that "
+                         "the bridge can use to\nreliably identify what is a plaintext highlight.")
+        if "bridge.highlight_edits" not in self:
+            self["bridge.highlight_edits"] = False
+            self.comment("bridge.highlight_edits",
+                         "Highlight changed/added parts in edits. Requires lxml.")
+        if "bridge.relaybot" not in self:
+            self["bridge.relaybot.authless_portals"] = bool(
+                self["bridge.authless_relaybot_portals"]) or True
+            del self["bridge.authless_relaybot_portals"]
+            self["bridge.relaybot.whitelist_group_admins"] = True
+            self["bridge.relaybot.whitelist"] = []
+            self.comment("bridge.relaybot", "Options related to the message relay Telegram bot.")
+            self.comment("bridge.relaybot.authless_portals",
+                         "Whether or not to allow creating portals from Telegram.")
+            self.comment("bridge.relaybot.whitelist_group_admins",
+                         "Whether or not to allow Telegram group admins to use the bot commands.")
+            self.comment("bridge.relaybot.whitelist",
+                         "List of usernames/user IDs who are also allowed to use the bot commands.")
+        self["version"] = 3
+        return self["version"]
+
     def check_updates(self):
         version = self.get("version", 0)
         new_version = version
@@ -225,6 +252,8 @@ class Config(DictWithRecursion):
             new_version = self.update_0_1()
         if version < 2:
             new_version = self.update_1_2()
+        if version < 3:
+            new_version = self.update_2_3()
         if new_version != version:
             self.save()
 

From 42e33ab54d08ef0868f037c10e24d4af3bee0fc2 Mon Sep 17 00:00:00 2001
From: Tulir Asokan 
Date: Sat, 10 Mar 2018 14:55:37 +0200
Subject: [PATCH 39/41] Add temporary patch for TypeMessageEntity

---
 mautrix_telegram/bot.py                     |  2 +-
 mautrix_telegram/formatter/from_matrix.py   | 15 ++++++++++++---
 mautrix_telegram/formatter/from_telegram.py | 13 +++++++++++--
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/mautrix_telegram/bot.py b/mautrix_telegram/bot.py
index b1d9e2ca..3e0b7cb9 100644
--- a/mautrix_telegram/bot.py
+++ b/mautrix_telegram/bot.py
@@ -229,7 +229,7 @@ class Bot(AbstractUser):
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
             self.remove_chat(to_id)
 
-    async def update(self, update: TypeUpdate):
+    async def update(self, update):
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
             return
         if isinstance(update.message, MessageService):
diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py
index aadc7b26..768643a6 100644
--- a/mautrix_telegram/formatter/from_matrix.py
+++ b/mautrix_telegram/formatter/from_matrix.py
@@ -17,16 +17,25 @@
 from html import unescape
 from html.parser import HTMLParser
 from collections import deque
-from typing import Optional, List, Tuple, Type, Callable, Dict
+from typing import Optional, List, Tuple, Type, Callable, Dict, Union
 import math
 import re
 import logging
 
-from telethon_aio.tl.types import (TypeMessageEntity, MessageEntityMention,
+from telethon_aio.tl.types import (MessageEntityMention,
                                    InputMessageEntityMentionName, MessageEntityEmail,
                                    MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
                                    MessageEntityItalic, MessageEntityCode, MessageEntityPre,
-                                   MessageEntityBotCommand, InputUser)
+                                   MessageEntityBotCommand, MessageEntityHashtag,
+                                   MessageEntityMentionName, InputUser)
+
+try:
+    from telethon_aio.tl.types import TypeMessageEntity
+except ImportError:
+    TypeMessageEntity = Union[
+        MessageEntityMention, MessageEntityHashtag, MessageEntityBotCommand, MessageEntityUrl,
+        MessageEntityEmail, MessageEntityBold, MessageEntityItalic, MessageEntityCode,
+        MessageEntityPre, MessageEntityTextUrl, MessageEntityMentionName]
 
 from ..context import Context
 from .. import user as u, puppet as pu, portal as po
diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py
index 2ec2d8f9..4cd6de00 100644
--- a/mautrix_telegram/formatter/from_telegram.py
+++ b/mautrix_telegram/formatter/from_telegram.py
@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see .
 from html import escape
-from typing import Optional, List, Tuple
+from typing import Optional, List, Tuple, Union
 
 try:
     from lxml.html.diff import htmldiff
@@ -28,7 +28,16 @@ from telethon_aio.tl.types import (MessageEntityMention, MessageEntityMentionNam
                                    MessageEntityEmail, MessageEntityUrl, MessageEntityTextUrl,
                                    MessageEntityBold, MessageEntityItalic, MessageEntityCode,
                                    MessageEntityPre, MessageEntityBotCommand, Message, PeerChannel,
-                                   MessageEntityHashtag, TypeMessageEntity)
+                                   MessageEntityHashtag)
+
+try:
+    from telethon_aio.tl.types import TypeMessageEntity
+except ImportError:
+    TypeMessageEntity = Union[
+        MessageEntityMention, MessageEntityHashtag, MessageEntityBotCommand, MessageEntityUrl,
+        MessageEntityEmail, MessageEntityBold, MessageEntityItalic, MessageEntityCode,
+        MessageEntityPre, MessageEntityTextUrl, MessageEntityMentionName]
+
 from mautrix_appservice import MatrixRequestError
 from mautrix_appservice.intent_api import IntentAPI
 

From 7837f035327851a49b7449d6d43e37e8946cbbd2 Mon Sep 17 00:00:00 2001
From: Tulir Asokan 
Date: Sat, 10 Mar 2018 15:45:41 +0200
Subject: [PATCH 40/41] Add Matrix->Telegram message pinning and show user in
 Telegram->Matrix pinning. Fixes #90

---
 mautrix_telegram/abstract_user.py |  2 +-
 mautrix_telegram/matrix.py        | 19 ++++++++++++
 mautrix_telegram/portal.py        | 48 ++++++++++++++++++++++++++-----
 3 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py
index 04554a32..4199b5d2 100644
--- a/mautrix_telegram/abstract_user.py
+++ b/mautrix_telegram/abstract_user.py
@@ -136,7 +136,7 @@ class AbstractUser:
     async def update_pinned_messages(self, update):
         portal = po.Portal.get_by_tgid(update.channel_id)
         if portal and portal.mxid:
-            await portal.update_telegram_pin(self, update.id)
+            await portal.receive_telegram_pin_id(update.id)
 
     async def update_participants(self, update):
         portal = po.Portal.get_by_tgid(update.participants.chat_id)
diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py
index 42c079b8..aca86ec7 100644
--- a/mautrix_telegram/matrix.py
+++ b/mautrix_telegram/matrix.py
@@ -222,6 +222,18 @@ class MatrixHandler:
                 return
             await handler(sender, content[content_key])
 
+    async def handle_room_pin(self, room, sender, new_events, old_events):
+        portal = Portal.get_by_mxid(room)
+        sender = await User.get_by_mxid(sender).ensure_started()
+        if sender.has_full_access and portal:
+            events = new_events - old_events
+            if len(events) > 0:
+                # New event pinned, set that as pinned in Telegram.
+                await portal.handle_matrix_pin(sender, events.pop())
+            elif len(new_events) == 0:
+                # All pinned events removed, remove pinned event in Telegram.
+                await portal.handle_matrix_pin(sender, None)
+
     def filter_matrix_event(self, event):
         return (event["sender"] == self.az.bot_mxid
                 or Puppet.get_id_from_mxid(event["sender"]) is not None)
@@ -250,3 +262,10 @@ class MatrixHandler:
                                            evt["prev_content"])
         elif type in ("m.room.name", "m.room.avatar", "m.room.topic"):
             await self.handle_room_meta(type, evt["room_id"], evt["sender"], evt["content"])
+        elif type == "m.room.pinned_events":
+            new_events = set(evt["content"]["pinned"])
+            try:
+                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+            except KeyError:
+                old_events = set()
+            await self.handle_room_pin(evt["room_id"], evt["sender"], new_events, old_events)
diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py
index 0c248de0..778dc8ee 100644
--- a/mautrix_telegram/portal.py
+++ b/mautrix_telegram/portal.py
@@ -66,6 +66,8 @@ class Portal:
 
         self._main_intent = None
         self._room_create_lock = asyncio.Lock()
+        self._temp_pinned_message_id = None
+        self._temp_pinned_message_sender = None
 
         self._dedup = deque()
         self._dedup_mxid = {}
@@ -637,6 +639,20 @@ class Portal:
             mxid=event_id))
         self.db.commit()
 
+    async def handle_matrix_pin(self, sender, pinned_message):
+        if self.peer_type != "channel":
+            return
+        try:
+            if not pinned_message:
+                await sender.client(UpdatePinnedMessageRequest(channel=self.peer, id=0))
+            else:
+                message = DBMessage.query.filter(DBMessage.mxid == pinned_message,
+                                                 DBMessage.tg_space == self.tgid,
+                                                 DBMessage.mx_room == self.mxid).one_or_none()
+                await sender.client(UpdatePinnedMessageRequest(channel=self.peer, id=message.tgid))
+        except ChatNotModifiedError:
+            pass
+
     async def handle_matrix_deletion(self, deleter, event_id):
         space = self.tgid if self.peer_type == "channel" else deleter.tgid
         message = DBMessage.query.filter(DBMessage.mxid == event_id,
@@ -657,7 +673,7 @@ class Portal:
                                         edit_messages=moderator, delete_messages=moderator,
                                         ban_users=moderator, invite_users=moderator,
                                         invite_link=moderator, pin_messages=moderator,
-                                        add_admins=admin, manage_call=moderator)
+                                        add_admins=admin)
             await sender.client(
                 EditAdminRequest(channel=await self.get_input_entity(sender),
                                  user_id=user_id, admin_rights=rights))
@@ -1036,7 +1052,6 @@ class Portal:
                          or self.is_duplicate_action(update))
         if should_ignore:
             return
-
         # TODO figure out how to see changes to about text / channel username
         if isinstance(action, MessageActionChatEditTitle):
             await self.update_title(action.title, save=True)
@@ -1054,6 +1069,8 @@ class Portal:
             self.peer_type = "channel"
             self.migrate_and_save(action.channel_id)
             await sender.intent.send_emote(self.mxid, "upgraded this group to a supergroup.")
+        elif isinstance(action, MessageActionPinMessage):
+            await self.receive_telegram_pin_sender(sender)
         else:
             self.log.debug("Unhandled Telegram action in %s: %s", self.title, action)
 
@@ -1068,13 +1085,30 @@ class Portal:
             levels["users"][puppet.mxid] = 50
         await self.main_intent.set_power_levels(self.mxid, levels)
 
-    async def update_telegram_pin(self, source, id):
-        space = self.tgid if self.peer_type == "channel" else source.tgid
-        message = DBMessage.query.get((id, space))
+    async def receive_telegram_pin_sender(self, sender):
+        self._temp_pinned_message_sender = sender
+        if self._temp_pinned_message_id:
+            await self.update_telegram_pin()
+
+    async def update_telegram_pin(self):
+        intent = (self._temp_pinned_message_sender.intent
+                  if self._temp_pinned_message_sender else self.main_intent)
+        id = self._temp_pinned_message_id
+        self._temp_pinned_message_id = None
+        self._temp_pinned_message_sender = None
+
+        message = DBMessage.query.get((id, self.tgid))
         if message:
-            await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
+            await intent.set_pinned_messages(self.mxid, [message.mxid])
         else:
-            await self.main_intent.set_pinned_messages(self.mxid, [])
+            await intent.set_pinned_messages(self.mxid, [])
+
+    async def receive_telegram_pin_id(self, id):
+        if id == 0:
+            return await self.update_telegram_pin()
+        self._temp_pinned_message_id = id
+        if self._temp_pinned_message_sender:
+            await self.update_telegram_pin()
 
     @staticmethod
     def _get_level_from_participant(participant, _):

From b06e7932f0fd6748856cc955c1e8a41bd76663c3 Mon Sep 17 00:00:00 2001
From: Tulir Asokan 
Date: Sat, 10 Mar 2018 19:53:08 +0200
Subject: [PATCH 41/41] Add Matrix->Telegram location bridging and add user to
 relaybot files. Fixes #89

---
 mautrix_telegram/formatter/from_matrix.py |  1 -
 mautrix_telegram/portal.py                | 87 ++++++++++++++++-------
 mautrix_telegram/tgclient.py              |  5 ++
 3 files changed, 68 insertions(+), 25 deletions(-)

diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py
index 768643a6..9ee7ead0 100644
--- a/mautrix_telegram/formatter/from_matrix.py
+++ b/mautrix_telegram/formatter/from_matrix.py
@@ -273,7 +273,6 @@ def matrix_to_telegram(html: str) -> Tuple[str, List[TypeMessageEntity]]:
         if should_bridge_plaintext_highlights:
             html = plain_mention_regex.sub(plain_mention_to_html, html)
         parser.feed(add_surrogates(html))
-        print([str(e) for e in parser.entities])
         return remove_surrogates(parser.text.strip()), parser.entities
     except Exception:
         log.exception("Failed to convert Matrix format:\nhtml=%s", html)
diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py
index 778dc8ee..7286fcf9 100644
--- a/mautrix_telegram/portal.py
+++ b/mautrix_telegram/portal.py
@@ -516,12 +516,10 @@ class Portal:
         try:
             current_extension = body[body.rindex("."):]
             if mimetypes.types_map[current_extension] == mime:
-                file_name = body
-            else:
-                file_name = f"matrix_upload{mimetypes.guess_extension(mime)}"
+                return body
         except (ValueError, KeyError):
-            file_name = f"matrix_upload{mimetypes.guess_extension(mime)}"
-        return file_name, None if file_name == body else body
+            pass
+        return f"matrix_upload{mimetypes.guess_extension(mime)}"
 
     async def leave_matrix(self, user, source, event_id):
         if not user.logged_in:
@@ -570,32 +568,53 @@ class Portal:
 
     @staticmethod
     def _preprocess_matrix_message(sender, message):
-        if message["msgtype"] == "m.emote":
+        msgtype = message["msgtype"]
+        if msgtype == "m.emote":
             if "formatted_body" in message:
                 message["formatted_body"] = f"* {sender.displayname} {message['formatted_body']}"
             message["body"] = f"* {sender.displayname} {message['body']}"
             message["msgtype"] = "m.text"
         elif not sender.logged_in:
-            if "formatted_body" in message:
-                html = message["formatted_body"]
-                message["formatted_body"] = f"<{sender.displayname}> {html}"
+            html = message["formatted_body"] if "formatted_body" in message else None
             text = message["body"]
-            message["body"] = f"<{sender.displayname}> {text}"
-        return type
+            if msgtype == "m.text":
+                if html:
+                    html = f"<{sender.displayname}> {html}"
+                text = f"<{sender.displayname}> {text}"
+            else:
+                msgtype = msgtype[len("m."):]
+                prefix = {
+                    "file": "a ",
+                    "image": "an ",
+                    "audio": "",
+                    "video": "a ",
+                    "location": "a ",
+                }.get(msgtype, "")
+                if html:
+                    html = f"{sender.displayname} sent {prefix}{msgtype}: {html}"
+                text = ": " + text if text else ""
+                text = f"{sender.displayname} sent {prefix}{msgtype}{text}"
+            if html:
+                message["formatted_body"] = html
+            message["body"] = text
+
+    async def _matrix_event_to_entities(self, client, event):
+        try:
+            if event.get("format", None) == "org.matrix.custom.html":
+                message, entities = formatter.matrix_to_telegram(event["formatted_body"])
+
+                # TODO remove this crap
+                for entity in entities:
+                    if isinstance(entity, InputMessageEntityMentionName):
+                        entity.user_id = await client.get_input_entity(entity.user_id.user_id)
+            else:
+                message, entities = formatter.matrix_text_to_telegram(event["body"])
+        except KeyError:
+            message, entities = None, None
+        return message, entities
 
     async def _handle_matrix_text(self, client, message, reply_to):
-        is_formatted = ("format" in message
-                        and message["format"] == "org.matrix.custom.html"
-                        and "formatted_body" in message)
-        if is_formatted:
-            message, entities = formatter.matrix_to_telegram(message["formatted_body"])
-
-            # TODO remove this crap
-            for entity in entities:
-                if isinstance(entity, InputMessageEntityMentionName):
-                    entity.user_id = await client.get_input_entity(entity.user_id.user_id)
-        else:
-            message, entities = formatter.matrix_text_to_telegram(message["body"])
+        message, entities = await self._matrix_event_to_entities(client, message)
         return await client.send_message(self.peer, message, entities=entities, reply_to=reply_to)
 
     async def _handle_matrix_file(self, client, message, reply_to):
@@ -604,32 +623,52 @@ class Portal:
         info = message["info"]
         mime = info["mimetype"]
 
-        file_name, caption = self._get_file_meta(message["body"], mime)
+        file_name = self._get_file_meta(message["mxtg_filename"], mime)
 
         attributes = [DocumentAttributeFilename(file_name=file_name)]
         if "w" in info and "h" in info:
             attributes.append(DocumentAttributeImageSize(w=info["w"], h=info["h"]))
 
+        caption = message["body"] if message["body"] != file_name else None
         return await client.send_file(self.peer, file, mime, caption=caption,
                                       attributes=attributes, file_name=file_name,
                                       reply_to=reply_to)
 
+    async def _handle_matrix_location(self, client, message, reply_to):
+        try:
+            lat, long = message["geo_uri"][len("geo:"):].split(",")
+            lat, long = float(lat), float(long)
+        except (KeyError, ValueError):
+            self.log.exception("Failed to parse location")
+            return None
+        message, entities = await self._matrix_event_to_entities(client, message)
+        media = MessageMediaGeo(geo=GeoPoint(lat, long))
+        return await client.send_media(self.peer, media, reply_to=reply_to, caption=message,
+                                       entities=entities)
+
     async def handle_matrix_message(self, sender, message, event_id):
         client = sender.client if sender.logged_in else self.bot.client
         space = (self.tgid if self.peer_type == "channel"  # Channels have their own ID space
                  else (sender.tgid if sender.logged_in else self.bot.tgid))
         reply_to = formatter.matrix_reply_to_telegram(message, space, room_id=self.mxid)
 
+        message["mxtg_filename"] = message["body"]
         self._preprocess_matrix_message(sender, message)
         type = message["msgtype"]
 
         if type == "m.text" or (self.bridge_notices and type == "m.notice"):
             response = await self._handle_matrix_text(client, message, reply_to)
+        elif type == "m.location":
+            response = await self._handle_matrix_location(client, message, reply_to)
         elif type in ("m.image", "m.file", "m.audio", "m.video"):
             response = await self._handle_matrix_file(client, message, reply_to)
         else:
             self.log.debug("Unhandled Matrix event: %s", message)
+            response = None
+
+        if not response:
             return
+
         self.log.debug("Handled Matrix message: %s", response)
         self.is_duplicate(response, (event_id, space))
         self.db.add(DBMessage(
diff --git a/mautrix_telegram/tgclient.py b/mautrix_telegram/tgclient.py
index 86bbd711..1b7aea85 100644
--- a/mautrix_telegram/tgclient.py
+++ b/mautrix_telegram/tgclient.py
@@ -73,6 +73,11 @@ class MautrixTelegramClient(TelegramClient):
                                    reply_to_msg_id=reply_to)
         return self._get_response_message(request, await self(request))
 
+    async def send_media(self, entity, media, caption=None, entities=None, reply_to=None):
+        request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],
+                                   reply_to_msg_id=reply_to)
+        return self._get_response_message(request, await self(request))
+
     async def download_file_bytes(self, location):
         if isinstance(location, Document):
             location = InputDocumentFileLocation(location.id, location.access_hash,