Bump version to 0.5.0rc2

Bump minimum telethon-session-sqlalchemy version for core mode support on non-postgres engines
Fixes #263

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/alpine:3.8
+FROM docker.io/alpine:3.9
 
 ENV UID=1337 \
     GID=1337 \
@@ -7,22 +7,33 @@ ENV UID=1337 \
 COPY . /opt/mautrix-telegram
 WORKDIR /opt/mautrix-telegram
 RUN apk add --no-cache \
-      python3-dev \
-      build-base \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-lxml \
       py3-magic \
-      py3-numpy \
-      py3-asn1crypto \
       py3-sqlalchemy \
       py3-markdown \
       py3-psycopg2 \
+      # Indirect dependencies
+      py3-numpy \
+      py3-asn1crypto \
+      py3-future \
+      py3-markupsafe \
+      py3-mako \
+      py3-decorator \
+      py3-dateutil \
+      py3-idna \
+      py3-six \
+      py3-asn1 \
+      py3-rsa \
+      # Other dependencies
+      python3-dev \
+      build-base \
       ffmpeg \
       ca-certificates \
       su-exec \
- && pip3 install -r requirements.txt -r optional-requirements.txt
+ && pip3 install .[all]
 
 VOLUME /data
 

alembic/env.py

@@ -7,10 +7,9 @@ from os.path import abspath, dirname
 
 sys.path.insert(0, dirname(dirname(abspath(__file__))))
 
-from mautrix_telegram.base import Base
+from mautrix_telegram.db import Base
 from mautrix_telegram.config import Config
 from alchemysession import AlchemySessionContainer
-import mautrix_telegram.db
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -12,7 +12,7 @@ import json
 import re
 
 from mautrix_telegram.config import Config
-from mautrix_telegram.base import Base
+from mautrix_telegram.db import Base
 
 # revision identifiers, used by Alembic.
 revision = "6ca3d74d51e4"

example-config.yaml

@@ -27,9 +27,6 @@ appservice:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
     database: sqlite:///mautrix-telegram.db
-    # Whether or not to use SQLAlchemy Core for common database actions. Use if the bridge is
-    # being bottlenecked on ORM commits. Only supported with PostgreSQL.
-    sqlalchemy_core_mode: false
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in
@@ -107,9 +104,21 @@ bridge:
     # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
     # list regardless of this setting.
     sync_channel_members: true
+    # Whether or not to skip deleted members when syncing members.
+    skip_deleted_members: true
+    # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
+    # their Telegram account at startup.
+    startup_sync: true
+    # Number of most recently active dialogs to check when syncing chats.
+    # Dialogs include groups and private chats, but only groups are synced.
+    # Set to 0 to remove limit.
+    sync_dialog_limit: 30
     # The maximum number of simultaneous Telegram deletions to handle.
     # A large number of simultaneous redactions could put strain on your homeserver.
     max_telegram_delete: 10
+    # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
+    # at startup and when creating a bridge.
+    sync_matrix_state: true
     # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
     # login website (see appservice.public config section)
     allow_matrix_login: true
@@ -117,6 +126,9 @@ bridge:
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
+    # Show message editing as a reply to the original message.
+    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
+    edits_as_replies: true
     # Highlight changed/added parts in edits. Requires lxml.
     highlight_edits: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
@@ -127,6 +139,22 @@ bridge:
     # Whether or not to use /sync to get presence, read receipts and typing notifications when using
     # your own Matrix account as the Matrix puppet for your Telegram account.
     sync_with_custom_puppets: true
+    # Set to false to disable link previews in messages sent to Telegram.
+    telegram_link_preview: true
+    # Use inline images instead of a separate message for the caption.
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    inline_images: false
+
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    bridge_notices:
+        # Whether or not Matrix bot messages (type m.notice) should be bridged.
+        default: false
+        # List of user IDs for whom the previous flag is flipped.
+        # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
+        #      notices from users listed here will be bridged.
+        exceptions:
+        - "@importantbot:example.com"
 
     # Some config options related to Telegram message deduplication.
     # The default values are usually fine, but some debug messages/warnings might recommend you
@@ -138,26 +166,6 @@ bridge:
         # You might need to increase this on high-traffic bridge instances.
         cache_queue_length: 20
 
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: false
-    bridge_notices:
-        # Whether or not Matrix bot messages (type m.notice) should be bridged.
-        default: false
-        # List of user IDs for whom the previous flag is flipped.
-        # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
-        #      notices from users listed here will be bridged.
-        exceptions:
-        - "@importantbot:example.com"
-    # Whether to bridge Telegram bot messages as m.notices or m.texts.
-    bot_messages_as_notices: true
-    # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
-    inline_images: false
-    # Whether to send stickers as the new native m.sticker type or normal m.images.
-    # Old versions of Riot don't support the new type at all.
-    # Remember that proper sticker support always requires Pillow to convert webp into png.
-    native_stickers: true
 
     # The formats to use when sending messages to Telegram via the relay bot.
     #
@@ -241,6 +249,17 @@ telegram:
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
+    # Custom server to connect to.
+    server:
+        # Set to true to use these server settings. If false, will automatically
+        # use production server assigned by Telegram. Set to false in production.
+        enabled: false
+        # The DC ID to connect to.
+        dc: 2
+        # The IP to connect to.
+        ip: 149.154.167.40
+        # The port to connect to. 443 may not work, 80 is better and both are equally secure.
+        port: 80
     # Telethon proxy configuration.
     # You must install PySocks from pip for proxies to work.
     proxy:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.4.0"
+__version__ = "0.5.0rc2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -14,7 +14,8 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Coroutine, List
+from typing import Awaitable, List, Any
+from time import time
 import argparse
 import asyncio
 import logging.config
@@ -31,11 +32,10 @@ from alchemysession import AlchemySessionContainer
 from .web.provisioning import ProvisioningAPI
 from .web.public import PublicBridgeWebsite
 from .abstract_user import init as init_abstract_user
-from .base import Base
 from .bot import init as init_bot
 from .config import Config
 from .context import Context
-from .db import init as init_db
+from .db import Base, init as init_db
 from .formatter import init as init_formatter
 from .matrix import MatrixHandler
 from .portal import init as init_portal
@@ -80,15 +80,11 @@ Base.metadata.bind = db_engine
 session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
                                             table_base=Base, table_prefix="telethon_",
                                             manage_tables=False)
-if config["appservice.sqlalchemy_core_mode"]:
-    try:
-        session_container.core_mode = True
-    except AttributeError:
-        log.error("Current version of teleton-session-sqlalchemy does not support core mode")
+session_container.core_mode = True
 
 loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
 
-state_store = SQLStateStore(db_session)
+state_store = SQLStateStore()
 mebibyte = 1024 ** 2
 appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      config["appservice.as_token"], config["appservice.hs_token"],
@@ -113,7 +109,8 @@ if config["appservice.provisioning.enabled"]:
     context.provisioning_api = provisioning_api
 
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
-    init_db(db_session, db_engine)
+    start_ts = time()
+    init_db(db_engine)
     init_abstract_user(context)
     context.bot = init_bot(context)
     context.mx = MatrixHandler(context)
@@ -121,8 +118,7 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st
     init_portal(context)
     startup_actions = (init_puppet(context) +
                        init_user(context) +
-                       [start,
-                        context.mx.init_as_bot()])  # type: List[Coroutine]
+                       [start, context.mx.init_as_bot()])  # type: List[Awaitable[Any]]
 
     if context.bot:
         startup_actions.append(context.bot.start())
@@ -130,10 +126,15 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st
     signal.signal(signal.SIGINT, signal.default_int_handler)
     signal.signal(signal.SIGTERM, signal.default_int_handler)
 
+    end_ts = time()
     try:
-        log.debug("Initialization complete, running startup actions")
+        log.debug(f"Initialization complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " running startup actions")
+        start_ts = time()
         loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
-        log.debug("Startup actions complete, now running forever")
+        end_ts = time()
+        log.debug(f"Startup actions complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " now running forever")
         loop.run_forever()
     except KeyboardInterrupt:
         log.debug("Interrupt received, stopping clients")

mautrix_telegram/abstract_user.py

@@ -21,14 +21,14 @@ import logging
 import platform
 
 from sqlalchemy import orm
-from telethon.tl.types import Channel, ChannelForbidden, Chat, ChatForbidden, Message, \
-    MessageActionChannelMigrateFrom, MessageService, PeerUser, TypeUpdate, \
-    UpdateChannelPinnedMessage, UpdateChatAdmins, UpdateChatParticipantAdmin, \
-    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, \
-    UpdateDeleteMessages, UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, \
-    UpdateNewMessage, UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, \
-    UpdateUserName, UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, \
-    UserStatusOnline
+from telethon.tl.patched import MessageService, Message
+from telethon.tl.types import (
+    Channel, ChannelForbidden, Chat, ChatForbidden, MessageActionChannelMigrateFrom, PeerUser,
+    TypeUpdate, UpdateChannelPinnedMessage, UpdateChatPinnedMessage, UpdateChatParticipantAdmin,
+    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateDeleteMessages,
+    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateNewMessage,
+    UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, UpdateUserName,
+    UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, UserStatusOnline)
 
 from mautrix_appservice import MatrixRequestError, AppService
 from alchemysession import AlchemySessionContainer
@@ -100,6 +100,14 @@ class AbstractUser(ABC):
         device = f"{platform.system()} {platform.release()}"
         sysversion = MautrixTelegramClient.__version__
         self.session = self.session_container.new_session(self.name)
+        if config["telegram.server.enabled"]:
+            self.session.set_dc(config["telegram.server.dc"],
+                                config["telegram.server.ip"],
+                                config["telegram.server.port"])
+        if self.is_relaybot:
+            base_logger = logging.getLogger("telethon.relaybot")
+        else:
+            base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
         self.client = MautrixTelegramClient(session=self.session,
                                             api_id=config["telegram.api_id"],
                                             api_hash=config["telegram.api_hash"],
@@ -108,6 +116,7 @@ class AbstractUser(ABC):
                                             system_version=sysversion,
                                             device_model=device,
                                             timeout=120,
+                                            base_logger=base_logger,
                                             proxy=self._proxy_settings)
         self.client.add_event_handler(self._update_catch)
 
@@ -164,16 +173,13 @@ class AbstractUser(ABC):
         return self
 
     async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
-        if not self.puppet_whitelisted:
+        if not self.puppet_whitelisted or self.connected:
             return self
-        self.log.debug("ensure_started(%s, connected=%s, even_if_no_session=%s, session_count=%s)",
-                       self.mxid, self.connected, even_if_no_session,
-                       self.session_container.Session.query.filter(
-                           self.session_container.Session.session_id == self.mxid).count())
-        should_connect = (even_if_no_session or
-                          self.session_container.Session.query.filter(
-                              self.session_container.Session.session_id == self.mxid).count() > 0)
-        if not self.connected and should_connect:
+        session_count = self.session_container.Session.query.filter(
+            self.session_container.Session.session_id == self.mxid).count()
+        self.log.debug("ensure_started(%s, even_if_no_session=%s, session_count=%s)",
+                       self.mxid, even_if_no_session, session_count)
+        if even_if_no_session or session_count > 0:
             await self.start(delete_unless_authenticated=not even_if_no_session)
         return self
 
@@ -195,11 +201,11 @@ class AbstractUser(ABC):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
-        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
+        elif isinstance(update, UpdateChatParticipantAdmin):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, UpdateChannelPinnedMessage):
+        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
@@ -208,11 +214,14 @@ class AbstractUser(ABC):
         else:
             self.log.debug("Unhandled update: %s", update)
 
-    @staticmethod
-    async def update_pinned_messages(update: UpdateChannelPinnedMessage) -> None:
-        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
+                                                         UpdateChatPinnedMessage]) -> None:
+        if isinstance(update, UpdateChatPinnedMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        else:
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id)
+            await portal.receive_telegram_pin_id(update.id, self.tgid)
 
     @staticmethod
     async def update_participants(update: UpdateChatParticipants) -> None:
@@ -230,26 +239,20 @@ class AbstractUser(ABC):
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.get_by_tgid(update.max_id, self.tgid)
+        message = DBMessage.get_by_tgid(TelegramID(update.max_id), self.tgid)
         if not message:
             return
 
         puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
-    async def update_admin(self,
-                           update: Union[UpdateChatAdmins, UpdateChatParticipantAdmin]) -> None:
+    async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
         # TODO duplication not checked
         portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
         if not portal or not portal.mxid:
             return
 
-        if isinstance(update, UpdateChatAdmins):
-            await portal.set_telegram_admins_enabled(update.enabled)
-        elif isinstance(update, UpdateChatParticipantAdmin):
-            await portal.set_telegram_admin(TelegramID(update.user_id))
-        else:
-            self.log.warning("Unexpected admin status update: %s", update)
+        await portal.set_telegram_admin(TelegramID(update.user_id))
 
     async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
         if isinstance(update, UpdateUserTyping):
@@ -267,6 +270,7 @@ class AbstractUser(ABC):
         # TODO duplication not checked
         puppet = pu.Puppet.get(TelegramID(update.user_id))
         if isinstance(update, UpdateUserName):
+            puppet.username = update.username
             if await puppet.update_displayname(self, update):
                 puppet.save()
         elif isinstance(update, UpdateUserPhoto):
@@ -331,7 +335,6 @@ class AbstractUser(ABC):
             if number_left == 0:
                 portal = po.Portal.get_by_mxid(message.mx_room)
                 await self._try_redact(portal, message)
-        self.db.commit()
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
@@ -347,7 +350,6 @@ class AbstractUser(ABC):
                 continue
             message.delete()
             await self._try_redact(portal, message)
-        self.db.commit()
 
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)

mautrix_telegram/base.py

@@ -1,2 +0,0 @@
-from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()  # type: declarative_base

mautrix_telegram/bot.py

@@ -18,11 +18,12 @@ from typing import Awaitable, Callable, Dict, List, Optional, Pattern, TYPE_CHEC
 import logging
 import re
 
+from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
     ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
-    ChatParticipantCreator, InputChannel, InputUser, Message, MessageActionChatAddUser,
-    MessageActionChatDeleteUser, MessageEntityBotCommand, MessageService, PeerChannel, PeerChat,
-    TypePeer, UpdateNewChannelMessage, UpdateNewMessage)
+    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
+    MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
@@ -30,6 +31,7 @@ from telethon.errors import ChannelInvalidError, ChannelPrivateError
 from .types import MatrixUserID
 from .abstract_user import AbstractUser
 from .db import BotChat
+from .types import TelegramID
 from . import puppet as pu, portal as po, user as u
 
 if TYPE_CHECKING:
@@ -54,7 +56,7 @@ class Bot(AbstractUser):
         self.username = None  # type: str
         self.is_relaybot = True  # type: bool
         self.is_bot = True  # type: bool
-        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}  # type: Dict[int, str]
+        self.chats = {chat.id: chat.type for chat in BotChat.all()}  # type: Dict[int, str]
         self.tg_whitelist = []  # type: List[int]
         self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
                                        or False)  # type: bool
@@ -89,7 +91,7 @@ class Bot(AbstractUser):
         response = await self.client(GetChatsRequest(chat_ids))
         for chat in response.chats:
             if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
-                self.remove_chat(chat.id)
+                self.remove_chat(TelegramID(chat.id))
 
         channel_ids = [InputChannel(chat_id, 0)
                        for chat_id, chat_type in self.chats.items()
@@ -98,7 +100,7 @@ class Bot(AbstractUser):
             try:
                 await self.client(GetChannelsRequest([channel_id]))
             except (ChannelPrivateError, ChannelInvalidError):
-                self.remove_chat(channel_id.channel_id)
+                self.remove_chat(TelegramID(channel_id.channel_id))
 
         if config["bridge.catch_up"]:
             try:
@@ -112,23 +114,19 @@ class Bot(AbstractUser):
     def unregister_portal(self, portal: po.Portal) -> None:
         self.remove_chat(portal.tgid)
 
-    def add_chat(self, chat_id: int, chat_type: str) -> None:
+    def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
         if chat_id not in self.chats:
             self.chats[chat_id] = chat_type
-            self.db.add(BotChat(id=chat_id, type=chat_type))
-            self.db.commit()
+            BotChat(id=TelegramID(chat_id), type=chat_type).insert()
 
-    def remove_chat(self, chat_id: int) -> None:
+    def remove_chat(self, chat_id: TelegramID) -> None:
         try:
             del self.chats[chat_id]
         except KeyError:
             pass
-        existing_chat = BotChat.query.get(chat_id)
-        if existing_chat:
-            self.db.delete(existing_chat)
-            self.db.commit()
+        BotChat.delete(chat_id)
 
-    async def _can_use_commands(self, chat: TypePeer, tgid: int) -> bool:
+    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
@@ -155,7 +153,7 @@ class Bot(AbstractUser):
             return False
         return True
 
-    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> None:
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> Message:
         if not config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
@@ -221,7 +219,8 @@ class Bot(AbstractUser):
         text = message.message
 
         if self.match_command(text, "id"):
-            return await self.handle_command_id(message, reply)
+            await self.handle_command_id(message, reply)
+            return
 
         portal = po.Portal.get_by_entity(message.to_id)
 
@@ -239,7 +238,7 @@ class Bot(AbstractUser):
             await self.handle_command_invite(portal, reply, mxid_input=mxid)
 
     def handle_service_message(self, message: MessageService) -> None:
-        to_id = message.to_id
+        to_id = message.to_id  # type: TelegramID
         if isinstance(to_id, PeerChannel):
             to_id = to_id.channel_id
             chat_type = "channel"
@@ -254,6 +253,9 @@ class Bot(AbstractUser):
             self.add_chat(to_id, chat_type)
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
             self.remove_chat(to_id)
+        elif isinstance(action, MessageActionChatMigrateTo):
+            self.remove_chat(to_id)
+            self.add_chat(TelegramID(action.channel_id), "channel")
 
     async def update(self, update) -> bool:
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):

mautrix_telegram/commands/__init__.py

@@ -2,4 +2,4 @@ from .handler import (command_handler, command_handlers as _command_handlers,
                       CommandHandler, CommandProcessor, CommandEvent,
                       SECTION_GENERAL, SECTION_AUTH, SECTION_CREATING_PORTALS,
                       SECTION_PORTAL_MANAGEMENT, SECTION_MISC, SECTION_ADMIN)
-from . import clean_rooms, auth, meta, telegram, portal
+from . import portal, telegram, clean_rooms, matrix_auth, meta

mautrix_telegram/commands/clean_rooms.py

@@ -70,7 +70,7 @@ async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
                for n, (room, other_member) in enumerate(management_rooms)]
               or ["No management rooms found."])
     reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [P{n+1}](https://matrix.to/#/{portal.mxid}) "
+    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
                f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(portals)]
               or ["No active portal rooms found."])
@@ -79,7 +79,7 @@ async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
                for n, room in enumerate(unidentified_rooms)]
               or ["No unidentified rooms found."])
     reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [E{n}](https://matrix.to/#/{portal.mxid}) "
+    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
                f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(empty_portals)]
               or ["No inactive portal rooms found."])
@@ -93,7 +93,7 @@ async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
               "",
               ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
                "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name."),
+               "the group name. (e.g. `I2-6`)"),
               "",
               ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
                "between each use of the commands above.")]
@@ -118,7 +118,7 @@ async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
     elif command == "clean-groups":
         if len(evt.args) < 2:
             return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1]
+        groups_to_clean = evt.args[1].upper()
         if "M" in groups_to_clean:
             rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
         if "A" in groups_to_clean:

mautrix_telegram/commands/handler.py

@@ -15,9 +15,11 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Awaitable, Callable, Dict, List, NamedTuple, Optional
-import commonmark
+import traceback
 import logging
 
+import commonmark
+
 from telethon.errors import FloodWaitError
 
 from ..types import MatrixRoomID
@@ -126,8 +128,7 @@ class CommandHandler:
                 (not self.needs_admin or is_admin) and
                 (not self.needs_auth or is_logged_in))
 
-    async def __call__(self, evt: CommandEvent
-                       ) -> Dict:
+    async def __call__(self, evt: CommandEvent) -> Dict:
         error = await self.get_permission_error(evt)
         if error is not None:
             return await evt.reply(error)
@@ -154,12 +155,11 @@ def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] =
                     help_section: HelpSection = None
                     ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
                                   CommandHandler]:
-    input_name = name
 
     def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:
-        name = input_name or func.__name__.replace("_", "-")
+        actual_name = name or func.__name__.replace("_", "-")
         handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
-                                 needs_admin, management_only, name, help_text, help_args,
+                                 needs_admin, management_only, actual_name, help_text, help_args,
                                  help_section)
         command_handlers[handler.name] = handler
         return handler
@@ -196,6 +196,11 @@ class CommandProcessor:
         except Exception:
             self.log.exception("Unhandled error while handling command "
                                f"{evt.command} {' '.join(args)} from {sender.mxid}")
+            if evt.sender.is_admin and evt.is_management:
+                return await evt.reply("Unhandled error while handling command:\n\n"
+                                       "```traceback\n"
+                                       f"{traceback.format_exc()}"
+                                       "```")
             return await evt.reply("Unhandled error while handling command. "
                                    "Check logs for more details.")
         return None

mautrix_telegram/commands/matrix_auth.py

@@ -0,0 +1,101 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from . import command_handler, CommandEvent, SECTION_AUTH
+from .. import puppet as pu
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
+                           "account.")
+async def logout_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
+
+
+@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account.")
+async def login_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        token = evt.public_website.make_token(evt.sender.mxid, "/matrix-login")
+        url = f"{prefix}/matrix-login?token={token}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Pings the server with the stored matrix authentication.")
+async def ping_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    resp = await puppet.init_custom_mxid()
+    if resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Your access token is invalid.")
+    elif resp == pu.PuppetError.Success:
+        return await evt.reply("Your Matrix login is working.")
+    return await evt.reply(f"Unknown response while checking your Matrix login: {resp}.")
+
+
+async def enter_matrix_token(evt: CommandEvent) -> Dict:
+    evt.sender.command_status = None
+
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+
+    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    if resp == pu.PuppetError.OnlyLoginSelf:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    elif resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Failed to verify access token.")
+    assert resp == pu.PuppetError.Success, "Encountered an unhandled PuppetError."
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")

mautrix_telegram/commands/meta.py

@@ -51,8 +51,8 @@ async def _get_help_text(evt: CommandEvent) -> str:
                 help_sections.setdefault(handler.help_section, [])
                 help_sections[handler.help_section].append(handler.help + "  ")
         help_sorted = sorted(help_sections.items(), key=lambda item: item[0].order)
-        help = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help_sorted]
-        help_cache[cache_key] = "\n".join(help)
+        helps = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help_sorted]
+        help_cache[cache_key] = "\n".join(helps)
     return help_cache[cache_key]
 
 

mautrix_telegram/commands/portal.py

@@ -1,614 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, Callable, Optional, Tuple, Coroutine, Awaitable
-from io import StringIO
-import asyncio
-
-from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
-                             UsernameNotModifiedError, UsernameOccupiedError)
-from telethon.tl.types import ChatForbidden, ChannelForbidden
-from mautrix_appservice import MatrixRequestError, IntentAPI
-
-from ..types import MatrixRoomID, TelegramID
-from ..config import yaml
-from .. import portal as po, user as u, util
-from . import (command_handler, CommandEvent,
-               SECTION_ADMIN, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT)
-
-
-@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
-                 help_section=SECTION_ADMIN,
-                 help_args="<_level_> [_mxid_]",
-                 help_text="Set a temporary power level without affecting Telegram.")
-async def set_power_level(evt: CommandEvent) -> Dict:
-    try:
-        level = int(evt.args[0])
-    except KeyError:
-        return await evt.reply("**Usage:** `$cmdprefix+sp set-power <level> [mxid]`")
-    except ValueError:
-        return await evt.reply("The level must be an integer.")
-    levels = await evt.az.intent.get_power_levels(evt.room_id)
-    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
-    levels["users"][mxid] = level
-    try:
-        await evt.az.intent.set_power_levels(evt.room_id, levels)
-    except MatrixRequestError:
-        evt.log.exception("Failed to set power level.")
-        return await evt.reply("Failed to set power level.")
-    return {}
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Get a Telegram invite link to the current chat.")
-async def invite_link(evt: CommandEvent) -> Dict:
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
-        return await evt.reply("You can't invite users to private chats.")
-
-    try:
-        link = await portal.get_invite_link(evt.sender)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to create an invite link.")
-
-
-async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50
-                               ) -> bool:
-    if sender.is_admin:
-        return True
-    # Make sure the state store contains the power levels.
-    try:
-        await intent.get_power_levels(room)
-    except MatrixRequestError:
-        return False
-    return intent.state_store.has_power_level(room, sender.mxid,
-                                              event=f"net.maunium.telegram.{event}",
-                                              default=default)
-
-
-async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
-                                           action: Optional[str] = None
-                                           ) -> Optional[po.Portal]:
-    room_id = MatrixRoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if not portal:
-        that_this = "This" if room_id == evt.room_id else "That"
-        await evt.reply(f"{that_this} is not a portal room.")
-        return None
-
-    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
-        action = action or f"{permission.replace('_', ' ')}s"
-        await evt.reply(f"You do not have the permissions to {action} that portal.")
-        return None
-    return portal
-
-
-def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
-                                completed_message: str) -> Dict:
-    async def post_confirm(confirm) -> Optional[Dict]:
-        confirm.sender.command_status = None
-        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
-            await function()
-            if confirm.room_id != room_id:
-                return await confirm.reply(completed_message)
-        else:
-            return await confirm.reply(f"{action} cancelled.")
-        return None
-
-    return {
-        "next": post_confirm,
-        "action": action,
-    }
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Remove all users from the current portal room and forget the portal. "
-                           "Only works for group chats; to delete a private chat portal, simply "
-                           "leave the room.")
-async def delete_portal(evt: CommandEvent) -> Optional[Dict]:
-    portal = await _get_portal_and_check_permission(evt, "unbridge")
-    if not portal:
-        return None
-
-    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
-                                                            portal.cleanup_and_delete, "delete",
-                                                            "Portal successfully deleted.")
-    return await evt.reply("Please confirm deletion of portal "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           f"to Telegram chat \"{portal.title}\" "
-                           "by typing `$cmdprefix+sp confirm-delete`"
-                           "\n\n"
-                           "**WARNING:** If the bridge bot has the power level to do so, **this "
-                           "will kick ALL users** in the room. If you just want to remove the "
-                           "bridge, use `$cmdprefix+sp unbridge` instead.")
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Remove puppets from the current portal room and forget the portal.")
-async def unbridge(evt: CommandEvent) -> Optional[Dict]:
-    portal = await _get_portal_and_check_permission(evt, "unbridge")
-    if not portal:
-        return None
-
-    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
-                                                            portal.unbridge, "unbridge",
-                                                            "Room successfully unbridged.")
-    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           "by typing `$cmdprefix+sp confirm-unbridge`")
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_args="[_id_]",
-                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
-                           "ID. The ID must be the prefixed version that you get with the `/id` "
-                           "command of the Telegram-side bot.")
-async def bridge(evt: CommandEvent) -> Dict:
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** "
-                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
-    room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
-    that_this = "This" if room_id == evt.room_id else "That"
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if portal:
-        return await evt.reply(f"{that_this} room is already a portal room.")
-
-    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
-
-    # The /id bot command provides the prefixed ID, so we assume
-    tgid_str = evt.args[0]
-    if tgid_str.startswith("-100"):
-        tgid = TelegramID(int(tgid_str[4:]))
-        peer_type = "channel"
-    elif tgid_str.startswith("-"):
-        tgid = TelegramID(-int(tgid_str))
-        peer_type = "chat"
-    else:
-        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
-                               "If you did not get the ID using the `/id` bot command, please "
-                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
-                               "Bridging private chats to existing rooms is not allowed.")
-
-    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
-    if not portal.allow_bridging():
-        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
-                               "If you're the bridge admin, try "
-                               "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
-    if portal.mxid:
-        has_portal_message = (
-            "That Telegram chat already has a portal at "
-            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
-        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-            return await evt.reply(f"{has_portal_message}"
-                                   "Additionally, you do not have the permissions to unbridge "
-                                   "that room.")
-        evt.sender.command_status = {
-            "next": confirm_bridge,
-            "action": "Room bridging",
-            "mxid": portal.mxid,
-            "bridge_to_mxid": room_id,
-            "tgid": portal.tgid,
-            "peer_type": portal.peer_type,
-        }
-        return await evt.reply(f"{has_portal_message}"
-                               "However, you have the permissions to unbridge that room.\n\n"
-                               "To delete that portal completely and continue bridging, use "
-                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
-                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
-                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
-    evt.sender.command_status = {
-        "next": confirm_bridge,
-        "action": "Room bridging",
-        "bridge_to_mxid": room_id,
-        "tgid": portal.tgid,
-        "peer_type": portal.peer_type,
-    }
-    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
-                           "chat to this room, use `$cmdprefix+sp continue`")
-
-
-async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
-                                            ) -> Tuple[bool, Optional[Coroutine[None, None, None]]]:
-    if not portal.mxid:
-        await evt.reply("The portal seems to have lost its Matrix room between you"
-                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                        "Continuing without touching previous Matrix room...")
-        return True, None
-    elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Portal deleted (moving to another room)")
-    elif evt.args[0] == "unbridge-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Room unbridged (portal moving to another room)",
-                                         puppets_only=True)
-    else:
-        await evt.reply(
-            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
-            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
-            "continue` to either delete or unbridge the existing room (respectively) and "
-            "continue with the bridging.\n\n"
-            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
-        return False, None
-
-
-async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
-    status = evt.sender.command_status
-    try:
-        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
-        bridge_to_mxid = status["bridge_to_mxid"]
-    except KeyError:
-        evt.sender.command_status = None
-        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
-                               "This shouldn't happen unless you're messing with the command "
-                               "handler code.")
-    if "mxid" in status:
-        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
-        if not ok:
-            return None
-        elif coro:
-            asyncio.ensure_future(coro, loop=evt.loop)
-            await evt.reply("Cleaning up previous portal room...")
-    elif portal.mxid:
-        evt.sender.command_status = None
-        return await evt.reply("The portal seems to have created a Matrix room between you "
-                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                               "Please start over by calling the bridge command again.")
-    elif evt.args[0] != "continue":
-        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
-                               "`$cmdprefix+sp cancel` to cancel.")
-
-    evt.sender.command_status = None
-    is_logged_in = await evt.sender.is_logged_in()
-    user = evt.sender if is_logged_in else evt.tgbot
-    try:
-        entity = await user.client.get_entity(portal.peer)
-    except Exception:
-        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
-        if is_logged_in:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You are logged in, are you in that chat?")
-        else:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You're not logged in, is the relay bot in the chat?")
-    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
-        if is_logged_in:
-            return await evt.reply("You don't seem to be in that chat.")
-        else:
-            return await evt.reply("The bot doesn't seem to be in that chat.")
-
-    direct = False
-
-    portal.mxid = bridge_to_mxid
-    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
-    portal.photo_id = ""
-    portal.save()
-
-    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
-                          loop=evt.loop)
-
-    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")
-
-
-async def get_initial_state(intent: IntentAPI, room_id: str) -> Tuple[str, str, Dict]:
-    state = await intent.get_room_state(room_id)
-    title = None
-    about = None
-    levels = None
-    for event in state:
-        try:
-            if event["type"] == "m.room.name":
-                title = event["content"]["name"]
-            elif event["type"] == "m.room.topic":
-                about = event["content"]["topic"]
-            elif event["type"] == "m.room.power_levels":
-                levels = event["content"]
-            elif event["type"] == "m.room.canonical_alias":
-                title = title or event["content"]["alias"]
-        except KeyError:
-            # Some state event probably has empty content
-            pass
-    return title, about, levels
-
-
-@command_handler(help_section=SECTION_CREATING_PORTALS,
-                 help_args="[_type_]",
-                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
-                           "The type is either `group`, `supergroup` or `channel` (defaults to "
-                           "`group`).")
-async def create(evt: CommandEvent) -> Dict:
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
-
-    if po.Portal.get_by_mxid(evt.room_id):
-        return await evt.reply("This is already a portal room.")
-
-    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply("You do not have the permissions to bridge this room.")
-
-    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
-    if not title:
-        return await evt.reply("Please set a title before creating a Telegram chat.")
-
-    supergroup = type == "supergroup"
-    type = {
-        "supergroup": "channel",
-        "channel": "channel",
-        "chat": "chat",
-        "group": "chat",
-    }[type]
-
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
-    try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
-    except ValueError as e:
-        portal.delete()
-        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Upgrade a normal Telegram group to a supergroup.")
-async def upgrade(evt: CommandEvent) -> Dict:
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type == "channel":
-        return await evt.reply("This is already a supergroup or a channel.")
-    elif portal.peer_type == "user":
-        return await evt.reply("You can't upgrade private chats.")
-
-    try:
-        await portal.upgrade_telegram_chat(evt.sender)
-        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to upgrade this group.")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="View or change per-portal settings.",
-                 help_args="<`help`|_subcommand_> [...]")
-async def config(evt: CommandEvent) -> None:
-    cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
-    if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
-        await config_help(evt)
-        return
-    elif cmd == "defaults":
-        await config_defaults(evt)
-        return
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        await evt.reply("This is not a portal room.")
-        return
-    elif cmd == "view":
-        await config_view(evt, portal)
-        return
-
-    key = evt.args[1] if len(evt.args) > 1 else None
-    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
-    if cmd == "set":
-        await config_set(evt, portal, key, value)
-    elif cmd == "unset":
-        await config_unset(evt, portal, key)
-    elif cmd == "add" or cmd == "del":
-        await config_add_del(evt, portal, key, value, cmd)
-    else:
-        return
-    portal.save()
-
-
-def config_help(evt: CommandEvent) -> Awaitable[Dict]:
-    return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
-
-* **help** - View this help text.
-* **view** - View the current config data.
-* **defaults** - View the default config values.
-* **set** <_key_> <_value_> - Set a config value.
-* **unset** <_key_> - Remove a config value.
-* **add** <_key_> <_value_> - Add a value to an array.
-* **del** <_key_> <_value_> - Remove a value from an array.
-""")
-
-
-def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
-    stream = StringIO()
-    yaml.dump(portal.local_config, stream)
-    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
-
-
-def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
-    stream = StringIO()
-    yaml.dump({
-        "edits_as_replies": evt.config["bridge.edits_as_replies"],
-        "bridge_notices": {
-            "default": evt.config["bridge.bridge_notices.default"],
-            "exceptions": evt.config["bridge.bridge_notices.exceptions"],
-        },
-        "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
-        "inline_images": evt.config["bridge.inline_images"],
-        "native_stickers": evt.config["bridge.native_stickers"],
-        "message_formats": evt.config["bridge.message_formats"],
-        "state_event_formats": evt.config["bridge.state_event_formats"],
-    }, stream)
-    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
-
-
-def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[Dict]:
-    if not key or value is None:
-        return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
-    elif util.recursive_set(portal.local_config, key, value):
-        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
-    else:
-        return evt.reply(f"Failed to set value of `{key}`. "
-                         "Does the path contain non-map types?")
-
-
-def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Dict]:
-    if not key:
-        return evt.reply(f"**Usage:** `$cmdprefix+sp config unset <key>`")
-    elif util.recursive_del(portal.local_config, key):
-        return evt.reply(f"Successfully deleted `{key}` from config.")
-    else:
-        return evt.reply(f"`{key}` not found in config.")
-
-
-def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
-                   ) -> Awaitable[Dict]:
-    if not key or value is None:
-        return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
-
-    arr = util.recursive_get(portal.local_config, key)
-    if not arr:
-        return evt.reply(f"`{key}` not found in config. "
-                         f"Maybe do `$cmdprefix+sp config set {key} []` first?")
-    elif not isinstance(arr, list):
-        return evt.reply("`{key}` does not seem to be an array.")
-    elif cmd == "add":
-        if value in arr:
-            return evt.reply(f"The array at `{key}` already contains `{value}`.")
-        arr.append(value)
-        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
-    else:
-        if value not in arr:
-            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
-        arr.remove(value)
-        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_args="<_name_|`-`>",
-                 help_text="Change the username of a supergroup/channel. "
-                           "To disable, use a dash (`-`) as the name.")
-async def group_name(evt: CommandEvent) -> Dict:
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type != "channel":
-        return await evt.reply("Only channels and supergroups have usernames.")
-
-    try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
-        if portal.username:
-            return await evt.reply(f"Username of channel changed to {portal.username}.")
-        else:
-            return await evt.reply(f"Channel is now private.")
-    except ChatAdminRequiredError:
-        return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
-    except UsernameNotModifiedError:
-        if portal.username:
-            return await evt.reply("That is already the username of this channel.")
-        else:
-            return await evt.reply("This channel is already private")
-    except UsernameOccupiedError:
-        return await evt.reply("That username is already in use.")
-    except UsernameInvalidError:
-        return await evt.reply("Invalid username")
-
-
-@command_handler(needs_admin=True,
-                 help_section=SECTION_ADMIN,
-                 help_args="<`whitelist`|`blacklist`>",
-                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
-                           "default.")
-async def filter_mode(evt: CommandEvent) -> Dict:
-    try:
-        mode = evt.args[0]
-        if mode not in ("whitelist", "blacklist"):
-            raise ValueError()
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
-
-    evt.config["bridge.filter.mode"] = mode
-    evt.config.save()
-    po.Portal.filter_mode = mode
-    if mode == "whitelist":
-        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
-                               "To allow bridging a specific chat, use"
-                               "`!filter whitelist <chat ID>`.")
-    else:
-        return await evt.reply("The bridge will now allow bridging chats by default.\n"
-                               "To disallow bridging a specific chat, use"
-                               "`!filter blacklist <chat ID>`.")
-
-
-@command_handler(needs_admin=True,
-                 help_section=SECTION_ADMIN,
-                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
-                 help_text="Allow or disallow bridging a specific chat.")
-async def filter(evt: CommandEvent) -> Optional[Dict]:
-    try:
-        action = evt.args[0]
-        if action not in ("whitelist", "blacklist", "add", "remove"):
-            raise ValueError()
-
-        id_str = evt.args[1]
-        if id_str.startswith("-100"):
-            id = int(id_str[4:])
-        elif id_str.startswith("-"):
-            id = int(id_str[1:])
-        else:
-            id = int(id_str)
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
-
-    mode = evt.config["bridge.filter.mode"]
-    if mode not in ("blacklist", "whitelist"):
-        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
-
-    list = evt.config["bridge.filter.list"]
-
-    if action in ("blacklist", "whitelist"):
-        action = "add" if mode == action else "remove"
-
-    def save() -> None:
-        evt.config["bridge.filter.list"] = list
-        evt.config.save()
-        po.Portal.filter_list = list
-
-    if action == "add":
-        if id in list:
-            return await evt.reply(f"That chat is already {mode}ed.")
-        list.append(id)
-        save()
-        return await evt.reply(f"Chat ID added to {mode}.")
-    elif action == "remove":
-        if id not in list:
-            return await evt.reply(f"That chat is not {mode}ed.")
-        list.remove(id)
-        save()
-        return await evt.reply(f"Chat ID removed from {mode}.")
-    return None

mautrix_telegram/commands/portal/__init__.py

@@ -0,0 +1 @@
+from . import admin, bridge, config, create_chat, filter, misc, unbridge

mautrix_telegram/commands/portal/admin.py

@@ -0,0 +1,101 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+import asyncio
+
+from mautrix_appservice import MatrixRequestError
+
+from ... import portal as po, puppet as pu, user as u
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
+                 help_section=SECTION_ADMIN,
+                 help_args="<_level_> [_mxid_]",
+                 help_text="Set a temporary power level without affecting Telegram.")
+async def set_power_level(evt: CommandEvent) -> Dict:
+    try:
+        level = int(evt.args[0])
+    except KeyError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp set-pl <level> [mxid]`")
+    except ValueError:
+        return await evt.reply("The level must be an integer.")
+    levels = await evt.az.intent.get_power_levels(evt.room_id)
+    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
+    levels["users"][mxid] = level
+    try:
+        await evt.az.intent.set_power_levels(evt.room_id, levels)
+    except MatrixRequestError:
+        evt.log.exception("Failed to set power level.")
+        return await evt.reply("Failed to set power level.")
+    return {}
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`portal`|`puppet`|`user`>",
+                 help_text="Clear internal bridge caches")
+async def clear_db_cache(evt: CommandEvent) -> Dict:
+    try:
+        section = evt.args[0].lower()
+    except KeyError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+    if section == "portal":
+        po.Portal.by_tgid = {}
+        po.Portal.by_mxid = {}
+        await evt.reply("Cleared portal cache")
+    elif section == "puppet":
+        pu.Puppet.cache = {}
+        for puppet in pu.Puppet.by_custom_mxid.values():
+            puppet.sync_task.cancel()
+        pu.Puppet.by_custom_mxid = {}
+        await asyncio.gather(
+            *[puppet.init_custom_mxid() for puppet in pu.Puppet.all_with_custom_mxid()],
+            loop=evt.loop)
+        await evt.reply("Cleared puppet cache and restarted custom puppet syncers")
+    elif section == "user":
+        u.User.by_mxid = {
+            user.mxid: user
+            for user in u.User.by_tgid.values()
+        }
+        await evt.reply("Cleared non-logged-in user cache")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="[_mxid_]",
+                 help_text="Reload and reconnect a user")
+async def reload_user(evt: CommandEvent) -> Dict:
+    if len(evt.args) > 0:
+        mxid = evt.args[0]
+    else:
+        mxid = evt.sender.mxid
+    user = u.User.get_by_mxid(mxid, create=False)
+    if not user:
+        return await evt.reply("User not found")
+    puppet = pu.Puppet.get_by_custom_mxid(mxid)
+    if puppet:
+        puppet.sync_task.cancel()
+    await user.stop()
+    user.delete(delete_db=False)
+    user = u.User.get_by_mxid(mxid)
+    await user.ensure_started()
+    if puppet:
+        await puppet.init_custom_mxid()
+    await evt.reply(f"Reloaded and reconnected {user.mxid} (telegram: {user.human_tg_id})")

mautrix_telegram/commands/portal/bridge.py

@@ -0,0 +1,181 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional, Tuple, Coroutine
+import asyncio
+
+from telethon.tl.types import ChatForbidden, ChannelForbidden
+
+from ...types import MatrixRoomID, TelegramID
+from ...util import ignore_coro
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_id_]",
+                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
+                           "ID. The ID must be the prefixed version that you get with the `/id` "
+                           "command of the Telegram-side bot.")
+async def bridge(evt: CommandEvent) -> Dict:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** "
+                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
+    that_this = "This" if room_id == evt.room_id else "That"
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if portal:
+        return await evt.reply(f"{that_this} room is already a portal room.")
+
+    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
+
+    # The /id bot command provides the prefixed ID, so we assume
+    tgid_str = evt.args[0]
+    if tgid_str.startswith("-100"):
+        tgid = TelegramID(int(tgid_str[4:]))
+        peer_type = "channel"
+    elif tgid_str.startswith("-"):
+        tgid = TelegramID(-int(tgid_str))
+        peer_type = "chat"
+    else:
+        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
+                               "If you did not get the ID using the `/id` bot command, please "
+                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+                               "Bridging private chats to existing rooms is not allowed.")
+
+    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    if not portal.allow_bridging():
+        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
+                               "If you're the bridge admin, try "
+                               "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
+    if portal.mxid:
+        has_portal_message = (
+            "That Telegram chat already has a portal at "
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
+        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+            return await evt.reply(f"{has_portal_message}"
+                                   "Additionally, you do not have the permissions to unbridge "
+                                   "that room.")
+        evt.sender.command_status = {
+            "next": confirm_bridge,
+            "action": "Room bridging",
+            "mxid": portal.mxid,
+            "bridge_to_mxid": room_id,
+            "tgid": portal.tgid,
+            "peer_type": portal.peer_type,
+        }
+        return await evt.reply(f"{has_portal_message}"
+                               "However, you have the permissions to unbridge that room.\n\n"
+                               "To delete that portal completely and continue bridging, use "
+                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
+    evt.sender.command_status = {
+        "next": confirm_bridge,
+        "action": "Room bridging",
+        "bridge_to_mxid": room_id,
+        "tgid": portal.tgid,
+        "peer_type": portal.peer_type,
+    }
+    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
+                           "chat to this room, use `$cmdprefix+sp continue`")
+
+
+async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
+                                            ) -> Tuple[bool, Optional[Coroutine[None, None, None]]]:
+    if not portal.mxid:
+        await evt.reply("The portal seems to have lost its Matrix room between you"
+                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                        "Continuing without touching previous Matrix room...")
+        return True, None
+    elif evt.args[0] == "delete-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Portal deleted (moving to another room)")
+    elif evt.args[0] == "unbridge-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Room unbridged (portal moving to another room)",
+                                         puppets_only=True)
+    else:
+        await evt.reply(
+            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
+            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
+            "continue` to either delete or unbridge the existing room (respectively) and "
+            "continue with the bridging.\n\n"
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
+        return False, None
+
+
+async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
+    status = evt.sender.command_status
+    try:
+        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        bridge_to_mxid = status["bridge_to_mxid"]
+    except KeyError:
+        evt.sender.command_status = None
+        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
+                               "This shouldn't happen unless you're messing with the command "
+                               "handler code.")
+    if "mxid" in status:
+        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
+        if not ok:
+            return None
+        elif coro:
+            ignore_coro(asyncio.ensure_future(coro, loop=evt.loop))
+            await evt.reply("Cleaning up previous portal room...")
+    elif portal.mxid:
+        evt.sender.command_status = None
+        return await evt.reply("The portal seems to have created a Matrix room between you "
+                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                               "Please start over by calling the bridge command again.")
+    elif evt.args[0] != "continue":
+        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
+                               "`$cmdprefix+sp cancel` to cancel.")
+
+    evt.sender.command_status = None
+    is_logged_in = await evt.sender.is_logged_in()
+    user = evt.sender if is_logged_in else evt.tgbot
+    try:
+        entity = await user.client.get_entity(portal.peer)
+    except Exception:
+        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+        if is_logged_in:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You are logged in, are you in that chat?")
+        else:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You're not logged in, is the relay bot in the chat?")
+    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
+        if is_logged_in:
+            return await evt.reply("You don't seem to be in that chat.")
+        else:
+            return await evt.reply("The bot doesn't seem to be in that chat.")
+
+    direct = False
+
+    portal.mxid = bridge_to_mxid
+    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    portal.photo_id = ""
+    portal.save()
+
+    ignore_coro(asyncio.ensure_future(portal.update_matrix_room(user, entity, direct,
+                                                                levels=levels),
+                                      loop=evt.loop))
+
+    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -0,0 +1,132 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Awaitable
+from io import StringIO
+
+from ...config import yaml
+from ... import portal as po, user as u, util
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="View or change per-portal settings.",
+                 help_args="<`help`|_subcommand_> [...]")
+async def config(evt: CommandEvent) -> None:
+    cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
+    if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
+        await config_help(evt)
+        return
+    elif cmd == "defaults":
+        await config_defaults(evt)
+        return
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        await evt.reply("This is not a portal room.")
+        return
+    elif cmd == "view":
+        await config_view(evt, portal)
+        return
+
+    key = evt.args[1] if len(evt.args) > 1 else None
+    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    if cmd == "set":
+        await config_set(evt, portal, key, value)
+    elif cmd == "unset":
+        await config_unset(evt, portal, key)
+    elif cmd == "add" or cmd == "del":
+        await config_add_del(evt, portal, key, value, cmd)
+    else:
+        return
+    portal.save()
+
+
+def config_help(evt: CommandEvent) -> Awaitable[Dict]:
+    return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
+
+* **help** - View this help text.
+* **view** - View the current config data.
+* **defaults** - View the default config values.
+* **set** <_key_> <_value_> - Set a config value.
+* **unset** <_key_> - Remove a config value.
+* **add** <_key_> <_value_> - Add a value to an array.
+* **del** <_key_> <_value_> - Remove a value from an array.
+""")
+
+
+def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump(portal.local_config, stream)
+    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
+
+
+def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump({
+        "edits_as_replies": evt.config["bridge.edits_as_replies"],
+        "bridge_notices": {
+            "default": evt.config["bridge.bridge_notices.default"],
+            "exceptions": evt.config["bridge.bridge_notices.exceptions"],
+        },
+        "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
+        "inline_images": evt.config["bridge.inline_images"],
+        "message_formats": evt.config["bridge.message_formats"],
+        "state_event_formats": evt.config["bridge.state_event_formats"],
+        "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
+    }, stream)
+    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
+
+
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[Dict]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
+    elif util.recursive_set(portal.local_config, key, value):
+        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
+    else:
+        return evt.reply(f"Failed to set value of `{key}`. "
+                         "Does the path contain non-map types?")
+
+
+def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Dict]:
+    if not key:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config unset <key>`")
+    elif util.recursive_del(portal.local_config, key):
+        return evt.reply(f"Successfully deleted `{key}` from config.")
+    else:
+        return evt.reply(f"`{key}` not found in config.")
+
+
+def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
+                   ) -> Awaitable[Dict]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
+
+    arr = util.recursive_get(portal.local_config, key)
+    if not arr:
+        return evt.reply(f"`{key}` not found in config. "
+                         f"Maybe do `$cmdprefix+sp config set {key} []` first?")
+    elif not isinstance(arr, list):
+        return evt.reply("`{key}` does not seem to be an array.")
+    elif cmd == "add":
+        if value in arr:
+            return evt.reply(f"The array at `{key}` already contains `{value}`.")
+        arr.append(value)
+        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
+    else:
+        if value not in arr:
+            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
+        arr.remove(value)
+        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -0,0 +1,59 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_type_]",
+                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
+                           "The type is either `group`, `supergroup` or `channel` (defaults to "
+                           "`group`).")
+async def create(evt: CommandEvent) -> Dict:
+    type = evt.args[0] if len(evt.args) > 0 else "group"
+    if type not in {"chat", "group", "supergroup", "channel"}:
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
+
+    if po.Portal.get_by_mxid(evt.room_id):
+        return await evt.reply("This is already a portal room.")
+
+    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply("You do not have the permissions to bridge this room.")
+
+    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    if not title:
+        return await evt.reply("Please set a title before creating a Telegram chat.")
+
+    supergroup = type == "supergroup"
+    type = {
+        "supergroup": "channel",
+        "channel": "channel",
+        "chat": "chat",
+        "group": "chat",
+    }[type]
+
+    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
+    try:
+        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+    except ValueError as e:
+        portal.delete()
+        return await evt.reply(e.args[0])
+    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/filter.py

@@ -0,0 +1,95 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`>",
+                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
+                           "default.")
+async def filter_mode(evt: CommandEvent) -> Dict:
+    try:
+        mode = evt.args[0]
+        if mode not in ("whitelist", "blacklist"):
+            raise ValueError()
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
+
+    evt.config["bridge.filter.mode"] = mode
+    evt.config.save()
+    po.Portal.filter_mode = mode
+    if mode == "whitelist":
+        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
+                               "To allow bridging a specific chat, use"
+                               "`!filter whitelist <chat ID>`.")
+    else:
+        return await evt.reply("The bridge will now allow bridging chats by default.\n"
+                               "To disallow bridging a specific chat, use"
+                               "`!filter blacklist <chat ID>`.")
+
+
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+                 help_text="Allow or disallow bridging a specific chat.")
+async def filter(evt: CommandEvent) -> Optional[Dict]:
+    try:
+        action = evt.args[0]
+        if action not in ("whitelist", "blacklist", "add", "remove"):
+            raise ValueError()
+
+        id_str = evt.args[1]
+        if id_str.startswith("-100"):
+            id = int(id_str[4:])
+        elif id_str.startswith("-"):
+            id = int(id_str[1:])
+        else:
+            id = int(id_str)
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
+
+    mode = evt.config["bridge.filter.mode"]
+    if mode not in ("blacklist", "whitelist"):
+        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
+
+    list = evt.config["bridge.filter.list"]
+
+    if action in ("blacklist", "whitelist"):
+        action = "add" if mode == action else "remove"
+
+    def save() -> None:
+        evt.config["bridge.filter.list"] = list
+        evt.config.save()
+        po.Portal.filter_list = list
+
+    if action == "add":
+        if id in list:
+            return await evt.reply(f"That chat is already {mode}ed.")
+        list.append(id)
+        save()
+        return await evt.reply(f"Chat ID added to {mode}.")
+    elif action == "remove":
+        if id not in list:
+            return await evt.reply(f"That chat is not {mode}ed.")
+        list.remove(id)
+        save()
+        return await evt.reply(f"Chat ID removed from {mode}.")
+    return None

mautrix_telegram/commands/portal/misc.py

@@ -0,0 +1,127 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+
+from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
+                             UsernameNotModifiedError, UsernameOccupiedError)
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT, SECTION_MISC
+from .util import user_has_power_level
+
+
+@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.")
+async def sync_state(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to synchronize this room.")
+
+    await portal.sync_matrix_members()
+    await evt.reply("Synchronization complete")
+
+
+@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Get the ID of the Telegram chat where this room is bridged.")
+async def id(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    tgid = portal.tgid
+    if portal.peer_type == "chat":
+        tgid = -tgid
+    elif portal.peer_type == "channel":
+        tgid = f"-100{tgid}"
+    await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Get a Telegram invite link to the current chat.")
+async def invite_link(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if portal.peer_type == "user":
+        return await evt.reply("You can't invite users to private chats.")
+
+    try:
+        link = await portal.get_invite_link(evt.sender)
+        return await evt.reply(f"Invite link to {portal.title}: {link}")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to create an invite link.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Upgrade a normal Telegram group to a supergroup.")
+async def upgrade(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type == "channel":
+        return await evt.reply("This is already a supergroup or a channel.")
+    elif portal.peer_type == "user":
+        return await evt.reply("You can't upgrade private chats.")
+
+    try:
+        await portal.upgrade_telegram_chat(evt.sender)
+        return await evt.reply(f"Group upgraded to supergroup. New ID: -100{portal.tgid}")
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to upgrade this group.")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="<_name_|`-`>",
+                 help_text="Change the username of a supergroup/channel. "
+                           "To disable, use a dash (`-`) as the name.")
+async def group_name(evt: CommandEvent) -> Dict:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type != "channel":
+        return await evt.reply("Only channels and supergroups have usernames.")
+
+    try:
+        await portal.set_telegram_username(evt.sender,
+                                           evt.args[0] if evt.args[0] != "-" else "")
+        if portal.username:
+            return await evt.reply(f"Username of channel changed to {portal.username}.")
+        else:
+            return await evt.reply(f"Channel is now private.")
+    except ChatAdminRequiredError:
+        return await evt.reply(
+            "You don't have the permission to set the username of this channel.")
+    except UsernameNotModifiedError:
+        if portal.username:
+            return await evt.reply("That is already the username of this channel.")
+        else:
+            return await evt.reply("This channel is already private")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username")

mautrix_telegram/commands/portal/unbridge.py

@@ -0,0 +1,97 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Callable, Optional
+
+from ...types import MatrixRoomID
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+from .util import user_has_power_level, get_initial_state
+
+
+async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
+                                           action: Optional[str] = None
+                                           ) -> Optional[po.Portal]:
+    room_id = MatrixRoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if not portal:
+        that_this = "This" if room_id == evt.room_id else "That"
+        await evt.reply(f"{that_this} is not a portal room.")
+        return None
+
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
+        action = action or f"{permission.replace('_', ' ')}s"
+        await evt.reply(f"You do not have the permissions to {action} that portal.")
+        return None
+    return portal
+
+
+def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
+                                completed_message: str) -> Dict:
+    async def post_confirm(confirm) -> Optional[Dict]:
+        confirm.sender.command_status = None
+        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
+            await function()
+            if confirm.room_id != room_id:
+                return await confirm.reply(completed_message)
+        else:
+            return await confirm.reply(f"{action} cancelled.")
+        return None
+
+    return {
+        "next": post_confirm,
+        "action": action,
+    }
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove all users from the current portal room and forget the portal. "
+                           "Only works for group chats; to delete a private chat portal, simply "
+                           "leave the room.")
+async def delete_portal(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
+                                                            portal.cleanup_and_delete, "delete",
+                                                            "Portal successfully deleted.")
+    return await evt.reply("Please confirm deletion of portal "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           f"to Telegram chat \"{portal.title}\" "
+                           "by typing `$cmdprefix+sp confirm-delete`"
+                           "\n\n"
+                           "**WARNING:** If the bridge bot has the power level to do so, **this "
+                           "will kick ALL users** in the room. If you just want to remove the "
+                           "bridge, use `$cmdprefix+sp unbridge` instead.")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove puppets from the current portal room and forget the portal.")
+async def unbridge(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
+                                                            portal.unbridge, "unbridge",
+                                                            "Room successfully unbridged.")
+    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           "by typing `$cmdprefix+sp confirm-unbridge`")

mautrix_telegram/commands/portal/util.py

@@ -0,0 +1,56 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Tuple
+
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from ... import user as u
+
+
+async def get_initial_state(intent: IntentAPI, room_id: str) -> Tuple[str, str, Dict]:
+    state = await intent.get_room_state(room_id)
+    title = None
+    about = None
+    levels = None
+    for event in state:
+        try:
+            if event["type"] == "m.room.name":
+                title = event["content"]["name"]
+            elif event["type"] == "m.room.topic":
+                about = event["content"]["topic"]
+            elif event["type"] == "m.room.power_levels":
+                levels = event["content"]
+            elif event["type"] == "m.room.canonical_alias":
+                title = title or event["content"]["alias"]
+        except KeyError:
+            # Some state event probably has empty content
+            pass
+    return title, about, levels
+
+
+async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50
+                               ) -> bool:
+    if sender.is_admin:
+        return True
+    # Make sure the state store contains the power levels.
+    try:
+        await intent.get_power_levels(room)
+    except MatrixRequestError:
+        return False
+    return intent.state_store.has_power_level(room, sender.mxid,
+                                              event=f"net.maunium.telegram.{event}",
+                                              default=default)

mautrix_telegram/commands/telegram/__init__.py

@@ -0,0 +1 @@
+from . import account, auth, misc

mautrix_telegram/commands/telegram/account.py

@@ -0,0 +1,102 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
+                             HashInvalidError)
+from telethon.tl.types import Authorization
+from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
+                                           ResetAuthorizationRequest)
+
+from mautrix_telegram.commands import command_handler, CommandEvent, SECTION_AUTH
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new username_>",
+                 help_text="Change your Telegram username.")
+async def username(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp username <new username>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own username.")
+    new_name = evt.args[0]
+    if new_name == "-":
+        new_name = ""
+    try:
+        await evt.sender.client(UpdateUsernameRequest(username=new_name))
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username. Usernames must be between 5 and 30 alphanumeric "
+                               "characters.")
+    except UsernameNotModifiedError:
+        return await evt.reply("That is your current username.")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    await evt.sender.update_info()
+    if not evt.sender.username:
+        await evt.reply("Username removed")
+    else:
+        await evt.reply(f"Username changed to {evt.sender.username}")
+
+
+def _format_session(sess: Authorization) -> str:
+    return (f"**{sess.app_name} {sess.app_version}**  \n"
+            f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
+            f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
+            f"  **From:** {sess.ip} - {sess.region}, {sess.country}")
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<`list`|`terminate`> [_hash_]",
+                 help_text="View or delete other Telegram sessions.")
+async def session(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't manage their sessions")
+    cmd = evt.args[0].lower()
+    if cmd == "list":
+        res = await evt.sender.client(GetAuthorizationsRequest())
+        session_list = res.authorizations
+        current = [s for s in session_list if s.current][0]
+        current_text = _format_session(current)
+        other_text = "\n".join(f"* {_format_session(sess)}  \n"
+                               f"  **Hash:** {sess.hash}"
+                               for sess in session_list if not sess.current)
+        return await evt.reply(f"### Current session\n"
+                               f"{current_text}\n"
+                               f"\n"
+                               f"### Other active sessions\n"
+                               f"{other_text}")
+    elif cmd == "terminate" and len(evt.args) > 1:
+        try:
+            session_hash = int(evt.args[1])
+        except ValueError:
+            return await evt.reply("Hash must be a positive integer")
+        if session_hash <= 0:
+            return await evt.reply("Hash must be a positive integer")
+        try:
+            ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
+        except HashInvalidError:
+            return await evt.reply("Invalid session hash.")
+        if ok:
+            return await evt.reply("Session terminated successfully.")
+        else:
+            return await evt.reply("Session not found.")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")

mautrix_telegram/commands/telegram/auth.py

@@ -23,9 +23,9 @@ from telethon.errors import (
     PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
     PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError)
 
-from . import command_handler, CommandEvent, SECTION_AUTH
-from .. import puppet as pu, user as u
-from ..util import format_duration
+from mautrix_telegram.commands import command_handler, CommandEvent, SECTION_AUTH
+from mautrix_telegram import puppet as pu, user as u
+from mautrix_telegram.util import format_duration, ignore_coro
 
 
 @command_handler(needs_auth=False,
@@ -33,8 +33,9 @@ from ..util import format_duration
                  help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> Optional[Dict]:
     me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
+    human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
     if me:
-        return await evt.reply(f"You're logged in as @{me.username}")
+        return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")
 
@@ -53,71 +54,6 @@ async def ping_bot(evt: CommandEvent) -> Optional[Dict]:
                            "To use the bot, simply invite it to a portal room.")
 
 
-@command_handler(needs_auth=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
-                           "account.")
-async def logout_matrix(evt: CommandEvent) -> Optional[Dict]:
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if not puppet.is_real_user:
-        return await evt.reply("You are not logged in with your Matrix account.")
-    await puppet.switch_mxid(None, None)
-    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
-
-
-@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
-                           "account")
-async def login_matrix(evt: CommandEvent) -> Optional[Dict]:
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if puppet.is_real_user:
-        return await evt.reply("You have already logged in with your Matrix account. "
-                               "Log out with `$cmdprefix+sp logout-matrix` first.")
-    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
-    if allow_matrix_login:
-        evt.sender.command_status = {
-            "next": enter_matrix_token,
-            "action": "Matrix login",
-        }
-    if evt.config["appservice.public.enabled"]:
-        prefix = evt.config["appservice.public.external"]
-        url = f"{prefix}/matrix-login?token={evt.public_website.make_token(evt.sender.mxid, '/matrix-login')}"
-        if allow_matrix_login:
-            return await evt.reply(
-                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
-                "If you would like to log in within Matrix, please send your Matrix access token "
-                "here.\n"
-                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
-                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
-                "your access token in the message history.")
-        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
-                               f"Please visit [the login page]({url}) to log in.")
-    elif allow_matrix_login:
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your Matrix access token here to log in.")
-    return await evt.reply("This bridge instance has been configured to not allow logging in.")
-
-
-async def enter_matrix_token(evt: CommandEvent) -> Dict:
-    evt.sender.command_status = None
-
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if puppet.is_real_user:
-        return await evt.reply("You have already logged in with your Matrix account. "
-                               "Log out with `$cmdprefix+sp logout-matrix` first.")
-
-    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
-    if resp == pu.PuppetError.OnlyLoginSelf:
-        return await evt.reply("You can only log in as your own Matrix user.")
-    elif resp == pu.PuppetError.InvalidAccessToken:
-        return await evt.reply("Failed to verify access token.")
-    assert resp == pu.PuppetError.Success, "Encountered an unhandled PuppetError."
-    return await evt.reply(
-        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")
-
-
 @command_handler(needs_auth=False, management_only=True,
                  help_section=SECTION_AUTH,
                  help_args="<_phone_> <_full name_>",
@@ -134,7 +70,7 @@ async def register(evt: CommandEvent) -> Optional[Dict]:
     else:
         full_name = " ".join(evt.args[1:-1]), evt.args[-1]
 
-    await request_code(evt, phone_number, {
+    await _request_code(evt, phone_number, {
         "next": enter_code_register,
         "action": "Register",
         "full_name": full_name,
@@ -149,7 +85,7 @@ async def enter_code_register(evt: CommandEvent) -> Dict:
         await evt.sender.ensure_started(even_if_no_session=True)
         first_name, last_name = evt.sender.command_status["full_name"]
         user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
         evt.sender.command_status = None
         return await evt.reply(f"Successfully registered to Telegram.")
     except PhoneNumberOccupiedError:
@@ -172,38 +108,64 @@ async def enter_code_register(evt: CommandEvent) -> Dict:
                  help_section=SECTION_AUTH,
                  help_text="Get instructions on how to log in.")
 async def login(evt: CommandEvent) -> Optional[Dict]:
+    override_sender = False
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        evt.sender = await u.User.get_by_mxid(evt.args[0]).ensure_started()
+        override_sender = True
     if await evt.sender.is_logged_in():
         return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 
     allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
-    if allow_matrix_login:
+    if allow_matrix_login and not override_sender:
         evt.sender.command_status = {
             "next": enter_phone_or_token,
             "action": "Login",
         }
 
+    nb = "**N.B. Logging in grants the bridge full access to your Telegram account.**"
     if evt.config["appservice.public.enabled"]:
         prefix = evt.config["appservice.public.external"]
         url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
         if allow_matrix_login:
+            if override_sender:
+                return await evt.reply(
+                    "This bridge instance allows you to log in inside or outside of Matrix, but "
+                    "logging in as another user is only possible via the web interface.\n\n"
+                    f"Please visit [the login page]({url}) to log in as "
+                    f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).\n\n")
             return await evt.reply(
                 "This bridge instance allows you to log in inside or outside Matrix.\n\n"
                 "If you would like to log in within Matrix, please send your phone number or bot "
                 "auth token here.\n"
-                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "If you would like to log in outside of Matrix, please visit [the login page]"
+                f"({url}).\n\n"
                 "Logging in outside of Matrix is recommended if you have two-factor authentication "
-                "enabled, because in-Matrix login would save your password in the message history.")
-        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
-                               f"Please visit [the login page]({url}) to log in.")
+                "enabled, because in-Matrix login would save your password in the message history."
+                f"\n\n{nb}")
+        if override_sender:
+            return await evt.reply(
+                "This bridge instance does not allow logging in inside Matrix, and logging in as "
+                "another user inside Matrix isn't possible anyway.\n\n"
+                f"Please visit [the login page]({url}) to log in as "
+                f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+        return await evt.reply(
+            "This bridge instance does not allow logging in inside Matrix.\n\n"
+            f"Please visit [the login page]({url}) to log in.\n\n"
+            f"{nb}")
     elif allow_matrix_login:
+        if override_sender:
+            return await evt.reply(
+                "This bridge instance does not allow you to log in outside of Matrix. "
+                "Logging in as another user inside Matrix is not currently possible.")
         return await evt.reply(
             "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number or bot auth token here to start the login process.")
+            "Please send your phone number or bot auth token here to start the login process.\n\n"
+            f"{nb}")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 
-async def request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
-                       ) -> Dict:
+async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
+                        ) -> Dict:
     ok = False
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
@@ -245,13 +207,13 @@ async def enter_phone_or_token(evt: CommandEvent) -> Optional[Dict]:
     # phone numbers don't contain colons but telegram bot auth tokens do
     if evt.args[0].find(":") > 0:
         try:
-            await sign_in(evt, bot_token=evt.args[0])
+            await _sign_in(evt, bot_token=evt.args[0])
         except Exception:
             evt.log.exception("Error sending auth token")
             return await evt.reply("Unhandled exception while sending auth token. "
                                    "Check console for more details.")
     else:
-        await request_code(evt, evt.args[0], {
+        await _request_code(evt, evt.args[0], {
             "next": enter_code,
             "action": "Login",
         })
@@ -266,7 +228,7 @@ async def enter_code(evt: CommandEvent) -> Optional[Dict]:
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await sign_in(evt, code=evt.args[0])
+        await _sign_in(evt, code=evt.args[0])
     except Exception:
         evt.log.exception("Error sending phone code")
         return await evt.reply("Unhandled exception while sending code. "
@@ -282,7 +244,7 @@ async def enter_password(evt: CommandEvent) -> Optional[Dict]:
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await sign_in(evt, password=" ".join(evt.args))
+        await _sign_in(evt, password=" ".join(evt.args))
     except AccessTokenInvalidError:
         return await evt.reply("That bot token is not valid.")
     except AccessTokenExpiredError:
@@ -294,7 +256,7 @@ async def enter_password(evt: CommandEvent) -> Optional[Dict]:
     return None
 
 
-async def sign_in(evt: CommandEvent, **sign_in_info) -> Dict:
+async def _sign_in(evt: CommandEvent, **sign_in_info) -> Dict:
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
         user = await evt.sender.client.sign_in(**sign_in_info)
@@ -304,7 +266,7 @@ async def sign_in(evt: CommandEvent, **sign_in_info) -> Dict:
             await evt.reply(f"[{existing_user.displayname}]"
                             f"(https://matrix.to/#/{existing_user.mxid})"
                             " was logged out from the account.")
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
         evt.sender.command_status = None
         name = f"@{user.username}" if user.username else f"+{user.phone}"
         return await evt.reply(f"Successfully logged in as {name}")

mautrix_telegram/commands/telegram/misc.py

@@ -14,18 +14,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
+import codecs
+import base64
 import re
 
-from telethon.errors import (
-    InviteHashInvalidError, InviteHashExpiredError, UserAlreadyParticipantError)
-from telethon.tl.types import User as TLUser
-from telethon.tl.types import TypeUpdates
-from telethon.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest
+from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError,
+                             UserAlreadyParticipantError)
+from telethon.tl.types import User as TLUser, TypeUpdates, MessageMediaGame
+from telethon.tl.types.messages import BotCallbackAnswer
+from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
+                                            GetBotCallbackAnswerRequest)
 from telethon.tl.functions.channels import JoinChannelRequest
 
-from .. import puppet as pu, portal as po
-from . import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
+from mautrix_telegram import puppet as pu, portal as po
+from mautrix_telegram.db import Message as DBMessage
+from mautrix_telegram.types import TelegramID
+from mautrix_telegram.commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
 
 
 @command_handler(help_section=SECTION_MISC,
@@ -66,14 +71,13 @@ async def search(evt: CommandEvent) -> Optional[Dict]:
     return await evt.reply("\n".join(reply))
 
 
-@command_handler(name="pm",
-                 help_section=SECTION_CREATING_PORTALS,
+@command_handler(help_section=SECTION_CREATING_PORTALS,
                  help_args="<_identifier_>",
                  help_text="Open a private chat with the given Telegram user. The identifier is "
                            "either the internal user ID, the username or the phone number. "
                            "**N.B.** The phone numbers you start chats with must already be in "
                            "your contacts.")
-async def private_message(evt: CommandEvent) -> Optional[Dict]:
+async def pm(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
 
@@ -158,3 +162,55 @@ async def sync(evt: CommandEvent) -> Optional[Dict]:
     if not sync_only or sync_only == "me":
         await evt.sender.update_info()
     return await evt.reply("Synchronization complete.")
+
+
+PEER_TYPE_CHAT = b"g"
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_play ID_>",
+                 help_text="Play a Telegram game.")
+async def play(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp play <play ID>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to play games.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't play games :(")
+
+    try:
+        play_id = evt.args[0]
+        play_id += (4 - len(play_id) % 4) * "="
+        play_id = base64.b64decode(play_id)
+        peer_type, play_id = bytes([play_id[0]]), play_id[1:]
+        tgid = TelegramID(int(codecs.encode(play_id[0:5], "hex_codec"), 16))
+        msg_id = TelegramID(int(codecs.encode(play_id[5:10], "hex_codec"), 16))
+        space = None
+        if peer_type == PEER_TYPE_CHAT:
+            space = TelegramID(int(codecs.encode(play_id[10:15], "hex_codec"), 16))
+    except ValueError:
+        return await evt.reply("Invalid play ID (format)")
+
+    if peer_type == PEER_TYPE_CHAT:
+        orig_msg = DBMessage.get_by_tgid(msg_id, space)
+        if not orig_msg:
+            return await evt.reply("Invalid play ID (original message not found in db)")
+        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, evt.sender.tgid)
+        if not new_msg:
+            return await evt.reply("Invalid play ID (your copy of message not found in db)")
+        msg_id = new_msg.tgid
+    try:
+        peer = await evt.sender.client.get_input_entity(tgid)
+    except ValueError:
+        return await evt.reply("Invalid play ID (chat not found)")
+
+    msg = await evt.sender.client.get_messages(entity=peer, ids=msg_id)
+    if not msg or not isinstance(msg.media, MessageMediaGame):
+        return await evt.reply("Invalid play ID (message doesn't look like a game)")
+
+    game = await evt.sender.client(GetBotCallbackAnswerRequest(peer=peer, msg_id=msg_id, game=True))
+    if not isinstance(game, BotCallbackAnswer):
+        return await evt.reply("Game request response invalid")
+
+    await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
+                    f"{msg.media.game.description}")

mautrix_telegram/config.py

@@ -151,8 +151,8 @@ class Config(DictWithRecursion):
                     base[to_path][key] = value
 
         copy("homeserver.address")
-        copy("homeserver.verify_ssl")
         copy("homeserver.domain")
+        copy("homeserver.verify_ssl")
 
         if "appservice.protocol" in self and "appservice.address" not in self:
             protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
@@ -165,7 +165,6 @@ class Config(DictWithRecursion):
         copy("appservice.max_body_size")
 
         copy("appservice.database")
-        copy("appservice.sqlalchemy_core_mode")
 
         copy("appservice.public.enabled")
         copy("appservice.public.prefix")
@@ -191,8 +190,24 @@ class Config(DictWithRecursion):
 
         copy("bridge.displayname_preference")
 
+        copy("bridge.max_initial_member_sync")
+        copy("bridge.sync_channel_members")
+        copy("bridge.skip_deleted_members")
+        copy("bridge.startup_sync")
+        copy("bridge.sync_dialog_limit")
+        copy("bridge.max_telegram_delete")
+        copy("bridge.sync_matrix_state")
+        copy("bridge.allow_matrix_login")
+        copy("bridge.plaintext_highlights")
         copy("bridge.edits_as_replies")
         copy("bridge.highlight_edits")
+        copy("bridge.public_portals")
+        copy("bridge.catch_up")
+        copy("bridge.sync_with_custom_puppets")
+        copy("bridge.telegram_link_preview")
+        copy("bridge.inline_images")
+
+        copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):
             base["bridge.bridge_notices"] = {
                 "default": self["bridge.bridge_notices"],
@@ -200,17 +215,6 @@ class Config(DictWithRecursion):
             }
         else:
             copy("bridge.bridge_notices")
-        copy("bridge.bot_messages_as_notices")
-        copy("bridge.max_initial_member_sync")
-        copy("bridge.sync_channel_members")
-        copy("bridge.max_telegram_delete")
-        copy("bridge.allow_matrix_login")
-        copy("bridge.inline_images")
-        copy("bridge.plaintext_highlights")
-        copy("bridge.public_portals")
-        copy("bridge.native_stickers")
-        copy("bridge.catch_up")
-        copy("bridge.sync_with_custom_puppets")
 
         copy("bridge.deduplication.pre_db_check")
         copy("bridge.deduplication.cache_queue_length")
@@ -218,6 +222,7 @@ class Config(DictWithRecursion):
         if "bridge.message_formats.m_text" in self:
             del self["bridge.message_formats"]
         copy_dict("bridge.message_formats", override_existing_map=False)
+
         copy("bridge.state_event_formats.join")
         copy("bridge.state_event_formats.leave")
         copy("bridge.state_event_formats.name_change")
@@ -251,6 +256,10 @@ class Config(DictWithRecursion):
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
+        copy("telegram.server.enabled")
+        copy("telegram.server.dc")
+        copy("telegram.server.ip")
+        copy("telegram.server.port")
         copy("telegram.proxy.type")
         copy("telegram.proxy.address")
         copy("telegram.proxy.port")

mautrix_telegram/context.py

@@ -31,8 +31,8 @@ if TYPE_CHECKING:
 
 
 class Context:
-    def __init__(self, az: "AppService", db: "scoped_session", config: "Config",
-                 loop: "asyncio.AbstractEventLoop", session_container: "AlchemySessionContainer"
+    def __init__(self, az: 'AppService', db: 'scoped_session', config: 'Config',
+                 loop: 'asyncio.AbstractEventLoop', session_container: 'AlchemySessionContainer'
                  ) -> None:
         self.az = az  # type: AppService
         self.db = db  # type: scoped_session

mautrix_telegram/db.py

@@ -1,258 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
-                        BigInteger, String, Boolean, Text, Table,
-                        and_, func, select)
-from sqlalchemy.engine import Engine, RowProxy
-from sqlalchemy.sql import expression
-from sqlalchemy.orm import relationship, Query
-from sqlalchemy.sql.base import ImmutableColumnCollection
-from typing import Dict, Optional, List
-import json
-
-from mautrix_telegram.types import MatrixUserID, MatrixRoomID, MatrixEventID
-from .types import TelegramID
-from .base import Base
-
-
-class Portal(Base):
-    query = None  # type: Query
-    __tablename__ = "portal"
-
-    # Telegram chat information
-    tgid = Column(Integer, primary_key=True)  # type: TelegramID
-    tg_receiver = Column(Integer, primary_key=True)  # type: TelegramID
-    peer_type = Column(String, nullable=False)
-    megagroup = Column(Boolean)
-
-    # Matrix portal information
-    mxid = Column(String, unique=True, nullable=True)  # type: Optional[MatrixRoomID]
-
-    config = Column(Text, nullable=True)
-
-    # Telegram chat metadata
-    username = Column(String, nullable=True)
-    title = Column(String, nullable=True)
-    about = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-
-
-class Message(Base):
-    db = None  # type: Engine
-    t = None  # type: Table
-    c = None  # type: ImmutableColumnCollection
-    __tablename__ = "message"
-
-    mxid = Column(String)  # type: MatrixEventID
-    mx_room = Column(String)  # type: MatrixRoomID
-    tgid = Column(Integer, primary_key=True)  # type: TelegramID
-    tg_space = Column(Integer, primary_key=True)  # type: TelegramID
-
-    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
-
-    @staticmethod
-    def _one_or_none(rows: RowProxy) -> Optional['Message']:
-        try:
-            mxid, mx_room, tgid, tg_space = next(rows)
-            return Message(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space)
-        except StopIteration:
-            return None
-
-    @staticmethod
-    def _all(rows: RowProxy) -> List['Message']:
-        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3])
-                for row in rows]
-
-    @classmethod
-    def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']:
-        rows = cls.db.execute(cls.t.select()
-                              .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space)))
-        return cls._one_or_none(rows)
-
-    @classmethod
-    def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
-        rows = cls.db.execute(select([func.count(cls.c.tg_space)])
-                              .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
-        try:
-            count, = next(rows)
-            return count
-        except StopIteration:
-            return 0
-
-    @classmethod
-    def get_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID, tg_space: TelegramID
-                    ) -> Optional['Message']:
-        rows = cls.db.execute(cls.t.select().where(
-            and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room, cls.c.tg_space == tg_space)))
-        return cls._one_or_none(rows)
-
-    @classmethod
-    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
-        cls.db.execute(cls.t.update()
-                       .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
-                       .values(**values))
-
-    @classmethod
-    def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values) -> None:
-        cls.db.execute(cls.t.update()
-                       .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
-                       .values(**values))
-
-    def update(self, **values) -> None:
-        for key, value in values.items():
-            setattr(self, key, value)
-        self.update_by_tgid(self.tgid, self.tg_space, **values)
-
-    def delete(self) -> None:
-        self.db.execute(self.t.delete().where(
-            and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)))
-
-    def insert(self) -> None:
-        self.db.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room, tgid=self.tgid,
-                                               tg_space=self.tg_space))
-
-
-class UserPortal(Base):
-    query = None  # type: Query
-    __tablename__ = "user_portal"
-
-    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
-                  primary_key=True)  # type: TelegramID
-    portal = Column(Integer, primary_key=True)  # type: TelegramID
-    portal_receiver = Column(Integer, primary_key=True)  # type: TelegramID
-
-    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
-                                           ("portal.tgid", "portal.tg_receiver"),
-                                           onupdate="CASCADE", ondelete="CASCADE"),)
-
-
-class User(Base):
-    query = None  # type: Query
-    __tablename__ = "user"
-
-    mxid = Column(String, primary_key=True)  # type: MatrixUserID
-    tgid = Column(Integer, nullable=True, unique=True)  # type: Optional[TelegramID]
-    tg_username = Column(String, nullable=True)
-    tg_phone = Column(String, nullable=True)
-    saved_contacts = Column(Integer, default=0, nullable=False)
-    contacts = relationship("Contact", uselist=True,
-                            cascade="save-update, merge, delete, delete-orphan"
-                            )  # type: List[Contact]
-    portals = relationship("Portal", secondary="user_portal")
-
-
-class RoomState(Base):
-    query = None  # type: Query
-    __tablename__ = "mx_room_state"
-
-    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
-    _power_levels_text = Column("power_levels", Text, nullable=True)
-    _power_levels_json = {}  # type: Dict
-
-    @property
-    def has_power_levels(self) -> bool:
-        return bool(self._power_levels_text)
-
-    @property
-    def power_levels(self) -> Dict:
-        if not self._power_levels_json and self._power_levels_text:
-            self._power_levels_json = json.loads(self._power_levels_text)
-        return self._power_levels_json
-
-    @power_levels.setter
-    def power_levels(self, val: Dict) -> None:
-        self._power_levels_json = val
-        self._power_levels_text = json.dumps(val)
-
-
-class UserProfile(Base):
-    query = None  # type: Query
-    __tablename__ = "mx_user_profile"
-
-    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
-    user_id = Column(String, primary_key=True)  # type: MatrixUserID
-    membership = Column(String, nullable=False, default="leave")
-    displayname = Column(String, nullable=True)
-    avatar_url = Column(String, nullable=True)
-
-    def dict(self) -> Dict[str, str]:
-        return {
-            "membership": self.membership,
-            "displayname": self.displayname,
-            "avatar_url": self.avatar_url,
-        }
-
-
-class Contact(Base):
-    query = None  # type: Query
-    __tablename__ = "contact"
-
-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)  # type: TelegramID
-    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)  # type: TelegramID
-
-
-class Puppet(Base):
-    query = None  # type: Query
-    __tablename__ = "puppet"
-
-    id = Column(Integer, primary_key=True)  # type: TelegramID
-    custom_mxid = Column(String, nullable=True)  # type: Optional[MatrixUserID]
-    access_token = Column(String, nullable=True)
-    displayname = Column(String, nullable=True)
-    displayname_source = Column(Integer, nullable=True)  # type: Optional[TelegramID]
-    username = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-    is_bot = Column(Boolean, nullable=True)
-    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
-
-
-# Fucking Telegram not telling bots what chats they are in 3:<
-class BotChat(Base):
-    query = None  # type: Query
-    __tablename__ = "bot_chat"
-    id = Column(Integer, primary_key=True)  # type: TelegramID
-    type = Column(String, nullable=False)
-
-
-class TelegramFile(Base):
-    query = None  # type: Query
-    __tablename__ = "telegram_file"
-
-    id = Column(String, primary_key=True)
-    mxc = Column(String)
-    mime_type = Column(String)
-    was_converted = Column(Boolean)
-    timestamp = Column(BigInteger)
-    size = Column(Integer, nullable=True)
-    width = Column(Integer, nullable=True)
-    height = Column(Integer, nullable=True)
-    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
-    thumbnail = relationship("TelegramFile", uselist=False)
-
-
-def init(db_session, db_engine) -> None:
-    Portal.query = db_session.query_property()
-    Message.db = db_engine
-    Message.t = Message.__table__
-    Message.c = Message.t.c
-    UserPortal.query = db_session.query_property()
-    User.query = db_session.query_property()
-    Puppet.query = db_session.query_property()
-    BotChat.query = db_session.query_property()
-    TelegramFile.query = db_session.query_property()
-    UserProfile.query = db_session.query_property()
-    RoomState.query = db_session.query_property()

mautrix_telegram/db/__init__.py

@@ -0,0 +1,33 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from .base import Base
+from .bot_chat import BotChat
+from .message import Message
+from .portal import Portal
+from .puppet import Puppet
+from .room_state import RoomState
+from .telegram_file import TelegramFile
+from .user import User, UserPortal, Contact
+from .user_profile import UserProfile
+
+
+def init(db_engine) -> None:
+    for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
+                  RoomState, BotChat):
+        table.db = db_engine
+        table.t = table.__table__
+        table.c = table.t.c

mautrix_telegram/db/base.py

@@ -0,0 +1,57 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+
+class BaseBase:
+    db = None  # type: Engine
+    t = None  # type: Table
+    __table__ = None  # type: Table
+    c = None  # type: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy):
+        pass
+
+    @classmethod
+    def _select_one_or_none(cls, *args):
+        return cls._one_or_none(cls.db.execute(cls.t.select().where(*args)))
+
+    @property
+    @abstractmethod
+    def _edit_identity(self):
+        pass
+
+    def update(self, **values) -> None:
+        self.db.execute(self.t.update()
+                        .where(self._edit_identity)
+                        .values(**values))
+        for key, value in values.items():
+            setattr(self, key, value)
+
+    def delete(self) -> None:
+        self.db.execute(self.t.delete().where(self._edit_identity))
+
+
+Base = declarative_base(cls=BaseBase)

mautrix_telegram/db/base.pyi

@@ -0,0 +1,26 @@
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+class Base(declarative_base):
+	db: Engine
+	t: Table
+	__table__: Table
+	c: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy): ...
+
+	@classmethod
+	def _select_one_or_none(cls, *args): ...
+
+    def _edit_identity(self): ...
+
+    def update(self, **values) -> None: ...
+
+    def delete(self) -> None: ...

mautrix_telegram/db/bot_chat.py

@@ -14,23 +14,30 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import re
-from typing import List, Tuple, Pattern
-from telethon.tl.types import TypeMessageEntity
+from typing import Iterable
+
+from sqlalchemy import Column, Integer, String
+
+from ..types import TelegramID
+from .base import Base
 
 
-class MatrixParserCommon:
-    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")  # type: Pattern
-    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")  # type: Pattern
-    block_tags = ("p", "pre", "blockquote",
-                  "ol", "ul", "li",
-                  "h1", "h2", "h3", "h4", "h5", "h6",
-                  "div", "hr", "table")  # type: Tuple[str, ...]
-    list_bullets = ("●", "○", "■", "‣")  # type: Tuple[str, ...]
+# Fucking Telegram not telling bots what chats they are in 3:<
+class BotChat(Base):
+    __tablename__ = "bot_chat"
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    type = Column(String, nullable=False)
 
     @classmethod
-    def list_bullet(cls, depth: int) -> str:
-        return cls.list_bullets[(depth - 1) % len(cls.list_bullets)] + " "
+    def delete(cls, id: TelegramID) -> None:
+        cls.db.execute(cls.t.delete().where(cls.c.id == id))
 
+    @classmethod
+    def all(cls) -> Iterable['BotChat']:
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            id, type = row
+            yield cls(id=id, type=type)
 
-ParsedMessage = Tuple[str, List[TypeMessageEntity]]
+    def insert(self) -> None:
+        self.db.execute(self.t.insert().values(id=self.id, type=self.type))

mautrix_telegram/db/message.py

@@ -0,0 +1,87 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, select
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional, List
+
+from ..types import MatrixRoomID, MatrixEventID, TelegramID
+from .base import Base
+
+
+class Message(Base):
+    __tablename__ = "message"
+
+    mxid = Column(String)  # type: MatrixEventID
+    mx_room = Column(String)  # type: MatrixRoomID
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+
+    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
+        try:
+            mxid, mx_room, tgid, tg_space = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space)
+        except StopIteration:
+            return None
+
+    @staticmethod
+    def _all(rows: RowProxy) -> List['Message']:
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3])
+                for row in rows]
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']:
+        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+
+    @classmethod
+    def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
+        rows = cls.db.execute(select([func.count(cls.c.tg_space)])
+                              .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
+        try:
+            count, = next(rows)
+            return count
+        except StopIteration:
+            return 0
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID, tg_space: TelegramID
+                    ) -> Optional['Message']:
+        return cls._select_one_or_none(and_(cls.c.mxid == mxid,
+                                            cls.c.mx_room == mx_room,
+                                            cls.c.tg_space == tg_space))
+
+    @classmethod
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
+        cls.db.execute(cls.t.update()
+                       .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
+                       .values(**values))
+
+    @classmethod
+    def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values) -> None:
+        cls.db.execute(cls.t.update()
+                       .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
+                       .values(**values))
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)
+
+    def insert(self) -> None:
+        self.db.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room, tgid=self.tgid,
+                                               tg_space=self.tg_space))

mautrix_telegram/db/portal.py

@@ -0,0 +1,80 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, Integer, String, Boolean, Text, and_
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional
+
+from ..types import MatrixRoomID, TelegramID
+from .base import Base
+
+
+class Portal(Base):
+    __tablename__ = "portal"
+
+    # Telegram chat information
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_receiver = Column(Integer, primary_key=True)  # type: TelegramID
+    peer_type = Column(String, nullable=False)
+    megagroup = Column(Boolean)
+
+    # Matrix portal information
+    mxid = Column(String, unique=True, nullable=True)  # type: Optional[MatrixRoomID]
+
+    config = Column(Text, nullable=True)
+
+    # Telegram chat metadata
+    username = Column(String, nullable=True)
+    title = Column(String, nullable=True)
+    about = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+
+    @classmethod
+    def scan(cls, row) -> Optional['Portal']:
+        (tgid, tg_receiver, peer_type, megagroup, mxid, config, username, title, about,
+         photo_id) = row
+        return cls(tgid=tgid, tg_receiver=tg_receiver, peer_type=peer_type, megagroup=megagroup,
+                   mxid=mxid, config=config, username=username, title=title, about=about,
+                   photo_id=photo_id)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Portal']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
+        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver))
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixRoomID) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.username == username)
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_receiver == self.tg_receiver)
+
+    def insert(self) -> None:
+        self.db.execute(self.t.insert().values(
+            tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
+            megagroup=self.megagroup, mxid=self.mxid, config=self.config, username=self.username,
+            title=self.title, about=self.about, photo_id=self.photo_id))

mautrix_telegram/db/puppet.py

@@ -0,0 +1,86 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql import expression
+from typing import Optional, Iterable
+
+from ..types import MatrixUserID, MatrixRoomID, TelegramID
+from .base import Base
+
+
+class Puppet(Base):
+    __tablename__ = "puppet"
+
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    custom_mxid = Column(String, nullable=True)  # type: Optional[MatrixUserID]
+    access_token = Column(String, nullable=True)
+    displayname = Column(String, nullable=True)
+    displayname_source = Column(Integer, nullable=True)  # type: Optional[TelegramID]
+    username = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+    is_bot = Column(Boolean, nullable=True)
+    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
+
+    @classmethod
+    def scan(cls, row) -> Optional['Puppet']:
+        (id, custom_mxid, access_token, displayname, displayname_source, username, photo_id,
+         is_bot, matrix_registered) = row
+        return cls(id=id, custom_mxid=custom_mxid, access_token=access_token,
+                   displayname=displayname, displayname_source=displayname_source,
+                   username=username, photo_id=photo_id, is_bot=is_bot,
+                   matrix_registered=matrix_registered)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Puppet']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
+
+    @classmethod
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.custom_mxid != None))
+        for row in rows:
+            yield cls.scan(row)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.id == tgid)
+
+    @classmethod
+    def get_by_custom_mxid(cls, mxid: MatrixUserID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.custom_mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.username == username)
+
+    @classmethod
+    def get_by_displayname(cls, displayname: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.displayname == displayname)
+
+    @property
+    def _edit_identity(self):
+        return self.c.id == self.id
+
+    def insert(self) -> None:
+        self.db.execute(self.t.insert().values(
+            id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
+            displayname=self.displayname, displayname_source=self.displayname_source,
+            username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
+            matrix_registered=self.matrix_registered))

mautrix_telegram/db/room_state.py

@@ -0,0 +1,60 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, Text
+from typing import Dict, Optional
+import json
+
+from ..types import MatrixRoomID
+from .base import Base
+
+
+class RoomState(Base):
+    __tablename__ = "mx_room_state"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    power_levels = Column("power_levels", Text, nullable=True)  # type: Optional[Dict]
+
+    @property
+    def _power_levels_text(self) -> Optional[str]:
+        return json.dumps(self.power_levels) if self.power_levels else None
+
+    @property
+    def has_power_levels(self) -> bool:
+        return bool(self.power_levels)
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID) -> Optional['RoomState']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.room_id == room_id))
+        try:
+            room_id, power_levels_text = next(rows)
+            return cls(room_id=room_id, power_levels=(json.loads(power_levels_text)
+                                                      if power_levels_text else None))
+        except StopIteration:
+            return None
+
+    def update(self) -> None:
+        self.db.execute(self.t.update()
+                        .where(self.c.room_id == self.room_id)
+                        .values(power_levels=self._power_levels_text))
+
+    @property
+    def _edit_identity(self):
+        return self.c.room_id == self.room_id
+
+    def insert(self) -> None:
+        self.db.execute(self.t.insert().values(room_id=self.room_id,
+                                               power_levels=self._power_levels_text))

mautrix_telegram/db/telegram_file.py

@@ -0,0 +1,55 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
+from sqlalchemy.orm import relationship
+from typing import Optional
+
+from .base import Base
+
+
+class TelegramFile(Base):
+    __tablename__ = "telegram_file"
+
+    id = Column(String, primary_key=True)
+    mxc = Column(String)
+    mime_type = Column(String)
+    was_converted = Column(Boolean)
+    timestamp = Column(BigInteger)
+    size = Column(Integer, nullable=True)
+    width = Column(Integer, nullable=True)
+    height = Column(Integer, nullable=True)
+    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+    thumbnail = relationship("TelegramFile", uselist=False)
+
+    @classmethod
+    def get(cls, id: str) -> Optional['TelegramFile']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.id == id))
+        try:
+            id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
+            thumb = None
+            if thumb_id:
+                thumb = cls.get(thumb_id)
+            return cls(id=id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
+                       size=s, width=w, height=h, thumbnail_id=thumb_id, thumbnail=thumb)
+        except StopIteration:
+            return None
+
+    def insert(self) -> None:
+        self.db.execute(self.t.insert().values(
+            id=self.id, mxc=self.mxc, mime_type=self.mime_type, was_converted=self.was_converted,
+            timestamp=self.timestamp, size=self.size, width=self.width, height=self.height,
+            thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/user.py

@@ -0,0 +1,128 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional, Iterable, Tuple
+
+from ..types import MatrixUserID, MatrixRoomID, TelegramID
+from .base import Base
+
+
+class User(Base):
+    __tablename__ = "user"
+
+    mxid = Column(String, primary_key=True)  # type: MatrixUserID
+    tgid = Column(Integer, nullable=True, unique=True)  # type: Optional[TelegramID]
+    tg_username = Column(String, nullable=True)
+    tg_phone = Column(String, nullable=True)
+    saved_contacts = Column(Integer, default=0, nullable=False)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['User']:
+        try:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = next(rows)
+            return cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                       saved_contacts=saved_contacts)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def all(cls) -> Iterable['User']:
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = row
+            yield cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                      saved_contacts=saved_contacts)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tgid == tgid)
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixUserID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tg_username == username)
+
+    @property
+    def _edit_identity(self):
+        return self.c.mxid == self.mxid
+
+    def insert(self) -> None:
+        self.db.execute(self.t.insert().values(
+            mxid=self.mxid, tgid=self.tgid, tg_username=self.tg_username, tg_phone=self.tg_phone,
+            saved_contacts=self.saved_contacts))
+
+    @property
+    def contacts(self) -> Iterable[TelegramID]:
+        rows = self.db.execute(Contact.t.select().where(Contact.c.user == self.tgid))
+        for row in rows:
+            user, contact = row
+            yield contact
+
+    @contacts.setter
+    def contacts(self, puppets: Iterable[TelegramID]) -> None:
+        self.db.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
+        if puppets:
+            self.db.execute(Contact.t.insert(), [{"user": self.tgid, "contact": tgid}
+                                                 for tgid in puppets])
+
+    @property
+    def portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        rows = self.db.execute(UserPortal.t.select().where(UserPortal.c.user == self.tgid))
+        for row in rows:
+            user, portal, portal_receiver = row
+            yield (portal, portal_receiver)
+
+    @portals.setter
+    def portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
+        self.db.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
+        if portals:
+            self.db.execute(UserPortal.t.insert(),
+                            [{
+                                "user": self.tgid,
+                                "portal": tgid,
+                                "portal_receiver": tg_receiver
+                            } for tgid, tg_receiver in portals])
+
+    def delete(self) -> None:
+        super().delete()
+        self.portals = None
+        self.contacts = None
+
+
+class UserPortal(Base):
+    __tablename__ = "user_portal"
+
+    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
+                  primary_key=True)  # type: TelegramID
+    portal = Column(Integer, primary_key=True)  # type: TelegramID
+    portal_receiver = Column(Integer, primary_key=True)  # type: TelegramID
+
+    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
+                                           ("portal.tgid", "portal.tg_receiver"),
+                                           onupdate="CASCADE", ondelete="CASCADE"),)
+
+
+class Contact(Base):
+    __tablename__ = "contact"
+
+    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)  # type: TelegramID
+    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)  # type: TelegramID
+

mautrix_telegram/db/user_profile.py

@@ -0,0 +1,68 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, and_
+from typing import Dict, Optional
+
+from ..types import MatrixUserID, MatrixRoomID
+from .base import Base
+
+
+class UserProfile(Base):
+    __tablename__ = "mx_user_profile"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    user_id = Column(String, primary_key=True)  # type: MatrixUserID
+    membership = Column(String, nullable=False, default="leave")
+    displayname = Column(String, nullable=True)
+    avatar_url = Column(String, nullable=True)
+
+    def dict(self) -> Dict[str, str]:
+        return {
+            "membership": self.membership,
+            "displayname": self.displayname,
+            "avatar_url": self.avatar_url,
+        }
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID, user_id: MatrixUserID) -> Optional['UserProfile']:
+        rows = cls.db.execute(
+            cls.t.select().where(and_(cls.c.room_id == room_id, cls.c.user_id == user_id)))
+        try:
+            room_id, user_id, membership, displayname, avatar_url = next(rows)
+            return cls(room_id=room_id, user_id=user_id, membership=membership,
+                       displayname=displayname, avatar_url=avatar_url)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def delete_all(cls, room_id: MatrixRoomID) -> None:
+        cls.db.execute(cls.t.delete().where(cls.c.room_id == room_id))
+
+    def update(self) -> None:
+        super().update(membership=self.membership, displayname=self.displayname,
+                       avatar_url=self.avatar_url)
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.room_id == self.room_id, self.c.user_id == self.user_id)
+
+    def insert(self) -> None:
+        self.db.execute(self.t.insert().values(room_id=self.room_id, user_id=self.user_id,
+                                               membership=self.membership,
+                                               displayname=self.displayname,
+                                               avatar_url=self.avatar_url))
+

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -26,12 +26,7 @@ from ...types import TelegramID, MatrixRoomID
 from ...db import Message as DBMessage
 from ..util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
                     trim_reply_fallback_text)
-from .parser_common import ParsedMessage
-
-try:
-    from mautrix_telegram.formatter.from_matrix.parser_lxml import parse_html
-except ImportError:
-    from mautrix_telegram.formatter.from_matrix.parser_htmlparser import parse_html
+from .parser import ParsedMessage, parse_html
 
 if TYPE_CHECKING:
     from ...context import Context
@@ -94,7 +89,9 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
 def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
                              room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
     try:
-        reply = content["m.relates_to"]["m.in_reply_to"]
+        reply = content.get("m.relates_to", {}).get("m.in_reply_to", {})
+        if not reply:
+            return None
         room_id = room_id or reply["room_id"]
         event_id = reply["event_id"]
 

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -0,0 +1,4 @@
+try:
+    from .html_reader_lxml import HTMLNode, read_html
+except ImportError:
+    from .html_reader_htmlparser import HTMLNode, read_html

mautrix_telegram/formatter/from_matrix/html_reader.pyi

@@ -0,0 +1,11 @@
+from typing import Dict, List
+
+
+class HTMLNode(List['HTMLNode']):
+    tag: str
+    text: str
+    tail: str
+    attrib: Dict[str, str]
+
+
+def read_html(data: str) -> HTMLNode: ...

mautrix_telegram/formatter/from_matrix/html_reader_htmlparser.py

@@ -0,0 +1,58 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Tuple
+
+from html.parser import HTMLParser
+
+
+class HTMLNode(list):
+    def __init__(self, tag: str, attrs: List[Tuple[str, str]]):
+        super().__init__()
+        self.tag = tag  # type: str
+        self.text = ""  # type: str
+        self.tail = ""  # type: str
+        self.attrib = dict(attrs)  # type: Dict[str, str]
+
+
+class NodeifyingParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
+
+    def handle_starttag(self, tag, attrs):
+        node = HTMLNode(tag, attrs)
+        self.stack[-1].append(node)
+        self.stack.append(node)
+
+    def handle_endtag(self, tag):
+        if tag == self.stack[-1].tag:
+            self.stack.pop()
+
+    def handle_data(self, data):
+        if len(self.stack[-1]) > 0:
+            self.stack[-1][-1].tail += data
+        else:
+            self.stack[-1].text += data
+
+    def error(self, message):
+        pass
+
+
+def read_html(data: str) -> HTMLNode:
+    parser = NodeifyingParser()
+    parser.feed(data)
+    return parser.stack[0]

mautrix_telegram/formatter/from_matrix/html_reader_lxml.py

@@ -0,0 +1,23 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from lxml import html
+
+HTMLNode = html.HtmlElement
+
+
+def read_html(data: str) -> HTMLNode:
+    return html.fromstring(data)

mautrix_telegram/formatter/from_matrix/parser.py

@@ -14,21 +14,25 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Tuple
-from lxml import html
+from typing import List, Tuple, Pattern
+import re
 
 from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
                                MessageEntityMentionName as MentionName, MessageEntityEmail as Email,
                                MessageEntityUrl as URL, MessageEntityTextUrl as TextURL,
                                MessageEntityBold as Bold, MessageEntityItalic as Italic,
-                               MessageEntityCode as Code, MessageEntityPre as Pre)
+                               MessageEntityCode as Code, MessageEntityPre as Pre,
+                               TypeMessageEntity)
 
 from ... import user as u, puppet as pu, portal as po
 from ...types import MatrixUserID
 from ..util import html_to_unicode
-from .parser_common import MatrixParserCommon, ParsedMessage
 from .telegram_message import TelegramMessage, Entity, offset_length_multiply
 
+from .html_reader import HTMLNode, read_html
+
+ParsedMessage = Tuple[str, List[TypeMessageEntity]]
+
 
 def parse_html(input_html: str) -> ParsedMessage:
     return MatrixParser.parse(input_html)
@@ -52,9 +56,21 @@ class RecursionContext:
         return RecursionContext(strip_linebreaks=False, ul_depth=self.ul_depth)
 
 
-class MatrixParser(MatrixParserCommon):
+class MatrixParser:
+    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")  # type: Pattern
+    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")  # type: Pattern
+    block_tags = ("p", "pre", "blockquote",
+                  "ol", "ul", "li",
+                  "h1", "h2", "h3", "h4", "h5", "h6",
+                  "div", "hr", "table")  # type: Tuple[str, ...]
+    list_bullets = ("●", "○", "■", "‣")  # type: Tuple[str, ...]
+
     @classmethod
-    def list_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def list_bullet(cls, depth: int) -> str:
+        return cls.list_bullets[(depth - 1) % len(cls.list_bullets)] + " "
+
+    @classmethod
+    def list_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         ordered = node.tag == "ol"
         tagged_children = cls.node_to_tagged_tmessages(node, ctx)
         counter = 1
@@ -86,23 +102,21 @@ class MatrixParser(MatrixParserCommon):
         return TelegramMessage.join(children, "\n")
 
     @classmethod
-    def blockquote_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext
-                               ) -> TelegramMessage:
+    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         msg = cls.tag_aware_parse_node(node, ctx)
         children = msg.trim().split("\n")
         children = [child.prepend("> ") for child in children]
         return TelegramMessage.join(children, "\n")
 
     @classmethod
-    def header_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def header_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         children = cls.node_to_tmessages(node, ctx)
         length = int(node.tag[1])
         prefix = "#" * length + " "
         return TelegramMessage.join(children, "").prepend(prefix).format(Bold)
 
     @classmethod
-    def basic_format_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext
-                                 ) -> TelegramMessage:
+    def basic_format_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         msg = cls.tag_aware_parse_node(node, ctx)
         if node.tag in ("b", "strong"):
             msg.format(Bold)
@@ -121,7 +135,7 @@ class MatrixParser(MatrixParserCommon):
         return msg
 
     @classmethod
-    def link_to_tstring(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def link_to_tstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         msg = cls.tag_aware_parse_node(node, ctx)
         href = node.attrib.get("href", "")
         if not href:
@@ -140,8 +154,8 @@ class MatrixParser(MatrixParserCommon):
             if user.username:
                 return TelegramMessage(f"@{user.username}").format(Mention)
             elif user.tgid:
-                return TelegramMessage(user.displayname or msg.text).format(MentionName,
-                                                                            user_id=user.tgid)
+                displayname = user.plain_displayname or msg.text
+                return TelegramMessage(displayname).format(MentionName, user_id=user.tgid)
             return msg
 
         room = cls.room_regex.match(href)
@@ -156,7 +170,7 @@ class MatrixParser(MatrixParserCommon):
                 else msg.format(TextURL, url=href))
 
     @classmethod
-    def node_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def node_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         if node.tag == "blockquote":
             return cls.blockquote_to_tmessage(node, ctx)
         elif node.tag == "ol":
@@ -193,7 +207,7 @@ class MatrixParser(MatrixParserCommon):
         return TelegramMessage(text)
 
     @classmethod
-    def node_to_tagged_tmessages(cls, node: html.HtmlElement, ctx: RecursionContext
+    def node_to_tagged_tmessages(cls, node: HTMLNode, ctx: RecursionContext
                                  ) -> List[Tuple[TelegramMessage, str]]:
         output = []
 
@@ -206,12 +220,12 @@ class MatrixParser(MatrixParserCommon):
         return output
 
     @classmethod
-    def node_to_tmessages(cls, node: html.HtmlElement, ctx: RecursionContext
+    def node_to_tmessages(cls, node: HTMLNode, ctx: RecursionContext
                           ) -> List[TelegramMessage]:
         return [msg for (msg, tag) in cls.node_to_tagged_tmessages(node, ctx)]
 
     @classmethod
-    def tag_aware_parse_node(cls, node: html.HtmlElement, ctx: RecursionContext
+    def tag_aware_parse_node(cls, node: HTMLNode, ctx: RecursionContext
                              ) -> TelegramMessage:
         msgs = cls.node_to_tagged_tmessages(node, ctx)
         output = TelegramMessage()
@@ -226,11 +240,10 @@ class MatrixParser(MatrixParserCommon):
         return output.trim()
 
     @classmethod
-    def parse_node(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def parse_node(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         return TelegramMessage.join(cls.node_to_tmessages(node, ctx))
 
     @classmethod
     def parse(cls, data: str) -> ParsedMessage:
-        document = html.fromstring(f"<html>{data}</html>")
-        msg = cls.parse_node(document, RecursionContext())
+        msg = cls.node_to_tmessage(read_html(f"<body>{data}</body>"), RecursionContext())
         return msg.text, msg.entities

mautrix_telegram/formatter/from_matrix/parser_htmlparser.py

@@ -1,241 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import (Optional, List, Tuple, Type, Dict, Any, TYPE_CHECKING, Match)
-from html import unescape
-from html.parser import HTMLParser
-from collections import deque
-import math
-
-from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityEmail,
-                               MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
-                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
-                               MessageEntityBotCommand, TypeMessageEntity)
-
-from ... import user as u, puppet as pu, portal as po
-from ...types import MatrixUserID
-from ..util import html_to_unicode
-from .parser_common import MatrixParserCommon, ParsedMessage
-
-if TYPE_CHECKING:
-    from typing import Deque
-
-
-def parse_html(html: str) -> ParsedMessage:
-    parser = MatrixParser()
-    parser.feed(html)
-    return parser.text, parser.entities
-
-
-class MatrixParser(HTMLParser, MatrixParserCommon):
-    def __init__(self):
-        super(MatrixParser, self).__init__()
-        self.text = ""  # type: str
-        self.entities = []  # type: List[TypeMessageEntity]
-        self._building_entities = {}  # type: Dict[str, TypeMessageEntity]
-        self._list_counter = 0  # type: int
-        self._open_tags = deque()  # type: Deque[str]
-        self._open_tags_meta = deque()  # type: Deque[Any]
-        self._line_is_new = True  # type: bool
-        self._list_entry_is_new = False  # type: bool
-
-    def _parse_url(self, url: str, args: Dict[str, Any]
-                   ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]:
-        mention = self.mention_regex.match(url)  # type: Match
-        if mention:
-            mxid = MatrixUserID(mention.group(1))
-            user = (pu.Puppet.get_by_mxid(mxid)
-                    or u.User.get_by_mxid(mxid, create=False))
-            if not user:
-                return None, None
-            if user.username:
-                return MessageEntityMention, f"@{user.username}"
-            elif user.tgid:
-                args["user_id"] = user.tgid
-                return MessageEntityMentionName, user.displayname or None
-            else:
-                return None, None
-
-        room = self.room_regex.match(url)  # type: Match
-        if room:
-            username = po.Portal.get_username_from_mx_alias(room.group(1))
-            portal = po.Portal.find_by_username(username)
-            if portal and portal.username:
-                return MessageEntityMention, f"@{portal.username}"
-
-        if url.startswith("mailto:"):
-            return MessageEntityEmail, url[len("mailto:"):]
-        elif self.get_starttag_text() == url:
-            return MessageEntityUrl, url
-        else:
-            args["url"] = url
-            return MessageEntityTextUrl, None
-
-    def handle_starttag(self, tag: str, attrs_list: List[Tuple[str, str]]):
-        self._open_tags.appendleft(tag)
-        self._open_tags_meta.appendleft(0)
-
-        attrs = dict(attrs_list)
-        entity_type = None  # type: Optional[Type[TypeMessageEntity]]
-        args = {}  # type: Dict[str, Any]
-        if tag in ("strong", "b"):
-            entity_type = MessageEntityBold
-        elif tag in ("em", "i"):
-            entity_type = MessageEntityItalic
-        elif tag == "code":
-            try:
-                pre = self._building_entities["pre"]
-                try:
-                    # Pre tag and language found, add language to MessageEntityPre
-                    pre.language = attrs["class"][len("language-"):]
-                except KeyError:
-                    # Pre tag found, but language not found, keep pre as-is
-                    pass
-            except KeyError:
-                # No pre tag found, this is inline code
-                entity_type = MessageEntityCode
-        elif tag == "pre":
-            entity_type = MessageEntityPre
-            args["language"] = ""
-        elif tag == "command":
-            entity_type = MessageEntityBotCommand
-        elif tag == "li":
-            self._list_entry_is_new = True
-        elif tag == "a":
-            try:
-                url = attrs["href"]
-            except KeyError:
-                return
-            entity_type, url = self._parse_url(url, args)
-            self._open_tags_meta.popleft()
-            self._open_tags_meta.appendleft(url)
-
-        if (tag in self.block_tags and ("blockquote" not in self._open_tags)) or tag == "br":
-            self._newline()
-
-        if entity_type and tag not in self._building_entities:
-            offset = len(self.text)
-            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
-
-    @property
-    def _list_indent(self) -> int:
-        indent = 0
-        first_skipped = False
-        for index, tag in enumerate(self._open_tags):
-            if not first_skipped and tag in ("ol", "ul"):
-                # The first list level isn't indented, so skip it.
-                first_skipped = True
-                continue
-            if tag == "ol":
-                n = self._open_tags_meta[index]
-                extra_length_for_long_index = (int(math.log(n, 10)) - 1) * 3
-                indent += 4 + extra_length_for_long_index
-            elif tag == "ul":
-                indent += 3
-        return indent
-
-    def _newline(self, allow_multi: bool = False):
-        if self._line_is_new and not allow_multi:
-            return
-        self.text += "\n"
-        self._line_is_new = True
-        for entity in self._building_entities.values():
-            entity.length += 1
-
-    def _handle_special_previous_tags(self, text: str) -> str:
-        if "pre" not in self._open_tags and "code" not in self._open_tags:
-            text = text.replace("\n", "")
-        else:
-            text = text.strip()
-
-        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
-        if previous_tag == "a":
-            url = self._open_tags_meta[0]
-            if url:
-                text = url
-        elif previous_tag == "command":
-            text = f"/{text}"
-        return text
-
-    def _html_to_unicode(self, text: str) -> str:
-        strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags
-        if strikethrough and underline:
-            text = html_to_unicode(text, "\u0336\u0332")
-        elif strikethrough:
-            text = html_to_unicode(text, "\u0336")
-        elif underline:
-            text = html_to_unicode(text, "\u0332")
-        return text
-
-    def _handle_tags_for_data(self, text: str) -> Tuple[str, int]:
-        extra_offset = 0
-        list_entry_handled_once = False
-        # In order to maintain order of things like blockquotes in lists or lists in blockquotes,
-        # we can't just have ifs/elses and we need to actually loop through the open tags in order.
-        for index, tag in enumerate(self._open_tags):
-            if tag == "blockquote" and self._line_is_new:
-                text = f"> {text}"
-                extra_offset += 2
-            elif tag == "li" and not list_entry_handled_once:
-                list_type_index = index + 1
-                list_type = self._open_tags[list_type_index]
-                indent = self._list_indent * " " if self._line_is_new else ""
-                if list_type == "ol":
-                    n = self._open_tags_meta[list_type_index]
-                    if self._list_entry_is_new:
-                        n += 1
-                        self._open_tags_meta[list_type_index] = n
-                        prefix = f"{n}. "
-                    else:
-                        prefix = int(math.log(n, 10)) * 3 * " " + 4 * " "
-                else:
-                    prefix = (self.list_bullet(self._open_tags.count('ul'))
-                              if self._list_entry_is_new else 3 * " ")
-                if not self._list_entry_is_new and not self._line_is_new:
-                    prefix = ""
-                extra_offset += len(indent) + len(prefix)
-                text = indent + prefix + text
-                self._list_entry_is_new = False
-                list_entry_handled_once = True
-        return text, extra_offset
-
-    def _extend_entities_in_construction(self, text: str, extra_offset: int):
-        for tag, entity in self._building_entities.items():
-            entity.length += len(text) - extra_offset
-            entity.offset += extra_offset
-
-    def handle_data(self, text: str):
-        text = unescape(text)
-        text = self._handle_special_previous_tags(text)
-        text = self._html_to_unicode(text)
-        text, extra_offset = self._handle_tags_for_data(text)
-        self._extend_entities_in_construction(text, extra_offset)
-        self._line_is_new = False
-        self.text += text
-
-    def handle_endtag(self, tag: str):
-        try:
-            self._open_tags.popleft()
-            self._open_tags_meta.popleft()
-        except IndexError:
-            pass
-
-        entity = self._building_entities.pop(tag, None)
-        if entity:
-            self.entities.append(entity)
-
-        if tag in self.block_tags and tag != "br" and "blockquote" not in self._open_tags:
-            self._newline(allow_multi=tag == "br")

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -153,5 +153,6 @@ class TelegramMessage:
                 msg = TelegramMessage(text=msg)
             main.entities += Entity.adjust(msg.entities, offset_diff(len(main.text)))
             main.text += msg.text + separator
-        main.text = main.text[:-len(separator)]
+        if len(separator) > 0:
+            main.text = main.text[:-len(separator)]
         return main

mautrix_telegram/formatter/from_telegram.py

@@ -87,6 +87,19 @@ async def _add_forward_header(source, text: str, html: Optional[str],
             if user:
                 fwd_from_text = pu.Puppet.get_displayname(user, False)
                 fwd_from_html = f"<b>{fwd_from_text}</b>"
+    else:
+        portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
+        if portal:
+            fwd_from_text = portal.title
+            if portal.alias:
+                fwd_from_html = f"<a href='https://matrix.to/#/{portal.alias}'>{fwd_from_text}</a>"
+            else:
+                fwd_from_html = f"<b>{fwd_from_text}</b>"
+        else:
+            channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
+            if channel:
+                fwd_from_text = channel.title
+                fwd_from_html = f"<b>{fwd_from_text}</b>"
 
     if not fwd_from_text:
         if fwd_from.from_id:
@@ -179,9 +192,12 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 async def telegram_to_matrix(evt: Message, source: "AbstractUser",
                              main_intent: Optional[IntentAPI] = None,
                              is_edit: bool = False, prefix_text: Optional[str] = None,
-                             prefix_html: Optional[str] = None) -> Tuple[str, str, Dict]:
-    text = add_surrogates(evt.message)
-    html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None
+                             prefix_html: Optional[str] = None, override_text: str = None,
+                             override_entities: List[TypeMessageEntity] = None
+                             ) -> Tuple[str, str, Dict]:
+    text = add_surrogates(override_text or evt.message)
+    entities = override_entities or evt.entities
+    html = _telegram_entities_to_matrix_catch(text, entities) if entities else None
     relates_to = {}  # type: Dict
 
     if prefix_html:

mautrix_telegram/matrix.py

@@ -222,7 +222,7 @@ class MatrixHandler:
         sender = await u.User.get_by_mxid(sender_id).ensure_started()
         if not sender.relaybot_whitelisted:
             self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:"
-                           " u.User is not whitelisted.")
+                           " User is not whitelisted.")
             return
         self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}")
 
@@ -296,11 +296,17 @@ class MatrixHandler:
             events = new_events - old_events
             if len(events) > 0:
                 # New event pinned, set that as pinned in Telegram.
-                await portal.handle_matrix_pin(sender, events.pop())
+                await portal.handle_matrix_pin(sender, MatrixEventID(events.pop()))
             elif len(new_events) == 0:
                 # All pinned events removed, remove pinned event in Telegram.
                 await portal.handle_matrix_pin(sender, None)
 
+    @staticmethod
+    async def handle_room_upgrade(room_id: MatrixRoomID, new_room_id: MatrixRoomID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if portal:
+            await portal.handle_matrix_upgrade(new_room_id)
+
     @staticmethod
     async def handle_name_change(room_id: MatrixRoomID, user_id: MatrixUserID, displayname: str,
                                  prev_displayname: str, event_id: MatrixEventID) -> None:
@@ -416,6 +422,8 @@ class MatrixHandler:
             except KeyError:
                 old_events = set()
             await self.handle_room_pin(room_id, sender, new_events, old_events)
+        elif evt_type == "m.room.tombstone":
+            await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
         elif evt_type == "m.receipt":
             await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
         elif evt_type == "m.presence":

mautrix_telegram/portal.py

@@ -22,50 +22,52 @@ from html import escape as escape_html
 import asyncio
 import random
 import mimetypes
+import codecs
 import unicodedata
+import base64
 import hashlib
 import logging
 import json
 import re
 
 import magic
-from sqlalchemy import orm
-from sqlalchemy.exc import IntegrityError, InvalidRequestError
-from sqlalchemy.orm.exc import FlushError
+from sqlalchemy.exc import IntegrityError
 
 from telethon.tl.functions.messages import (
     AddChatUserRequest, CreateChatRequest, DeleteChatUserRequest, EditChatAdminRequest,
     EditChatPhotoRequest, EditChatTitleRequest, ExportChatInviteRequest, GetFullChatRequest,
-    MigrateChatRequest, SetTypingRequest)
+    UpdatePinnedMessageRequest, MigrateChatRequest, SetTypingRequest, EditChatAboutRequest)
 from telethon.tl.functions.channels import (
-    CreateChannelRequest, EditAboutRequest, EditAdminRequest, EditBannedRequest, EditPhotoRequest,
-    EditTitleRequest, ExportInviteRequest, GetParticipantsRequest, InviteToChannelRequest,
-    JoinChannelRequest, LeaveChannelRequest, UpdatePinnedMessageRequest, UpdateUsernameRequest)
+    CreateChannelRequest, EditAdminRequest, EditBannedRequest, EditPhotoRequest,
+    EditTitleRequest, GetParticipantsRequest, InviteToChannelRequest,
+    JoinChannelRequest, LeaveChannelRequest, UpdateUsernameRequest)
 from telethon.tl.functions.messages import ReadHistoryRequest as ReadMessageHistoryRequest
 from telethon.tl.functions.channels import ReadHistoryRequest as ReadChannelHistoryRequest
 from telethon.errors import ChatAdminRequiredError, ChatNotModifiedError
+from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
-    Channel, ChannelAdminRights, ChannelBannedRights, ChannelFull, ChannelParticipantAdmin,
+    Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin,
     ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat,
     ChatFull, ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto,
     DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker,
     DocumentAttributeVideo, FileLocation, GeoPoint, InputChannel, InputChatUploadedPhoto,
-    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf, Message,
+    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf,
     MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate,
     MessageActionChatDeletePhoto, MessageActionChatDeleteUser, MessageActionChatEditPhoto,
     MessageActionChatEditTitle, MessageActionChatJoinedByLink, MessageActionChatMigrateTo,
-    MessageActionPinMessage, MessageMediaContact, MessageMediaDocument, MessageMediaGeo,
-    MessageMediaPhoto, MessageService, PeerChannel, PeerChat, PeerUser, Photo, PhotoCachedSize,
-    SendMessageCancelAction, SendMessageTypingAction, TypeChannelParticipant, TypeChat,
-    TypeChatParticipant, TypeDocumentAttribute, TypeInputPeer, TypeMessageAction,
-    TypeMessageEntity, TypePeer, TypePhotoSize, TypeUpdates, TypeUser, TypeUserFull,
-    UpdateChatUserTyping, UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping, User,
-    UserFull)
+    MessageActionPinMessage, MessageActionGameScore, MessageMediaContact, MessageMediaDocument,
+    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, PeerChannel,
+    PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction, SendMessageTypingAction,
+    TypeChannelParticipant, TypeChat, TypeChatParticipant, TypeDocumentAttribute, TypeInputPeer,
+    TypeMessageAction, TypeMessageEntity, TypePeer, TypePhotoSize, TypeUpdates, TypeUser, PhotoSize,
+    TypeUserFull, UpdateChatUserTyping, UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping,
+    User, UserFull, MessageEntityPre)
 from mautrix_appservice import MatrixRequestError, IntentError, AppService, IntentAPI
 
 from .types import MatrixEventID, MatrixRoomID, MatrixUserID, TelegramID
 from .context import Context
 from .db import Portal as DBPortal, Message as DBMessage, TelegramFile as DBTelegramFile
+from .util import ignore_coro
 from . import puppet as p, user as u, formatter, util
 
 if TYPE_CHECKING:
@@ -85,8 +87,7 @@ InviteList = Union[MatrixUserID, List[MatrixUserID]]
 
 
 class Portal:
-    log = logging.getLogger("mau.portal")  # type: logging.Logger
-    db = None  # type: orm.Session
+    base_log = logging.getLogger("mau.portal")  # type: logging.Logger
     az = None  # type: AppService
     bot = None  # type: Bot
     loop = None  # type: asyncio.AbstractEventLoop
@@ -98,6 +99,7 @@ class Portal:
     public_portals = False  # type: bool
     max_initial_member_sync = -1  # type: int
     sync_channel_members = True  # type: bool
+    sync_matrix_state = True  # type: bool
 
     dedup_pre_db_check = False  # type: bool
     dedup_cache_queue_length = 20  # type: int
@@ -114,7 +116,7 @@ class Portal:
                  mxid: Optional[MatrixRoomID] = None, username: Optional[str] = None,
                  megagroup: Optional[bool] = False, title: Optional[str] = None,
                  about: Optional[str] = None, photo_id: Optional[str] = None,
-                 config: Optional[str] = None, db_instance: DBPortal = None) -> None:
+                 local_config: Optional[str] = None, db_instance: DBPortal = None) -> None:
         self.mxid = mxid  # type: Optional[MatrixRoomID]
         self.tgid = tgid  # type: TelegramID
         self.tg_receiver = tg_receiver or tgid  # type: TelegramID
@@ -124,13 +126,15 @@ class Portal:
         self.title = title  # type: Optional[str]
         self.about = about  # type: str
         self.photo_id = photo_id  # type: str
-        self.local_config = json.loads(config or "{}")  # type: Dict[str, Any]
+        self.local_config = json.loads(local_config or "{}")  # type: Dict[str, Any]
         self._db_instance = db_instance  # type: DBPortal
         self.deleted = False  # type: bool
+        self.log = self.base_log.getChild(self.tgid_log) if self.tgid else self.base_log
 
         self._main_intent = None  # type: IntentAPI
         self._room_create_lock = asyncio.Lock()  # type: asyncio.Lock
         self._temp_pinned_message_id = None  # type: Optional[int]
+        self._temp_pinned_message_id_space = None  # type: Optional[TelegramID]
         self._temp_pinned_message_sender = None  # type: Optional[p.Puppet]
 
         self._dedup = deque()  # type: deque
@@ -279,15 +283,19 @@ class Portal:
             del self._dedup_mxid[self._dedup.popleft()]
         return None
 
-    def get_input_entity(self, user: 'u.User') -> Awaitable[TypeInputPeer]:
+    def get_input_entity(self, user: 'AbstractUser') -> Awaitable[TypeInputPeer]:
         return user.client.get_input_entity(self.peer)
 
-    async def get_entity(self, user: 'u.User') -> TypeChat:
+    async def get_entity(self, user: 'AbstractUser') -> TypeChat:
         try:
             return await user.client.get_entity(self.peer)
         except ValueError:
-            self.log.warning(f"Could not find entity for {self.tgid_log} with user {user.tgid} "
-                             f"falling back to get_dialogs.")
+            if user.is_bot:
+                self.log.warning(f"Could not find entity with bot {user.tgid}. "
+                                 "Failing...")
+                raise
+            self.log.warning(f"Could not find entity with user {user.tgid}. "
+                             "falling back to get_dialogs.")
             async for dialog in user.client.iter_dialogs():
                 if dialog.entity.id == self.tgid:
                     return dialog.entity
@@ -320,8 +328,10 @@ class Portal:
                 puppet = p.Puppet.get(self.tgid)
             await puppet.update_info(user, entity)
             await puppet.intent.join_room(self.mxid)
+        if self.sync_matrix_state:
+            await self.sync_matrix_members()
 
-    async def create_matrix_room(self, user: "AbstractUser", entity: TypeChat = None,
+    async def create_matrix_room(self, user: 'AbstractUser', entity: TypeChat = None,
                                  invites: InviteList = None, update_if_exists: bool = True,
                                  synchronous: bool = False) -> Optional[str]:
         if self.mxid:
@@ -332,7 +342,7 @@ class Portal:
                 if synchronous:
                     await update
                 else:
-                    asyncio.ensure_future(update, loop=self.loop)
+                    ignore_coro(asyncio.ensure_future(update, loop=self.loop))
                 await self.invite_to_matrix(invites or [])
             return self.mxid
         async with self._room_create_lock:
@@ -352,7 +362,7 @@ class Portal:
             entity = await self.get_entity(user)
             self.log.debug("Fetched data: %s", entity)
 
-        self.log.debug(f"Creating room for {self.tgid_log}")
+        self.log.debug(f"Creating room")
 
         try:
             self.title = entity.title
@@ -392,37 +402,52 @@ class Portal:
                                                      is_direct=direct, invitees=invites or [],
                                                      name=self.title, initial_state=initial_state)
         if not room_id:
-            raise Exception(f"Failed to create room for {self.tgid_log}")
+            raise Exception(f"Failed to create room")
 
         self.mxid = MatrixRoomID(room_id)
         self.by_mxid[self.mxid] = self
         self.save()
         self.az.state_store.set_power_levels(self.mxid, power_levels)
         user.register_portal(self)
-        asyncio.ensure_future(self.update_matrix_room(user, entity, direct, puppet,
-                                                      levels=power_levels, users=users,
-                                                      participants=participants),
-                              loop=self.loop)
+        ignore_coro(asyncio.ensure_future(self.update_matrix_room(user, entity, direct, puppet,
+                                                                  levels=power_levels, users=users,
+                                                                  participants=participants),
+                                          loop=self.loop))
 
         return self.mxid
 
     def _get_base_power_levels(self, levels: dict = None, entity: TypeChat = None) -> dict:
         levels = levels or {}
-        power_level_requirement = (0 if self.peer_type == "chat" and not entity.admins_enabled
-                                   else 50)
-        levels["ban"] = 99
-        levels["invite"] = power_level_requirement if self.peer_type == "chat" else 75
-        if "events" not in levels:
-            levels["events"] = {}
-        levels["events"]["m.room.name"] = power_level_requirement
-        levels["events"]["m.room.avatar"] = power_level_requirement
-        levels["events"]["m.room.topic"] = 50 if self.peer_type == "channel" else 99
-        levels["events"]["m.room.power_levels"] = 75
-        levels["events"]["m.room.history_visibility"] = 75
-        levels["state_default"] = 50
-        levels["users_default"] = 0
-        levels["events_default"] = (50 if self.peer_type == "channel" and not entity.megagroup
-                                    else 0)
+        if self.peer_type == "user":
+            levels["ban"] = 100
+            levels["kick"] = 100
+            levels["invite"] = 100
+            levels.setdefault("events", {})
+            levels["events"]["m.room.name"] = 0
+            levels["events"]["m.room.avatar"] = 0
+            levels["events"]["m.room.topic"] = 0
+            levels["state_default"] = 0
+            levels["users_default"] = 0
+            levels["events_default"] = 0
+        else:
+            dbr = entity.default_banned_rights
+            levels["ban"] = 99
+            levels["kick"] = 50
+            levels["invite"] = 50 if dbr.invite_users else 0
+            levels.setdefault("events", {})
+            levels["events"]["m.room.name"] = 50 if dbr.change_info else 0
+            levels["events"]["m.room.avatar"] = 50 if dbr.change_info else 0
+            levels["events"]["m.room.topic"] = 50 if dbr.change_info else 0
+            levels["events"][
+                "m.room.pinned_events"] = 50 if dbr.pin_messages else 0
+            levels["events"]["m.sticker"] = 50 if dbr.send_stickers else 0
+            levels["events"]["m.room.power_levels"] = 75
+            levels["events"]["m.room.history_visibility"] = 75
+            levels["state_default"] = 50
+            levels["users_default"] = 0
+            levels["events_default"] = (50 if (self.peer_type == "channel" and not entity.megagroup
+                                               or entity.default_banned_rights.send_messages)
+                                        else 0)
         if "users" not in levels:
             levels["users"] = {
                 self.main_intent.mxid: 100
@@ -448,13 +473,16 @@ class Portal:
             self.bot.add_chat(self.tgid, self.peer_type)
             return
 
-        user = u.User.get_by_tgid(bot.id)
+        user = u.User.get_by_tgid(TelegramID(bot.id))
         if user and user.is_bot:
             user.register_portal(self)
 
-    async def sync_telegram_users(self, source: "AbstractUser", users: List[User]) -> None:
+    async def sync_telegram_users(self, source: 'AbstractUser', users: List[User]) -> None:
         allowed_tgids = set()
+        skip_deleted = config["bridge.skip_deleted_members"]
         for entity in users:
+            if skip_deleted and entity.deleted:
+                continue
             puppet = p.Puppet.get(TelegramID(entity.id))
             if entity.bot:
                 self.add_bot_chat(entity)
@@ -462,7 +490,7 @@ class Portal:
             await puppet.intent.ensure_joined(self.mxid)
             await puppet.update_info(source, entity)
 
-            user = u.User.get_by_tgid(entity.id)
+            user = u.User.get_by_tgid(TelegramID(entity.id))
             if user:
                 await self.invite_to_matrix(user.mxid)
 
@@ -522,12 +550,12 @@ class Portal:
             user.unregister_portal(self)
             await self.main_intent.kick(self.mxid, user.mxid, kick_message)
 
-    async def update_info(self, user: "AbstractUser", entity: TypeChat = None) -> None:
+    async def update_info(self, user: 'AbstractUser', entity: TypeChat = None) -> None:
         if self.peer_type == "user":
-            self.log.warning(f"Called update_info() for direct chat portal {self.tgid_log}")
+            self.log.warning(f"Called update_info() for direct chat portal")
             return
 
-        self.log.debug(f"Updating info of {self.tgid_log}")
+        self.log.debug(f"Updating info")
         if not entity:
             entity = await self.get_entity(user)
             self.log.debug("Fetched data: %s", entity)
@@ -582,22 +610,21 @@ class Portal:
         return False
 
     @staticmethod
-    def _get_largest_photo_size(photo: Photo) -> TypePhotoSize:
-        return max(photo.sizes, key=(lambda photo2: (
-            len(photo2.bytes) if isinstance(photo2, PhotoCachedSize) else photo2.size)))
+    def _get_largest_photo_size(photo: Union[Photo, List[TypePhotoSize]]) -> TypePhotoSize:
+        return max(photo.sizes if isinstance(photo, Photo) else photo, key=(lambda photo2: (
+            len(photo2.bytes) if not isinstance(photo2, PhotoSize) else photo2.size)))
 
-    async def remove_avatar(self, _: "AbstractUser", save: bool = False) -> None:
+    async def remove_avatar(self, _: 'AbstractUser', save: bool = False) -> None:
         await self.main_intent.set_room_avatar(self.mxid, None)
         self.photo_id = None
         if save:
             self.save()
 
-    async def update_avatar(self, user: "AbstractUser", photo: FileLocation,
+    async def update_avatar(self, user: 'AbstractUser', photo: FileLocation,
                             save: bool = False) -> bool:
         photo_id = f"{photo.volume_id}-{photo.local_id}"
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(self.db, user.client, self.main_intent,
-                                                      photo)
+            file = await util.transfer_file_to_matrix(user.client, self.main_intent, photo)
             if file:
                 await self.main_intent.set_room_avatar(self.mxid, file.mxc)
                 self.photo_id = photo_id
@@ -626,7 +653,8 @@ class Portal:
                         entity, ChannelParticipantsRecent(), offset=0, limit=limit, hash=0))
                     return response.users, response.participants
                 elif limit > 200 or limit == -1:
-                    users, participants = [], []  # type: Tuple[List[TypeUser], List[TypeParticipant]]
+                    users = []  # type: List[TypeUser]
+                    participants = []  # type: List[TypeParticipant]
                     offset = 0
                     remaining_quota = limit if limit > 0 else 1000000
                     query = (ChannelParticipantsSearch("") if limit == -1
@@ -652,19 +680,11 @@ class Portal:
     async def get_invite_link(self, user: 'u.User') -> str:
         if self.peer_type == "user":
             raise ValueError("You can't invite users to private chats.")
-        elif self.peer_type == "chat":
-            link = await user.client(ExportChatInviteRequest(chat_id=self.tgid))
-        elif self.peer_type == "channel":
-            if self.username:
-                return f"https://t.me/{self.username}"
-            link = await user.client(
-                ExportInviteRequest(channel=await self.get_input_entity(user)))
-        else:
-            raise ValueError(f"Invalid peer type '{self.peer_type}' for invite link.")
-
+        if self.username:
+            return f"https://t.me/{self.username}"
+        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user)))
         if isinstance(link, ChatInviteEmpty):
             raise ValueError("Failed to get invite link.")
-
         return link.link
 
     async def get_authenticated_matrix_users(self) -> List['u.User']:
@@ -672,13 +692,13 @@ class Portal:
             members = await self.main_intent.get_room_members(self.mxid)
         except MatrixRequestError:
             return []
-        authenticated = []
+        authenticated = []  # type: List[u.User]
         has_bot = self.has_bot
         for member_str in members:
             member = MatrixUserID(member_str)
             if p.Puppet.get_id_from_mxid(member) or member == self.main_intent.mxid:
                 continue
-            user = await u.User.get_by_mxid(member).ensure_started()
+            user = await u.User.get_by_mxid(member).ensure_started()  # type: u.User
             authenticated_through_bot = has_bot and user.relaybot_whitelisted
             if authenticated_through_bot or await user.has_full_access(allow_bot=True):
                 authenticated.append(user)
@@ -774,6 +794,19 @@ class Portal:
         return (await self.main_intent.get_displayname(self.mxid, user.mxid)
                 or user.mxid)
 
+    async def sync_matrix_members(self) -> None:
+        resp = await self.main_intent.get_room_joined_memberships(self.mxid)
+        members = resp["joined"]
+        for mxid, info in members.items():
+            member = {
+                "membership": "join",
+            }
+            if "display_name" in info:
+                member["displayname"] = info["display_name"]
+            if "avatar_url" in info:
+                member["avatar_url"] = info["avatar_url"]
+            self.az.state_store.set_member(self.mxid, mxid, member)
+
     def set_typing(self, user: 'u.User', typing: bool = True,
                    action: type = SendMessageTypingAction) -> Awaitable[bool]:
         return user.client(SetTypingRequest(
@@ -801,7 +834,7 @@ class Portal:
             await source.client(DeleteChatUserRequest(chat_id=self.tgid, user_id=user.tgid))
         elif self.peer_type == "channel":
             channel = await self.get_input_entity(source)
-            rights = ChannelBannedRights(datetime.fromtimestamp(0), True)
+            rights = ChatBannedRights(datetime.fromtimestamp(0), True)
             await source.client(EditBannedRequest(channel=channel,
                                                   user_id=user.tgid,
                                                   banned_rights=rights))
@@ -881,8 +914,8 @@ class Portal:
             await self._apply_msg_format(sender, msgtype, message)
 
     @staticmethod
-    def _matrix_event_to_entities(event: Dict[str, Any]) -> Tuple[
-        str, Optional[List[TypeMessageEntity]]]:
+    def _matrix_event_to_entities(event: Dict[str, Any]
+                                  ) -> Tuple[str, Optional[List[TypeMessageEntity]]]:
         try:
             if event.get("format", None) == "org.matrix.custom.html":
                 message, entities = formatter.matrix_to_telegram(event.get("formatted_body", ""))
@@ -914,8 +947,10 @@ class Portal:
                                   reply_to: TelegramID) -> None:
         lock = self.require_send_lock(sender_id)
         async with lock:
+            lp = self.get_config("telegram_link_preview")
             response = await client.send_message(self.peer, message, reply_to=reply_to,
-                                                 parse_mode=self._matrix_event_to_entities)
+                                                 parse_mode=self._matrix_event_to_entities,
+                                                 link_preview=lp)
             self._add_telegram_message_to_db(event_id, space, response)
 
     async def _handle_matrix_file(self, msgtype: str, sender_id: TelegramID,
@@ -976,13 +1011,17 @@ class Portal:
         self.log.debug("Handled Matrix message: %s", response)
         self.is_duplicate(response, (event_id, space))
         DBMessage(
-            tgid=response.id,
+            tgid=TelegramID(response.id),
             tg_space=space,
             mx_room=self.mxid,
             mxid=event_id).insert()
 
     async def handle_matrix_message(self, sender: 'u.User', message: Dict[str, Any],
                                     event_id: MatrixEventID) -> None:
+        if "body" not in message or "msgtype" not in message:
+            self.log.debug(f"Ignoring message {event_id} in {self.mxid} without body or msgtype")
+            return
+
         puppet = p.Puppet.get_by_custom_mxid(sender.mxid)
         if puppet and message.get("net.maunium.telegram.puppet", False):
             self.log.debug("Ignoring puppet-sent message by confirmed puppet user %s", sender.mxid)
@@ -1018,17 +1057,18 @@ class Portal:
 
     async def handle_matrix_pin(self, sender: 'u.User',
                                 pinned_message: Optional[MatrixEventID]) -> None:
-        if self.peer_type != "channel":
+        if self.peer_type != "chat" and self.peer_type != "channel":
             return
         try:
             if not pinned_message:
-                await sender.client(UpdatePinnedMessageRequest(channel=self.peer, id=0))
+                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=0))
             else:
-                message = DBMessage.get_by_mxid(pinned_message, self.mxid, self.tgid)
+                tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
+                message = DBMessage.get_by_mxid(pinned_message, self.mxid, tg_space)
                 if message is None:
                     self.log.warning(f"Could not find pinned {pinned_message} in {self.mxid}")
                     return
-                await sender.client(UpdatePinnedMessageRequest(channel=self.peer, id=message.tgid))
+                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=message.tgid))
         except ChatNotModifiedError:
             pass
 
@@ -1048,11 +1088,10 @@ class Portal:
         elif self.peer_type == "channel":
             moderator = level >= 50
             admin = level >= 75
-            rights = ChannelAdminRights(change_info=moderator, post_messages=moderator,
-                                        edit_messages=moderator, delete_messages=moderator,
-                                        ban_users=moderator, invite_users=moderator,
-                                        invite_link=moderator, pin_messages=moderator,
-                                        add_admins=admin)
+            rights = ChatAdminRights(change_info=moderator, post_messages=moderator,
+                                     edit_messages=moderator, delete_messages=moderator,
+                                     ban_users=moderator, invite_users=moderator,
+                                     pin_messages=moderator, add_admins=admin)
             await sender.client(
                 EditAdminRequest(channel=await self.get_input_entity(sender),
                                  user_id=user_id, admin_rights=rights))
@@ -1076,15 +1115,15 @@ class Portal:
                 await self._update_telegram_power_level(sender, user_id, level)
 
     async def handle_matrix_about(self, sender: 'u.User', about: str) -> None:
-        if self.peer_type not in {"channel"}:
+        if self.peer_type not in ("chat", "channel"):
             return
-        channel = await self.get_input_entity(sender)
-        await sender.client(EditAboutRequest(channel=channel, about=about))
+        peer = await self.get_input_entity(sender)
+        await sender.client(EditChatAboutRequest(peer=peer, about=about))
         self.about = about
         self.save()
 
     async def handle_matrix_title(self, sender: 'u.User', title: str) -> None:
-        if self.peer_type not in {"chat", "channel"}:
+        if self.peer_type not in ("chat", "channel"):
             return
 
         if self.peer_type == "chat":
@@ -1097,14 +1136,14 @@ class Portal:
         self.save()
 
     async def handle_matrix_avatar(self, sender: 'u.User', url: str) -> None:
-        if self.peer_type not in {"chat", "channel"}:
+        if self.peer_type not in ("chat", "channel"):
             # Invalid peer type
             return
 
         file = await self.main_intent.download_file(url)
         mime = magic.from_buffer(file, mime=True)
         ext = mimetypes.guess_extension(mime)
-        uploaded = await sender.client.upload_file_direct(file, file_name=f"avatar{ext}")
+        uploaded = await sender.client.upload_file(file, file_name=f"avatar{ext}", use_cache=False)
         photo = InputChatUploadedPhoto(file=uploaded)
 
         if self.peer_type == "chat":
@@ -1123,6 +1162,36 @@ class Portal:
                 self.save()
                 break
 
+    async def handle_matrix_upgrade(self, new_room: MatrixRoomID) -> None:
+        old_room = self.mxid
+        self.migrate_and_save_matrix(new_room)
+        await self.main_intent.join_room(new_room)
+        entity = None  # type: TypeInputPeer
+        user = None  # type: AbstractUser
+        if self.bot and self.has_bot:
+            user = self.bot
+            entity = await self.get_input_entity(self.bot)
+        if not entity:
+            user_mxids = await self.main_intent.get_room_members(self.mxid)
+            for user_str in user_mxids:
+                user_id = MatrixUserID(user_str)
+                if user_id == self.az.bot_mxid:
+                    continue
+                user = u.User.get_by_mxid(user_id, create=False)
+                if user and user.tgid:
+                    entity = await self.get_input_entity(user)
+                    if entity:
+                        break
+        if not entity:
+            self.log.error(
+                "Failed to fully migrate to upgraded Matrix room: no Telegram user found.")
+            return
+        users, participants = await self._get_users(self.bot, entity)
+        await self.sync_telegram_users(user, users)
+        levels = await self.main_intent.get_power_levels(self.mxid)
+        await self.update_telegram_participants(participants, levels)
+        self.log.info(f"Upgraded room from {old_room} to {self.mxid}")
+
     def _register_outgoing_actions_for_dedup(self, response: TypeUpdates) -> None:
         for update in response.updates:
             check_dedup = (isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage))
@@ -1161,7 +1230,7 @@ class Portal:
         if not entity:
             raise ValueError("Upgrade may have failed: output channel not found.")
         self.peer_type = "channel"
-        self.migrate_and_save(TelegramID(entity.id))
+        self.migrate_and_save_telegram(TelegramID(entity.id))
         await self.update_info(source, entity)
 
     async def set_telegram_username(self, source: 'u.User', username: str) -> None:
@@ -1200,8 +1269,8 @@ class Portal:
         self.tg_receiver = self.tgid
         self.by_tgid[self.tgid_full] = self
         await self.update_info(source, entity)
-        self.db.add(self.db_instance)
-        self.save()
+        self.db_instance.insert()
+        self.log = self.base_log.getChild(str(self.tgid))
 
         if self.bot and self.bot.tgid in invites:
             self.bot.add_chat(self.tgid, self.peer_type)
@@ -1214,7 +1283,7 @@ class Portal:
         await self.handle_matrix_power_levels(source, levels["users"], {})
 
     async def invite_telegram(self, source: 'u.User',
-                              puppet: Union[p.Puppet, "AbstractUser"]) -> None:
+                              puppet: Union[p.Puppet, 'AbstractUser']) -> None:
         if self.peer_type == "chat":
             await source.client(
                 AddChatUserRequest(chat_id=self.tgid, user_id=puppet.tgid, fwd_limit=0))
@@ -1235,11 +1304,10 @@ class Portal:
             return f"https://t.me/{self.username}/{evt.id}"
         return None
 
-    async def handle_telegram_photo(self, source: "AbstractUser", intent: IntentAPI, evt: Message,
+    async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                     relates_to: Dict = None) -> Optional[Dict]:
         largest_size = self._get_largest_photo_size(evt.media.photo)
-        file = await util.transfer_file_to_matrix(self.db, source.client, intent,
-                                                  largest_size.location)
+        file = await util.transfer_file_to_matrix(source.client, intent, largest_size.location)
         if not file:
             return None
         if self.get_config("inline_images") and (evt.message
@@ -1296,14 +1364,17 @@ class Portal:
         return attrs
 
     @staticmethod
-    def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: Dict
-                                      ) -> Tuple[Dict, str]:
+    def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: Dict,
+                                      thumb: TypePhotoSize) -> Tuple[Dict, str]:
         document = evt.media.document
         name = evt.message or attrs["name"]
         if attrs["is_sticker"]:
             alt = attrs["sticker_alt"]
             if len(alt) > 0:
-                name = f"{alt} ({unicodedata.name(alt[0]).lower()})"
+                try:
+                    name = f"{alt} ({unicodedata.name(alt[0]).lower()})"
+                except ValueError:
+                    name = alt
 
         mime_type = document.mime_type or file.mime_type
         info = {
@@ -1322,25 +1393,28 @@ class Portal:
             info["thumbnail_url"] = file.thumbnail.mxc
             info["thumbnail_info"] = {
                 "mimetype": file.thumbnail.mime_type,
-                "h": file.thumbnail.height or document.thumb.h,
-                "w": file.thumbnail.width or document.thumb.w,
+                "h": file.thumbnail.height or thumb.h,
+                "w": file.thumbnail.width or thumb.w,
                 "size": file.thumbnail.size,
             }
 
         return info, name
 
-    async def handle_telegram_document(self, source: "AbstractUser", intent: IntentAPI,
-                                       evt: Message,
-                                       relates_to: dict = None) -> Optional[Dict]:
+    async def handle_telegram_document(self, source: 'AbstractUser', intent: IntentAPI,
+                                       evt: Message, relates_to: dict = None) -> Optional[Dict]:
         document = evt.media.document
         attrs = self._parse_telegram_document_attributes(document.attributes)
 
-        file = await util.transfer_file_to_matrix(self.db, source.client, intent, document,
-                                                  document.thumb, is_sticker=attrs["is_sticker"])
+        thumb = self._get_largest_photo_size(document.thumbs)
+        if not isinstance(thumb, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb)}")
+            thumb = None
+        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb,
+                                                  is_sticker=attrs["is_sticker"])
         if not file:
             return None
 
-        info, name = self._parse_telegram_document_meta(evt, file, attrs)
+        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb)
 
         await intent.set_typing(self.mxid, is_typing=False)
 
@@ -1354,7 +1428,7 @@ class Portal:
             "external_url": self.get_external_url(evt)
         }
 
-        if attrs["is_sticker"] and self.get_config("native_stickers"):
+        if attrs["is_sticker"]:
             return await intent.send_sticker(**kwargs)
 
         mime_type = info["mimetype"]
@@ -1368,15 +1442,15 @@ class Portal:
             kwargs["file_type"] = "m.file"
         return await intent.send_file(**kwargs)
 
-    def handle_telegram_location(self, _: "AbstractUser", intent: IntentAPI, evt: Message,
+    def handle_telegram_location(self, _: 'AbstractUser', intent: IntentAPI, evt: Message,
                                  relates_to: dict = None) -> Awaitable[dict]:
         location = evt.media.geo
         long = location.long
         lat = location.lat
         long_char = "E" if long > 0 else "W"
         lat_char = "N" if lat > 0 else "S"
-        rounded_long = abs(round(long * 100000) / 100000)
-        rounded_lat = abs(round(lat * 100000) / 100000)
+        rounded_long = round(long, 5)
+        rounded_lat = round(lat, 5)
 
         body = f"{rounded_lat}° {lat_char}, {rounded_long}° {long_char}"
 
@@ -1396,7 +1470,7 @@ class Portal:
             "m.relates_to": relates_to or None,
         }, timestamp=evt.date, external_url=self.get_external_url(evt))
 
-    async def handle_telegram_text(self, source: "AbstractUser", intent: IntentAPI, is_bot: bool,
+    async def handle_telegram_text(self, source: 'AbstractUser', intent: IntentAPI, is_bot: bool,
                                    evt: Message) -> dict:
         self.log.debug(f"Sending {evt.message} to {self.mxid} by {intent.mxid}")
         text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent)
@@ -1406,13 +1480,73 @@ class Portal:
                                       msgtype=msgtype, timestamp=evt.date,
                                       external_url=self.get_external_url(evt))
 
-    async def handle_telegram_edit(self, source: "AbstractUser", sender: p.Puppet,
+    async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
+                                          evt: Message, _: dict = None) -> dict:
+        override_text = ("This message is not supported on your version of Mautrix-Telegram. "
+                         "Please check https://github.com/tulir/mautrix-telegram or ask your "
+                         "bridge administrator about possible updates.")
+        text, html, relates_to = await formatter.telegram_to_matrix(
+            evt, source, self.main_intent, override_text=override_text)
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await intent.send_message(self.mxid, {
+            "body": text,
+            "msgtype": "m.notice",
+            "format": "org.matrix.custom.html",
+            "formatted_body": html,
+            "m.relates_to": relates_to,
+            "net.maunium.telegram.unsupported": True,
+        }, timestamp=evt.date, external_url=self.get_external_url(evt))
+
+    @staticmethod
+    def _int_to_bytes(i: int) -> bytes:
+        hex_value = "{0:010x}".format(i)
+        return codecs.decode(hex_value, "hex_codec")
+
+    async def handle_telegram_game(self, source: 'AbstractUser', intent: IntentAPI,
+                                   evt: Message, _: dict = None):
+        game = evt.media.game
+        if self.peer_type == "channel":
+            play_id = base64.b64encode(b"c"
+                                       + self._int_to_bytes(self.tgid)
+                                       + self._int_to_bytes(evt.id))
+        elif self.peer_type == "chat":
+            play_id = base64.b64encode(b"g"
+                                       + self._int_to_bytes(self.tgid)
+                                       + self._int_to_bytes(evt.id)
+                                       + self._int_to_bytes(source.tgid))
+        elif self.peer_type == "user":
+            play_id = base64.b64encode(b"u"
+                                       + self._int_to_bytes(self.tgid)
+                                       + self._int_to_bytes(evt.id))
+        else:
+            raise ValueError("Portal has invalid peer type")
+        play_id = play_id.decode("utf-8").rstrip("=")
+        command = f"!tg play {play_id}"
+        override_text = f"Run {command} in your bridge management room to play {game.title}"
+        override_entities = [MessageEntityPre(offset=len("Run "), length=len(command), language="")]
+        text, html, relates_to = await formatter.telegram_to_matrix(
+            evt, source, self.main_intent,
+            override_text=override_text, override_entities=override_entities)
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await intent.send_message(self.mxid, {
+            "body": text,
+            "msgtype": "m.notice",
+            "format": "org.matrix.custom.html",
+            "formatted_body": html,
+            "m.relates_to": relates_to,
+            "net.maunium.telegram.game": play_id,
+        }, timestamp=evt.date, external_url=self.get_external_url(evt))
+
+    async def handle_telegram_edit(self, source: 'AbstractUser', sender: p.Puppet,
                                    evt: Message) -> None:
         if not self.mxid:
             return
         elif not self.get_config("edits_as_replies"):
             self.log.debug("Edits as replies disabled, ignoring edit event...")
             return
+        elif hasattr(evt, "media") and isinstance(evt.media, (MessageMediaGame,)):
+            self.log.debug("Ignoring game message edit event")
+            return
 
         lock = self.optional_send_lock(sender.tgid if sender else None)
         if lock:
@@ -1422,12 +1556,12 @@ class Portal:
         tg_space = self.tgid if self.peer_type == "channel" else source.tgid
 
         temporary_identifier = MatrixEventID(
-            f"${random.randint(1000000000000,9999999999999)}TGBRIDGEDITEMP")
+            f"${random.randint(1000000000000, 9999999999999)}TGBRIDGEDITEMP")
         duplicate_found = self.is_duplicate(evt, (temporary_identifier, tg_space), force_hash=True)
         if duplicate_found:
             mxid, other_tg_space = duplicate_found
             if tg_space != other_tg_space:
-                DBMessage.update_by_tgid(evt.id, tg_space,
+                DBMessage.update_by_tgid(TelegramID(evt.id), tg_space,
                                          mxid=mxid,
                                          mx_room=self.mxid)
             return
@@ -1442,7 +1576,7 @@ class Portal:
 
         mxid = response["event_id"]
 
-        msg = DBMessage.get_by_tgid(evt.id, tg_space)
+        msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
         if not msg:
             self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) "
                           "in database.")
@@ -1451,7 +1585,7 @@ class Portal:
         msg.update(mxid=mxid, mx_room=self.mxid)
         DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
 
-    async def handle_telegram_message(self, source: "AbstractUser", sender: p.Puppet,
+    async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
                                       evt: Message) -> None:
         if not self.mxid:
             await self.create_matrix_room(source, invites=[source.mxid], update_if_exists=False)
@@ -1464,18 +1598,19 @@ class Portal:
         tg_space = self.tgid if self.peer_type == "channel" else source.tgid
 
         temporary_identifier = MatrixEventID(
-            f"${random.randint(1000000000000,9999999999999)}TGBRIDGETEMP")
+            f"${random.randint(1000000000000, 9999999999999)}TGBRIDGETEMP")
         duplicate_found = self.is_duplicate(evt, (temporary_identifier, tg_space))
         if duplicate_found:
             mxid, other_tg_space = duplicate_found
             self.log.debug(f"Ignoring message {evt.id}@{tg_space} (src {source.tgid}) "
                            f"as it was already handled (in space {other_tg_space})")
             if tg_space != other_tg_space:
-                DBMessage(tgid=evt.id, mx_room=self.mxid, mxid=mxid, tg_space=tg_space).insert()
+                DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
+                          tg_space=tg_space).insert()
             return
 
         if self.dedup_pre_db_check and self.peer_type == "channel":
-            msg = DBMessage.get_by_tgid(evt.id, tg_space)
+            msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
             if msg:
                 self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
                                f"handled into {msg.mxid}. This duplicate was catched in the db "
@@ -1489,7 +1624,8 @@ class Portal:
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
 
-        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo)
+        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo, MessageMediaGame,
+                         MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
         intent = sender.intent if sender else self.main_intent
@@ -1497,16 +1633,14 @@ class Portal:
             is_bot = sender.is_bot if sender else False
             response = await self.handle_telegram_text(source, intent, is_bot, evt)
         elif media:
-            relates_to = formatter.telegram_reply_to_matrix(evt, source)
-            if isinstance(media, MessageMediaPhoto):
-                response = await self.handle_telegram_photo(source, intent, evt, relates_to)
-            elif isinstance(media, MessageMediaDocument):
-                response = await self.handle_telegram_document(source, intent, evt, relates_to)
-            elif isinstance(media, MessageMediaGeo):
-                response = await self.handle_telegram_location(source, intent, evt, relates_to)
-            else:
-                self.log.debug("Unhandled Telegram media: %s", media)
-                return
+            response = await {
+                MessageMediaPhoto: self.handle_telegram_photo,
+                MessageMediaDocument: self.handle_telegram_document,
+                MessageMediaGeo: self.handle_telegram_location,
+                MessageMediaUnsupported: self.handle_telegram_unsupported,
+                MessageMediaGame: self.handle_telegram_game,
+            }[type(media)](source, intent, evt,
+                           relates_to=formatter.telegram_reply_to_matrix(evt, source))
         else:
             self.log.debug("Unhandled Telegram message: %s", evt)
             return
@@ -1529,23 +1663,17 @@ class Portal:
 
         self.log.debug("Handled Telegram message: %s", evt)
         try:
-            DBMessage(tgid=evt.id, mx_room=self.mxid, mxid=mxid, tg_space=tg_space).insert()
+            DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
+                      tg_space=tg_space).insert()
             DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
-        except FlushError as e:
+        except IntegrityError as e:
             self.log.exception(f"{e.__class__.__name__} while saving message mapping. "
                                "This might mean that an update was handled after it left the "
                                "dedup cache queue. You can try enabling bridge.deduplication."
                                "pre_db_check in the config.")
             await intent.redact(self.mxid, mxid)
-        except (IntegrityError, InvalidRequestError) as e:
-            self.log.exception(f"{e.__class__.__name__} while saving message mapping. "
-                               "This might mean that an update was handled after it left the "
-                               "dedup cache queue. You can try enabling bridge.deduplication."
-                               "pre_db_check in the config.")
-            self.db.rollback()
-            await intent.redact(self.mxid, mxid)
 
-    async def _create_room_on_action(self, source: "AbstractUser",
+    async def _create_room_on_action(self, source: 'AbstractUser',
                                      action: TypeMessageAction) -> bool:
         if source.is_relaybot:
             return False
@@ -1558,7 +1686,7 @@ class Portal:
             return False
         return True
 
-    async def handle_telegram_action(self, source: "AbstractUser", sender: p.Puppet,
+    async def handle_telegram_action(self, source: 'AbstractUser', sender: p.Puppet,
                                      update: MessageService) -> None:
         action = update.action
         should_ignore = ((not self.mxid and not await self._create_room_on_action(source, action))
@@ -1582,10 +1710,13 @@ class Portal:
             await self.delete_telegram_user(TelegramID(action.user_id), sender)
         elif isinstance(action, MessageActionChatMigrateTo):
             self.peer_type = "channel"
-            self.migrate_and_save(TelegramID(action.channel_id))
+            self.migrate_and_save_telegram(TelegramID(action.channel_id))
             await sender.intent.send_emote(self.mxid, "upgraded this group to a supergroup.")
         elif isinstance(action, MessageActionPinMessage):
             await self.receive_telegram_pin_sender(sender)
+        elif isinstance(action, MessageActionGameScore):
+            # TODO handle game score
+            pass
         else:
             self.log.debug("Unhandled Telegram action in %s: %s", self.title, action)
 
@@ -1612,16 +1743,17 @@ class Portal:
         self._temp_pinned_message_id = None
         self._temp_pinned_message_sender = None
 
-        message = DBMessage.get_by_tgid(msg_id, self.tgid)
+        message = DBMessage.get_by_tgid(msg_id, self._temp_pinned_message_id_space)
         if message:
             await intent.set_pinned_messages(self.mxid, [message.mxid])
         else:
             await intent.set_pinned_messages(self.mxid, [])
 
-    async def receive_telegram_pin_id(self, msg_id: int) -> None:
+    async def receive_telegram_pin_id(self, msg_id: int, receiver: TelegramID) -> None:
         if msg_id == 0:
             return await self.update_telegram_pin()
         self._temp_pinned_message_id = msg_id
+        self._temp_pinned_message_id_space = receiver if self.peer_type != "channel" else self.tgid
         if self._temp_pinned_message_sender:
             await self.update_telegram_pin()
 
@@ -1680,7 +1812,7 @@ class Portal:
 
         for participant in participants:
             puppet = p.Puppet.get(TelegramID(participant.user_id))
-            user = u.User.get_by_tgid(participant.user_id)
+            user = u.User.get_by_tgid(TelegramID(participant.user_id))
             new_level = self._get_level_from_participant(participant, levels)
 
             if user:
@@ -1723,27 +1855,37 @@ class Portal:
                         title=self.title, about=self.about, photo_id=self.photo_id,
                         config=json.dumps(self.local_config))
 
-    def migrate_and_save(self, new_id: TelegramID) -> None:
-        existing = DBPortal.query.get(self.tgid_full)
-        if existing:
-            self.db.delete(existing)
+    def migrate_and_save_telegram(self, new_id: TelegramID) -> None:
         try:
             del self.by_tgid[self.tgid_full]
         except KeyError:
             pass
+        try:
+            existing = self.by_tgid[(new_id, new_id)]
+            existing.delete()
+        except KeyError:
+            pass
+        self.db_instance.update(tgid=new_id, tg_receiver=new_id)
+        old_id = self.tgid
         self.tgid = new_id
         self.tg_receiver = new_id
         self.by_tgid[self.tgid_full] = self
-        self.save()
+        self.log = self.base_log.getChild(str(self.tgid))
+        self.log.info(f"Telegram chat upgraded from {old_id}")
+
+    def migrate_and_save_matrix(self, new_id: MatrixRoomID) -> None:
+        try:
+            del self.by_mxid[self.mxid]
+        except KeyError:
+            pass
+        self.mxid = new_id
+        self.db_instance.update(mxid=self.mxid)
+        self.by_mxid[self.mxid] = self
 
     def save(self) -> None:
-        self.db_instance.mxid = self.mxid
-        self.db_instance.username = self.username
-        self.db_instance.title = self.title
-        self.db_instance.about = self.about
-        self.db_instance.photo_id = self.photo_id
-        self.db_instance.config = json.dumps(self.local_config)
-        self.db.commit()
+        self.db_instance.update(mxid=self.mxid, username=self.username, title=self.title,
+                                about=self.about, photo_id=self.photo_id,
+                                config=json.dumps(self.local_config))
 
     def delete(self) -> None:
         try:
@@ -1755,8 +1897,7 @@ class Portal:
         except KeyError:
             pass
         if self._db_instance:
-            self.db.delete(self._db_instance)
-            self.db.commit()
+            self._db_instance.delete()
         self.deleted = True
 
     @classmethod
@@ -1765,7 +1906,7 @@ class Portal:
                       peer_type=db_portal.peer_type, mxid=db_portal.mxid,
                       username=db_portal.username, megagroup=db_portal.megagroup,
                       title=db_portal.title, about=db_portal.about, photo_id=db_portal.photo_id,
-                      config=db_portal.config, db_instance=db_portal)
+                      local_config=db_portal.config, db_instance=db_portal)
 
     # endregion
     # region Class instance lookup
@@ -1777,7 +1918,7 @@ class Portal:
         except KeyError:
             pass
 
-        portal = DBPortal.query.filter(DBPortal.mxid == mxid).one_or_none()
+        portal = DBPortal.get_by_mxid(mxid)
         if portal:
             return cls.from_db(portal)
 
@@ -1799,7 +1940,7 @@ class Portal:
             if portal.username and portal.username.lower() == username.lower():
                 return portal
 
-        dbportal = DBPortal.query.filter(DBPortal.username == username).one_or_none()
+        dbportal = DBPortal.get_by_username(username)
         if dbportal:
             return cls.from_db(dbportal)
 
@@ -1815,14 +1956,13 @@ class Portal:
         except KeyError:
             pass
 
-        portal = DBPortal.query.get(tgid_full)
-        if portal:
-            return cls.from_db(portal)
+        db_portal = DBPortal.get_by_tgid(tgid, tg_receiver)
+        if db_portal:
+            return cls.from_db(db_portal)
 
         if peer_type:
             portal = Portal(tgid, peer_type=peer_type, tg_receiver=tg_receiver)
-            cls.db.add(portal.db_instance)
-            cls.db.commit()
+            portal.db_instance.insert()
             return portal
 
         return None
@@ -1862,9 +2002,10 @@ class Portal:
 
 def init(context: Context) -> None:
     global config
-    Portal.az, Portal.db, config, Portal.loop, Portal.bot = context.core
+    Portal.az, _, config, Portal.loop, Portal.bot = context.core
     Portal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
     Portal.sync_channel_members = config["bridge.sync_channel_members"]
+    Portal.sync_matrix_state = config["bridge.sync_matrix_state"]
     Portal.public_portals = config["bridge.public_portals"]
     Portal.filter_mode = config["bridge.filter.mode"]
     Portal.filter_list = config["bridge.filter.list"]

mautrix_telegram/puppet.py

@@ -14,7 +14,8 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Coroutine, Dict, List, Optional, Pattern, TYPE_CHECKING
+from typing import (Awaitable, Coroutine, Dict, List, Iterable, Optional, Pattern, Union,
+                    TYPE_CHECKING)
 from difflib import SequenceMatcher
 from enum import Enum
 from aiohttp import ServerDisconnectedError
@@ -24,7 +25,7 @@ import re
 
 from sqlalchemy import orm
 
-from telethon.tl.types import UserProfilePhoto, User, FileLocation
+from telethon.tl.types import UserProfilePhoto, User, FileLocation, UpdateUserName, PeerUser
 from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
 from .types import MatrixUserID, TelegramID
@@ -35,7 +36,6 @@ if TYPE_CHECKING:
     from .matrix import MatrixHandler
     from .config import Config
     from .context import Context
-    from . import user as u
     from .abstract_user import AbstractUser
 
 PuppetError = Enum('PuppetError', 'Success OnlyLoginSelf InvalidAccessToken')
@@ -81,6 +81,7 @@ class Puppet:
 
         self.default_mxid_intent = self.az.intent.user(self.default_mxid)
         self.intent = self._fresh_intent()  # type: IntentAPI
+        self.sync_task = None  # type: Optional[asyncio.Future]
 
         self.cache[id] = self
         if self.custom_mxid:
@@ -104,6 +105,16 @@ class Puppet:
         """ Is True if the puppet is logged in. """
         return True
 
+    @property
+    def plain_displayname(self) -> str:
+        tpl = config["bridge.displayname_template"]
+        if tpl == "{displayname}":
+            # Template has no extra stuff, no need to parse.
+            return self.displayname
+        regex = re.compile("^" + re.escape(tpl).replace(re.escape("{displayname}"), "(.+?)") + "$")
+        match = regex.match(self.displayname)
+        return match.group(1) or self.displayname
+
     # region Custom puppet management
     def _fresh_intent(self) -> IntentAPI:
         return (self.az.intent.user(self.custom_mxid, self.access_token)
@@ -143,7 +154,7 @@ class Puppet:
                 return PuppetError.OnlyLoginSelf
             return PuppetError.InvalidAccessToken
         if config["bridge.sync_with_custom_puppets"]:
-            asyncio.ensure_future(self.sync(), loop=self.loop)
+            self.sync_task = asyncio.ensure_future(self.sync(), loop=self.loop)
         return PuppetError.Success
 
     async def leave_rooms_with_default_user(self) -> None:
@@ -225,6 +236,8 @@ class Puppet:
     async def sync(self) -> None:
         try:
             await self._sync()
+        except asyncio.CancelledError:
+            self.log.info("Syncing cancelled")
         except Exception:
             self.log.exception("Fatal error syncing")
 
@@ -282,15 +295,10 @@ class Puppet:
                       db_instance=db_puppet)
 
     def save(self) -> None:
-        self.db_instance.access_token = self.access_token
-        self.db_instance.custom_mxid = self.custom_mxid
-        self.db_instance.username = self.username
-        self.db_instance.displayname = self.displayname
-        self.db_instance.displayname_source = self.displayname_source
-        self.db_instance.photo_id = self.photo_id
-        self.db_instance.is_bot = self.is_bot
-        self.db_instance.matrix_registered = self.is_registered
-        self.db.commit()
+        self.db_instance.update(access_token=self.access_token, custom_mxid=self.custom_mxid,
+                                username=self.username, displayname=self.displayname,
+                                displayname_source=self.displayname_source, photo_id=self.photo_id,
+                                is_bot=self.is_bot, matrix_registered=self.is_registered)
 
     # endregion
     # region Info updating
@@ -312,22 +320,21 @@ class Puppet:
             "first name": info.first_name,
             "last name": info.last_name,
         }
-        preferences = config.get("bridge.displayname_preference",
-                                 ["full name", "username", "phone"])
+        preferences = config["bridge.displayname_preference"]
         name = None
         for preference in preferences:
             name = data[preference]
             if name:
                 break
 
-        if info.deleted:
+        if isinstance(info, User) and info.deleted:
             name = f"Deleted account {info.id}"
         elif not name:
             name = info.id
 
         if not enable_format:
             return name
-        return config.get("bridge.displayname_template", "{displayname} (Telegram)").format(
+        return config["bridge.displayname_template"].format(
             displayname=name)
 
     async def update_info(self, source: 'AbstractUser', info: User) -> None:
@@ -345,12 +352,15 @@ class Puppet:
         if changed:
             self.save()
 
-    async def update_displayname(self, source: 'AbstractUser', info: User) -> bool:
+    async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
+                                 ) -> bool:
         ignore_source = (not source.is_relaybot
                          and self.displayname_source is not None
                          and self.displayname_source != source.tgid)
         if ignore_source:
             return False
+        if isinstance(info, UpdateUserName):
+            info = await source.client.get_entity(PeerUser(self.tgid))
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
@@ -366,8 +376,8 @@ class Puppet:
     async def update_avatar(self, source: 'AbstractUser', photo: FileLocation) -> bool:
         photo_id = f"{photo.volume_id}-{photo.local_id}"
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(self.db, source.client,
-                                                      self.default_mxid_intent, photo)
+            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent,
+                                                      photo)
             if file:
                 await self.default_mxid_intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
@@ -384,7 +394,7 @@ class Puppet:
         except KeyError:
             pass
 
-        puppet = DBPuppet.query.get(tgid)
+        puppet = DBPuppet.get_by_tgid(tgid)
         if puppet:
             return cls.from_db(puppet)
 
@@ -414,7 +424,7 @@ class Puppet:
         except KeyError:
             pass
 
-        puppet = DBPuppet.query.filter(DBPuppet.custom_mxid == mxid).one_or_none()
+        puppet = DBPuppet.get_by_custom_mxid(mxid)
         if puppet:
             puppet = cls.from_db(puppet)
             return puppet
@@ -422,11 +432,11 @@ class Puppet:
         return None
 
     @classmethod
-    def get_all_with_custom_mxid(cls) -> List['Puppet']:
-        return [cls.by_custom_mxid[puppet.mxid]
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        return (cls.by_custom_mxid[puppet.mxid]
                 if puppet.custom_mxid in cls.by_custom_mxid
                 else cls.from_db(puppet)
-                for puppet in DBPuppet.query.filter(DBPuppet.custom_mxid is not None).all()]
+                for puppet in DBPuppet.all_with_custom_mxid())
 
     @classmethod
     def get_id_from_mxid(cls, mxid: MatrixUserID) -> Optional[TelegramID]:
@@ -448,7 +458,7 @@ class Puppet:
             if puppet.username and puppet.username.lower() == username.lower():
                 return puppet
 
-        dbpuppet = DBPuppet.query.filter(DBPuppet.username == username).one_or_none()
+        dbpuppet = DBPuppet.get_by_username(username)
         if dbpuppet:
             return cls.from_db(dbpuppet)
 
@@ -463,7 +473,7 @@ class Puppet:
             if puppet.displayname and puppet.displayname == displayname:
                 return puppet
 
-        dbpuppet = DBPuppet.query.filter(DBPuppet.displayname == displayname).one_or_none()
+        dbpuppet = DBPuppet.get_by_displayname(displayname)
         if dbpuppet:
             return cls.from_db(dbpuppet)
 
@@ -479,4 +489,4 @@ def init(context: 'Context') -> List[Coroutine]:  # [None, None, PuppetError]
     Puppet.hs_domain = config["homeserver"]["domain"]
     Puppet.mxid_regex = re.compile(
         f"@{Puppet.username_template.format(userid='([0-9]+)')}:{Puppet.hs_domain}")
-    return [puppet.init_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()]
+    return [puppet.init_custom_mxid() for puppet in Puppet.all_with_custom_mxid()]

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -1,10 +1,27 @@
-import argparse
-import sqlalchemy as sql
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
 from sqlalchemy import orm
+import sqlalchemy as sql
+import argparse
 
-from mautrix_telegram.base import Base
+from mautrix_telegram.db import Base, Portal, Message, Puppet, BotChat
 from mautrix_telegram.config import Config
-from mautrix_telegram.db import Portal, Message, Puppet, BotChat
+
 from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as TelematrixBase
 
 parser = argparse.ArgumentParser(
@@ -21,7 +38,8 @@ args = parser.parse_args()
 config = Config(args.config, None, None)
 config.load()
 
-mxtg_db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+mxtg_db_engine = sql.create_engine(
+    config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
 mxtg = orm.sessionmaker(bind=mxtg_db_engine)()
 Base.metadata.bind = mxtg_db_engine
 
@@ -37,10 +55,11 @@ tm_messages = telematrix.query(TMMessage).all()
 telematrix.close()
 telematrix_db_engine.dispose()
 
-portals = {}  # Dict[int, Portal]
-chats = {}  # Dict[int, BotChat]
-messages = {}  # Dict[str, Message]
-puppets = {}  # Dict[int, Puppet]
+portals_by_tgid = {}  # type: Dict[int, Portal]
+portals_by_mxid = {}  # type: Dict[str, Portal]
+chats = {}  # type: Dict[int, BotChat]
+messages = {}  # type: Dict[str, Message]
+puppets = {}  # type: Dict[int, Puppet]
 
 for chat_link in chat_links:
     if type(chat_link.tg_room) is str:
@@ -61,16 +80,28 @@ for chat_link in chat_links:
 
     portal = Portal(tgid=tgid, tg_receiver=tgid, peer_type=peer_type, megagroup=megagroup,
                     mxid=chat_link.matrix_room)
-    bot_chat = BotChat(id=tgid, type=peer_type)
-    portals[chat_link.tg_room] = portal
-    chats[tgid] = bot_chat
+    chats[tgid] = BotChat(id=tgid, type=peer_type)
+    if chat_link.tg_room in portals_by_tgid:
+        print(f"Warning: Ignoring bridge from {portal.tgid} to {portal.mxid} "
+              f"in favor of {portals_by_tgid[portal.tgid].mxid}")
+        continue
+    elif chat_link.matrix_room in portals_by_mxid:
+        print(f"Warning: Ignoring bridge from {portal.mxid} to {portal.tgid} "
+              f"in favor of {portals_by_mxid[portal.mxid].tgid}")
+        continue
+    portals_by_tgid[portal.tgid] = portal
+    portals_by_mxid[portal.mxid] = portal
 
 for tm_msg in tm_messages:
     try:
-        portal = portals[tm_msg.tg_group_id]
+        portal = portals_by_tgid[tm_msg.tg_group_id]
     except KeyError:
-        print("Found message entry %d in unlinked chat %d, ignoring..." % (tm_msg.tg_message_id,
-                                                                           tm_msg.tg_group_id))
+        print(f"Found message entry {tm_msg.tg_message_id} in unlinked chat {tm_msg.tg_group_id},"
+              " ignoring...")
+        continue
+    if tm_msg.matrix_room_id != portal.mxid:
+        print(f"Found message entry {tm_msg.tg_message_id} with "
+              f"mismatching matrix room ID {tm_msg.matrix_room_id} (expected {portal.mxid})")
         continue
     tg_space = portal.tgid if portal.peer_type == "channel" else args.bot_id
     message = Message(mxid=tm_msg.matrix_event_id, mx_room=tm_msg.matrix_room_id,
@@ -81,7 +112,7 @@ for user in tg_users:
     puppets[user.tg_id] = Puppet(id=user.tg_id, displayname=user.name,
                                  displayname_source=args.bot_id)
 
-for k, v in portals.items():
+for k, v in portals_by_tgid.items():
     mxtg.add(v)
 for k, v in chats.items():
     mxtg.add(v)

mautrix_telegram/sqlstatestore.py

@@ -16,8 +16,6 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Dict, Tuple
 
-from sqlalchemy import orm
-
 from mautrix_appservice import StateStore
 
 from .types import MatrixUserID, MatrixRoomID
@@ -26,9 +24,8 @@ from .db import RoomState, UserProfile
 
 
 class SQLStateStore(StateStore):
-    def __init__(self, db: orm.Session) -> None:
+    def __init__(self) -> None:
         super().__init__()
-        self.db = db  # type: orm.Session
         self.profile_cache = {}  # type: Dict[Tuple[str, str], UserProfile]
         self.room_state_cache = {}  # type: Dict[str, RoomState]
 
@@ -59,13 +56,12 @@ class SQLStateStore(StateStore):
         except KeyError:
             pass
 
-        profile = UserProfile.query.get(key)
+        profile = UserProfile.get(*key)
         if profile:
             self.profile_cache[key] = profile
         elif create:
-            profile = UserProfile(room_id=room_id, user_id=user_id)
-            self.db.add(profile)
-            self.db.commit()
+            profile = UserProfile(room_id=room_id, user_id=user_id, membership="leave")
+            profile.insert()
             self.profile_cache[key] = profile
         return profile
 
@@ -77,7 +73,7 @@ class SQLStateStore(StateStore):
         profile.membership = member.get("membership", profile.membership or "leave")
         profile.displayname = member.get("displayname", profile.displayname)
         profile.avatar_url = member.get("avatar_url", profile.avatar_url)
-        self.db.commit()
+        profile.update()
 
     def set_membership(self, room: MatrixRoomID, user: MatrixUserID, membership: str) -> None:
         self.set_member(room, user, {
@@ -90,16 +86,17 @@ class SQLStateStore(StateStore):
         except KeyError:
             pass
 
-        room = RoomState.query.get(room_id)
+        room = RoomState.get(room_id)
         if room:
             self.room_state_cache[room_id] = room
         elif create:
             room = RoomState(room_id=room_id)
+            room.insert()
             self.room_state_cache[room_id] = room
         return room
 
     def has_power_levels(self, room: MatrixRoomID) -> bool:
-        return self._get_room_state(room).has_power_levels
+        return bool(self._get_room_state(room).power_levels)
 
     def get_power_levels(self, room: MatrixRoomID) -> Dict:
         return self._get_room_state(room).power_levels
@@ -114,9 +111,9 @@ class SQLStateStore(StateStore):
             }
         power_levels[room]["users"][user] = level
         room_state.power_levels = power_levels
-        self.db.commit()
+        room_state.update()
 
     def set_power_levels(self, room: MatrixRoomID, content: Dict) -> None:
         state = self._get_room_state(room)
         state.power_levels = content
-        self.db.commit()
+        state.update()

mautrix_telegram/tgclient.py

@@ -21,7 +21,7 @@ from telethon.tl.functions.messages import SendMediaRequest
 from telethon.tl.types import (
     InputMediaUploadedDocument, InputMediaUploadedPhoto, TypeDocumentAttribute, TypeInputMedia,
     TypeInputPeer, TypeMessageEntity, TypeMessageMedia, TypePeer)
-from telethon.tl import custom
+from telethon.tl.patched import Message
 
 
 class MautrixTelegramClient(TelegramClient):
@@ -45,7 +45,7 @@ class MautrixTelegramClient(TelegramClient):
     async def send_media(self, entity: Union[TypeInputPeer, TypePeer],
                          media: Union[TypeInputMedia, TypeMessageMedia],
                          caption: str = None, entities: List[TypeMessageEntity] = None,
-                         reply_to: int = None) -> Optional[custom.Message]:
+                         reply_to: int = None) -> Optional[Message]:
         entity = await self.get_input_entity(entity)
         reply_to = utils.get_message_id(reply_to)
         request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],

mautrix_telegram/user.py

@@ -14,22 +14,21 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Coroutine, Dict, List, Match, NewType, Optional, Tuple, cast, TYPE_CHECKING
+from typing import Awaitable, Dict, List, Iterable, Match, NewType, Optional, Tuple, TYPE_CHECKING
 import logging
 import asyncio
 import re
 
 from telethon.tl.types import (
     TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
-    UpdateShortChatMessage, UpdateShortMessage)
-from telethon.tl.types import User as TLUser
+    UpdateShortChatMessage, UpdateShortMessage, User as TLUser)
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.account import UpdateStatusRequest
 from mautrix_appservice import MatrixRequestError
 
 from .types import MatrixUserID, TelegramID
-from .db import User as DBUser, Contact as DBContact, Portal as DBPortal
+from .db import User as DBUser
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
@@ -49,9 +48,9 @@ class User(AbstractUser):
 
     def __init__(self, mxid: MatrixUserID, tgid: Optional[TelegramID] = None,
                  username: Optional[str] = None, phone: Optional[str] = None,
-                 db_contacts: Optional[List[DBContact]] = None,
+                 db_contacts: Optional[Iterable[TelegramID]] = None,
                  saved_contacts: int = 0, is_bot: bool = False,
-                 db_portals: Optional[List[DBPortal]] = None,
+                 db_portals: Optional[Iterable[Tuple[TelegramID, TelegramID]]] = None,
                  db_instance: Optional[DBUser] = None) -> None:
         super().__init__()
         self.mxid = mxid  # type: MatrixUserID
@@ -61,9 +60,9 @@ class User(AbstractUser):
         self.phone = phone  # type: str
         self.contacts = []  # type: List[pu.Puppet]
         self.saved_contacts = saved_contacts  # type: int
-        self.db_contacts = db_contacts  # type: List[DBContact]
-        self.portals = {}  # type: Dict[Tuple[int, int], po.Portal]
-        self.db_portals = db_portals or []  # type: List[DBPortal]
+        self.db_contacts = db_contacts
+        self.portals = {}  # type: Dict[Tuple[TelegramID, TelegramID], po.Portal]
+        self.db_portals = db_portals or []
         self._db_instance = db_instance  # type: Optional[DBUser]
 
         self.command_status = None  # type: Dict
@@ -98,24 +97,26 @@ class User(AbstractUser):
         return self.mxid_localpart
 
     @property
-    def db_contacts(self) -> List[DBContact]:
-        return [self.db.merge(DBContact(user=self.tgid, contact=puppet.id))
-                for puppet in self.contacts]
-
-    @db_contacts.setter
-    def db_contacts(self, contacts: List[DBContact]) -> None:
-        self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts] if contacts else []
+    def plain_displayname(self) -> str:
+        return self.displayname
 
     @property
-    def db_portals(self) -> List[DBPortal]:
-        return [portal.db_instance for portal in self.portals.values() if not portal.deleted]
+    def db_contacts(self) -> Iterable[TelegramID]:
+        return (puppet.id for puppet in self.contacts)
+
+    @db_contacts.setter
+    def db_contacts(self, contacts: Iterable[TelegramID]) -> None:
+        self.contacts = [pu.Puppet.get(entry) for entry in contacts] if contacts else []
+
+    @property
+    def db_portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        return (portal.tgid_full for portal in self.portals.values() if not portal.deleted)
 
     @db_portals.setter
-    def db_portals(self, portals: List[DBPortal]) -> None:
+    def db_portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
         self.portals = {
-            (portal.tgid, portal.tg_receiver): po.Portal.get_by_tgid(portal.tgid,
-                                                                     portal.tg_receiver)
-            for portal in portals
+            tgid_full: po.Portal.get_by_tgid(*tgid_full)
+            for tgid_full in portals
         } if portals else {}
 
     # region Database conversion
@@ -132,23 +133,17 @@ class User(AbstractUser):
                       portals=self.db_portals)
 
     def save(self) -> None:
-        self.db_instance.tgid = self.tgid
-        self.db_instance.tg_username = self.username
-        self.db_instance.tg_phone = self.phone
-        self.db_instance.contacts = self.db_contacts
-        self.db_instance.saved_contacts = self.saved_contacts
-        self.db_instance.portals = self.db_portals
-        self.db.commit()
+        self.db_instance.update(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
+                                saved_contacts=self.saved_contacts)
 
-    def delete(self) -> None:
+    def delete(self, delete_db: bool = True) -> None:
         try:
             del self.by_mxid[self.mxid]
             del self.by_tgid[self.tgid]
         except KeyError:
             pass
-        if self._db_instance:
-            self.db.delete(self._db_instance)
-            self.db.commit()
+        if delete_db and self._db_instance:
+            self._db_instance.delete()
 
     @classmethod
     def from_db(cls, db_user: DBUser) -> 'User':
@@ -159,6 +154,9 @@ class User(AbstractUser):
     # endregion
     # region Telegram connection management
 
+    def ensure_started(self, even_if_no_session=False) -> Awaitable['User']:
+        return super().ensure_started(even_if_no_session)
+
     async def start(self, delete_unless_authenticated: bool = False) -> 'User':
         await super().start()
         if await self.is_logged_in():
@@ -173,7 +171,7 @@ class User(AbstractUser):
     async def post_login(self, info: TLUser = None) -> None:
         try:
             await self.update_info(info)
-            if not self.is_bot:
+            if not self.is_bot and config["bridge.startup_sync"]:
                 await self.sync_dialogs()
                 await self.sync_contacts()
             if config["bridge.catch_up"]:
@@ -207,9 +205,6 @@ class User(AbstractUser):
     # endregion
     # region Telegram actions that need custom methods
 
-    def ensure_started(self, even_if_no_session: bool = False) -> Coroutine[None, None, 'User']:
-        return cast(Coroutine[None, None, 'User'], super().ensure_started(even_if_no_session))
-
     async def set_presence(self, online: bool = True) -> None:
         if not self.is_bot:
             await self.client(UpdateStatusRequest(offline=not online))
@@ -294,7 +289,7 @@ class User(AbstractUser):
 
     async def sync_dialogs(self, synchronous_create: bool = False) -> None:
         creators = []
-        for entity in await self.get_dialogs(limit=30):
+        for entity in await self.get_dialogs(limit=config["bridge.sync_dialog_limit"] or None):
             portal = po.Portal.get_by_entity(entity)
             self.portals[portal.tgid_full] = portal
             creators.append(
@@ -321,19 +316,19 @@ class User(AbstractUser):
 
     async def needs_relaybot(self, portal: po.Portal) -> bool:
         return not await self.is_logged_in() or (
-            self.is_bot and portal.tgid_full not in self.portals)
+            (portal.has_bot or self.bot) and portal.tgid_full not in self.portals)
 
     def _hash_contacts(self) -> int:
         acc = 0
-        for id in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
-            acc = (acc * 20261 + id) & 0xffffffff
+        for contact in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
+            acc = (acc * 20261 + contact) & 0xffffffff
         return acc & 0x7fffffff
 
     async def sync_contacts(self) -> None:
         response = await self.client(GetContactsRequest(hash=self._hash_contacts()))
         if isinstance(response, ContactsNotModified):
             return
-        self.log.debug("Updating contacts...")
+        self.log.debug(f"Updating contacts of {self.name}...")
         self.contacts = []
         self.saved_contacts = response.saved_count
         for user in response.users:
@@ -355,27 +350,26 @@ class User(AbstractUser):
         except KeyError:
             pass
 
-        user = DBUser.query.get(mxid)
+        user = DBUser.get_by_mxid(mxid)
         if user:
             user = cls.from_db(user)
             return user
 
         if create:
             user = cls(mxid)
-            cls.db.add(user.db_instance)
-            cls.db.commit()
+            user.db_instance.insert()
             return user
 
         return None
 
     @classmethod
-    def get_by_tgid(cls, tgid: int) -> Optional['User']:
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
         try:
             return cls.by_tgid[tgid]
         except KeyError:
             pass
 
-        user = DBUser.query.filter(DBUser.tgid == tgid).one_or_none()
+        user = DBUser.get_by_tgid(tgid)
         if user:
             user = cls.from_db(user)
             return user
@@ -391,7 +385,7 @@ class User(AbstractUser):
             if user.username and user.username.lower() == username.lower():
                 return user
 
-        puppet = DBUser.query.filter(DBUser.tg_username == username).one_or_none()
+        puppet = DBUser.get_by_username(username)
         if puppet:
             return cls.from_db(puppet)
 
@@ -399,9 +393,9 @@ class User(AbstractUser):
     # endregion
 
 
-def init(context: 'Context') -> List[Coroutine]:  # [None, None, AbstractUser]
+def init(context: 'Context') -> List[Awaitable['User']]:
     global config
     config = context.config
 
-    users = [User.from_db(user) for user in DBUser.query.all()]
-    return [user.ensure_started() for user in users]
+    users = [User.from_db(user) for user in DBUser.all()]
+    return [user.ensure_started() for user in users if user.tgid]

mautrix_telegram/util/__init__.py

@@ -2,3 +2,7 @@ from .file_transfer import transfer_file_to_matrix, convert_image
 from .format_duration import format_duration
 from .signed_token import sign_token, verify_token
 from .recursive_dict import recursive_del, recursive_set, recursive_get
+
+
+def ignore_coro(coro):
+    pass

mautrix_telegram/util/file_transfer.py

@@ -21,12 +21,10 @@ import logging
 import asyncio
 
 import magic
-from sqlalchemy import orm
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
-from sqlalchemy.orm.exc import FlushError
 
-from telethon.tl.types import (Document, FileLocation, InputFileLocation,
-                               InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
+from telethon.tl.types import (Document, FileLocation, InputFileLocation, InputDocumentFileLocation,
+                               TypePhotoSize, PhotoSize, PhotoCachedSize)
 from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
                              SecurityError)
 from mautrix_appservice import IntentAPI
@@ -102,7 +100,7 @@ def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str =
 
 def _location_to_id(location: TypeLocation) -> str:
     if isinstance(location, (Document, InputDocumentFileLocation)):
-        return f"{location.id}-{location.version}"
+        return f"{location.id}-{location.access_hash}"
     elif isinstance(location, (FileLocation, InputFileLocation)):
         return f"{location.volume_id}-{location.local_id}"
 
@@ -117,6 +115,10 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if not loc_id:
         return None
 
+    db_file = DBTelegramFile.get(loc_id)
+    if db_file:
+        return db_file
+
     video_ext = mimetypes.guess_extension(mime)
     if VideoFileClip and video_ext:
         try:
@@ -131,22 +133,29 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
 
     content_uri = await intent.upload_file(file, mime_type)
 
-    return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
-                          was_converted=False, timestamp=int(time.time()), size=len(file),
-                          width=width, height=height)
+    db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
+                             was_converted=False, timestamp=int(time.time()), size=len(file),
+                             width=width, height=height)
+    try:
+        db_file.insert()
+    except (IntegrityError, InvalidRequestError) as e:
+        log.exception(f"{e.__class__.__name__} while saving transferred file thumbnail data. "
+                      "This was probably caused by two simultaneous transfers of the same file, "
+                      "and might (but probably won't) cause problems with thumbnails or something.")
+    return db_file
 
 
 transfer_locks = {}  # type: Dict[str, asyncio.Lock]
 
 
-async def transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: Optional[TypeLocation] = None,
+async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                  location: TypeLocation, thumbnail: Optional[Union[TypeLocation, TypePhotoSize]] = None,
                                   is_sticker: bool = False) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
         return None
 
-    db_file = DBTelegramFile.query.get(location_id)
+    db_file = DBTelegramFile.get(location_id)
     if db_file:
         return db_file
 
@@ -156,15 +165,15 @@ async def transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient
         lock = asyncio.Lock()
         transfer_locks[location_id] = lock
     async with lock:
-        return await _unlocked_transfer_file_to_matrix(db, client, intent, location_id, location,
+        return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
                                                        thumbnail, is_sticker)
 
 
-async def _unlocked_transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient,
-                                            intent: IntentAPI, loc_id: str, location: TypeLocation,
-                                            thumbnail: Optional[TypeLocation],
+async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                            loc_id: str, location: TypeLocation,
+                                            thumbnail: Optional[Union[TypeLocation, TypePhotoSize]],
                                             is_sticker: bool) -> Optional[DBTelegramFile]:
-    db_file = DBTelegramFile.query.get(loc_id)
+    db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
@@ -201,16 +210,9 @@ async def _unlocked_transfer_file_to_matrix(db: orm.Session, client: MautrixTele
                                                                mime_type)
 
     try:
-        db.add(db_file)
-        db.commit()
-    except FlushError as e:
-        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
-                      "This was probably caused by two simultaneous transfers of the same file, "
-                      "and should not cause any problems.")
+        db_file.insert()
     except (IntegrityError, InvalidRequestError) as e:
-        db.rollback()
         log.exception(f"{e.__class__.__name__} while saving transferred file data. "
                       "This was probably caused by two simultaneous transfers of the same file, "
                       "and should not cause any problems.")
-
     return db_file

mautrix_telegram/web/common/auth_api.py

@@ -24,8 +24,8 @@ import logging
 
 from telethon.errors import *
 
-from ...commands.auth import enter_password
-from ...util import format_duration
+from ...commands.telegram.auth import enter_password
+from ...util import format_duration, ignore_coro
 from ...puppet import Puppet, PuppetError
 from ...user import User
 
@@ -112,7 +112,7 @@ class AuthAPI(abc.ABC):
         existing_user = User.get_by_tgid(user_info.id)
         if existing_user and existing_user != user:
             await existing_user.log_out()
-        asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+        ignore_coro(asyncio.ensure_future(user.post_login(user_info), loop=self.loop))
         if user.command_status and user.command_status["action"] == "Login":
             user.command_status = None
 

mautrix_telegram/web/provisioning/__init__.py

@@ -27,7 +27,8 @@ from mautrix_appservice import AppService, MatrixRequestError, IntentError
 from ...types import MatrixUserID, TelegramID
 from ...user import User
 from ...portal import Portal
-from ...commands.portal import user_has_power_level, get_initial_state
+from ...util import ignore_coro
+from ...commands.portal.util import user_has_power_level, get_initial_state
 from ..common import AuthAPI
 
 if TYPE_CHECKING:
@@ -190,8 +191,9 @@ class ProvisioningAPI(AuthAPI):
         portal.photo_id = ""
         portal.save()
 
-        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
-                              loop=self.loop)
+        ignore_coro(asyncio.ensure_future(portal.update_matrix_room(user, entity, direct,
+                                                                    levels=levels),
+                                          loop=self.loop))
 
         return web.Response(status=202, body="{}")
 
@@ -285,7 +287,7 @@ class ProvisioningAPI(AuthAPI):
                 self.log.exception("Failed to disconnect chat")
                 return self.get_error_response(500, "exception", "Failed to disconnect chat")
         else:
-            asyncio.ensure_future(coro, loop=self.loop)
+            ignore_coro(asyncio.ensure_future(coro, loop=self.loop))
         return web.json_response({}, status=200 if sync else 202)
 
     async def get_user_info(self, request: web.Request) -> web.Response:

setup.py

@@ -6,10 +6,15 @@ extras = {
     "highlight_edits": ["lxml>=4.1.1,<5"],
     "better_formatter": ["lxml>=4.1.1,<5"],
     "fast_crypto": ["cryptg>=0.1,<0.2"],
-    "webp_convert": ["Pillow>=5.0.0,<6"],
-    "hq_thumbnails": ["moviepy>=0.2,<0.3"],
+    "webp_convert": ["Pillow>=4.3.0,<6"],
+    "hq_thumbnails": ["moviepy>=1.0,<2.0"],
 }
-extras["all"] = list(set(deps[0] for deps in extras.values()))
+extras["all"] = list({dep for deps in extras.values() for dep in deps})
+
+try:
+    long_desc = open("README.md").read()
+except IOError:
+    long_desc = "Failed to read README.md"
 
 setuptools.setup(
     name="mautrix-telegram",
@@ -20,22 +25,22 @@ setuptools.setup(
     author_email="tulir@maunium.net",
 
     description="A Matrix-Telegram hybrid puppeting/relaybot bridge.",
-    long_description=open("README.md").read(),
+    long_description=long_desc,
     long_description_content_type="text/markdown",
 
     packages=setuptools.find_packages(),
 
     install_requires=[
         "aiohttp>=3.0.1,<4",
-        "mautrix-appservice>=0.3.7,<0.4.0",
+        "mautrix-appservice>=0.3.8,<0.4.0",
         "SQLAlchemy>=1.2.3,<2",
         "alembic>=1.0.0,<2",
         "commonmark>=0.8.1,<1",
         "ruamel.yaml>=0.15.35,<0.16",
         "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
-        "telethon>=1.0,<1.3",
-        "telethon-session-sqlalchemy>=0.2.3,<0.3",
+        "telethon>=1.5.5,<1.6",
+        "telethon-session-sqlalchemy>=0.2.8,<0.3",
     ],
     extras_require=extras,
 
@@ -62,4 +67,3 @@ setuptools.setup(
         ("alembic/versions", glob.glob("alembic/versions/*.py"))
     ],
 )
-