Bump version to 0.10.2

SQLite is now supported for the crypto store instead of pickle (via aiosqlite)

.gitlab-ci.yml

@@ -19,10 +19,28 @@ build amd64:
   - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
   after_script:
   - |
-    if [ "$CI_COMMIT_BRANCH" = "master" ]; then
-      apk add --update curl
+    if [[ "$CI_COMMIT_BRANCH" == "master" && "$CI_JOB_STATUS" == "success" ]]; then
+      apk add --update curl jq
       rm -rf /var/cache/apk/*
-      curl "$NOVA_ADMIN_API_URL" -H "Content-Type: application/json" -d '{"password":"'"$NOVA_ADMIN_NIGHTLY_PASS"'","bridge":"'$NOVA_BRIDGE_TYPE'","image":"'$CI_REGISTRY_IMAGE':'$CI_COMMIT_SHA'-amd64"}'
+
+      jq -n '
+         {
+           password:  env.BEEPER_DEV_ADMIN_NIGHTLY_PASS,
+           bridge:    env.BEEPER_BRIDGE_TYPE,
+           image:     "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
+           channel:   "STABLE"
+         }
+        ' | curl "$BEEPER_DEV_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
+
+      jq -n '
+         {
+           password:    env.BEEPER_PROD_ADMIN_NIGHTLY_PASS,
+           bridge:      env.BEEPER_BRIDGE_TYPE,
+           image:       "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
+           channel:     "INTERNAL",
+           deployNext:  true
+         }
+         ' | curl "$BEEPER_PROD_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
     fi
 
 build arm64:

Dockerfile

@@ -54,7 +54,8 @@ RUN apk add --virtual .build-deps \
       libffi-dev \
       build-base \
  && sed -Ei 's/psycopg2-binary.+//' optional-requirements.txt \
- && pip3 install -r requirements.txt -r optional-requirements.txt \
+ # TODO: unpin Pillow here after it's updated in Alpine
+ && pip3 install -r requirements.txt -r optional-requirements.txt 'pillow==8.2' \
  && apk del .build-deps
 
 COPY . /opt/mautrix-telegram

ROADMAP.md

@@ -59,5 +59,5 @@
   * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
   * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
 
-† Information not automatically sent from source, i.e. implementation may not be possible
+† Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.10.1"
+__version__ = "0.10.2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/abstract_user.py

@@ -50,6 +50,7 @@ if TYPE_CHECKING:
     from .context import Context
     from .config import Config
     from .bot import Bot
+    from .__main__ import TelegramBridge
 
 config: Optional['Config'] = None
 # Value updated from config in init()
@@ -71,6 +72,7 @@ class AbstractUser(ABC):
     loop: asyncio.AbstractEventLoop = None
     log: TraceLogger
     az: AppService
+    bridge: 'TelegramBridge'
     relaybot: Optional['Bot']
     ignore_incoming_bot_events: bool = True
 
@@ -196,7 +198,7 @@ class AbstractUser(ABC):
             if not await self.update(update):
                 await self._update(update)
         except Exception:
-            self.log.exception(f"Failed to handle Telegram update {update}")
+            self.log.exception("Failed to handle Telegram update")
             UPDATE_ERRORS.labels(update_type=update_type).inc()
         UPDATE_TIME.labels(update_type=update_type).observe(time.time() - start_time)
 
@@ -513,6 +515,7 @@ class AbstractUser(ABC):
 def init(context: 'Context') -> None:
     global config, MAX_DELETIONS
     AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
+    AbstractUser.bridge = context.bridge
     AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
     AbstractUser.session_container = context.session_container
     MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/commands/telegram/auth.py

@@ -46,10 +46,13 @@ except ImportError:
                  help_section=SECTION_AUTH,
                  help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> EventID:
-    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
-    if me:
-        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
-        return await evt.reply(f"You're logged in as {human_tg_id}")
+    if await evt.sender.is_logged_in():
+        me = await evt.sender.get_me()
+        if me:
+            human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
+            return await evt.reply(f"You're logged in as {human_tg_id}")
+        else:
+            return await evt.reply("You were logged in, but there appears to have been an error.")
     else:
         return await evt.reply("You're not logged in.")
 
@@ -346,10 +349,12 @@ async def _finish_sign_in(evt: CommandEvent, user: User, login_as: 'u.User' = No
     return await evt.reply(msg)
 
 
-@command_handler(needs_auth=True,
+@command_handler(needs_auth=False,
                  help_section=SECTION_AUTH,
                  help_text="Log out from Telegram.")
 async def logout(evt: CommandEvent) -> EventID:
+    if not evt.sender.tgid:
+        return await evt.reply("You're not logged in")
     if await evt.sender.log_out():
         return await evt.reply("Logged out successfully.")
     return await evt.reply("Failed to log out.")

mautrix_telegram/example-config.yaml

@@ -244,14 +244,7 @@ bridge:
         # Default to encryption, force-enable encryption in all portals the bridge creates
         # This will cause the bridge bot to be in private chats for the encryption to work properly.
         default: false
-        # Database for the encryption data. Currently only supports Postgres and an in-memory
-        # store that's persisted as a pickle.
-        # If set to `default`, will use the appservice postgres database
-        # or a pickle file if the appservice database is sqlite.
-        #
-        # Format examples:
-        #   Pickle:   pickle:///filename.pickle
-        #   Postgres: postgres://username:password@hostname/dbname
+        # Database for the encryption data. If set to `default`, will use the appservice database.
         database: default
         # Options for automatic key sharing.
         key_sharing:
@@ -394,6 +387,21 @@ bridge:
     # The prefix for commands. Only required in non-management rooms.
     command_prefix: "!tg"
 
+    # Messages sent upon joining a management room.
+    # Markdown is supported. The defaults are listed below.
+    management_room_text:
+        # Sent when joining a room.
+        welcome: "Hello, I'm a Telegram bridge bot."
+        # Sent when joining a management room and the user is already logged in.
+        welcome_connected: "Use `help` for help."
+        # Sent when joining a management room and the user is not logged in.
+        welcome_unconnected: "Use `help` for help or `login` to log in."
+        # Optional extra text sent when joining a management room.
+        additional_help: ""
+
+    # Send each message separately (for readability in some clients)
+    management_room_multiple_messages: false
+
     # Permissions for using the bridge.
     # Permitted values:
     #   relaybot - Only use the bridge via the relaybot, no access to commands.

mautrix_telegram/formatter/from_telegram.py

@@ -32,7 +32,7 @@ from telethon.helpers import add_surrogate, del_surrogate
 from mautrix.errors import MatrixRequestError
 from mautrix.appservice import IntentAPI
 from mautrix.types import (TextMessageEventContent, RelatesTo, RelationType, Format, MessageType,
-                           MessageEvent)
+                           MessageEvent, EventType)
 
 from .. import user as u, puppet as pu, portal as po
 from ..types import TelegramID
@@ -129,12 +129,14 @@ async def _add_reply_header(source: 'AbstractUser', content: TextMessageEventCon
     content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
 
     try:
-        event: MessageEvent = await main_intent.get_event(msg.mx_room, msg.mxid)
+        event = await main_intent.get_event(msg.mx_room, msg.mxid)
+        if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
+            event = await source.bridge.matrix.e2ee.decrypt(event)
         if isinstance(event.content, TextMessageEventContent):
             event.content.trim_reply_fallback()
         puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
         content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
-    except MatrixRequestError:
+    except Exception:
         log.exception("Failed to get event to add reply fallback")
 
 

mautrix_telegram/matrix.py

@@ -115,23 +115,6 @@ class MatrixHandler(BaseMatrixHandler):
             await intent.send_notice(room_id, "This puppet will remain inactive until a "
                                               "Telegram chat is created for this room.")
 
-    async def send_welcome_message(self, room_id: RoomID, inviter: 'u.User') -> None:
-        try:
-            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
-        except MatrixError:
-            # The AS bot is not in the room.
-            return
-        cmd_prefix = self.commands.command_prefix
-        text = html = "Hello, I'm a Telegram bridge bot. "
-        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
-            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
-            html += (f"Use <code>{cmd_prefix} help</code> for help"
-                     f" or <code>{cmd_prefix} login</code> to log in.")
-        else:
-            text += f"Use `{cmd_prefix} help` for help."
-            html += f"Use <code>{cmd_prefix} help</code> for help."
-        await self.az.intent.send_notice(room_id, text=text, html=html)
-
     async def handle_invite(self, room_id: RoomID, user_id: UserID, inviter: 'u.User',
                             event_id: EventID) -> None:
         user = u.User.get_by_mxid(user_id, create=False)

mautrix_telegram/portal/matrix.py

@@ -421,7 +421,8 @@ class PortalMatrix(BasePortal, ABC):
             await self._handle_matrix_file(sender_id, event_id, space, client, content, reply_to,
                                            caption_content)
         else:
-            self.log.trace("Unhandled Matrix event: %s", content)
+            self.log.debug("Didn't handle Matrix event {event_id} due to unknown msgtype {content.msgtype}")
+            self.log.trace("Unhandled Matrix event content: %s", content)
 
     async def handle_matrix_unpin_all(self, sender: 'u.User', pin_event_id: EventID) -> None:
         await sender.client(UnpinAllMessagesRequest(peer=self.peer))

mautrix_telegram/portal/metadata.py

@@ -27,7 +27,7 @@ from telethon.tl.types import (
     PhotoEmpty, InputChannel, InputUser, ChatPhotoEmpty, PeerUser, Photo, TypeChat, TypeInputPeer,
     TypeUser, User, InputPeerPhotoFileLocation, ChatParticipantAdmin, ChannelParticipantAdmin,
     ChatParticipantCreator, ChannelParticipantCreator, UserProfilePhoto, UserProfilePhotoEmpty,
-    InputPeerUser)
+    InputPeerUser, ChannelParticipantBanned)
 
 from mautrix.errors import MForbidden
 from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership,
@@ -503,14 +503,15 @@ class PortalMetadata(BasePortal, ABC):
             levels.users[self.main_intent.mxid] = 100
         return levels
 
-    @staticmethod
-    def _get_level_from_participant(participant: TypeParticipant) -> int:
+    @classmethod
+    def _get_level_from_participant(cls, participant: TypeParticipant,
+                                    levels: PowerLevelStateEventContent) -> int:
         # TODO use the power level requirements to get better precision in channels
         if isinstance(participant, (ChatParticipantAdmin, ChannelParticipantAdmin)):
-            return 50
+            return levels.state_default or 50
         elif isinstance(participant, (ChatParticipantCreator, ChannelParticipantCreator)):
-            return 95
-        return 0
+            return levels.get_user_level(cls.az.bot_mxid) - 5
+        return levels.users_default or 0
 
     @staticmethod
     def _participant_to_power_levels(levels: PowerLevelStateEventContent,
@@ -541,7 +542,7 @@ class PortalMetadata(BasePortal, ABC):
 
             puppet = p.Puppet.get(TelegramID(participant.user_id))
             user = u.User.get_by_tgid(TelegramID(participant.user_id))
-            new_level = self._get_level_from_participant(participant)
+            new_level = self._get_level_from_participant(participant, levels)
 
             if user:
                 await user.register_portal(self)
@@ -796,7 +797,8 @@ class PortalMetadata(BasePortal, ABC):
     @staticmethod
     def _filter_participants(users: List[TypeUser], participants: List[TypeParticipant]
                              ) -> Iterable[TypeUser]:
-        participant_map = {part.user_id: part for part in participants}
+        participant_map = {part.user_id: part for part in participants
+                           if not isinstance(part, ChannelParticipantBanned)}
         for user in users:
             try:
                 user.participant = participant_map[user.id]
@@ -832,15 +834,16 @@ class PortalMetadata(BasePortal, ABC):
     async def _get_users(self, user: 'AbstractUser',
                          entity: Union[TypeInputPeer, InputUser, TypeChat, TypeUser, InputChannel]
                          ) -> List[TypeUser]:
+        limit = self.max_initial_member_sync
         if self.peer_type == "chat":
             chat = await user.client(GetFullChatRequest(chat_id=self.tgid))
-            return list(self._filter_participants(chat.users,
-                                                  chat.full_chat.participants.participants))
+            return list(
+                self._filter_participants(chat.users, chat.full_chat.participants.participants)
+            )[:limit]
         elif self.peer_type == "channel":
             if not self.megagroup and not self.sync_channel_members:
                 return []
 
-            limit = self.max_initial_member_sync
             if limit == 0:
                 return []
 

mautrix_telegram/portal/telegram.py

@@ -193,13 +193,13 @@ class PortalTelegram(BasePortal, ABC):
                                                 height=file.thumbnail.height or thumb_size.h,
                                                 width=file.thumbnail.width or thumb_size.w,
                                                 size=file.thumbnail.size)
-        else:
+        elif attrs.is_sticker:
             # This is a hack for bad clients like Element iOS that require a thumbnail
+            info.thumbnail_info = ImageInfo.deserialize(info.serialize())
             if file.decryption_info:
                 info.thumbnail_file = file.decryption_info
             else:
                 info.thumbnail_url = file.mxc
-            info.thumbnail_info = ImageInfo.deserialize(info.serialize())
 
         return info, name
 
@@ -258,9 +258,12 @@ class PortalTelegram(BasePortal, ABC):
             info["fi.mau.autoplay"] = True
             info["fi.mau.hide_controls"] = True
             info["fi.mau.no_audio"] = True
+        if not name:
+            ext = sane_mimetypes.guess_extension(file.mime_type)
+            name = "unnamed_file" + ext
 
         content = MediaMessageEventContent(
-            body=name or "unnamed file", info=info, relates_to=relates_to,
+            body=name, info=info, relates_to=relates_to,
             external_url=self._get_external_url(evt),
             msgtype={
                 "video/": MessageType.VIDEO,

mautrix_telegram/user.py

@@ -22,12 +22,14 @@ import asyncio
 from telethon.tl.types import (TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage,
                                UpdateShortChatMessage, UpdateShortMessage, User as TLUser, Chat,
                                ChatForbidden, UpdateFolderPeers, UpdatePinnedDialogs,
-                               UpdateNotifySettings, NotifyPeer)
+                               UpdateNotifySettings, NotifyPeer, InputUserSelf)
 from telethon.tl.custom import Dialog
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.account import UpdateStatusRequest
-from telethon.errors import AuthKeyDuplicatedError
+from telethon.tl.functions.users import GetUsersRequest
+from telethon.errors import (AuthKeyDuplicatedError, UserDeactivatedError, UserDeactivatedBanError,
+                             SessionRevokedError, UnauthorizedError)
 
 from mautrix.client import Client
 from mautrix.errors import MatrixRequestError, MNotFound
@@ -56,6 +58,7 @@ METRIC_CONNECTED = Gauge('bridge_connected', 'Users connected to Telegram')
 BridgeState.human_readable_errors.update({
     "tg-not-connected": "Your Telegram connection failed",
     "tg-auth-key-duplicated": "The bridge accidentally logged you out",
+    "tg-not-authenticated": "The stored auth token did not work",
 })
 
 
@@ -226,13 +229,16 @@ class User(AbstractUser, BaseUser):
         elif delete_unless_authenticated:
             self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
             await self.client.disconnect()
+            if self.tgid:
+                await self.push_bridge_state(BridgeStateEvent.BAD_CREDENTIALS,
+                                             error="tg-not-authenticated")
             self.client.session.delete()
         return self
 
     @property
     def _is_connected(self) -> bool:
         return bool(self.client and self.client._sender
-                    and self.client._sender._transport_connected)
+                    and self.client._sender._transport_connected())
 
     async def _track_connection(self) -> None:
         self.log.debug("Starting loop to track connection state")
@@ -252,6 +258,18 @@ class User(AbstractUser, BaseUser):
         state.remote_id = str(self.tgid)
         state.remote_name = self.human_tg_id
 
+    async def get_bridge_states(self) -> List[BridgeState]:
+        if not self.tgid:
+            return []
+        if self._is_connected and await self.is_logged_in():
+            state_event = (BridgeStateEvent.BACKFILLING if self._is_backfilling
+                           else BridgeStateEvent.CONNECTED)
+            ttl = 3600
+        else:
+            state_event = BridgeStateEvent.UNKNOWN_ERROR
+            ttl = 240
+        return [BridgeState(state_event=state_event, ttl=ttl)]
+
     async def get_puppet(self) -> Optional['pu.Puppet']:
         if not self.tgid:
             return None
@@ -321,8 +339,22 @@ class User(AbstractUser, BaseUser):
         if not self.is_bot:
             await self.client(UpdateStatusRequest(offline=not online))
 
+    async def get_me(self) -> Optional[TLUser]:
+        try:
+            return (await self.client(GetUsersRequest([InputUserSelf()])))[0]
+        except UnauthorizedError as e:
+            self.log.error(f"Authorization error in get_me(): {e}")
+            await self.push_bridge_state(BridgeStateEvent.BAD_CREDENTIALS, error="tg-auth-error",
+                                         message=str(e), ttl=3600)
+            await self.stop()
+            return None
+
     async def update_info(self, info: TLUser = None) -> None:
-        info = info or await self.client.get_me()
+        if not info:
+            info = await self.get_me()
+            if not info:
+                self.log.warning("get_me() returned None, aborting update_info()")
+                return
         changed = False
         if self.is_bot != info.bot:
             self.is_bot = info.bot
@@ -366,12 +398,11 @@ class User(AbstractUser, BaseUser):
             self.tgid = None
             await self.save()
         ok = await self.client.log_out()
-        if not ok:
-            return False
+        self.client.session.delete()
         self.delete()
         await self.stop()
         self._track_metric(METRIC_LOGGED_IN, False)
-        return True
+        return ok
 
     def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
                       ) -> List[SearchResult]:

mautrix_telegram/web/provisioning/__init__.py

@@ -21,7 +21,10 @@ import json
 from aiohttp import web
 
 from telethon.utils import get_peer_id, resolve_id
-from telethon.tl.types import ChatForbidden, ChannelForbidden, TypeChat
+from telethon.tl.types import ChatForbidden, ChannelForbidden, TypeChat, InputUserSelf
+from telethon.tl.functions.users import GetUsersRequest
+from telethon.errors import (UserDeactivatedError, UserDeactivatedBanError, SessionRevokedError,
+                             UnauthorizedError)
 
 from mautrix.appservice import AppService
 from mautrix.errors import MatrixRequestError, IntentError
@@ -294,16 +297,17 @@ class ProvisioningAPI(AuthAPI):
 
         user_data = None
         if await user.is_logged_in():
-            me = await user.client.get_me()
-            await user.update_info(me)
-            user_data = {
-                "id": user.tgid,
-                "username": user.username,
-                "first_name": me.first_name,
-                "last_name": me.last_name,
-                "phone": me.phone,
-                "is_bot": user.is_bot,
-            }
+            me = await user.get_me()
+            if me:
+                await user.update_info(me)
+                user_data = {
+                    "id": user.tgid,
+                    "username": user.username,
+                    "first_name": me.first_name,
+                    "last_name": me.last_name,
+                    "phone": me.phone,
+                    "is_bot": user.is_bot,
+                }
         return web.json_response({
             "telegram": user_data,
             "mxid": user.mxid,
@@ -351,7 +355,7 @@ class ProvisioningAPI(AuthAPI):
         return await self.post_login_password(user, data.get("password", ""))
 
     async def logout(self, request: web.Request) -> web.Response:
-        _, user, err = await self.get_user_request_info(request, expect_logged_in=True,
+        _, user, err = await self.get_user_request_info(request, expect_logged_in=None,
                                                         require_puppeting=False,
                                                         want_data=False)
         if err is not None:
@@ -461,7 +465,7 @@ class ProvisioningAPI(AuthAPI):
                                                 Optional[web.Response]]):
         err = self.check_authorization(request)
         if err is not None:
-            return err
+            return None, None, err
 
         data = None
         if want_data and (request.method == "POST" or request.method == "PUT"):

optional-requirements.txt

@@ -15,13 +15,16 @@ qrcode>=6,<7
 moviepy>=1,<2
 
 #/metrics
-prometheus_client>=0.6,<0.12
+prometheus_client>=0.6,<0.13
 
 #/postgres
 psycopg2-binary>=2,<3
+asyncpg>=0.20,<0.25
+
+#/sqlite
+aiosqlite>=0.17,<0.18
 
 #/e2be
-asyncpg>=0.20,<0.25
 python-olm>=3,<4
 pycryptodome>=3,<4
 unpaddedbase64>=1,<2

requirements.txt

@@ -5,6 +5,11 @@ python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.10.4,<0.11
-telethon>=1.22,<1.23
+mautrix>=0.11.3,<0.12
+#telethon>=1.22,<1.24
+# Temporary patch for 64-bit IDs until upstream telethon 2.0 is ready
+tulir-telethon==1.24.0a2
 telethon-session-sqlalchemy>=0.2.14,<0.3
+# Temporarily always depend on aiosqlite to prevent breaking old installs
+# Will be removed in v0.12 (after which you need to choose the [sqlite] optional dependency)
+aiosqlite>=0.17,<0.18