Bump version to 0.14.1

* Replace pysocks with python-socks

* Log proxy settings on init

* Rename extra requirement group

Co-authored-by: Tulir Asokan <tulir@maunium.net>

---------

Co-authored-by: Tulir Asokan <tulir@maunium.net>

.gitignore

@@ -14,6 +14,7 @@ __pycache__
 /registration.yaml
 *.log*
 *.db
+*.db-*
 /*.pickle
 *.bak
 /*.session

CHANGELOG.md

@@ -1,3 +1,24 @@
+# v0.14.1 (2023-06-26)
+
+### Added
+* Added option to delete megolm sessions that were received before the
+  automatic ratcheting options were introduced.
+* Added config option to use IPv6 for Telegram connection
+  (thanks to [@exciler] in [#920]).
+
+### Improved
+* Dropped support for Python 3.8.
+* Updated Docker image to Alpine 3.18.
+* Added timeout for forward backfills to prevent it from getting stuck
+  permanently.
+
+### Fixed
+* Fixed `bridge.filter.users` config option not being read correctly.
+* Fixed proxy support to use python-socks instead of pysocks.
+
+[@exciler]: https://github.com/exciler
+[#920]: https://github.com/mautrix/telegram/pull/920
+
 # v0.14.0 (2023-05-26)
 
 ### Added

Dockerfile

@@ -1,4 +1,4 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.17
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.18
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
@@ -14,8 +14,6 @@ RUN apk add --no-cache \
       py3-idna \
       py3-rsa \
       #py3-telethon \ (outdated)
-        # Optional for socks proxies
-        py3-pysocks \
         py3-pyaes \
         # cryptg
           py3-cffi \

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.14.0"
+__version__ = "0.14.1"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/abstract_user.py

@@ -208,6 +208,8 @@ class AbstractUser(ABC):
         sysversion = self.config["telegram.device_info.system_version"]
         appversion = self.config["telegram.device_info.app_version"]
         connection, proxy = self._proxy_settings
+        if proxy:
+            self.log.debug(f"Using proxy setting: {proxy}")
 
         assert isinstance(session, Session)
 
@@ -235,6 +237,7 @@ class AbstractUser(ABC):
             loop=self.loop,
             base_logger=base_logger,
             update_error_callback=self._telethon_update_error_callback,
+            use_ipv6=self.config["telegram.connection.use_ipv6"],
         )
         self.client.add_event_handler(self._update_catch)
 
@@ -708,6 +711,22 @@ class AbstractUser(ABC):
             self.log.debug("Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
             return
 
+        task = self._call_portal_message_handler(update, original_update, portal, sender)
+        if portal.backfill_lock.locked:
+            self.log.debug(
+                f"{portal.tgid_log} is backfill locked, moving incoming message to async task"
+            )
+            background_task.create(task)
+        else:
+            await task
+
+    async def _call_portal_message_handler(
+        self,
+        update: UpdateMessageContent,
+        original_update: UpdateMessage,
+        portal: po.Portal,
+        sender: pu.Puppet,
+    ) -> None:
         await portal.backfill_lock.wait(f"update {update.id}")
 
         if isinstance(update, MessageService):

mautrix_telegram/config.py

@@ -268,6 +268,7 @@ class Config(BaseBridgeConfig):
         copy("telegram.connection.retry_delay")
         copy("telegram.connection.flood_sleep_threshold")
         copy("telegram.connection.request_retries")
+        copy("telegram.connection.use_ipv6")
 
         copy("telegram.device_info.device_model")
         copy("telegram.device_info.system_version")

mautrix_telegram/example-config.yaml

@@ -40,7 +40,7 @@ appservice:
 
     # The full URI to the database. SQLite and Postgres are supported.
     # Format examples:
-    #   SQLite:   sqlite:///filename.db
+    #   SQLite:   sqlite:filename.db
     #   Postgres: postgres://username:password@hostname/dbname
     database: postgres://username:password@hostname/dbname
     # Additional arguments for asyncpg.create_pool() or sqlite3.connect()
@@ -291,6 +291,10 @@ bridge:
             delete_on_device_delete: false
             # Periodically delete megolm sessions when 2x max_age has passed since receiving the session.
             periodically_delete_expired: false
+            # Delete inbound megolm sessions that don't have the received_at field used for
+            # automatic ratcheting and expired session deletion. This is meant as a migration
+            # to delete old keys prior to the bridge update.
+            delete_outdated_inbound: false
         # What level of device verification should be required from users?
         #
         # Valid levels:
@@ -326,6 +330,10 @@ bridge:
             # default.
             messages: 100
 
+            # Disable rotating keys when a user's devices change?
+            # You should not enable this option unless you understand all the implications.
+            disable_device_change_key_rotation: false
+
     # Whether to explicitly set the avatar and room name for private chat portal rooms.
     # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms.
     # If set to `always`, all DM rooms will have explicit names and avatars set.
@@ -604,6 +612,8 @@ telegram:
         # is not recommended, since some requests can always trigger a call fail (such as searching
         # for messages).
         request_retries: 5
+        # Use IPv6 for Telethon connection
+        use_ipv6: false
 
     # Device info sent to Telegram.
     device_info:

mautrix_telegram/portal.py

@@ -488,7 +488,7 @@ class Portal(DBPortal, BasePortal):
         cls.private_chat_portal_meta = cls.config["bridge.private_chat_portal_meta"]
         cls.filter_mode = cls.config["bridge.filter.mode"]
         cls.filter_list = cls.config["bridge.filter.list"]
-        cls.filter_users = cls.config["bridge.filter.filter_users"]
+        cls.filter_users = cls.config["bridge.filter.users"]
         cls.hs_domain = cls.config["homeserver.domain"]
         cls.backfill_msc2716 = cls.config["bridge.backfill.msc2716"]
         cls.backfill_enable = cls.config["bridge.backfill.enable"]
@@ -2863,8 +2863,11 @@ class Portal(DBPortal, BasePortal):
         if limit == 0:
             return "Limit is zero, not backfilling"
         with self.backfill_lock:
-            output = await self.backfill(
-                source, client, forward=True, forward_limit=limit, last_tgid=last_tgid
+            output = await asyncio.wait_for(
+                self.backfill(
+                    source, client, forward=True, forward_limit=limit, last_tgid=last_tgid
+                ),
+                timeout=15 * 60,
             )
             self.log.debug(f"Forward backfill complete, status: {output}")
             return output
@@ -3129,9 +3132,16 @@ class Portal(DBPortal, BasePortal):
             anchor_id = 2**31 - 1
             minmax = {}
         self.log.debug(f"Iterating messages through {source.tgid} with {limit=}, {minmax}")
+        delay_warn_handle = self.loop.call_later(
+            5 * 60, lambda: self.log.warning("Iterating messages is taking long")
+        )
         # Iterate messages newest to oldest and collect the results
         async for msg in client.iter_messages(entity, limit=limit, **minmax):
             message_count += 1
+            if message_count == 1:
+                self.log.debug(f"Backfill iter: got first message {msg.id}")
+            elif message_count % 50 == 0:
+                self.log.debug(f"Backfill iter: got {message_count} messages so far (at {msg.id})")
             if (forward and msg.id <= anchor_id) or (not forward and msg.id >= anchor_id):
                 continue
             elif isinstance(msg, MessageService):
@@ -3156,6 +3166,7 @@ class Portal(DBPortal, BasePortal):
                 events.append(await self._wrap_batch_msg(intent, msg, converted, caption=True))
                 intents.append(intent)
                 metas.append(None)
+        delay_warn_handle.cancel()
         if len(events) == 0:
             self.log.debug(
                 f"Didn't get any events to send out of {message_count} messages fetched "

optional-requirements.txt

@@ -14,7 +14,7 @@ qrcode>=6,<8
 phonenumbers>=8,<9
 
 #/metrics
-prometheus_client>=0.6,<0.17
+prometheus_client>=0.6,<0.18
 
 #/e2be
 python-olm>=3,<4
@@ -23,3 +23,6 @@ unpaddedbase64>=1,<3
 
 #/sqlite
 aiosqlite>=0.16,<0.20
+
+#/proxy
+python-socks[asyncio]

requirements.txt

@@ -3,8 +3,8 @@ python-magic>=0.4,<0.5
 commonmark>=0.8,<0.10
 aiohttp>=3,<4
 yarl>=1,<2
-mautrix>=0.19.14,<0.20
-tulir-telethon==1.28.0a9
+mautrix>=0.20.0,<0.21
+tulir-telethon==1.29.0a2
 asyncpg>=0.20,<0.28
 mako>=1,<2
 setuptools

setup.py

@@ -51,7 +51,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.8",
+    python_requires="~=3.9",
 
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -60,9 +60,9 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     package_data={"mautrix_telegram": [
         "web/public/*.mako", "web/public/*.png", "web/public/*.css",