Bump version to 0.11.0

Everyone on v0.10.x already has the latest alembic database reversion,
so it's not needed here anymore.

.codeclimate.yml

@@ -1,8 +0,0 @@
-engines:
-  sonar-python:
-    enabled: true
-    checks:
-      python:S107:
-        enabled: false
-exclude_patterns:
-- "alembic/"

.dockerignore

@@ -2,3 +2,9 @@
 .codeclimate.yml
 *.png
 *.md
+logs
+.venv
+start
+config.yaml
+registration.yaml
+*.db

.editorconfig

@@ -13,3 +13,6 @@ max_line_length = 99
 
 [*.{yaml,yml,py}]
 indent_style = space
+
+[.gitlab-ci.yml]
+indent_size = 2

.github/workflows/python-lint.yml

@@ -0,0 +1,18 @@
+name: Python lint
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: "3.10"
+    - uses: isort/isort-action@master
+      with:
+        sortPaths: "./mautrix_telegram"
+    - uses: psf/black@21.12b0
+      with:
+        src: "./mautrix_telegram"

.gitignore

@@ -1,11 +1,18 @@
-.idea/
+/.idea/
 
-.venv
+/.venv
+/env/
 pip-selfcheck.json
 *.pyc
 __pycache__
+/build
+/dist
+/*.egg-info
+/.eggs
 
-config.yaml
-registration.yaml
-*.log
+/config.yaml
+/registration.yaml
+*.log*
 *.db
+*.pickle
+*.bak

.gitlab-ci.yml

@@ -0,0 +1,66 @@
+image: docker:stable
+
+stages:
+- build
+- manifest
+
+default:
+  before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+
+build amd64:
+  stage: build
+  tags:
+  - amd64
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  after_script:
+  - |
+    if [[ "$CI_COMMIT_BRANCH" == "master" && "$CI_JOB_STATUS" == "success" ]]; then
+      apk add --update curl jq
+      rm -rf /var/cache/apk/*
+
+      jq -n '
+         {
+           password:  env.BEEPER_DEV_ADMIN_NIGHTLY_PASS,
+           bridge:    env.BEEPER_BRIDGE_TYPE,
+           image:     "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
+           channel:   "STABLE"
+         }
+        ' | curl "$BEEPER_DEV_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
+
+      jq -n '
+         {
+           password:    env.BEEPER_PROD_ADMIN_NIGHTLY_PASS,
+           bridge:      env.BEEPER_BRIDGE_TYPE,
+           image:       "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
+           channel:     "INTERNAL",
+           deployNext:  true
+         }
+         ' | curl "$BEEPER_PROD_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
+    fi
+
+build arm64:
+  stage: build
+  tags:
+  - arm64
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+
+manifest:
+  stage: manifest
+  before_script:
+  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
+  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
+  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64

CHANGELOG.md

@@ -0,0 +1,704 @@
+# (unreleased)
+
+* Added support for message reactions
+* Added support for spoiler text.
+
+# v0.11.0 (2021-12-28)
+
+* Switched from SQLAlchemy to asyncpg/aiosqlite.
+  * The default database is now Postgres. If using SQLite, make sure you install
+    the `sqlite` [optional dependency](https://docs.mau.fi/bridges/python/optional-dependencies.html).
+  * **Alembic is no longer used**, schema migrations happen automatically on startup.
+  * **The automatic database migration requires you to be on the latest legacy
+    database version.** If you were running any v0.10.x version, you should be on
+    the latest version already. Otherwise, update to v0.10.2 first, upgrade the
+    database with `alembic`, then upgrade to v0.11.0 (or higher).
+* Added support for contact messages.
+* Added support for Telegram sponsored messages in channels.
+  * Only applies to broadcast channels with 1000+ members
+    (as per <https://t.me/durov/172>).
+  * Only applies if you're using puppeting with a normal user account,
+    because bots can't get sponsored messages.
+* Fixed non-supergroup member sync incorrectly kicking one user from the Matrix
+  side if there was no limit on the number of members to sync (broke in v0.10.2).
+* Updated animated sticker conversion to support [lottieconverter r0.2]
+  (thanks to [@sot-tech] in [#694]).
+* Updated Docker image to Alpine 3.15.
+* Formatted all code using [black](https://github.com/psf/black)
+  and [isort](https://github.com/PyCQA/isort).
+
+[lottieconverter r0.2]: https://github.com/sot-tech/LottieConverter/releases/tag/r0.2
+[#694]: https://github.com/mautrix/telegram/pull/694
+
+# v0.10.2 (2021-11-13)
+
+### Added
+* Added extensions when bridging unnamed files from Telegram.
+* Added support for custom bridge bot welcome messages
+  (thanks to [@justinbot] in [#676]).
+
+### Improved
+* Improved handling authorization errors if the bridge was logged out remotely.
+* Updated room syncer to use existing power levels to find appropriate levels
+  for admins and normal users instead of hardcoding 50 and 0.
+* Updated to Telegram API layer 133 to handle 64-bit user/chat/channel IDs.
+* Stopped logging message contents when message handling failed
+  (thanks to [@justinbot] in [#681]).
+* Removed Element iOS compatibility hack from non-sticker files.
+* Made `max_initial_member_sync` work for non-supergroups too
+  (thanks to [@tadzik] in [#680]).
+* SQLite is now supported for the crypto database. Pickle is no longer supported.
+  If you were using pickle, the bridge will create a new e2ee session and store
+  the data in SQLite this time.
+
+### Fixed
+* Fixed generating reply fallbacks to encrypted messages.
+* Fixed chat sync failing if the member list contained banned users.
+
+[@justinbot]: https://github.com/justinbot
+[@tadzik]: https://github.com/tadzik
+[#676]: https://github.com/mautrix/telegram/pull/676
+[#680]: https://github.com/mautrix/telegram/pull/680
+[#681]: https://github.com/mautrix/telegram/pull/681
+
+# v0.10.1 (2021-08-19)
+
+**N.B.** Docker images have moved from `dock.mau.dev/tulir/mautrix-telegram`
+to `dock.mau.dev/mautrix/telegram`. New versions are only available at the new
+path.
+
+### Added
+
+* Warning when bridging existing room if bridge bot doesn't have redaction
+  permissions.
+* Custom flag to invite events that will be auto-accepted using double puppeting.
+* Custom flags for animated stickers (same as what gifs already had).
+
+### Improved
+* Updated to Telethon 1.22.
+* Updated Docker image to Alpine 3.14.
+
+### Fixed
+* Fixed Bridging Matrix location messages with additional flags in `geo_uri`.
+* Editing encrypted messages will no longer add an asterisk on Telegram.
+* Matrix typing notifications won't be echoed back for double puppeted users anymore.
+* `AuthKeyDuplicatedError` is now handled properly instead of making the user
+  get stuck.
+* Fixed `public_portals` setting not being respected on room creation.
+
+# v0.10.0 (2021-06-14)
+
+* Added options to bridge archive, pin and mute status from Telegram to Matrix.
+* Added custom fields in Matrix events indicating Telegram gifs.
+* Allowed zero-width joiners in displaynames so things like multi-part emoji
+  would work correctly.
+* Fixed Telegram->Matrix typing notifications.
+
+## rc1 (2021-04-05)
+
+### Added
+* Support for multiple pins from/to Telegram.
+* Option to resolve redirects when joining invite links, for people who use
+  custom URLs as invite links.
+* Command to update about section in Telegram profile info
+  (thanks to [@MadhuranS] in [#599]).
+* Own read marker/unread status from Telegram is now synced to Matrix after backfilling.
+* Support for showing the individual slots in 🎰 dice rolls from Telegram.
+
+### Improved
+* Improved invite link regex to allow joining with less precise invite links.
+* Invite links can be customized with the `--uses=<amount>` and
+  `--expire=<delta>` flags for `!tg invite-link`.
+* Read receipts where the target message is unknown will now cause the chat to
+  be marked as fully read instead of the read receipt event being ignored.
+* WebP stickers are now sent as-is without converting to png.
+* Default power levels in rooms now allow enabling encryption with PL 50 if
+  e2be is enabled in config (thanks to [@Rafaeltheraven] in [#550]).
+* Updated Docker image to Alpine 3.13 and removed all edge repo stuff.
+
+### Fixed
+* Matrix->Telegram location message bridging no longer flips the coordinates.
+* Fixed some user displaynames constantly changing between contact/non-contact
+  names and other similar cases.
+
+[@Rafaeltheraven]: https://github.com/Rafaeltheraven
+[@MadhuranS]: https://github.com/MadhuranS
+[#550]: https://github.com/mautrix/telegram/pull/550
+[#599]: https://github.com/mautrix/telegram/pull/599
+
+# v0.9.0 (2020-11-17)
+
+* Fixed cleaning unidentified rooms.
+
+## rc3 (2020-11-12)
+
+### Added
+* Added retrying message sending if server returns 502.
+
+### Fixed
+* Fixed Matrix → Telegram name mentions.
+* Fixed some bugs with replies.
+
+## rc2 (2020-11-06)
+
+### Improved
+* Ephemeral event handling should be faster by not checking the database for
+  user existence.
+* Using the register command now sends a link to the Telegram terms of service.
+* The `bridge_connected` metric is now only set for users who are logged in.
+
+### Fixed
+* Fixed bug where syncing members sometimes kicked ghosts of users who were
+  actually still in the chat.
+* Fixed sending captions to Telegram with `!tg caption` (broken in rc1).
+* Logging out will now delete private chat portals, instead of only kicking the
+  user and leaving the portal in a broken state.
+* Unbridging direct chat portals is now possible.
+
+## rc1 (2020-10-24)
+
+### Breaking changes
+* Prometheus metric names are now prefixed with `bridge_`.
+* An entrypoint script is no longer automatically generated. This won't affect
+  most users, as `python -m mautrix_telegram` has been the official way to start
+  the bridge for a long time.
+
+### Added
+* Support for logging in by scanning a QR code from another Telegram client.
+* Automatic backfilling of old messages when creating portals.
+* Automatic backfilling of missed messages when starting bridge.
+* Option to update `m.direct` list when using double puppeting.
+* PNG thumbnails for animated stickers when converted to webm.
+* Support for receiving ephemeral events pushed directly with [MSC2409]
+  (requires Synapse 1.22 or higher).
+
+### Improved
+* Switched end-to-bridge encryption to mautrix-python instead of a hacky
+  matrix-nio solution.
+* End-to-bridge encryption no longer requires `login_shared_secret`, it uses
+  [MSC2778] instead (requires Synapse 1.21 or higher).
+* The bridge info state event is now updated whenever the chat name or avatar changes.
+* Double puppeting is no longer limited to users on the same homeserver as the bridge.
+* Delivery receipts are no longer sent in unencrypted private chat portals, as
+  the bridge bot is usually not present in them.
+
+### Fixed
+* File captions are now sent as a separate message like photo captions.
+* The relaybot no longer drops Telegram messages with commands.
+* Bridging events of a user whose power level is malformed (i.e. a string
+  instead of an integer) now works.
+
+[MSC2409]: https://github.com/matrix-org/matrix-doc/pull/2409
+[MSC2778]: https://github.com/matrix-org/matrix-doc/pull/2778
+
+# v0.8.2 (2020-07-27)
+
+* Fixed deleting messages from Matrix.
+* Fixed Alpine edge dependencies in Docker image.
+
+Note: this release is not on PyPI, as the only changes were a mautrix-python
+update (v0.5.8) and a fix to the Docker image.
+
+# v0.8.1 (2020-06-08)
+
+* Fixed starting bridge for the first time failing due to not registering the bridge bot.
+* Updated Docker image to Alpine 3.12.
+
+# v0.8.0 (2020-06-03)
+
+* Updated to mautrix-python 0.5.0 and matrix-nio 0.12.0.
+
+## rc5 (2020-05-30)
+
+* Added option to disable removing avatars from Telegram ghosts.
+* Added option to send delivery error notices.
+* Added option to send delivery receipts.
+* Bumped maximum Telethon version to 1.14.
+* Possibly fixed infinite loop of avatar changes when using double puppeting.
+
+## rc3 (2020-05-22)
+
+* Moved private information to trace log level.
+* Added `private_chat_portal_meta` option. This is implicitly enabled when
+  encryption is enabled, it was only added as an option for instances with
+  encryption disabled.
+* Removed avatars are now synced properly from Telegram, instead of leaving the
+  last known avatar forever.
+* Fixed admin detection on Telegram-side relaybot commands
+  (thanks to [@davidmehren] in [#468]).
+* Fixed bug handling `ChatForbidden` when syncing chats.
+
+[@davidmehren]: https://github.com/davidmehren
+[#468]: https://github.com/mautrix/telegram/pull/468
+
+## rc2 (2020-05-20)
+
+* Implemented [MSC2346]: Bridge information state event for newly created rooms.
+* Fixed `sync_direct_chats` option creating non-working portals.
+* Fixed video thumbnailing sometimes leaving behind downloaded videos in `/tmp`.
+
+[MSC2346]: https://github.com/matrix-org/matrix-doc/pull/2346
+
+## rc1 (2020-04-25)
+
+### Added
+* Command for backfilling room history from Telegram.
+* arm64 support in docker images.
+* Optional end-to-bridge encryption support.
+  See [docs](https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html) for more info.
+* Bridging for Telegram dice roll messages.
+
+### Fixed
+* Riot iOS not showing stickers properly.
+* Updated to Telethon 1.13 to fix bugs like [#443].
+
+[#443]: https://github.com/mautrix/telegram/issues/443
+
+# v0.7.2 (2020-04-04)
+
+* No changes since rc1.
+
+## rc1 (2020-02-08)
+
+* Fixed enabling double puppeting causing saved messages to become unusable.
+* Fixed receiving channel messages when `ignore_own_incoming_events` was enabled.
+
+# v0.7.1 (2020-02-04)
+
+* Fixed missing responses in logout provisioning API.
+
+## rc2 (2020-01-25)
+
+* Fixed import in database migration script (thanks to [@cubesky] in [#409]).
+* Fixed relaybot messages being allowed through to Matrix even when
+  `ignore_own_incoming_events` was set to `true`.
+
+[@cubesky]: https://github.com/cubesky
+[#409]: https://github.com/mautrix/telegram/pull/409
+
+## rc1 (2020-01-11)
+
+* Fixed incorrect parameter name causing `!tg config set` to throw an error.
+* Fixed potential dictionary size changed during iteration crash.
+
+# v0.7.0 (2019-12-28)
+
+* No changes since rc4.
+
+## rc4 (2019-12-25)
+
+* Fixed handling of Matrix `m.emote` events.
+
+## rc3 (2019-12-25)
+
+* Added option to log in to custom puppet with shared secret
+  (<https://github.com/devture/matrix-synapse-shared-secret-auth>).
+* Updated Docker image to Alpine 3.11.
+* Improved displayname syncing by trusting any displayname if user is not a contact.
+* Fixed error when cleaning up rooms.
+* Fixed stack traces being printed to non-admin users.
+* Fixed invite rejections being handles as leaves.
+* Fixed `version` command output in CI docker builds not showing the correct
+  git commit hash.
+
+## rc2 (2019-12-01)
+
+* Added command to get bridge version.
+* Made bridge refuse to start if config contains example values.
+* Removed some debug stack traces.
+* Ignored `ChatForbidden` when syncing chats that was causing the sync to fail.
+* Fixed DB migration causing some incorrect values to be left behind.
+
+## rc1 (2019-11-30)
+
+### Important changes
+* Dropped Python 3.5 compatibility.
+* Moved docker registry to [dock.mau.dev](https://mau.dev/tulir/mautrix-telegram/container_registry).
+
+### Added
+* Support for bridging animated stickers (thanks to [@sot-tech] in [#366]).
+  * Requires [LottieConverter](https://github.com/sot-tech/LottieConverter),
+    which is included in the docker image.
+  * Can be [configured](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L174-L187).
+* Support for MTProxy (thanks to [@sot-tech] in [#344]).
+* [Config option](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L117-L118)
+  for max length of displayname, with the default being 100.
+* Separate [config option](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L232-L238)
+  for `m.emote` formatting of logged in users.
+* Streamed file transfers and parallel telegram file download/upload.
+  * Files are streamed from telegram servers to the media repo rather than
+    downloading the whole file into memory.
+  * File transfers use multiple connections to telegram servers to transfer faster.
+  * Parallel and streamed file transfers can be enabled in the
+    [config](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L166-L169).
+* Command to set caption for files and images when sending to telegram.
+* Bridging bans to telegram.
+* Helm chart.
+
+### Improved
+* Switched from mautrix-appservice-python to [mautrix-python](https://github.com/mautrix/python).
+* Users with Matrix puppeting can now bridge their "Saved Messages" chat.
+* The bridge will refuse to start without access to the example config file.
+* Changed default port to 29317.
+* Mentions are now marked as read on Telegram when bridging read receipts using
+  double puppeting.
+* Kicking or banning the bridge bot will now unbridge the room.
+* Shrinked Docker image from 151mb to 77mb.
+
+### Fixed
+* The bridge will no longer crash if one user's startup fails.
+* (hopefully) Incorrect peer type being saved into database in some cases.
+* File names when bridging to Telegram.
+* Alembic config interpolating passwords with `%`.
+* A single chat failing to sync preventing any chat from syncing.
+* Users logged in as a bot not receiving any messages.
+* Username matching being case-sensitive in the database and preventing
+  telegram->matrix pillifying.
+* IndexError if running `!tg set-pl` with no parameters.
+
+[@sot-tech]: https://github.com/sot-tech
+[#344]: https://github.com/mautrix/telegram/pull/344
+[#366]: https://github.com/mautrix/telegram/pull/366
+
+# v0.6.0 (2019-07-09)
+
+* Fixed vulnerability in event handling.
+
+## rc2 (2019-07-06)
+
+* Nested formatting is now supported by Telegram, so the bridge also supports it.
+* Strikethrough and underline are now bridged into native Telegram formatting
+  rather than unicode hacks.
+* Fixed displayname not updating for users who the bridge only saw via a logged
+  in user who had the problematic user in their contacts.
+* Fixed handling unsupported media.
+* Added handling for `FileIdInvalidError` in file transfers that could disrupt
+  `sync`s.
+
+## rc1 (2019-06-22)
+
+### Added
+* Native Matrix edit support and new fallback format.
+* Config options for `retry_delay` and other TelegramClient constructor fields.
+* Config option for maximum document size to let through the bridge.
+* External URL field for chat and private channel messages.
+* Telegram user info (puppet displayname & avatar) is now updated every time
+  the user sends a message.
+* Command to change Telegram displayname.
+* Possibility to override config fields with environment variables
+  (thanks to [@pacien] in [#332]).
+
+### Improved
+* Simplified bridged poll message.
+* Telegram user info updates are now accepted from any logged in user as long
+  as the logged in user doesn't see a phone number for the Telegram user.
+* Some image errors are now handled by resending the image as a document.
+* Made getting started more user-friendly.
+* Updated to Telethon 1.8.
+
+### Fixed
+* Portal peer type not being saved in database after Telegram chat upgrade.
+* Newlines in unformatted messages not being bridged when using relaybot.
+* Mime type info field for stickers converted to PNG.
+* Content after newlines being stripped in messages sent by some clients.
+* Potential `NoneType is not iterable` exception when logging out
+  (thanks to [@turt2live] in [#315]).
+* Handling of Matrix messages where `m.relates_to` is null.
+* Internal server error when logging in with an account on another DC.
+* Spaces between command and arguments are now trimmed.
+* Changed migrations to use `batch_alter_table` for adding columns to have less
+  warnings with SQLite.
+* Error when `ping`ing without being logged in.
+* Terminating sessions with negative hashes.
+* State cache not being updated when sending events, causing invalid cache if
+  the server doesn't echo the sent events.
+
+[@pacien]: https://github.com/pacien
+[#315]: https://github.com/mautrix/telegram/pull/315
+[#332]: https://github.com/mautrix/telegram/pull/332
+
+# v0.5.2 (2019-05-25)
+
+* Fixed null `m.relates_to`'s that break Synapse 0.99.5.
+
+# v0.5.1 (2019-03-21)
+
+* Fixed Python 3.5 compatibility.
+* Fixed DBMS migration script.
+
+# v0.5.0 (2019-03-19)
+
+* Replaced rawgit with cdnjs in public website as rawgit is deprecated.
+* Fixed login command throwing error when web login is enabled.
+* Updated telethon-session-sqlalchemy to fix logging into an account on another DC.
+* Stopped adding reply fallback to caption when sending caption and image as
+  separate messages.
+
+## rc4 (2019-03-16)
+
+* Added verbose flag to migration script.
+* Added pytest setup and some tests (thanks to [@V02460] in [#290]).
+* Fixed scripts (DBMS migration and Telematrix import) not being included in builds.
+* Fixed some database problems.
+* Removed remaining traces of ORM that might have been the causes of some other
+  database problems.
+* Removed option to use lxml in HTML parsing as it was messing up emoji offset
+  handling. The new HTML parser supports using the default python HTMLParser
+  class since 0.5.0rc1, so lxml wasn't really useful anway.
+
+[#290]: https://github.com/mautrix/telegram/pull/290
+
+## rc3 (2019-02-16)
+
+* Fixed bridging documents without thumbnails to Matrix.
+* Added option to set maximum size of image to send to Telegram. Images above
+  the size limit will be sent as documents without the compression Telegram
+  applies to images.
+* Fixed saving user portals and contacts.
+* Added Telegram -> Matrix poll bridging and a command to vote in polls.
+
+## rc2 (2019-02-15)
+
+* Added missing future-fstrings comments that caused the bridge to not start on
+  Python 3.5.
+* Fixed handling of document thumbnails.
+* Fixed private chat portals failing to be created.
+* Made relaybot handle Telegram chat upgrade events.
+
+## rc1 (2019-02-14)
+
+### Added
+* More config options
+  * Option to to use Telegram test servers.
+  * Option to disable link previews on Telegram.
+  * Option to disable startup sync.
+  * Option to skip deleted members when syncing member lists.
+  * Option to change number of dialogs to handle in startup sync.
+* More commands
+  * `username` for setting Telegram username.
+  * `sync-state` for updating Matrix room state cache.
+  * `matrix-ping` for checking Matrix login status (thanks to [@krombel] in [#271]).
+  * `clear-db-cache` for clearing internal database caches.
+  * `reload-user` for reloading and reconnecting a Telegram user.
+  * `session` for listing and terminating other Telegram sessions.
+  * Added argument to `login` to allow admins to log in for other users.
+* Added warning when logging in that it grants the bridge full access to the
+  telegram account.
+* Telegram->Matrix bridging:
+  * Telegram games
+  * Message pins in normal groups
+  * Custom message for unsupported media like polls
+* Added client ID in logs when making requests to telegram.
+* Added handling for Matrix room upgrades.
+
+### Improved
+* Removed lxml dependency from the new HTML parser and removed the old parser
+  completely.
+* Switched mautrix-appservice-python state store and most mautrix-telegram
+  tables to SQLAlchemy core. This should speed things up and reduce problems
+  with the ORM getting stuck.
+* `ensure_started` is now only called for logged in users, which should improve
+  performance for large instances.
+* Displayname template extras (e.g. the `(Telegram)` suffix) are now stripped
+  when mentioning Telegram users with no username.
+* Updated Telethon.
+* Switched Dockerfile to use setup.py for dependencies to avoid dependency
+  updates breaking stuff.
+* The telematrix import script will now warn about and skip over duplicate portals.
+* Relaybot will now be used for users who have logged in, but are not in the chat.
+
+### Fixed
+* Bug where stickers with an unidentified emoji failed to bridge.
+* Invalid letter prefixes in clean-rooms output.
+* Messages forwarded from channels showing up as "Unknown source".
+* Matrix->Telegram room avatar bridging.
+
+[@krombel]: https://github.com/krombel
+[#271]: https://github.com/mautrix/telegram/pull/271
+
+# v0.4.0 (2018-11-28)
+
+* No changes since rc2.
+
+## rc2 (2018-11-15)
+
+* Fixed kicking Telegram puppets from Matrix.
+
+## rc1 (2018-11-15)
+
+### Added
+* Flag to indicate if user can unbridge portal in provisioning API
+  (thanks to [@turt2live] in [#225]).
+* Option to send captions as second message (replaces option to send caption
+  in `body`.
+* Room-specific settings.
+
+### Improved
+* (internal) Added type hints everywhere (mostly thanks to [@V02460] in [#206]).
+* Telegram->Matrix formatter now uses `<pre>` tags for multiline code even if
+  said code was in the telegram equivalent of inline code tags.
+* Better bullets and linebreak handling in Matrix->Telegram formatter.
+* Logging in will now show your phone number instead of `@None` if you don't
+  have a username.
+* Significantly improved performance on high-load instances (t2bot.io) by
+  moving most used database tables to SQLAlchemy Core.
+
+### Fixed
+* Bugs that caused database migrations to fail in some cases.
+* Editing the config (e.g. whitelisting chats) corrupting the config.
+* Negative numbers (chat IDs) in `/connect` of the provisioning API
+  (thanks to [@turt2live] in [#223]).
+* Relaybot creating portals automatically when receiving message.
+* Not being able to use a bridge bot localpart that would also match the puppet
+  localpart format.
+* Matrix login sync failing completely if the homeserver stopped during a sync
+  response.
+* Errors when cleaning rooms.
+* Bridging code blocks without a language.
+* Error and lost messages when trying to bridge PM from new users in some cases.
+* Logging in with an account that someone has already logged in failing
+  silently and then breaking the bridge.
+* Relaybot message when adding/removing Matrix displaynames.
+
+[@V02460]: https://github.com/V02460
+[#206]: https://github.com/mautrix/telegram/pull/206
+[#223]: https://github.com/mautrix/telegram/pull/223
+[#225]: https://github.com/mautrix/telegram/pull/225
+
+# v0.3.0 (2018-08-15)
+
+* Added database URI format examples.
+* Bumped maximum Telethon version to 1.2, possibly fixing the catch_up option.
+
+## rc3 (2018-08-08)
+
+* Improved Telegram message deduplication options.
+  * Added pre-send message database check for deduplication.
+  * Made dedup cache queue length configurable.
+
+## rc2 (2018-08-06)
+
+* Added option to change max body size for AS API.
+* Fixed a minor error regarding power level changes (thanks to [@turt2live] in [#203]).
+* Updated minimum mautrix-appservice version to include some recent bugfixes.
+
+[@turt2live]: https://github.com/turt2live
+[#203]: https://github.com/mautrix/telegram/pull/203
+
+## rc1 (2018-08-05)
+
+### Added
+* Logging in with a bot
+  (see [docs](https://docs.mau.fi/bridges/python/telegram/authentication.html#bot-token) for usage).
+  * You can log in with a personal Telegram bot to appear almost like a real
+    user without logging in with a real Telegram account.
+* Replacing your Telegram account's Matrix puppet with your Matrix account
+  (see [docs](https://docs.mau.fi/bridges/general/double-puppeting.html) for usage).
+* Formatting options for relaybot messages.
+  * Real displaynames are now supported and enabled by default.
+  * State events (join/leave/name change) can be independently disabled by
+    setting the format to a blank string.
+* New config sections
+  * Proper log config, including logging to file (by default)
+  * Proxy support (requires installing PySocks)
+  * Separate field for appservice address for homeserver
+    (useful if using a reverse proxy).
+* New permission levels to allow initiating bridges without allowing puppeting
+  and to allow Telegram puppeting without allowing Matrix puppeting.
+* Telematrix import script (see [docs](https://docs.mau.fi/bridges/python/telegram/migrating-from-telematrix.html) for usage).
+* Provisioning API (see [docs](https://docs.mau.fi/bridges/python/telegram/provisioning-api.html) for more info).
+* DBMS migration script (see [docs](https://docs.mau.fi/bridges/python/telegram/dbms-migration.html) for usage).
+
+### Improved
+* Tabs are now replaced with 4 spaces so that Telegram servers wouldn't change
+  the message.
+* Help page now detects your permissions and only shows commands you can use.
+* Moved Matrix state cache to the main database. This means that the
+  `mx-state.json` file is no longer needed and all non-config data is
+  stored in the main database.
+* Better lxml-based HTML parser for Matrix->Telegram formatting bridging.
+  lxml is still optional, so the old parser is used as fallback if lxml is not
+  installed.
+* Disabled Telegram->Matrix bridging of messages sent by the relaybot.
+  Can be re-enabled in config if necessary.
+
+### Fixed
+* A `ValueError` in some cases when syncing power levels.
+* Telegram connections being created for unauthenticated users possibly
+  triggering spam protection connection delays in the Telegram servers.
+* Logging out if a portal had been deleted/unbridged.
+
+# v0.2.0 (2018-06-08)
+
+* No changes since rc6.
+
+## rc6 (2018-06-06)
+
+* Added warning about `delete-portal` kicking all room members.
+* Fixed error when upgrading/creating SQLite database.
+
+## rc5 (2018-06-01)
+
+* Fixed relaybot automatically creating portal rooms when invited to Telegram chat ([#145]).
+* Fixed kicking Telegram puppets and fix error message when bridging chats you've left.
+* Fixed integrity error deleting portals from database.
+
+[#145]: https://github.com/mautrix/telegram/issues/145
+
+## rc4 (2018-05-29)
+
+* ~~Fixed~~ Added Postgres compatibility.
+* Fixed manual bridging (`!tg bridge`) for unauthenticated users.
+* Fixed inviting unauthenticated Matrix users from Telegram (via `/invite <mxid>`).
+* Changed Alembic to read database path from the config, so editing `alembic.ini`
+  is no longer necessary. Use `alembic -x config=/path/to/config.yaml ...` to
+  specify the config path.
+
+## rc3 (2018-05-25)
+
+* Reworked Dockerfile to remove virtualenv and use Alpine packages (thanks to
+  [@jcgruenhage] in [#142]). This fixes webp->png conversion for stickers.
+
+[#142]: https://github.com/mautrix/telegram/pull/142
+
+## rc2 (2018-05-21)
+
+* Added Dockerfile (thanks to [@jcgruenhage] in [#136]).
+
+[#136]: https://github.com/mautrix/telegram/pull/136
+[@jcgruenhage]: https://github.com/jcgruenhage
+
+## rc1 (2018-05-19)
+
+* Added
+  * Option to exclude telegram chats from being bridged.
+  * Support for using a relay bot to relay messages for unauthenticated users
+  * Bridging for message pinning and room mentions/pills.
+  * Matrix->Telegram sticker bridging.
+  * `!command` to `/command` conversion at the start of Matrix message text.
+  * Conversion of t.me message links to matrix.to message links
+  * Timestamp massaging (bridge Telegram timestamps to Matrix)
+  * Support for out-of-Matrix login (useful if you don't want your 2FA password to be stored in the homeserver)
+  * Optional HQ gif/video thumbnails using moviepy.
+  * Option to send bot messages as `m.notice`
+* Improved deduplication
+  * Matrix file uploads are now reused if the same Telegram file (e.g. a sticker) is sent multiple times
+  * Room metadata changes and other non-message actions are now deduplicated
+* Improved formatting bridging
+* Improved Telegram user display name handling in cases where one or more users have set custom display names for other users.
+* Fixed Alembic setup and removed automatic database generation.
+* Fixed outgoing message deduplication in cases where message is sent to other clients before responding to the sender.
+* Moved mautrix-appservice-python to separate repository.
+* Switched to telethon-session-sqlalchemy to have the session databases in the main database.
+* Switched license from GPLv3 to AGPLv3
+* Probably a bunch of other stuff I forgot
+
+# v0.1.1 (2018-02-18)
+
+Fixed bridging formatted messages from Matrix to Telegram
+
+# v0.1.0 (2018-02-17)
+
+First release.
+
+Things work.

Dockerfile

@@ -1,29 +1,61 @@
-FROM docker.io/alpine:3.8
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.15
 
-ENV UID=1337 \
-    GID=1337 \
-    FFMPEG_BINARY=/usr/bin/ffmpeg
+ARG TARGETARCH=amd64
 
-COPY . /opt/mautrix-telegram
-WORKDIR /opt/mautrix-telegram
 RUN apk add --no-cache \
-      python3-dev \
-      build-base \
+      python3 py3-pip py3-setuptools py3-wheel \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
-      py3-lxml \
       py3-magic \
-      py3-numpy \
-      py3-asn1crypto \
-      py3-sqlalchemy \
-      py3-markdown \
-      py3-psycopg2 \
+      py3-ruamel.yaml \
+      py3-commonmark \
+      py3-prometheus-client \
+      # Indirect dependencies
+      py3-idna \
+      #moviepy
+        py3-decorator \
+        py3-tqdm \
+        py3-requests \
+        #imageio
+          py3-numpy \
+      #py3-telethon \ (outdated)
+        # Optional for socks proxies
+        py3-pysocks \
+        py3-pyaes \
+        # cryptg
+          py3-cffi \
+	  py3-qrcode \
+      py3-brotli \
+      # Other dependencies
       ffmpeg \
       ca-certificates \
       su-exec \
- && pip3 install -r requirements.txt -r optional-requirements.txt
+      netcat-openbsd \
+      # encryption
+      py3-olm \
+      py3-pycryptodome \
+      py3-unpaddedbase64 \
+      py3-future \
+      bash \
+      curl \
+      jq \
+      yq
+
+COPY requirements.txt /opt/mautrix-telegram/requirements.txt
+COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
+WORKDIR /opt/mautrix-telegram
+RUN apk add --virtual .build-deps python3-dev libffi-dev build-base \
+ && pip3 install -r requirements.txt -r optional-requirements.txt \
+ && apk del .build-deps
+
+COPY . /opt/mautrix-telegram
+RUN apk add git && pip3 install .[all] && apk del git \
+  # This doesn't make the image smaller, but it's needed so that the `version` command works properly
+  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
 
 VOLUME /data
+ENV UID=1337 GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
 
 CMD ["/opt/mautrix-telegram/docker-run.sh"]

MANIFEST.in

@@ -0,0 +1,5 @@
+include README.md
+include CHANGELOG.md
+include LICENSE
+include requirements.txt
+include optional-requirements.txt

README.md

@@ -1,9 +1,30 @@
 # mautrix-telegram
+![Languages](https://img.shields.io/github/languages/top/mautrix/telegram.svg)
+[![License](https://img.shields.io/github/license/mautrix/telegram.svg)](LICENSE)
+[![Release](https://img.shields.io/github/release/mautrix/telegram/all.svg)](https://github.com/mautrix/telegram/releases)
+[![GitLab CI](https://mau.dev/mautrix/telegram/badges/master/pipeline.svg)](https://mau.dev/mautrix/telegram/container_registry)
+[![Code style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Imports](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+
 A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
-### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)
+## Sponsors
+* [Joel Lehtonen / Zouppen](https://github.com/zouppen)
 
-### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+### Documentation
+All setup and usage instructions are located on
+[docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
+Some quick links:
+
+* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
+  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
+* Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
+  [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
+  [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)
+
+### Features & Roadmap
+[ROADMAP.md](https://github.com/mautrix/telegram/blob/master/ROADMAP.md)
+contains a general overview of what is supported by the bridge.
 
 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)
@@ -11,4 +32,4 @@ Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net
 Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
 
 ## Preview
-![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)
+![Preview](preview.png)

ROADMAP.md

@@ -3,10 +3,11 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message edits
   * [ ] ‡ Message history
-  * [ ] † Presence
-  * [ ] † Typing notifications
-  * [ ] † Read receipts
+  * [x] Presence
+  * [x] Typing notifications
+  * [x] Read receipts
   * [x] Pinning messages
   * [x] Power level
     * [x] Normal chats
@@ -21,9 +22,16 @@
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
+  * [ ] Advanced message content/media
+    * [x] Polls
+	* [x] Games
+	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
-  * [ ] Message history
+  * [x] Message history
+    * [x] Manually (`!tg backfill`)
+    * [x] Automatically when creating portal
+    * [x] Automatically for missed messages
   * [x] Avatars
   * [x] Presence
   * [x] Typing notifications
@@ -45,9 +53,11 @@
     * [x] At startup
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [x] Option to use bot to relay messages for unauthenticated Matrix users
-  * [ ] Option to use own Matrix account for messages sent from other Telegram clients
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
+  * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
+  * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic.ini

@@ -1,71 +0,0 @@
-# A generic, single database configuration.
-
-[alembic]
-# path to migration scripts
-script_location = alembic
-
-# template used to generate migration files
-# file_template = %%(rev)s_%%(slug)s
-
-# timezone to use when rendering the date
-# within the migration file as well as the filename.
-# string value is passed to dateutil.tz.gettz()
-# leave blank for localtime
-# timezone =
-
-# max length of characters to apply to the
-# "slug" field
-#truncate_slug_length = 40
-
-# set to 'true' to run the environment during
-# the 'revision' command, regardless of autogenerate
-# revision_environment = false
-
-# set to 'true' to allow .pyc and .pyo files without
-# a source .py file to be detected as revisions in the
-# versions/ directory
-# sourceless = false
-
-# version location specification; this defaults
-# to alembic/versions.  When using multiple version
-# directories, initial revisions must be specified with --version-path
-# version_locations = %(here)s/bar %(here)s/bat alembic/versions
-
-# the output encoding used when revision files
-# are written from script.py.mako
-# output_encoding = utf-8
-
-# Logging configuration
-[loggers]
-keys = root,sqlalchemy,alembic
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = WARN
-handlers = console
-qualname =
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-
-[logger_alembic]
-level = INFO
-handlers =
-qualname = alembic
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = NOTSET
-formatter = generic
-
-[formatter_generic]
-format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S

alembic/README

@@ -1 +0,0 @@
-Generic single-database configuration.

alembic/env.py

@@ -1,96 +0,0 @@
-from alembic import context
-from sqlalchemy import engine_from_config, pool
-from logging.config import fileConfig
-
-import sys
-from os.path import abspath, dirname
-
-sys.path.insert(0, dirname(dirname(abspath(__file__))))
-
-from mautrix_telegram.base import Base
-from mautrix_telegram.config import Config
-from alchemysession import AlchemySessionContainer
-import mautrix_telegram.db
-
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
-mxtg_config = Config(mxtg_config_path, None, None)
-mxtg_config.load()
-config.set_main_option("sqlalchemy.url",
-                       mxtg_config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
-
-
-class FakeDB:
-    @staticmethod
-    def query_property():
-        return None
-
-
-AlchemySessionContainer.create_table_classes(FakeDB(), "telethon_", Base)
-
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-fileConfig(config.config_file_name)
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-target_metadata = Base.metadata
-
-
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-
-def run_migrations_offline():
-    """Run migrations in 'offline' mode.
-
-    This configures the context with just a URL
-    and not an Engine, though an Engine is acceptable
-    here as well.  By skipping the Engine creation
-    we don't even need a DBAPI to be available.
-
-    Calls to context.execute() here emit the given string to the
-    script output.
-
-    """
-    url = config.get_main_option("sqlalchemy.url")
-    context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True)
-
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-def run_migrations_online():
-    """Run migrations in 'online' mode.
-
-    In this scenario we need to create an Engine
-    and associate a connection with the context.
-
-    """
-    connectable = engine_from_config(
-        config.get_section(config.config_ini_section),
-        prefix='sqlalchemy.',
-        poolclass=pool.NullPool)
-
-    with connectable.connect() as connection:
-        context.configure(
-            connection=connection,
-            target_metadata=target_metadata
-        )
-
-        with context.begin_transaction():
-            context.run_migrations()
-
-
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()

alembic/script.py.mako

@@ -1,24 +0,0 @@
-"""${message}
-
-Revision ID: ${up_revision}
-Revises: ${down_revision | comma,n}
-Create Date: ${create_date}
-
-"""
-from alembic import op
-import sqlalchemy as sa
-${imports if imports else ""}
-
-# revision identifiers, used by Alembic.
-revision = ${repr(up_revision)}
-down_revision = ${repr(down_revision)}
-branch_labels = ${repr(branch_labels)}
-depends_on = ${repr(depends_on)}
-
-
-def upgrade():
-    ${upgrades if upgrades else "pass"}
-
-
-def downgrade():
-    ${downgrades if downgrades else "pass"}

alembic/versions/1b241f7e8530_add_telegramfile_table.py

@@ -1,28 +0,0 @@
-"""Add TelegramFile table
-
-Revision ID: 1b241f7e8530
-Revises: 97d2a942bcf8
-Create Date: 2018-02-19 23:52:06.605741
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '1b241f7e8530'
-down_revision = '97d2a942bcf8'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table('telegram_file',
-                    sa.Column('id', sa.String(), nullable=False),
-                    sa.Column('mxc', sa.String(), nullable=True),
-                    sa.Column('mime_type', sa.String(), nullable=True),
-                    sa.Column('was_converted', sa.Boolean(), nullable=True),
-                    sa.PrimaryKeyConstraint('id'))
-
-
-def downgrade():
-    op.drop_table('telegram_file')

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -1,24 +0,0 @@
-"""Add is_bot field to puppets
-
-Revision ID: 1fa46383a9d3
-Revises: 30eca60587f1
-Create Date: 2018-04-29 23:44:40.102333
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '1fa46383a9d3'
-down_revision = '30eca60587f1'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
-
-
-def downgrade():
-    op.drop_column('puppet', 'is_bot')

alembic/versions/2228d49c383f_add_cascade_rules_to_userportal.py

@@ -1,41 +0,0 @@
-"""Add cascade rules to UserPortal
-
-Revision ID: 2228d49c383f
-Revises: bcfefa1f1299
-Create Date: 2018-05-31 11:11:59.482112
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '2228d49c383f'
-down_revision = 'bcfefa1f1299'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    try:
-        with op.batch_alter_table("user_portal") as batch_op:
-            batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
-            batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
-            batch_op.create_foreign_key("user_portal_user_fkey", "user", ["user"], ["tgid"],
-                                        onupdate="CASCADE", ondelete="CASCADE")
-            batch_op.create_foreign_key("user_portal_portal_fkey", "portal",
-                                        ["portal", "portal_receiver"], ["tgid", "tg_receiver"],
-                                        onupdate="CASCADE", ondelete="CASCADE")
-    except ValueError:
-        return
-
-
-def downgrade():
-    try:
-        with op.batch_alter_table("user_portal") as batch_op:
-                batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
-                batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
-                batch_op.create_foreign_key("user_portal_user_fkey", "portal",
-                                            ["portal", "portal_receiver"], ["tgid", "tg_receiver"])
-                batch_op.create_foreign_key("user_portal_portal_fkey", "user", ["user"], ["tgid"])
-    except ValueError:
-        return

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -1,23 +0,0 @@
-"""Add megagroup field to portals
-
-Revision ID: 30eca60587f1
-Revises: cfc972368e50
-Create Date: 2018-04-29 15:51:04.656605
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '30eca60587f1'
-down_revision = 'cfc972368e50'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
-
-
-def downgrade():
-    op.drop_column('portal', 'megagroup')

alembic/versions/501dad2868bc_move_sessions_to_main_database.py

@@ -1,111 +0,0 @@
-"""Move sessions to main database
-
-Revision ID: 501dad2868bc
-Revises: 7d47d84380b6
-Create Date: 2018-03-02 19:15:53.826985
-
-"""
-from alembic import op
-import sqlalchemy as sa
-import sqlite3
-import os
-
-# revision identifiers, used by Alembic.
-revision = '501dad2868bc'
-down_revision = '7d47d84380b6'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    Session = op.create_table('telethon_sessions',
-                              sa.Column('session_id', sa.String, nullable=False),
-                              sa.Column('dc_id', sa.Integer, nullable=False),
-                              sa.Column('server_address', sa.String, nullable=True),
-                              sa.Column('port', sa.Integer, nullable=True),
-                              sa.Column('auth_key', sa.LargeBinary, nullable=True),
-                              sa.PrimaryKeyConstraint('session_id', 'dc_id'))
-    SentFile = op.create_table('telethon_sent_files',
-                               sa.Column('session_id', sa.String, nullable=False),
-                               sa.Column('md5_digest', sa.LargeBinary, nullable=False),
-                               sa.Column('file_size', sa.Integer, nullable=False),
-                               sa.Column('type', sa.Integer, nullable=False),
-                               sa.Column('id', sa.BigInteger, nullable=True),
-                               sa.Column('hash', sa.BigInteger, nullable=True),
-                               sa.PrimaryKeyConstraint('session_id', 'md5_digest', 'file_size',
-                                                       'type'))
-    Entity = op.create_table('telethon_entities',
-                             sa.Column('session_id', sa.String, nullable=False),
-                             sa.Column('id', sa.Integer, nullable=False),
-                             sa.Column('hash', sa.Integer, nullable=False),
-                             sa.Column('username', sa.String, nullable=True),
-                             sa.Column('phone', sa.Integer, nullable=True),
-                             sa.Column('name', sa.String, nullable=True),
-                             sa.PrimaryKeyConstraint('session_id', 'id'))
-    Version = op.create_table('telethon_version',
-                              sa.Column('version', sa.Integer, nullable=False),
-                              sa.PrimaryKeyConstraint('version'))
-    conn = op.get_bind()
-    sessions = [os.path.basename(f) for f in os.listdir(".") if f.endswith(".session")]
-    for session in sessions:
-        session_to_sqlalchemy(conn, session, Session, SentFile, Entity)
-
-
-def session_to_sqlalchemy(conn, path, Session, SentFile, Entity):
-    session_conn = sqlite3.connect(path)
-    session_id = os.path.splitext(path)[0]
-    c = session_conn.cursor()
-
-    auth_data_tuples = c.execute("SELECT * FROM sessions").fetchall()
-    auth_data_dicts = []
-    for row in auth_data_tuples:
-        dc_id, server_address, port, auth_key = row
-        auth_data_dicts.append({
-            "session_id": session_id,
-            "dc_id": dc_id,
-            "server_address": server_address,
-            "port": port,
-            "auth_key": auth_key,
-        })
-    if auth_data_dicts:
-        conn.execute(Session.insert().values(auth_data_dicts))
-
-    sent_file_tuples = c.execute("SELECT * FROM sent_files").fetchall()
-    sent_file_dicts = []
-    for row in sent_file_tuples:
-        md5_digest, file_size, type, id, hash = row
-        sent_file_dicts.append({
-            "session_id": session_id,
-            "md5_digest": md5_digest,
-            "file_size": file_size,
-            "type": type,
-            "id": id,
-            "hash": hash,
-        })
-    if sent_file_dicts:
-        conn.execute(SentFile.insert().values(sent_file_dicts))
-
-    entity_tuples = c.execute("SELECT * FROM entities").fetchall()
-    entity_dicts = []
-    for row in entity_tuples:
-        id, hash, username, phone, name = row
-        entity_dicts.append({
-            "session_id": session_id,
-            "id": id,
-            "hash": hash,
-            "username": username,
-            "phone": phone,
-            "name": name,
-        })
-    if entity_dicts:
-        conn.execute(Entity.insert().values(entity_dicts))
-
-    c.close()
-    session_conn.close()
-
-
-def downgrade():
-    op.drop_table('telethon_version')
-    op.drop_table('telethon_entities')
-    op.drop_table('telethon_sent_files')
-    op.drop_table('telethon_sessions')

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -1,129 +0,0 @@
-"""Move state store to main database
-
-Revision ID: 6ca3d74d51e4
-Revises: 2228d49c383f
-Create Date: 2018-06-26 21:31:26.911307
-
-"""
-from alembic import context, op
-import sqlalchemy.orm as orm
-import sqlalchemy as sa
-import json
-import re
-
-from mautrix_telegram.config import Config
-from mautrix_telegram.base import Base
-
-# revision identifiers, used by Alembic.
-revision = "6ca3d74d51e4"
-down_revision = "2228d49c383f"
-branch_labels = None
-depends_on = None
-
-
-class RoomState(Base):
-    query = None
-    __tablename__ = "mx_room_state"
-    __table_args__ = {"extend_existing": True}
-
-    room_id = sa.Column(sa.String, primary_key=True)
-    power_levels = sa.Column("power_levels", sa.Text, nullable=True)
-
-
-class UserProfile(Base):
-    query = None
-    __tablename__ = "mx_user_profile"
-    __table_args__ = {"extend_existing": True}
-
-    room_id = sa.Column(sa.String, primary_key=True)
-    user_id = sa.Column(sa.String, primary_key=True)
-    membership = sa.Column(sa.String, nullable=False, default="leave")
-    displayname = sa.Column(sa.String, nullable=True)
-    avatar_url = sa.Column(sa.String, nullable=True)
-
-
-class Puppet(Base):
-    query = None
-    __tablename__ = "puppet"
-    __table_args__ = {"extend_existing": True}
-
-    id = sa.Column(sa.Integer, primary_key=True)
-    displayname = sa.Column(sa.String, nullable=True)
-    displayname_source = sa.Column(sa.Integer, nullable=True)
-    username = sa.Column(sa.String, nullable=True)
-    photo_id = sa.Column(sa.String, nullable=True)
-    is_bot = sa.Column(sa.Boolean, nullable=True)
-    matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False)
-
-
-def upgrade():
-    op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False,
-                                      server_default=sa.sql.expression.false()))
-    op.create_table("mx_room_state",
-                    sa.Column("room_id", sa.String(), nullable=False),
-                    sa.Column("power_levels", sa.Text(), nullable=True),
-                    sa.PrimaryKeyConstraint("room_id"))
-    op.create_table("mx_user_profile",
-                    sa.Column("room_id", sa.String(), nullable=False),
-                    sa.Column("user_id", sa.String(), nullable=False),
-                    sa.Column("membership", sa.String(), nullable=False,
-                              default="leave"),
-                    sa.Column("displayname", sa.String(), nullable=True),
-                    sa.Column("avatar_url", sa.String(), nullable=True),
-                    sa.PrimaryKeyConstraint("room_id", "user_id"))
-
-    conn = op.get_bind()
-    session = orm.sessionmaker(bind=conn)
-    session = orm.scoping.scoped_session(session)
-    Puppet.query = session.query_property()
-
-    try:
-        with open("mx-state.json") as file:
-            data = json.load(file)
-    except FileNotFoundError:
-        return
-    if not data:
-        return
-    registrations = data.get("registrations", [])
-
-    mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
-    mxtg_config = Config(mxtg_config_path, None, None)
-    mxtg_config.load()
-
-    username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}")
-    hs_domain = mxtg_config["homeserver.domain"]
-    localpart = username_template.format(userid="(.+)")
-    mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain))
-    for user in registrations:
-        match = mxid_regex.match(user)
-        if not match:
-            continue
-
-        puppet = Puppet.query.get(match.group(1))
-        if not puppet:
-            continue
-
-        puppet.matrix_registered = True
-        session.merge(puppet)
-    session.commit()
-
-    user_profiles = [UserProfile(room_id=room, user_id=user,
-                                 membership=member.get("membership", "leave"),
-                                 displayname=member.get("displayname", None),
-                                 avatar_url=member.get("avatar_url", None))
-                     for room, members in data.get("members", {}).items()
-                     for user, member in members.items()]
-    session.add_all(user_profiles)
-    session.commit()
-
-    room_state = [RoomState(room_id=room, power_levels=json.dumps(levels))
-                  for room, levels in data.get("power_levels", {}).items()]
-    session.add_all(room_state)
-    session.commit()
-
-
-def downgrade():
-    op.drop_table("mx_user_profile")
-    op.drop_table("mx_room_state")
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column("matrix_registered")

alembic/versions/7d47d84380b6_add_timestamp_to_telegramfile.py

@@ -1,25 +0,0 @@
-"""Add timestamp to TelegramFile
-
-Revision ID: 7d47d84380b6
-Revises: 1b241f7e8530
-Create Date: 2018-02-19 23:53:18.050871
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '7d47d84380b6'
-down_revision = '1b241f7e8530'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column('telegram_file',
-                  sa.Column('timestamp', sa.BigInteger(), nullable=True, default=0,
-                            server_default="0"))
-
-
-def downgrade():
-    op.drop_column('telegram_file', 'timestamp')

alembic/versions/97d2a942bcf8_initial_revision.py

@@ -1,80 +0,0 @@
-"""initial revision
-
-Revision ID: 97d2a942bcf8
-Revises:
-Create Date: 2018-02-11 18:40:55.483842
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '97d2a942bcf8'
-down_revision = None
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table('portal',
-                    sa.Column('tgid', sa.Integer),
-                    sa.Column('tg_receiver', sa.Integer),
-                    sa.Column('peer_type', sa.String, nullable=False, default=""),
-                    sa.Column('mxid', sa.String, nullable=True),
-                    sa.Column('username', sa.String, nullable=True),
-                    sa.Column('title', sa.String, nullable=True),
-                    sa.Column('about', sa.String, nullable=True),
-                    sa.Column('photo_id', sa.String, nullable=True),
-                    sa.PrimaryKeyConstraint('tgid', 'tg_receiver'),
-                    sa.UniqueConstraint('mxid'))
-    op.create_table('user',
-                    sa.Column('mxid', sa.String),
-                    sa.Column('tgid', sa.Integer, nullable=True, unique=True),
-                    sa.Column('tg_username', sa.String, nullable=True),
-                    sa.Column('saved_contacts', sa.Integer, nullable=False, default=0),
-                    sa.PrimaryKeyConstraint('mxid'))
-    op.create_table('puppet',
-                    sa.Column('id', sa.Integer),
-                    sa.Column('displayname', sa.String, nullable=True),
-                    sa.Column('username', sa.String, nullable=True),
-                    sa.Column('photo_id', sa.String, nullable=True),
-                    sa.PrimaryKeyConstraint('id'))
-    op.create_table('contact',
-                    sa.Column('user', sa.Integer),
-                    sa.Column('contact', sa.Integer),
-                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
-                    sa.ForeignKeyConstraint(("contact",), ("puppet.id",)),
-                    sa.PrimaryKeyConstraint('user', 'contact'))
-    op.create_table('user_portal',
-                    sa.Column('user', sa.Integer),
-                    sa.Column('portal', sa.Integer),
-                    sa.Column('portal_receiver', sa.Integer),
-                    sa.PrimaryKeyConstraint('user', 'portal', 'portal_receiver'),
-                    sa.ForeignKeyConstraint(("user",), ("user.tgid",),
-                                            name="user_portal_user_fkey",
-                                            onupdate="CASCADE", ondelete="CASCADE"),
-                    sa.ForeignKeyConstraint(("portal", "portal_receiver"),
-                                            ("portal.tgid", "portal.tg_receiver"),
-                                            name="user_portal_portal_fkey",
-                                            onupdate="CASCADE", ondelete="CASCADE"))
-    op.create_table('message',
-                    sa.Column('mxid', sa.String),
-                    sa.Column('mx_room', sa.String),
-                    sa.Column('tgid', sa.Integer),
-                    sa.Column('tg_space', sa.Integer),
-                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
-                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
-    op.create_table('bot_chat',
-                    sa.Column('id', sa.Integer),
-                    sa.Column('type', sa.String, nullable=False, default=""),
-                    sa.PrimaryKeyConstraint('id'))
-
-
-def downgrade():
-    op.drop_table('bot_chat')
-    op.drop_table('message')
-    op.drop_table('user_portal')
-    op.drop_table('contact')
-    op.drop_table('puppet')
-    op.drop_table('user')
-    op.drop_table('portal')

alembic/versions/bcfefa1f1299_add_displayname_source_fields_for_.py

@@ -1,23 +0,0 @@
-"""Add displayname source fields for puppets
-
-Revision ID: bcfefa1f1299
-Revises: bdadd173ee02
-Create Date: 2018-05-19 17:00:21.078098
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = 'bcfefa1f1299'
-down_revision = 'bdadd173ee02'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column('puppet', sa.Column('displayname_source', sa.Integer(), nullable=True))
-
-
-def downgrade():
-    op.drop_column('puppet', 'displayname_source')

alembic/versions/bdadd173ee02_update_telethon_update_state_table.py

@@ -1,43 +0,0 @@
-"""Update telethon update state table
-
-Revision ID: bdadd173ee02
-Revises: eeaf0dae87ce
-Create Date: 2018-05-13 10:42:59.395597
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = 'bdadd173ee02'
-down_revision = 'eeaf0dae87ce'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("telethon_entities") as batch_op:
-        batch_op.alter_column("id", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("hash", existing_type=sa.Integer, type_=sa.BigInteger)
-
-    with op.batch_alter_table("telethon_update_state") as batch_op:
-        batch_op.alter_column("entity_id", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("pts", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("qts", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("date", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("seq", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.add_column(sa.Column("unread_count", sa.Integer))
-
-
-def downgrade():
-    with op.batch_alter_table("telethon_entities") as batch_op:
-        batch_op.alter_column("id", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("hash", existing_type=sa.BigInteger, type_=sa.Integer)
-
-    with op.batch_alter_table("telethon_update_state") as batch_op:
-        batch_op.alter_column("entity_id", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("pts", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("qts", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("date", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("seq", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.drop_column("unread_count")

alembic/versions/cfc972368e50_add_metadata_to_telegramfile.py

@@ -1,35 +0,0 @@
-"""Add metadata to TelegramFile
-
-Revision ID: cfc972368e50
-Revises: 501dad2868bc
-Create Date: 2018-03-09 16:07:01.236712
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = 'cfc972368e50'
-down_revision = '501dad2868bc'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("telegram_file") as batch_op:
-        batch_op.add_column(sa.Column('size', sa.Integer(), nullable=True))
-        batch_op.add_column(sa.Column('width', sa.Integer(), nullable=True))
-        batch_op.add_column(sa.Column('height', sa.Integer(), nullable=True))
-        batch_op.add_column(sa.Column('thumbnail', sa.String(), nullable=True))
-        batch_op.create_foreign_key(constraint_name="fk_file_thumbnail",
-                                    referent_table="telegram_file",
-                                    local_cols=['thumbnail'],
-                                    remote_cols=['id'])
-
-
-def downgrade():
-    with op.batch_alter_table("telegram_file") as batch_op:
-        batch_op.drop_column('size')
-        batch_op.drop_column('width')
-        batch_op.drop_column('height')
-        batch_op.drop_column('thumbnail')

alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py

@@ -1,26 +0,0 @@
-"""Add access_token and custom_mxid fields for puppets
-
-Revision ID: d5f7b8b4b456
-Revises: 6ca3d74d51e4
-Create Date: 2018-07-20 12:09:30.277960
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = "d5f7b8b4b456"
-down_revision = "6ca3d74d51e4"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
-    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column("custom_mxid")
-        batch_op.drop_column("access_token")

alembic/versions/eeaf0dae87ce_add_telethon_update_state_table.py

@@ -1,34 +0,0 @@
-"""Add telethon update state table
-
-Revision ID: eeaf0dae87ce
-Revises: 1fa46383a9d3
-Create Date: 2018-04-30 17:30:59.610885
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = 'eeaf0dae87ce'
-down_revision = '1fa46383a9d3'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("telethon_entities") as batch_op:
-        batch_op.alter_column('phone', existing_type=sa.Integer, type_=sa.BigInteger)
-    op.create_table('telethon_update_state',
-                    sa.Column('session_id', sa.String, nullable=False),
-                    sa.Column('entity_id', sa.Integer, nullable=False),
-                    sa.Column('pts', sa.Integer, nullable=True),
-                    sa.Column('qts', sa.Integer, nullable=True),
-                    sa.Column('date', sa.Integer, nullable=True),
-                    sa.Column('seq', sa.Integer, nullable=True),
-                    sa.PrimaryKeyConstraint('session_id', 'entity_id'))
-
-
-def downgrade():
-    with op.batch_alter_table("telethon_entities") as batch_op:
-        batch_op.alter_column('phone', existing_type=sa.BigInteger, type_=sa.Integer)
-    op.drop_table('telethon_update_state')

docker-run.sh

@@ -7,15 +7,6 @@ function fixperms {
 
 cd /opt/mautrix-telegram
 
-# Replace database path in config.
-sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
-
-if [ -f /data/mx-state.json ]; then
-	ln -s /data/mx-state.json
-fi
-# Check that database is in the right state
-alembic -x config=/data/config.yaml upgrade head
-
 if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
 	echo "Didn't find a config file."
@@ -28,9 +19,6 @@ fi
 
 if [ ! -f /data/registration.yaml ]; then
 	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
-	echo "Didn't find a registration file."
-	echo "Generated one for you."
-	echo "Copy that over to synapses app service directory."
 	fixperms
 	exit
 fi

example-config.yaml

@@ -1,273 +0,0 @@
-# Homeserver details
-homeserver:
-    # The address that this appservice can use to connect to the homeserver.
-    address: https://matrix.org
-    # The domain of the homeserver (for MXIDs, etc).
-    domain: matrix.org
-    # Whether or not to verify the SSL certificate of the homeserver.
-    # Only applies if address starts with https://
-    verify_ssl: true
-
-# Application service host/registration related details
-# Changing these values requires regeneration of the registration.
-appservice:
-    # The address that the homeserver can use to connect to this appservice.
-    address: http://localhost:8080
-
-    # The hostname and port where this appservice should listen.
-    hostname: 0.0.0.0
-    port: 8080
-    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
-    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
-    max_body_size: 1
-
-    # The full URI to the database. SQLite and Postgres are fully supported.
-    # Other DBMSes supported by SQLAlchemy may or may not work.
-    database: sqlite:///mautrix-telegram.db
-
-    # Public part of web server for out-of-Matrix interaction with the bridge.
-    # Used for things like login if the user wants to make sure the 2FA password isn't stored in
-    # the HS database.
-    public:
-        # Whether or not the public-facing endpoints should be enabled.
-        enabled: true
-        # The prefix to use in the public-facing endpoints.
-        prefix: /public
-        # The base URL where the public-facing endpoints are available. The prefix is not added
-        # implicitly.
-        external: https://example.com/public
-
-    # Provisioning API part of the web server for automated portal creation and fetching information.
-    # Used by things like Dimension (https://dimension.t2bot.io/).
-    provisioning:
-        # Whether or not the provisioning API should be enabled.
-        enabled: true
-        # The prefix to use in the provisioning API endpoints.
-        prefix: /_matrix/provision/v1
-        # The shared secret to authorize users of the API.
-        # Set to "generate" to generate and save a new token.
-        shared_secret: generate
-
-    # The unique ID of this appservice.
-    id: telegram
-    # Username of the appservice bot.
-    bot_username: telegrambot
-    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
-    # to leave display name/avatar as-is.
-    bot_displayname: Telegram bridge bot
-    bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
-
-    # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
-    as_token: "This value is generated when generating the registration"
-    hs_token: "This value is generated when generating the registration"
-
-# Bridge config
-bridge:
-    # Localpart template of MXIDs for Telegram users.
-    # {userid} is replaced with the user ID of the Telegram user.
-    username_template: "telegram_{userid}"
-    # Localpart template of room aliases for Telegram portal rooms.
-    # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} )
-    alias_template: "telegram_{groupname}"
-    # Displayname template for Telegram users.
-    # {displayname} is replaced with the display name of the Telegram user.
-    displayname_template: "{displayname} (Telegram)"
-
-    # Set the preferred order of user identifiers which to use in the Matrix puppet display name.
-    # In the (hopefully unlikely) scenario that none of the given keys are found, the numeric user
-    # ID is used.
-    #
-    # If the bridge is working properly, a phone number or an username should always be known, but
-    # the other one can very well be empty.
-    #
-    # Valid keys:
-    #   "full name"          (First and/or last name)
-    #   "full name reversed" (Last and/or first name)
-    #   "first name"
-    #   "last name"
-    #   "username"
-    #   "phone number"
-    displayname_preference:
-    - full name
-    - username
-    - phone number
-
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: false
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
-    # Whether or not Matrix bot messages (type m.notice) should be bridged.
-    bridge_notices: true
-    # Whether to bridge Telegram bot messages as m.notices or m.texts.
-    bot_messages_as_notices: true
-    # Maximum number of members to sync per portal when starting up. Other members will be
-    # synced when they send messages. The maximum is 10000, after which the Telegram server
-    # will not send any more members.
-    # Defaults to no local limit (-> limited to 10000 by server)
-    max_initial_member_sync: -1
-    # Whether or not to sync the member list in channels.
-    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
-    # list regardless of this setting.
-    sync_channel_members: true
-    # The maximum number of simultaneous Telegram deletions to handle.
-    # A large number of simultaneous redactions could put strain on your homeserver.
-    max_telegram_delete: 10
-    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
-    # login website (see appservice.public config section)
-    allow_matrix_login: true
-    # Use inline images instead of m.image to make rich captions possible.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
-    inline_images: false
-    # Whether or not to bridge plaintext highlights.
-    # Only enable this if your displayname_template has some static part that the bridge can use to
-    # reliably identify what is a plaintext highlight.
-    plaintext_highlights: false
-    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
-    public_portals: true
-    # Whether to send stickers as the new native m.sticker type or normal m.images.
-    # Old versions of Riot don't support the new type at all.
-    # Remember that proper sticker support always requires Pillow to convert webp into png.
-    native_stickers: true
-    # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
-    # WARNING: Probably buggy, might get stuck in infinite loop.
-    catch_up: false
-    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
-    # your own Matrix account as the Matrix puppet for your Telegram account.
-    sync_with_custom_puppets: true
-
-    # Some config options related to Telegram message deduplication.
-    # The default values are usually fine, but some debug messages/warnings might recommend you
-    # change these.
-    deduplication:
-        # Whether or not to check the database if the message about to be sent is a duplicate.
-        pre_db_check: false
-        # The number of latest events to keep when checking for duplicates.
-        # You might need to increase this on high-traffic bridge instances.
-        cache_queue_length: 20
-
-    # The formats to use when sending messages to Telegram via the relay bot.
-    #
-    # Telegram doesn't have built-in emotes, so the m.emote format is also used for non-relaybot users.
-    #
-    # Available variables:
-    #   $sender_displayname    - The display name of the sender (e.g. Example User)
-    #   $sender_username       - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
-    #   $sender_mxid           - The Matrix ID of the sender (e.g. @exampleuser:example.com)
-    #   $message               - The message content as HTML
-    message_formats:
-        m.text: "<b>$sender_displayname</b>: $message"
-        m.emote: "* <b>$sender_displayname</b> $message"
-        m.file: "<b>$sender_displayname</b> sent a file: $message"
-        m.image: "<b>$sender_displayname</b> sent an image: $message"
-        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
-        m.video: "<b>$sender_displayname</b> sent a video: $message"
-        m.location: "<b>$sender_displayname</b> sent a location: $message"
-
-    # The formats to use when sending state events to Telegram via the relay bot.
-    #
-    # Variables from `message_formats` that have the `sender_` prefix are available without the prefix.
-    # In name_change events, `$prev_displayname` is the previous displayname.
-    #
-    # Set format to an empty string to disable the messages for that event.
-    state_event_formats:
-        join: "<b>$displayname</b> joined the room."
-        leave: "<b>$displayname</b> left the room."
-        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
-
-    # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
-    # `filter-mode` management commands.
-    #
-    # Filters do not affect direct chats.
-    # An empty blacklist will essentially disable the filter.
-    filter:
-        # Filter mode to use. Either "blacklist" or "whitelist".
-        # If the mode is "blacklist", the listed chats will never be bridged.
-        # If the mode is "whitelist", only the listed chats can be bridged.
-        mode: blacklist
-        # The list of group/channel IDs to filter.
-        list: []
-
-    # The prefix for commands. Only required in non-management rooms.
-    command_prefix: "!tg"
-
-    # Permissions for using the bridge.
-    # Permitted values:
-    #   relaybot - Only use the bridge via the relaybot, no access to commands.
-    #       user - Relaybot level + access to commands to create bridges.
-    #  puppeting - User level + logging in with a Telegram account.
-    #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
-    #      admin - Full access to use the bridge and some extra administration commands.
-    # Permitted keys:
-    #        * - All Matrix users
-    #   domain - All users on that homeserver
-    #     mxid - Specific user
-    permissions:
-        "*": "relaybot"
-        "public.example.com": "user"
-        "example.com": "full"
-        "@admin:example.com": "admin"
-
-    # Options related to the message relay Telegram bot.
-    relaybot:
-        # Whether or not to allow creating portals from Telegram.
-        authless_portals: true
-        # Whether or not to allow Telegram group admins to use the bot commands.
-        whitelist_group_admins: true
-        # Whether or not to ignore incoming events sent by the relay bot.
-        ignore_own_incoming_events: true
-        # List of usernames/user IDs who are also allowed to use the bot commands.
-        whitelist:
-        - myusername
-        - 12345678
-
-# Telegram config
-telegram:
-    # Get your own API keys at https://my.telegram.org/apps
-    api_id: 12345
-    api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
-    # (Optional) Create your own bot at https://t.me/BotFather
-    bot_token: disabled
-    # Telethon proxy configuration.
-    # You must install PySocks from pip for proxies to work.
-    proxy:
-        # Allowed types: disabled, socks4, socks5, http
-        type: disabled
-        # Proxy IP address and port.
-        address: 127.0.0.1
-        port: 1080
-        # Whether or not to perform DNS resolving remotely.
-        rdns: true
-        # Proxy authentication (optional).
-        username: ""
-        password: ""
-
-# Python logging configuration.
-#
-# See section 16.7.2 of the Python documentation for more info:
-# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
-logging:
-    version: 1
-    formatters:
-        precise:
-            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
-    handlers:
-        file:
-            class: logging.handlers.RotatingFileHandler
-            formatter: precise
-            filename: ./mautrix-telegram.log
-            maxBytes: 10485760
-            backupCount: 10
-        console:
-            class: logging.StreamHandler
-            formatter: precise
-    loggers:
-        mau:
-            level: DEBUG
-        telethon:
-            level: DEBUG
-        aiohttp:
-            level: INFO
-    root:
-        level: DEBUG
-        handlers: [file, console]

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.3.0rc3"
+__version__ = "0.11.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,120 +13,129 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
-import argparse
-import asyncio
-import logging.config
-import sys
+from __future__ import annotations
 
-from sqlalchemy import orm
-import sqlalchemy as sql
+from typing import Any
 
-from mautrix_appservice import AppService
-from alchemysession import AlchemySessionContainer
+from telethon import __version__ as __telethon_version__
 
+from mautrix.bridge import Bridge
+from mautrix.types import RoomID, UserID
+
+from .bot import Bot
+from .config import Config
+from .db import init as init_db, upgrade_table
+from .matrix import MatrixHandler
+from .portal import Portal
+from .puppet import Puppet
+from .user import User
+from .version import linkified_version, version
 from .web.provisioning import ProvisioningAPI
 from .web.public import PublicBridgeWebsite
-from .abstract_user import init as init_abstract_user
-from .base import Base
-from .bot import init as init_bot
-from .config import Config
-from .context import Context
-from .db import init as init_db
-from .formatter import init as init_formatter
-from .matrix import MatrixHandler
-from .portal import init as init_portal
-from .puppet import init as init_puppet
-from .sqlstatestore import SQLStateStore
-from .user import User, init as init_user
-from . import __version__
 
-parser = argparse.ArgumentParser(
-    description="A Matrix-Telegram puppeting bridge.",
-    prog="python -m mautrix-telegram")
-parser.add_argument("-c", "--config", type=str, default="config.yaml",
-                    metavar="<path>", help="the path to your config file")
-parser.add_argument("-b", "--base-config", type=str, default="example-config.yaml",
-                    metavar="<path>", help="the path to the example config "
-                                           "(for automatic config updates)")
-parser.add_argument("-g", "--generate-registration", action="store_true",
-                    help="generate registration and quit")
-parser.add_argument("-r", "--registration", type=str, default="registration.yaml",
-                    metavar="<path>", help="the path to save the generated registration to")
-args = parser.parse_args()
+from .abstract_user import AbstractUser  # isort: skip
 
-config = Config(args.config, args.registration, args.base_config)
-config.load()
-config.update()
 
-if args.generate_registration:
-    config.generate_registration()
-    config.save()
-    print(f"Registration generated and saved to {config.registration_path}")
-    sys.exit(0)
+class TelegramBridge(Bridge):
+    module = "mautrix_telegram"
+    name = "mautrix-telegram"
+    command = "python -m mautrix-telegram"
+    description = "A Matrix-Telegram puppeting bridge."
+    repo_url = "https://github.com/mautrix/telegram"
+    version = version
+    markdown_version = linkified_version
+    config_class = Config
+    matrix_class = MatrixHandler
+    upgrade_table = upgrade_table
 
-logging.config.dictConfig(config["logging"])
-log = logging.getLogger("mau.init")  # type: logging.Logger
-log.debug(f"Initializing mautrix-telegram {__version__}")
+    config: Config
+    bot: Bot | None
+    public_website: PublicBridgeWebsite | None
+    provisioning_api: ProvisioningAPI | None
 
-db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
-db_factory = orm.sessionmaker(bind=db_engine)
-db_session = orm.scoping.scoped_session(db_factory)
-Base.metadata.bind = db_engine
+    def prepare_db(self) -> None:
+        super().prepare_db()
+        init_db(self.db)
 
-session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
-                                            table_base=Base, table_prefix="telethon_",
-                                            manage_tables=False)
+    def _prepare_website(self) -> None:
+        if self.config["appservice.provisioning.enabled"]:
+            self.provisioning_api = ProvisioningAPI(self)
+            self.az.app.add_subapp(
+                self.config["appservice.provisioning.prefix"], self.provisioning_api.app
+            )
+        else:
+            self.provisioning_api = None
 
-loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
+        if self.config["appservice.public.enabled"]:
+            self.public_website = PublicBridgeWebsite(self.loop)
+            self.az.app.add_subapp(
+                self.config["appservice.public.prefix"], self.public_website.app
+            )
+        else:
+            self.public_website = None
 
-state_store = SQLStateStore(db_session)
-mebibyte = 1024 ** 2
-appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
-                     config["appservice.as_token"], config["appservice.hs_token"],
-                     config["appservice.bot_username"], log="mau.as", loop=loop,
-                     verify_ssl=config["homeserver.verify_ssl"], state_store=state_store,
-                     real_user_content_key="net.maunium.telegram.puppet",
-                     aiohttp_params={
-                         "client_max_size": config["appservice.max_body_size"] * mebibyte
-                     })
+    def prepare_bridge(self) -> None:
+        self._prepare_website()
+        AbstractUser.init_cls(self)
+        bot_token: str = self.config["telegram.bot_token"]
+        if bot_token and not bot_token.lower().startswith("disable"):
+            self.bot = AbstractUser.relaybot = Bot(bot_token)
+        else:
+            self.bot = AbstractUser.relaybot = None
+        self.matrix = MatrixHandler(self)
+        Portal.init_cls(self)
+        self.add_startup_actions(Puppet.init_cls(self))
+        self.add_startup_actions(User.init_cls(self))
+        if self.bot:
+            self.add_startup_actions(self.bot.start())
+        if self.config["bridge.resend_bridge_info"]:
+            self.add_startup_actions(self.resend_bridge_info())
 
-context = Context(appserv, db_session, config, loop, session_container)
+    async def resend_bridge_info(self) -> None:
+        self.config["bridge.resend_bridge_info"] = False
+        self.config.save()
+        self.log.info("Re-sending bridge info state event to all portals")
+        async for portal in Portal.all():
+            await portal.update_bridge_info()
+        self.log.info("Finished re-sending bridge info state events")
 
-if config["appservice.public.enabled"]:
-    public_website = PublicBridgeWebsite(loop)
-    appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public_website.app)
-    context.public_website = public_website
+    def prepare_stop(self) -> None:
+        for puppet in Puppet.by_custom_mxid.values():
+            puppet.stop()
+        self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
 
-if config["appservice.provisioning.enabled"]:
-    provisioning_api = ProvisioningAPI(context)
-    appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning",
-                           provisioning_api.app)
-    context.provisioning_api = provisioning_api
+    async def get_user(self, user_id: UserID, create: bool = True) -> User | None:
+        user = await User.get_by_mxid(user_id, create=create)
+        if user:
+            await user.ensure_started()
+        return user
 
-with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
-    init_db(db_session)
-    init_abstract_user(context)
-    context.bot = init_bot(context)
-    context.mx = MatrixHandler(context)
-    init_formatter(context)
-    init_portal(context)
-    startup_actions = (init_puppet(context) +
-                       init_user(context) +
-                       [start,
-                        context.mx.init_as_bot()])
+    async def get_portal(self, room_id: RoomID) -> Portal | None:
+        return await Portal.get_by_mxid(room_id)
 
-    if context.bot:
-        startup_actions.append(context.bot.start())
+    async def get_puppet(self, user_id: UserID, create: bool = False) -> Puppet | None:
+        return await Puppet.get_by_mxid(user_id, create=create)
 
-    try:
-        log.debug("Initialization complete, running startup actions")
-        loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
-        log.debug("Startup actions complete, now running forever")
-        loop.run_forever()
-    except KeyboardInterrupt:
-        log.debug("Keyboard interrupt received, stopping clients")
-        loop.run_until_complete(
-            asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
-        log.debug("Clients stopped, shutting down")
-        sys.exit(0)
+    async def get_double_puppet(self, user_id: UserID) -> Puppet | None:
+        return await Puppet.get_by_custom_mxid(user_id)
+
+    def is_bridge_ghost(self, user_id: UserID) -> bool:
+        return bool(Puppet.get_id_from_mxid(user_id))
+
+    async def count_logged_in_users(self) -> int:
+        return len([user for user in User.by_tgid.values() if user.tgid])
+
+    async def manhole_global_namespace(self, user_id: UserID) -> dict[str, Any]:
+        return {
+            **await super().manhole_global_namespace(user_id),
+            "User": User,
+            "Portal": Portal,
+            "Puppet": Puppet,
+        }
+
+    @property
+    def manhole_banner_program_version(self) -> str:
+        return f"{super().manhole_banner_program_version} and Telethon {__telethon_version__}"
+
+
+TelegramBridge().run()

mautrix_telegram/abstract_user.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,99 +13,217 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional, List, Union, TYPE_CHECKING
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Type, Union
 from abc import ABC, abstractmethod
 import asyncio
 import logging
 import platform
+import time
 
-from sqlalchemy import orm
-from telethon.tl.types import Channel, ChannelForbidden, Chat, ChatForbidden, Message, \
-    MessageActionChannelMigrateFrom, MessageService, PeerUser, TypeUpdate, \
-    UpdateChannelPinnedMessage, UpdateChatAdmins, UpdateChatParticipantAdmin, \
-    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, \
-    UpdateDeleteMessages, UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, \
-    UpdateNewMessage, UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, \
-    UpdateUserName, UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, \
-    UserStatusOnline
+from telethon.network import (
+    Connection,
+    ConnectionTcpFull,
+    ConnectionTcpMTProxyRandomizedIntermediate,
+)
+from telethon.sessions import Session
+from telethon.tl.patched import Message, MessageService
+from telethon.tl.types import (
+    Channel,
+    Chat,
+    MessageActionChannelMigrateFrom,
+    MessageEmpty,
+    PeerChat,
+    PeerUser,
+    TypeUpdate,
+    UpdateChannelUserTyping,
+    UpdateChatParticipantAdmin,
+    UpdateChatParticipants,
+    UpdateChatUserTyping,
+    UpdateDeleteChannelMessages,
+    UpdateDeleteMessages,
+    UpdateEditChannelMessage,
+    UpdateEditMessage,
+    UpdateFolderPeers,
+    UpdateNewChannelMessage,
+    UpdateNewMessage,
+    UpdateNotifySettings,
+    UpdatePinnedChannelMessages,
+    UpdatePinnedDialogs,
+    UpdatePinnedMessages,
+    UpdateReadChannelInbox,
+    UpdateReadHistoryInbox,
+    UpdateReadHistoryOutbox,
+    UpdateShortChatMessage,
+    UpdateShortMessage,
+    UpdateUserName,
+    UpdateUserPhoto,
+    UpdateUserStatus,
+    UpdateUserTyping,
+    User,
+    UserStatusOffline,
+    UserStatusOnline,
+)
 
-from mautrix_appservice import MatrixRequestError, AppService
-from alchemysession import AlchemySessionContainer
+from mautrix.appservice import AppService
+from mautrix.errors import MatrixError
+from mautrix.types import PresenceState, UserID
+from mautrix.util.logging import TraceLogger
+from mautrix.util.opt_prometheus import Counter, Histogram
 
-from . import portal as po, puppet as pu, __version__
-from .db import Message as DBMessage
+from . import __version__, portal as po, puppet as pu
+from .config import Config
+from .db import Message as DBMessage, PgSession
 from .tgclient import MautrixTelegramClient
+from .types import TelegramID
 
 if TYPE_CHECKING:
-    from .context import Context
-    from .config import Config
+    from .__main__ import TelegramBridge
     from .bot import Bot
 
-config = None  # type: Config
-# Value updated from config in init()
-MAX_DELETIONS = 10  # type: int
+UpdateMessage = Union[
+    UpdateShortChatMessage,
+    UpdateShortMessage,
+    UpdateNewChannelMessage,
+    UpdateNewMessage,
+    UpdateEditMessage,
+    UpdateEditChannelMessage,
+]
+UpdateMessageContent = Union[
+    UpdateShortMessage, UpdateShortChatMessage, Message, MessageService, MessageEmpty
+]
 
-UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
-                      UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
-UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
+UPDATE_TIME = Histogram(
+    name="bridge_telegram_update",
+    documentation="Time spent processing Telegram updates",
+    labelnames=("update_type",),
+)
+UPDATE_ERRORS = Counter(
+    name="bridge_telegram_update_error",
+    documentation="Number of fatal errors while handling Telegram updates",
+    labelnames=("update_type",),
+)
 
 
 class AbstractUser(ABC):
-    session_container = None  # type: AlchemySessionContainer
-    loop = None  # type: asyncio.AbstractEventLoop
-    log = None  # type: logging.Logger
-    db = None  # type: orm.Session
-    az = None  # type: AppService
-    bot = None  # type: Bot
-    ignore_incoming_bot_events = True  # type: bool
+    loop: asyncio.AbstractEventLoop = None
+    log: TraceLogger
+    az: AppService
+    bridge: "TelegramBridge"
+    config: Config
+    relaybot: "Bot"
+    ignore_incoming_bot_events: bool = True
+    max_deletions: int = 10
 
-    def __init__(self):
-        self.is_admin = False  # type: bool
-        self.matrix_puppet_whitelisted = False  # type: bool
-        self.puppet_whitelisted = False  # type: bool
-        self.whitelisted = False  # type: bool
-        self.relaybot_whitelisted = False  # type: bool
-        self.client = None  # type: MautrixTelegramClient
-        self.tgid = None  # type: int
-        self.mxid = None  # type: str
-        self.is_relaybot = False  # type: bool
-        self.is_bot = False  # type: bool
+    client: MautrixTelegramClient | None
+    mxid: UserID | None
+
+    tgid: TelegramID | None
+    username: str | None
+    is_bot: bool
+
+    is_relaybot: bool
+
+    puppet_whitelisted: bool
+    whitelisted: bool
+    relaybot_whitelisted: bool
+    matrix_puppet_whitelisted: bool
+    is_admin: bool
+
+    def __init__(self) -> None:
+        self.is_admin = False
+        self.matrix_puppet_whitelisted = False
+        self.puppet_whitelisted = False
+        self.whitelisted = False
+        self.relaybot_whitelisted = False
+        self.client = None
+        self.is_relaybot = False
+        self.is_bot = False
 
     @property
     def connected(self) -> bool:
         return self.client and self.client.is_connected()
 
     @property
-    def _proxy_settings(self) -> Optional[Tuple[int, str, str, str, str, str]]:
-        proxy_type = config["telegram.proxy.type"].lower()
+    def _proxy_settings(self) -> tuple[Type[Connection], tuple[Any, ...] | None]:
+        proxy_type = self.config["telegram.proxy.type"].lower()
+        connection = ConnectionTcpFull
+        connection_data = (
+            self.config["telegram.proxy.address"],
+            self.config["telegram.proxy.port"],
+            self.config["telegram.proxy.rdns"],
+            self.config["telegram.proxy.username"],
+            self.config["telegram.proxy.password"],
+        )
         if proxy_type == "disabled":
-            return None
+            connection_data = None
         elif proxy_type == "socks4":
-            proxy_type = 1
+            connection_data = (1,) + connection_data
         elif proxy_type == "socks5":
-            proxy_type = 2
+            connection_data = (2,) + connection_data
         elif proxy_type == "http":
-            proxy_type = 3
+            connection_data = (3,) + connection_data
+        elif proxy_type == "mtproxy":
+            connection = ConnectionTcpMTProxyRandomizedIntermediate
+            connection_data = (connection_data[0], connection_data[1], connection_data[4])
 
-        return (proxy_type,
-                config["telegram.proxy.address"], config["telegram.proxy.port"],
-                config["telegram.proxy.rdns"],
-                config["telegram.proxy.username"], config["telegram.proxy.password"])
+        return connection, connection_data
 
-    def _init_client(self):
+    @classmethod
+    def init_cls(cls, bridge: "TelegramBridge") -> None:
+        cls.bridge = bridge
+        cls.config = bridge.config
+        cls.loop = bridge.loop
+        cls.az = bridge.az
+        cls.ignore_incoming_bot_events = cls.config["bridge.relaybot.ignore_own_incoming_events"]
+        cls.max_deletions = cls.config["bridge.max_telegram_delete"]
+
+    async def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
-        self.session = self.session_container.new_session(self.name)
-        self.client = MautrixTelegramClient(session=self.session,
-                                            api_id=config["telegram.api_id"],
-                                            api_hash=config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device,
-                                            timeout=120,
-                                            proxy=self._proxy_settings)
+
+        session = await PgSession.get(self.name)
+        if self.config["telegram.server.enabled"]:
+            session.set_dc(
+                self.config["telegram.server.dc"],
+                self.config["telegram.server.ip"],
+                self.config["telegram.server.port"],
+            )
+
+        if self.is_relaybot:
+            base_logger = logging.getLogger("telethon.relaybot")
+        else:
+            base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
+
+        device = self.config["telegram.device_info.device_model"]
+        sysversion = self.config["telegram.device_info.system_version"]
+        appversion = self.config["telegram.device_info.app_version"]
+        connection, proxy = self._proxy_settings
+
+        assert isinstance(session, Session)
+
+        self.client = MautrixTelegramClient(
+            session=session,
+            api_id=self.config["telegram.api_id"],
+            api_hash=self.config["telegram.api_hash"],
+            app_version=__version__ if appversion == "auto" else appversion,
+            system_version=(
+                MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion
+            ),
+            device_model=(
+                f"{platform.system()} {platform.release()}" if device == "auto" else device
+            ),
+            timeout=self.config["telegram.connection.timeout"],
+            connection_retries=self.config["telegram.connection.retries"],
+            retry_delay=self.config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=self.config["telegram.connection.flood_sleep_threshold"],
+            request_retries=self.config["telegram.connection.request_retries"],
+            connection=connection,
+            proxy=proxy,
+            raise_last_call_error=True,
+            loop=self.loop,
+            base_logger=base_logger,
+        )
         self.client.add_event_handler(self._update_catch)
 
     @abstractmethod
@@ -114,32 +231,27 @@ class AbstractUser(ABC):
         return False
 
     @abstractmethod
-    async def post_login(self):
+    async def post_login(self) -> None:
         raise NotImplementedError()
 
     @abstractmethod
-    def register_portal(self, portal: po.Portal):
+    async def register_portal(self, portal: po.Portal) -> None:
         raise NotImplementedError()
 
     @abstractmethod
-    def unregister_portal(self, portal: po.Portal):
+    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
         raise NotImplementedError()
 
-    async def _update_catch(self, update: TypeUpdate):
+    async def _update_catch(self, update: TypeUpdate) -> None:
+        start_time = time.time()
+        update_type = type(update).__name__
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception("Failed to handle Telegram update")
-
-    async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
-        if self.is_bot:
-            return []
-        dialogs = await self.client.get_dialogs(limit=limit)
-        return [dialog.entity for dialog in dialogs if (
-            not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden))
-            and not (isinstance(dialog.entity, Chat)
-                     and (dialog.entity.deactivated or dialog.entity.left)))]
+            UPDATE_ERRORS.labels(update_type=update_type).inc()
+        UPDATE_TIME.labels(update_type=update_type).observe(time.time() - start_time)
 
     @property
     @abstractmethod
@@ -147,231 +259,358 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     async def is_logged_in(self) -> bool:
-        return self.client and await self.client.is_user_authorized()
+        return (
+            self.client and self.client.is_connected() and await self.client.is_user_authorized()
+        )
 
     async def has_full_access(self, allow_bot: bool = False) -> bool:
-        return (self.puppet_whitelisted
-                and (not self.is_bot or allow_bot)
-                and await self.is_logged_in())
+        return (
+            self.puppet_whitelisted
+            and (not self.is_bot or allow_bot)
+            and await self.is_logged_in()
+        )
 
-    async def start(self, delete_unless_authenticated: bool = False) -> "AbstractUser":
+    async def start(self, delete_unless_authenticated: bool = False) -> AbstractUser:
         if not self.client:
-            self._init_client()
+            await self._init_client()
         await self.client.connect()
-        self.log.debug("%s connected: %s", self.mxid, self.connected)
+        self.log.debug(f"{'Bot' if self.is_relaybot else self.mxid} connected: {self.connected}")
         return self
 
-    async def ensure_started(self, even_if_no_session=False) -> "AbstractUser":
-        if not self.puppet_whitelisted:
+    async def ensure_started(self, even_if_no_session=False) -> AbstractUser:
+        if self.connected:
             return self
-        self.log.debug("ensure_started(%s, connected=%s, even_if_no_session=%s, session_count=%s)",
-                       self.mxid, self.connected, even_if_no_session,
-                       self.session_container.Session.query.filter(
-                           self.session_container.Session.session_id == self.mxid).count())
-        should_connect = (even_if_no_session or
-                          self.session_container.Session.query.filter(
-                              self.session_container.Session.session_id == self.mxid).count() > 0)
-        if not self.connected and should_connect:
+        if even_if_no_session or await PgSession.has(self.mxid):
+            self.log.debug(
+                "Starting client due to ensure_started"
+                f"(even_if_no_session={even_if_no_session})"
+            )
             await self.start(delete_unless_authenticated=not even_if_no_session)
         return self
 
-    async def stop(self):
+    async def stop(self) -> None:
         await self.client.disconnect()
         self.client = None
 
     # region Telegram update handling
 
-    async def _update(self, update: TypeUpdate):
-        if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
-                               UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
+    async def _update(self, update: TypeUpdate) -> None:
+        asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
+        if isinstance(
+            update,
+            (
+                UpdateShortChatMessage,
+                UpdateShortMessage,
+                UpdateNewChannelMessage,
+                UpdateNewMessage,
+                UpdateEditMessage,
+                UpdateEditChannelMessage,
+            ),
+        ):
             await self.update_message(update)
         elif isinstance(update, UpdateDeleteMessages):
             await self.delete_message(update)
         elif isinstance(update, UpdateDeleteChannelMessages):
             await self.delete_channel_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
+        elif isinstance(update, (UpdateChatUserTyping, UpdateChannelUserTyping, UpdateUserTyping)):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
-        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
+        elif isinstance(update, UpdateChatParticipantAdmin):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, UpdateChannelPinnedMessage):
+        elif isinstance(update, (UpdatePinnedMessages, UpdatePinnedChannelMessages)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
         elif isinstance(update, UpdateReadHistoryOutbox):
             await self.update_read_receipt(update)
+        elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
+            await self.update_own_read_receipt(update)
+        elif isinstance(update, UpdateFolderPeers):
+            await self.update_folder_peers(update)
+        elif isinstance(update, UpdatePinnedDialogs):
+            await self.update_pinned_dialogs(update)
+        elif isinstance(update, UpdateNotifySettings):
+            await self.update_notify_settings(update)
         else:
-            self.log.debug("Unhandled update: %s", update)
+            self.log.trace("Unhandled update: %s", update)
+
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
+        pass
+
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        pass
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        pass
+
+    async def update_pinned_messages(
+        self, update: UpdatePinnedMessages | UpdatePinnedChannelMessages
+    ) -> None:
+        if isinstance(update, UpdatePinnedMessages):
+            portal = await po.Portal.get_by_entity(update.peer, tg_receiver=self.tgid)
+        else:
+            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        if portal and portal.mxid:
+            await portal.receive_telegram_pin_ids(
+                update.messages, self.tgid, remove=not update.pinned
+            )
 
     @staticmethod
-    async def update_pinned_messages(update: UpdateChannelPinnedMessage):
-        portal = po.Portal.get_by_tgid(update.channel_id)
+    async def update_participants(update: UpdateChatParticipants) -> None:
+        portal = await po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id)
+            await portal.update_power_levels(update.participants.participants)
 
-    @staticmethod
-    async def update_participants(update: UpdateChatParticipants):
-        portal = po.Portal.get_by_tgid(update.participants.chat_id)
-        if portal and portal.mxid:
-            await portal.update_telegram_participants(update.participants.participants)
-
-    async def update_read_receipt(self, update: UpdateReadHistoryOutbox):
+    async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
         if not isinstance(update.peer, PeerUser):
             self.log.debug("Unexpected read receipt peer: %s", update.peer)
             return
 
-        portal = po.Portal.get_by_tgid(update.peer.user_id, self.tgid)
+        portal = await po.Portal.get_by_tgid(
+            TelegramID(update.peer.user_id), tg_receiver=self.tgid
+        )
         if not portal or not portal.mxid:
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.query.get((update.max_id, self.tgid))
+        message = await DBMessage.get_one_by_tgid(
+            TelegramID(update.max_id), self.tgid, edit_index=-1
+        )
         if not message:
             return
 
-        puppet = pu.Puppet.get(update.peer.user_id)
+        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
-    async def update_admin(self, update: Union[UpdateChatAdmins, UpdateChatParticipantAdmin]):
-        # TODO duplication not checked
-        portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        if isinstance(update, UpdateChatAdmins):
-            await portal.set_telegram_admins_enabled(update.enabled)
-        elif isinstance(update, UpdateChatParticipantAdmin):
-            await portal.set_telegram_admin(update.user_id)
-        else:
-            self.log.warning("Unexpected admin status update: %s", update)
+    async def update_own_read_receipt(
+        self, update: UpdateReadHistoryInbox | UpdateReadChannelInbox
+    ) -> None:
+        puppet = await pu.Puppet.get_by_tgid(self.tgid)
+        if not puppet.is_real_user:
+            return
 
-    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]):
-        if isinstance(update, UpdateUserTyping):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+        if isinstance(update, UpdateReadChannelInbox):
+            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update.peer, PeerChat):
+            portal = await po.Portal.get_by_tgid(TelegramID(update.peer.chat_id))
+        elif isinstance(update.peer, PeerUser):
+            portal = await po.Portal.get_by_tgid(
+                TelegramID(update.peer.user_id), tg_receiver=self.tgid
+            )
         else:
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        sender = pu.Puppet.get(update.user_id)
+            self.log.debug("Unexpected own read receipt peer: %s", update.peer)
+            return
+
+        if not portal or not portal.mxid:
+            return
+
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        message = await DBMessage.get_one_by_tgid(
+            TelegramID(update.max_id), tg_space, edit_index=-1
+        )
+        if not message:
+            return
+
+        await puppet.intent.mark_read(portal.mxid, message.mxid)
+
+    async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
+        # TODO duplication not checked
+        portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        if not portal or not portal.mxid:
+            return
+
+        await portal.set_telegram_admin(TelegramID(update.user_id))
+
+    async def update_typing(
+        self, update: UpdateUserTyping | UpdateChatUserTyping | UpdateChannelUserTyping
+    ) -> None:
+        sender = None
+        if isinstance(update, UpdateUserTyping):
+            portal = await po.Portal.get_by_tgid(
+                TelegramID(update.user_id), tg_receiver=self.tgid, peer_type="user"
+            )
+            sender = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
+        elif isinstance(update, UpdateChannelUserTyping):
+            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update, UpdateChatUserTyping):
+            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        else:
+            return
+
+        if isinstance(update, (UpdateChannelUserTyping, UpdateChatUserTyping)):
+            # Can typing notifications come from non-user peers?
+            if not update.from_id.user_id:
+                return
+            sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id.user_id))
+
+        if not sender or not portal or not portal.mxid:
+            return
+
         await portal.handle_telegram_typing(sender, update)
 
-    async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]):
-        # TODO duplication not checked
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update, UpdateUserName):
-            if await puppet.update_displayname(self, update):
-                puppet.save()
-        elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
-                puppet.save()
-        else:
-            self.log.warning("Unexpected other user info update: %s", update)
+    async def _handle_entity_updates(self, entities: dict[int, User | Chat | Channel]) -> None:
+        try:
+            users = (entity for entity in entities.values() if isinstance(entity, User))
+            puppets = ((await pu.Puppet.get_by_tgid(TelegramID(user.id)), user) for user in users)
+            await asyncio.gather(
+                *[puppet.try_update_info(self, info) async for puppet, info in puppets if puppet]
+            )
+        except Exception:
+            self.log.exception("Failed to handle entity updates")
 
-    async def update_status(self, update: UpdateUserStatus):
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update.status, UserStatusOnline):
-            await puppet.default_mxid_intent.set_presence("online")
-        elif isinstance(update.status, UserStatusOffline):
-            await puppet.default_mxid_intent.set_presence("offline")
+    async def update_others_info(self, update: UpdateUserName | UpdateUserPhoto) -> None:
+        # TODO duplication not checked
+        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
+        if isinstance(update, UpdateUserName):
+            puppet.username = update.username
+            if await puppet.update_displayname(self, update):
+                await puppet.save()
+        elif isinstance(update, UpdateUserPhoto):
+            if await puppet.update_avatar(self, update.photo):
+                await puppet.save()
         else:
-            self.log.warning("Unexpected user status update: %s", update)
+            self.log.warning(f"Unexpected other user info update: {type(update)}")
+
+    async def update_status(self, update: UpdateUserStatus) -> None:
+        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
+        if isinstance(update.status, UserStatusOnline):
+            await puppet.default_mxid_intent.set_presence(PresenceState.ONLINE)
+        elif isinstance(update.status, UserStatusOffline):
+            await puppet.default_mxid_intent.set_presence(PresenceState.OFFLINE)
+        else:
+            self.log.warning(f"Unexpected user status update: type({update})")
         return
 
-    def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
-                                                                  Optional[pu.Puppet],
-                                                                  Optional[po.Portal]]:
+    async def get_message_details(
+        self, update: UpdateMessage
+    ) -> tuple[UpdateMessageContent, pu.Puppet | None, po.Portal | None]:
         if isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-            sender = pu.Puppet.get(update.from_id)
+            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
+            if not portal:
+                self.log.warning(f"Received message in chat with unknown type {update.chat_id}")
+            sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id))
         elif isinstance(update, UpdateShortMessage):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
-            sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
-        elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
-                                 UpdateEditMessage, UpdateEditChannelMessage)):
+            portal = await po.Portal.get_by_tgid(
+                TelegramID(update.user_id), tg_receiver=self.tgid, peer_type="user"
+            )
+            sender = await pu.Puppet.get_by_tgid(self.tgid if update.out else update.user_id)
+        elif isinstance(
+            update,
+            (
+                UpdateNewMessage,
+                UpdateNewChannelMessage,
+                UpdateEditMessage,
+                UpdateEditChannelMessage,
+            ),
+        ):
             update = update.message
-            if isinstance(update.to_id, PeerUser) and not update.out:
-                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
+            if isinstance(update, MessageEmpty):
+                return update, None, None
+            portal = await po.Portal.get_by_entity(update.peer_id, tg_receiver=self.tgid)
+            if update.out:
+                sender = await pu.Puppet.get_by_tgid(self.tgid)
+            elif isinstance(update.from_id, PeerUser):
+                sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id.user_id))
             else:
-                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
-            sender = pu.Puppet.get(update.from_id) if update.from_id else None
+                sender = None
         else:
             self.log.warning(
-                f"Unexpected message type in User#get_message_details: {type(update)}")
+                f"Unexpected message type in User#get_message_details: {type(update)}"
+            )
             return update, None, None
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal: po.Portal, message: DBMessage):
+    async def _try_redact(message: DBMessage) -> None:
+        portal = await po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
             await portal.main_intent.redact(message.mx_room, message.mxid)
-        except MatrixRequestError:
+        except MatrixError:
             pass
 
-    async def delete_message(self, update: UpdateDeleteMessages):
-        if len(update.messages) > MAX_DELETIONS:
+    async def delete_message(self, update: UpdateDeleteMessages) -> None:
+        if len(update.messages) > self.max_deletions:
             return
 
-        for message in update.messages:
-            message = DBMessage.query.get((message, self.tgid))
-            if not message:
-                continue
-            self.db.delete(message)
-            number_left = DBMessage.query.filter(DBMessage.mxid == message.mxid,
-                                                 DBMessage.mx_room == message.mx_room).count()
-            if number_left == 0:
-                portal = po.Portal.get_by_mxid(message.mx_room)
-                await self._try_redact(portal, message)
-        self.db.commit()
+        for message_id in update.messages:
+            for message in await DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
+                if message.redacted:
+                    continue
+                await message.delete()
+                number_left = await DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    await self._try_redact(message)
 
-    async def delete_channel_message(self, update: UpdateDeleteChannelMessages):
-        if len(update.messages) > MAX_DELETIONS:
+    async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
+        if len(update.messages) > self.max_deletions:
             return
 
-        portal = po.Portal.get_by_tgid(update.channel_id)
+        channel_id = TelegramID(update.channel_id)
+
+        for message_id in update.messages:
+            for message in await DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
+                if message.redacted:
+                    continue
+                await message.delete()
+                await self._try_redact(message)
+
+    async def update_message(self, original_update: UpdateMessage) -> None:
+        update, sender, portal = await self.get_message_details(original_update)
         if not portal:
             return
-
-        for message in update.messages:
-            message = DBMessage.query.get((message, portal.tgid))
-            if not message:
-                continue
-            self.db.delete(message)
-            await self._try_redact(portal, message)
-        self.db.commit()
-
-    async def update_message(self, original_update: UpdateMessage):
-        update, sender, portal = self.get_message_details(original_update)
-        if self.ignore_incoming_bot_events and self.bot and sender.id == self.bot.tgid:
-            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
+        elif portal and not portal.allow_bridging:
+            self.log.debug(f"Ignoring message in portal {portal.tgid_log} (bridging disallowed)")
             return
 
+        if self.is_relaybot:
+            if update.is_private:
+                if not self.config["bridge.relaybot.private_chat.invite"]:
+                    if sender:
+                        self.log.debug(f"Ignoring private message to bot from {sender.id}")
+                    return
+            elif not portal.mxid and self.config["bridge.relaybot.ignore_unbridged_group_chat"]:
+                self.log.debug(
+                    f"Ignoring message received by bot in unbridged chat {portal.tgid_log}"
+                )
+                return
+
+        if (
+            self.ignore_incoming_bot_events
+            and self.relaybot
+            and sender
+            and sender.id == self.relaybot.tgid
+        ):
+            self.log.debug("Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
+            return
+
+        await portal.backfill_lock.wait(f"update {update.id}")
+
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.debug(f"Ignoring action %s to %s by %d", update.action,
-                               portal.tgid_log,
-                               sender.id)
+                self.log.trace(
+                    "Received %s in %s by %d, unregistering portal...",
+                    update.action,
+                    portal.tgid_log,
+                    sender.id,
+                )
+                await self.unregister_portal(update.action.chat_id, update.action.chat_id)
+                await self.register_portal(portal)
                 return
-            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
-                           sender.id)
+            self.log.trace(
+                "Handling action %s to %s by %d",
+                update.action,
+                portal.tgid_log,
+                (sender.id if sender else 0),
+            )
             return await portal.handle_telegram_action(self, sender, update)
 
-        user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            if config["bridge.edits_as_replies"]:
-                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-                return await portal.handle_telegram_edit(self, sender, update)
-            return
-
-        self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
+            return await portal.handle_telegram_edit(self, sender, update)
         return await portal.handle_telegram_message(self, sender, update)
 
     # endregion
-
-
-def init(context: "Context"):
-    global config, MAX_DELETIONS
-    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, AbstractUser.relaybot = context
-    AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
-    AbstractUser.session_container = context.session_container
-    MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/base.py

@@ -1,2 +0,0 @@
-from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()  # type: declarative_base

mautrix_telegram/bot.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,119 +13,146 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Pattern, Dict, TYPE_CHECKING
+from typing import Awaitable, Callable, Dict, List, Optional, Tuple
 import logging
-import re
 
-from telethon.tl.types import *
-from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
-from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
+from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
+from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
+from telethon.tl.patched import Message, MessageService
+from telethon.tl.types import (
+    ChannelParticipantAdmin,
+    ChannelParticipantCreator,
+    ChatForbidden,
+    ChatParticipantAdmin,
+    ChatParticipantCreator,
+    InputChannel,
+    InputUser,
+    MessageActionChatAddUser,
+    MessageActionChatDeleteUser,
+    MessageActionChatMigrateTo,
+    MessageEntityBotCommand,
+    PeerChannel,
+    PeerChat,
+    PeerUser,
+    TypePeer,
+    UpdateNewChannelMessage,
+    UpdateNewMessage,
+    User,
+)
 
+from mautrix.types import UserID
+
+from . import portal as po, puppet as pu, user as u
 from .abstract_user import AbstractUser
 from .db import BotChat
-from . import puppet as pu, portal as po, user as u
-
-if TYPE_CHECKING:
-    from .config import Config
-
-config = None  # type: Config
+from .types import TelegramID
 
 ReplyFunc = Callable[[str], Awaitable[Message]]
 
 
 class Bot(AbstractUser):
-    log = logging.getLogger("mau.bot")  # type: logging.Logger
-    mxid_regex = re.compile("@.+:.+")  # type: Pattern
+    log: logging.Logger = logging.getLogger("mau.user.bot")
 
-    def __init__(self, token: str):
+    token: str
+    chats: Dict[int, str]
+    tg_whitelist: List[int]
+    whitelist_group_admins: bool
+    _me_info: Optional[User]
+    _me_mxid: Optional[UserID]
+
+    def __init__(self, token: str) -> None:
         super().__init__()
-        self.token = token  # type: str
-        self.puppet_whitelisted = True  # type: bool
-        self.whitelisted = True  # type: bool
-        self.relaybot_whitelisted = True  # type: bool
-        self.username = None  # type: str
-        self.is_relaybot = True  # type: bool
-        self.is_bot = True  # type: bool
-        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}  # type: Dict[int, str]
-        self.tg_whitelist = []  # type: List[int]
-        self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
-                                       or False)  # type: bool
+        self.token = token
+        self.tgid = None
+        self.mxid = None
+        self.puppet_whitelisted = True
+        self.whitelisted = True
+        self.relaybot_whitelisted = True
+        self.tg_username = None
+        self.is_relaybot = True
+        self.is_bot = True
+        self.chats = {}
+        self.tg_whitelist = []
+        self.whitelist_group_admins = (
+            self.config["bridge.relaybot.whitelist_group_admins"] or False
+        )
+        self._me_info = None
+        self._me_mxid = None
 
-    async def init_permissions(self):
-        whitelist = config["bridge.relaybot.whitelist"] or []
-        for id in whitelist:
-            if isinstance(id, str):
-                entity = await self.client.get_input_entity(id)
+    async def get_me(self, use_cache: bool = True) -> Tuple[User, UserID]:
+        if not use_cache or not self._me_mxid:
+            self._me_info = await self.client.get_me()
+            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
+        return self._me_info, self._me_mxid
+
+    async def init_permissions(self) -> None:
+        whitelist = self.config["bridge.relaybot.whitelist"] or []
+        for user_id in whitelist:
+            if isinstance(user_id, str):
+                entity = await self.client.get_input_entity(user_id)
                 if isinstance(entity, InputUser):
-                    id = entity.user_id
+                    user_id = entity.user_id
                 else:
-                    id = None
-            if isinstance(id, int):
-                self.tg_whitelist.append(id)
+                    user_id = None
+            if isinstance(user_id, int):
+                self.tg_whitelist.append(user_id)
 
     async def start(self, delete_unless_authenticated: bool = False) -> "Bot":
+        self.chats = {chat.id: chat.type for chat in await BotChat.all()}
         await super().start(delete_unless_authenticated)
         if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
         await self.post_login()
         return self
 
-    async def post_login(self):
+    async def post_login(self) -> None:
         await self.init_permissions()
         info = await self.client.get_me()
-        self.tgid = info.id
-        self.username = info.username
+        self.tgid = TelegramID(info.id)
+        self.tg_username = info.username
         self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
 
-        chat_ids = [id for id, type in self.chats.items() if type == "chat"]
+        chat_ids = [chat_id for chat_id, chat_type in self.chats.items() if chat_type == "chat"]
         response = await self.client(GetChatsRequest(chat_ids))
         for chat in response.chats:
             if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
-                self.remove_chat(chat.id)
+                await self.remove_chat(TelegramID(chat.id))
 
-        channel_ids = [InputChannel(id, 0)
-                       for id, type in self.chats.items()
-                       if type == "channel"]
-        for id in channel_ids:
+        channel_ids = [
+            InputChannel(chat_id, 0)
+            for chat_id, chat_type in self.chats.items()
+            if chat_type == "channel"
+        ]
+        for channel_id in channel_ids:
             try:
-                await self.client(GetChannelsRequest([id]))
+                await self.client(GetChannelsRequest([channel_id]))
             except (ChannelPrivateError, ChannelInvalidError):
-                self.remove_chat(id.channel_id)
+                await self.remove_chat(TelegramID(channel_id.channel_id))
 
-        if config["bridge.catch_up"]:
-            try:
-                await self.client.catch_up()
-            except Exception:
-                self.log.exception("Failed to run catch_up() for bot")
+    async def register_portal(self, portal: po.Portal) -> None:
+        await self.add_chat(portal.tgid, portal.peer_type)
 
-    def register_portal(self, portal: po.Portal):
-        self.add_chat(portal.tgid, portal.peer_type)
+    async def unregister_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
+        await self.remove_chat(tgid)
 
-    def unregister_portal(self, portal: po.Portal):
-        self.remove_chat(portal.tgid)
+    async def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
+        if chat_id not in self.chats:
+            self.chats[chat_id] = chat_type
+            await BotChat(id=chat_id, type=chat_type).insert()
 
-    def add_chat(self, id: int, type: str):
-        if id not in self.chats:
-            self.chats[id] = type
-            self.db.add(BotChat(id=id, type=type))
-            self.db.commit()
-
-    def remove_chat(self, id: int):
+    async def remove_chat(self, chat_id: TelegramID) -> None:
         try:
-            del self.chats[id]
+            del self.chats[chat_id]
         except KeyError:
             pass
-        existing_chat = BotChat.query.get(id)
-        if existing_chat:
-            self.db.delete(existing_chat)
-            self.db.commit()
+        await BotChat.delete_by_id(chat_id)
 
-    async def _can_use_commands(self, chat: TypePeer, tgid: int) -> bool:
+    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
-        user = u.User.get_by_tgid(tgid)
+        user = await u.User.get_by_tgid(tgid)
         if user and user.is_admin:
             self.tg_whitelist.append(user.tgid)
             return True
@@ -134,66 +160,79 @@ class Bot(AbstractUser):
         if self.whitelist_group_admins:
             if isinstance(chat, PeerChannel):
                 p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
+                return isinstance(
+                    p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin)
+                )
             elif isinstance(chat, PeerChat):
                 chat = await self.client(GetFullChatRequest(chat.chat_id))
                 participants = chat.full_chat.participants.participants
                 for p in participants:
                     if p.user_id == tgid:
                         return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+        return False
 
     async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
-        if not await self._can_use_commands(event.to_id, event.from_id):
+        # FIXME event.from_id is not int
+        if not await self._can_use_commands(event.to_id, TelegramID(event.from_id)):
             await reply("You do not have the permission to use that command.")
             return False
         return True
 
-    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc):
-        if not config["bridge.relaybot.authless_portals"]:
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> Message:
+        if not self.config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
-        if not portal.allow_bridging():
+        if not portal.allow_bridging:
             return await reply("This bridge doesn't allow bridging this chat.")
 
         await portal.create_matrix_room(self)
         if portal.mxid:
             if portal.username:
                 return await reply(
-                    f"Portal is public: [{portal.alias}](https://matrix.to/#/{portal.alias})")
+                    f"Portal is public: [{portal.alias}](https://matrix.to/#/{portal.alias})"
+                )
             else:
-                return await reply(
-                    "Portal is not public. Use `/invite <mxid>` to get an invite.")
+                return await reply("Portal is not public. Use `/invite <mxid>` to get an invite.")
 
-    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc, mxid: str):
-        if len(mxid) == 0:
+    async def handle_command_invite(
+        self, portal: po.Portal, reply: ReplyFunc, mxid_input: UserID
+    ) -> Message:
+        if len(mxid_input) == 0:
             return await reply("Usage: `/invite <mxid>`")
         elif not portal.mxid:
-            return await reply("Portal does not have Matrix room. "
-                               "Create one with /portal first.")
-        if not self.mxid_regex.match(mxid):
+            return await reply("Portal does not have Matrix room. Create one with /portal first.")
+        if mxid_input[0] != "@" or mxid_input.find(":") < 2:
             return await reply("That doesn't look like a Matrix ID.")
-        user = await u.User.get_by_mxid(mxid).ensure_started()
+        user = await u.User.get_and_start_by_mxid(mxid_input)
         if not user.relaybot_whitelisted:
             return await reply("That user is not whitelisted to use the bridge.")
         elif await user.is_logged_in():
-            displayname = f"@{user.username}" if user.username else user.displayname
-            return await reply("That user seems to be logged in. "
-                               f"Just invite [{displayname}](tg://user?id={user.tgid})")
+            displayname = f"@{user.tg_username}" if user.tg_username else user.displayname
+            return await reply(
+                "That user seems to be logged in. "
+                f"Just invite [{displayname}](tg://user?id={user.tgid})"
+            )
         else:
-            await portal.main_intent.invite(portal.mxid, user.mxid)
+            await portal.invite_to_matrix(user.mxid)
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
-    def handle_command_id(self, message: Message, reply: ReplyFunc):
+    @staticmethod
+    def handle_command_id(message: Message, reply: ReplyFunc) -> Awaitable[Message]:
         # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
         # chat is a normal group or a supergroup/channel when using the ID.
         if isinstance(message.to_id, PeerChannel):
             return reply(f"-100{message.to_id.channel_id}")
-        return reply(str(-message.to_id.chat_id))
+        elif isinstance(message.to_id, PeerChat):
+            return reply(str(-message.to_id.chat_id))
+        elif isinstance(message.to_id, PeerUser):
+            return reply(f"Your user ID is {message.to_id.user_id}.")
+        else:
+            return reply("Failed to find chat ID.")
 
     def match_command(self, text: str, command: str) -> bool:
         text = text.lower()
         command = f"/{command.lower()}"
-        command_targeted = f"{command}@{self.username.lower()}"
+        command_targeted = f"{command}@{self.tg_username.lower()}"
 
         is_plain_command = text == command or text == command_targeted
         if is_plain_command:
@@ -205,58 +244,76 @@ class Bot(AbstractUser):
 
         return False
 
-    async def handle_command(self, message: Message):
-        def reply(reply_text):
-            return self.client.send_message(message.to_id, reply_text, reply_to=message.id)
+    async def handle_command(self, message: Message) -> None:
+        def reply(reply_text: str) -> Awaitable[Message]:
+            return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)
 
         text = message.message
 
-        if self.match_command(text, "id"):
-            return await self.handle_command_id(message, reply)
+        if self.match_command(text, "start"):
+            pcm = self.config["bridge.relaybot.private_chat.message"]
+            if pcm:
+                await reply(pcm)
+            return
+        elif self.match_command(text, "id"):
+            await self.handle_command_id(message, reply)
+            return
+        elif message.is_private:
+            return
 
-        portal = po.Portal.get_by_entity(message.to_id)
+        portal = await po.Portal.get_by_entity(message.to_id)
 
-        if self.match_command(text, "portal"):
+        is_portal_cmd = self.match_command(text, "portal")
+        is_invite_cmd = self.match_command(text, "invite")
+        if is_portal_cmd or is_invite_cmd:
             if not await self.check_can_use_commands(message, reply):
                 return
-            await self.handle_command_portal(portal, reply)
-        elif self.match_command(text, "invite"):
-            if not await self.check_can_use_commands(message, reply):
-                return
-            try:
-                mxid = text[text.index(" ") + 1:]
-            except ValueError:
-                mxid = ""
-            await self.handle_command_invite(portal, reply, mxid=mxid)
+            if is_portal_cmd:
+                await self.handle_command_portal(portal, reply)
+            elif is_invite_cmd:
+                try:
+                    mxid = text[text.index(" ") + 1 :]
+                except ValueError:
+                    mxid = ""
+                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
 
-    def handle_service_message(self, message: MessageService):
-        to_id = message.to_id
-        if isinstance(to_id, PeerChannel):
-            to_id = to_id.channel_id
-            type = "channel"
-        elif isinstance(to_id, PeerChat):
-            to_id = to_id.chat_id
-            type = "chat"
+    async def handle_service_message(self, message: MessageService) -> None:
+        to_peer = message.to_id
+        if isinstance(to_peer, PeerChannel):
+            to_id = TelegramID(to_peer.channel_id)
+            chat_type = "channel"
+        elif isinstance(to_peer, PeerChat):
+            to_id = TelegramID(to_peer.chat_id)
+            chat_type = "chat"
         else:
             return
 
         action = message.action
         if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
-            self.add_chat(to_id, type)
+            await self.add_chat(to_id, chat_type)
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
-            self.remove_chat(to_id)
+            await self.remove_chat(to_id)
+        elif isinstance(action, MessageActionChatMigrateTo):
+            await self.remove_chat(to_id)
+            await self.add_chat(TelegramID(action.channel_id), "channel")
 
-    async def update(self, update):
+    async def update(self, update) -> bool:
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
-            return
+            return False
         if isinstance(update.message, MessageService):
-            return self.handle_service_message(update.message)
+            await self.handle_service_message(update.message)
+            return False
 
-        is_command = (isinstance(update.message, Message)
-                      and update.message.entities and len(update.message.entities) > 0
-                      and isinstance(update.message.entities[0], MessageEntityBotCommand))
+        is_command = (
+            isinstance(update.message, Message)
+            and update.message.entities
+            and len(update.message.entities) > 0
+            and isinstance(update.message.entities[0], MessageEntityBotCommand)
+            and update.message.entities[0].offset == 0
+        )
         if is_command:
-            return await self.handle_command(update.message)
+            await self.handle_command(update.message)
+        return False
 
     def is_in_chat(self, peer_id) -> bool:
         return peer_id in self.chats
@@ -264,12 +321,3 @@ class Bot(AbstractUser):
     @property
     def name(self) -> str:
         return "bot"
-
-
-def init(context) -> Optional[Bot]:
-    global config
-    config = context.config
-    token = config["telegram.bot_token"]
-    if token and not token.lower().startswith("disable"):
-        return Bot(token)
-    return None

mautrix_telegram/commands/__init__.py

@@ -1,5 +1,26 @@
-from .handler import (command_handler, command_handlers as _command_handlers,
-                      CommandHandler, CommandProcessor, CommandEvent,
-                      SECTION_GENERAL, SECTION_AUTH, SECTION_CREATING_PORTALS,
-                      SECTION_PORTAL_MANAGEMENT, SECTION_MISC, SECTION_ADMIN)
-from . import clean_rooms, auth, meta, telegram, portal
+from .handler import (
+    SECTION_ADMIN,
+    SECTION_AUTH,
+    SECTION_CREATING_PORTALS,
+    SECTION_MISC,
+    SECTION_PORTAL_MANAGEMENT,
+    CommandEvent,
+    CommandHandler,
+    CommandProcessor,
+    command_handler,
+)
+
+# This has to happen after the handler imports
+from . import matrix_auth, portal, telegram  # isort: skip
+
+__all__ = [
+    "command_handler",
+    "CommandHandler",
+    "CommandProcessor",
+    "CommandEvent",
+    "SECTION_AUTH",
+    "SECTION_MISC",
+    "SECTION_ADMIN",
+    "SECTION_CREATING_PORTALS",
+    "SECTION_PORTAL_MANAGEMENT",
+]

mautrix_telegram/commands/auth.py

@@ -1,315 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict
-import asyncio
-
-from telethon.errors import *
-
-from . import command_handler, CommandEvent, SECTION_AUTH
-from .. import puppet as pu
-from ..util import format_duration
-
-
-@command_handler(needs_auth=False,
-                 help_section=SECTION_AUTH,
-                 help_text="Check if you're logged into Telegram.")
-async def ping(evt: CommandEvent):
-    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
-    if me:
-        return await evt.reply(f"You're logged in as @{me.username}")
-    else:
-        return await evt.reply("You're not logged in.")
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_AUTH,
-                 help_text="Get the info of the message relay Telegram bot.")
-async def ping_bot(evt: CommandEvent):
-    if not evt.tgbot:
-        return await evt.reply("Telegram message relay bot not configured.")
-    bot_info = await evt.tgbot.client.get_me()
-    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
-    displayname = bot_info.first_name
-    return await evt.reply("Telegram message relay bot is active: "
-                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
-                           "To use the bot, simply invite it to a portal room.")
-
-
-@command_handler(needs_auth=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
-                           "account.")
-async def logout_matrix(evt: CommandEvent):
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if not puppet.is_real_user:
-        return await evt.reply("You are not logged in with your Matrix account.")
-    await puppet.switch_mxid(None, None)
-    await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
-
-
-@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
-                           "account")
-async def login_matrix(evt: CommandEvent):
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if puppet.is_real_user:
-        return await evt.reply("You have already logged in with your Matrix account. "
-                               "Log out with `$cmdprefix+sp logout-matrix` first.")
-    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
-    if allow_matrix_login:
-        evt.sender.command_status = {
-            "next": enter_matrix_token,
-            "action": "Matrix login",
-        }
-    if evt.config["appservice.public.enabled"]:
-        prefix = evt.config["appservice.public.external"]
-        url = f"{prefix}/matrix-login?token={evt.public_website.make_token(evt.sender.mxid, '/matrix-login')}"
-        if allow_matrix_login:
-            return await evt.reply(
-                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
-                "If you would like to log in within Matrix, please send your Matrix access token "
-                "here.\n"
-                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
-                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
-                "your access token in the message history.")
-        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
-                               f"Please visit [the login page]({url}) to log in.")
-    elif allow_matrix_login:
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your Matrix access token here to log in.")
-    return await evt.reply("This bridge instance has been configured to not allow logging in.")
-
-
-async def enter_matrix_token(evt: CommandEvent):
-    evt.sender.command_status = None
-
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if puppet.is_real_user:
-        return await evt.reply("You have already logged in with your Matrix account. "
-                               "Log out with `$cmdprefix+sp logout-matrix` first.")
-
-    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
-    if resp == 2:
-        return await evt.reply("You can only log in as your own Matrix user.")
-    elif resp == 1:
-        return await evt.reply("Failed to verify access token.")
-    return await evt.reply(
-        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")
-
-
-@command_handler(needs_auth=False, management_only=True,
-                 help_section=SECTION_AUTH,
-                 help_args="<_phone_> <_full name_>",
-                 help_text="Register to Telegram")
-async def register(evt: CommandEvent):
-    if await evt.sender.is_logged_in():
-        return await evt.reply("You are already logged in.")
-    elif len(evt.args) < 1:
-        return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
-
-    phone_number = evt.args[0]
-    if len(evt.args) == 2:
-        full_name = evt.args[1], ""
-    else:
-        full_name = " ".join(evt.args[1:-1]), evt.args[-1]
-
-    await request_code(evt, phone_number, {
-        "next": enter_code_register,
-        "action": "Register",
-        "full_name": full_name,
-    })
-
-
-async def enter_code_register(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
-    try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        first_name, last_name = evt.sender.command_status["full_name"]
-        user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully registered to Telegram.")
-    except PhoneNumberOccupiedError:
-        return await evt.reply("That phone number has already been registered. "
-                               "You can log in with `$cmdprefix+sp login`.")
-    except FirstNameInvalidError:
-        return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
-    except PhoneCodeExpiredError:
-        return await evt.reply(
-            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`.")
-    except PhoneCodeInvalidError:
-        return await evt.reply("Invalid phone code.")
-    except Exception:
-        evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code. "
-                               "Check console for more details.")
-
-
-@command_handler(needs_auth=False, management_only=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Get instructions on how to log in.")
-async def login(evt: CommandEvent):
-    if await evt.sender.is_logged_in():
-        return await evt.reply("You are already logged in.")
-
-    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
-    if allow_matrix_login:
-        evt.sender.command_status = {
-            "next": enter_phone_or_token,
-            "action": "Login",
-        }
-
-    if evt.config["appservice.public.enabled"]:
-        prefix = evt.config["appservice.public.external"]
-        url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
-        if allow_matrix_login:
-            return await evt.reply(
-                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
-                "If you would like to log in within Matrix, please send your phone number or bot "
-                "auth token here.\n"
-                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
-                "Logging in outside of Matrix is recommended if you have two-factor authentication "
-                "enabled, because in-Matrix login would save your password in the message history.")
-        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
-                               f"Please visit [the login page]({url}) to log in.")
-    elif allow_matrix_login:
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number or bot auth token here to start the login process.")
-    return await evt.reply("This bridge instance has been configured to not allow logging in.")
-
-
-async def request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, str]):
-    ok = False
-    try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        await evt.sender.client.sign_in(phone_number)
-        ok = True
-        return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
-    except PhoneNumberAppSignupForbiddenError:
-        return await evt.reply(
-            "Your phone number does not allow 3rd party apps to sign in.")
-    except PhoneNumberFloodError:
-        return await evt.reply(
-            "Your phone number has been temporarily blocked for flooding. "
-            "The ban is usually applied for around a day.")
-    except FloodWaitError as e:
-        return await evt.reply(
-            "Your phone number has been temporarily blocked for flooding. "
-            f"Please wait for {format_duration(e.seconds)} before trying again.")
-    except PhoneNumberBannedError:
-        return await  evt.reply("Your phone number has been banned from Telegram.")
-    except PhoneNumberUnoccupiedError:
-        return await  evt.reply("That phone number has not been registered. "
-                                "Please register with `$cmdprefix+sp register <phone>`.")
-    except Exception:
-        evt.log.exception("Error requesting phone code")
-        return await evt.reply("Unhandled exception while requesting code. "
-                               "Check console for more details.")
-    finally:
-        evt.sender.command_status = next_status if ok else None
-
-
-@command_handler(needs_auth=False)
-async def enter_phone_or_token(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
-    elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
-
-    # phone numbers don't contain colons but telegram bot auth tokens do
-    if evt.args[0].find(":") > 0:
-        try:
-            await sign_in(evt, bot_token=evt.args[0])
-        except Exception:
-            evt.log.exception("Error sending auth token")
-            return await evt.reply("Unhandled exception while sending auth token. "
-                                   "Check console for more details.")
-    else:
-        await request_code(evt, evt.args[0], {
-            "next": enter_code,
-            "action": "Login",
-        })
-
-
-@command_handler(needs_auth=False)
-async def enter_code(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
-    elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
-    try:
-        await sign_in(evt, code=evt.args[0])
-    except Exception:
-        evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code. "
-                               "Check console for more details.")
-
-
-@command_handler(needs_auth=False)
-async def enter_password(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
-    elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
-    try:
-        await sign_in(evt, password=" ".join(evt.args))
-    except AccessTokenInvalidError:
-        return await evt.reply("That bot token is not valid.")
-    except AccessTokenExpiredError:
-        return await evt.reply("That bot token has expired.")
-    except Exception:
-        evt.log.exception("Error sending password")
-        return await evt.reply("Unhandled exception while sending password. "
-                               "Check console for more details.")
-
-
-async def sign_in(evt: CommandEvent, **sign_in_info):
-    try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        user = await evt.sender.client.sign_in(**sign_in_info)
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PhoneCodeExpiredError:
-        return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
-    except PhoneCodeInvalidError:
-        return await evt.reply("Invalid phone code.")
-    except PasswordHashInvalidError:
-        return await evt.reply("Incorrect password.")
-    except SessionPasswordNeededError:
-        evt.sender.command_status = {
-            "next": enter_password,
-            "action": "Login (password entry)",
-        }
-        return await evt.reply("Your account has two-factor authentication. "
-                               "Please send your password here.")
-
-
-@command_handler(needs_auth=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Log out from Telegram.")
-async def logout(evt: CommandEvent):
-    if await evt.sender.log_out():
-        return await evt.reply("Logged out successfully.")
-    return await evt.reply("Failed to log out.")

mautrix_telegram/commands/clean_rooms.py

@@ -1,176 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, List
-
-from mautrix_appservice import MatrixRequestError, IntentAPI
-
-from . import command_handler, CommandEvent, SECTION_ADMIN
-from .. import puppet as pu, portal as po
-
-ManagementRoomList = List[Tuple[str, str]]
-RoomIDList = List[str]
-
-
-async def _find_rooms(intent: IntentAPI) -> Tuple[ManagementRoomList, RoomIDList,
-                                                  List["po.Portal"], List["po.Portal"]]:
-    management_rooms = []  # type: ManagementRoomList
-    unidentified_rooms = []  # type: RoomIDList
-    portals = []  # type: List[po.Portal]
-    empty_portals = []  # type: List[po.Portal]
-
-    rooms = await intent.get_joined_rooms()
-    for room in rooms:
-        portal = po.Portal.get_by_mxid(room)
-        if not portal:
-            try:
-                members = await intent.get_room_members(room)
-            except MatrixRequestError:
-                members = []
-            if len(members) == 2:
-                other_member = members[0] if members[0] != intent.mxid else members[1]
-                if pu.Puppet.get_id_from_mxid(other_member):
-                    unidentified_rooms.append(room)
-                else:
-                    management_rooms.append((room, other_member))
-            else:
-                unidentified_rooms.append(room)
-        else:
-            members = await portal.get_authenticated_matrix_users()
-            if len(members) == 0:
-                empty_portals.append(portal)
-            else:
-                portals.append(portal)
-
-    return management_rooms, unidentified_rooms, portals, empty_portals
-
-
-@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
-                 help_section=SECTION_ADMIN,
-                 help_text="Clean up unused portal/management rooms.")
-async def clean_rooms(evt: CommandEvent):
-    management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
-
-    reply = ["#### Management rooms (M)"]
-    reply += ([f"{n+1}. [M{n+1}](https://matrix.to/#/{room}) (with {other_member}"
-               for n, (room, other_member) in enumerate(management_rooms)]
-              or ["No management rooms found."])
-    reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [P{n+1}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(portals)]
-              or ["No active portal rooms found."])
-    reply.append("#### Unidentified rooms (U)")
-    reply += ([f"{n+1}. [U{n+1}](https://matrix.to/#/{room})"
-               for n, room in enumerate(unidentified_rooms)]
-              or ["No unidentified rooms found."])
-    reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [E{n}](https://matrix.to/#/{portal.mxid}) "
-               f"(to Telegram chat \"{portal.title}\")"
-               for n, portal in enumerate(empty_portals)]
-              or ["No inactive portal rooms found."])
-
-    reply += ["#### Usage",
-              ("To clean the recommended set of rooms (unidentified & inactive portals), "
-               "type `$cmdprefix+sp clean-recommended`"),
-              "",
-              ("To clean other groups of rooms, type `$cmdprefix+sp clean-groups <letters>` "
-               "where `letters` are the first letters of the group names (M, A, U, I)"),
-              "",
-              ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
-               "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name."),
-              "",
-              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
-               "between each use of the commands above.")]
-
-    evt.sender.command_status = {
-        "next": lambda clean_evt: set_rooms_to_clean(clean_evt, management_rooms,
-                                                     unidentified_rooms, portals, empty_portals),
-        "action": "Room cleaning",
-    }
-
-    return await evt.reply("\n".join(reply))
-
-
-async def set_rooms_to_clean(evt, management_rooms: ManagementRoomList,
-                             unidentified_rooms: RoomIDList, portals: List["po.Portal"],
-                             empty_portals: List["po.Portal"]):
-    command = evt.args[0]
-    rooms_to_clean = []
-    if command == "clean-recommended":
-        rooms_to_clean = empty_portals + unidentified_rooms
-    elif command == "clean-groups":
-        if len(evt.args) < 2:
-            return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1]
-        if "M" in groups_to_clean:
-            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
-        if "A" in groups_to_clean:
-            rooms_to_clean += portals
-        if "U" in groups_to_clean:
-            rooms_to_clean += unidentified_rooms
-        if "I" in groups_to_clean:
-            rooms_to_clean += empty_portals
-    elif command == "clean-range":
-        try:
-            range = evt.args[1]
-            group, range = range[0], range[1:]
-            start, end = range.split("-")
-            start, end = int(start), int(end)
-            if group == "M":
-                group = [room_id for (room_id, user_id) in management_rooms]
-            elif group == "A":
-                group = portals
-            elif group == "U":
-                group = unidentified_rooms
-            elif group == "I":
-                group = empty_portals
-            else:
-                raise ValueError("Unknown group")
-            rooms_to_clean = group[start - 1:end]
-        except (KeyError, ValueError):
-            return await evt.reply(
-                "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
-    else:
-        return await evt.reply(f"Unknown room cleaning action `{command}`. "
-                               "Use `$cmdprefix+sp cancel` to cancel room "
-                               "cleaning.")
-
-    evt.sender.command_status = {
-        "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
-        "action": "Room cleaning",
-    }
-    await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type"
-                    "`$cmdprefix+sp confirm-clean`.")
-
-
-async def execute_room_cleanup(evt, rooms_to_clean):
-    if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
-        await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
-                        "This might take a while.")
-        cleaned = 0
-        for room in rooms_to_clean:
-            if isinstance(room, po.Portal):
-                await room.cleanup_and_delete()
-                cleaned += 1
-            elif isinstance(room, str):
-                await po.Portal.cleanup_room(evt.az.intent, room, message="Room deleted")
-                cleaned += 1
-        evt.sender.command_status = None
-        await evt.reply(f"{cleaned} rooms cleaned up successfully.")
-    else:
-        await evt.reply("Room cleaning cancelled.")

mautrix_telegram/commands/handler.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,21 +13,38 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Dict, Callable, Optional
-from collections import namedtuple
-import markdown
-import logging
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, NamedTuple
 
 from telethon.errors import FloodWaitError
 
-from ..util import format_duration
-from .. import user as u, context as c
+from mautrix.bridge.commands import (
+    CommandEvent as BaseCommandEvent,
+    CommandHandler as BaseCommandHandler,
+    CommandHandlerFunc,
+    CommandProcessor as BaseCommandProcessor,
+    HelpSection,
+    command_handler as base_command_handler,
+)
+from mautrix.types import EventID, MessageEventContent, RoomID
+from mautrix.util.format_duration import format_duration
 
-command_handlers = {}  # type: Dict[str, CommandHandler]
+from .. import portal as po, user as u
+
+if TYPE_CHECKING:
+    from ..__main__ import TelegramBridge
+
+
+class HelpCacheKey(NamedTuple):
+    is_management: bool
+    is_portal: bool
+    puppet_whitelisted: bool
+    matrix_puppet_whitelisted: bool
+    is_admin: bool
+    is_logged_in: bool
 
-HelpSection = namedtuple("HelpSection", "name order description")
 
-SECTION_GENERAL = HelpSection("General", 0, "")
 SECTION_AUTH = HelpSection("Authentication", 10, "")
 SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
 SECTION_PORTAL_MANAGEMENT = HelpSection("Portal management", 30, "")
@@ -36,133 +52,143 @@ SECTION_MISC = HelpSection("Miscellaneous", 40, "")
 SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 
-class CommandEvent:
-    def __init__(self, processor: "CommandProcessor", room: str, sender: u.User, command: str,
-                 args: List[str], is_management: bool, is_portal: bool):
-        self.az = processor.az
-        self.log = processor.log
-        self.loop = processor.loop
+class CommandEvent(BaseCommandEvent):
+    sender: u.User
+    portal: po.Portal
+
+    def __init__(
+        self,
+        processor: CommandProcessor,
+        room_id: RoomID,
+        event_id: EventID,
+        sender: u.User,
+        command: str,
+        args: list[str],
+        content: MessageEventContent,
+        portal: po.Portal | None,
+        is_management: bool,
+        has_bridge_bot: bool,
+    ) -> None:
+        super().__init__(
+            processor,
+            room_id,
+            event_id,
+            sender,
+            command,
+            args,
+            content,
+            portal,
+            is_management,
+            has_bridge_bot,
+        )
+        self.bridge = processor.bridge
         self.tgbot = processor.tgbot
         self.config = processor.config
         self.public_website = processor.public_website
-        self.command_prefix = processor.command_prefix
-        self.room_id = room
-        self.sender = sender
-        self.command = command
-        self.args = args
-        self.is_management = is_management
-        self.is_portal = is_portal
 
-    def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True):
-        message = message.replace("$cmdprefix+sp ",
-                                  "" if self.is_management else f"{self.command_prefix} ")
-        message = message.replace("$cmdprefix", self.command_prefix)
-        html = None
-        if render_markdown:
-            html = markdown.markdown(message, safe_mode="escape" if allow_html else False)
-        elif allow_html:
-            html = message
-        return self.az.intent.send_notice(self.room_id, message, html=html)
+    @property
+    def print_error_traceback(self) -> bool:
+        return self.sender.is_admin
+
+    async def get_help_key(self) -> HelpCacheKey:
+        return HelpCacheKey(
+            self.is_management,
+            self.portal is not None,
+            self.sender.puppet_whitelisted,
+            self.sender.matrix_puppet_whitelisted,
+            self.sender.is_admin,
+            await self.sender.is_logged_in(),
+        )
 
 
-class CommandHandler:
-    def __init__(self, handler: Callable[[CommandEvent], None], needs_auth: bool,
-                 needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
-                 management_only: bool, name: str, help_text: str, help_args: str,
-                 help_section: HelpSection):
-        self._handler = handler
-        self.needs_auth = needs_auth
-        self.needs_puppeting = needs_puppeting
-        self.needs_matrix_puppeting = needs_matrix_puppeting
-        self.needs_admin = needs_admin
-        self.management_only = management_only
-        self.name = name
-        self._help_text = help_text
-        self._help_args = help_args
-        self.help_section = help_section
+class CommandHandler(BaseCommandHandler):
+    name: str
 
-    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
-        if self.management_only and not evt.is_management:
-            return (f"`{evt.command}` is a restricted command: "
-                    "you may only run it in management rooms.")
-        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
+    needs_puppeting: bool
+    needs_matrix_puppeting: bool
+
+    def __init__(
+        self,
+        handler: Callable[[CommandEvent], Awaitable[EventID]],
+        management_only: bool,
+        name: str,
+        help_text: str,
+        help_args: str,
+        help_section: HelpSection,
+        needs_auth: bool,
+        needs_puppeting: bool,
+        needs_matrix_puppeting: bool,
+        needs_admin: bool,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            handler,
+            management_only,
+            name,
+            help_text,
+            help_args,
+            help_section,
+            needs_auth=needs_auth,
+            needs_puppeting=needs_puppeting,
+            needs_matrix_puppeting=needs_matrix_puppeting,
+            needs_admin=needs_admin,
+            **kwargs,
+        )
+
+    async def get_permission_error(self, evt: CommandEvent) -> str | None:
+        if self.needs_puppeting and not evt.sender.puppet_whitelisted:
             return "This command requires puppeting privileges."
         elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
             return "This command requires Matrix puppeting privileges."
-        elif self.needs_admin and not evt.sender.is_admin:
-            return "This command requires administrator privileges."
-        elif self.needs_auth and not await evt.sender.is_logged_in():
-            return "This command requires you to be logged in."
-        return None
+        return await super().get_permission_error(evt)
 
-    def has_permission(self, is_management: bool, puppet_whitelisted: bool,
-                       matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
-        return ((not self.management_only or is_management) and
-                (not self.needs_puppeting or puppet_whitelisted) and
-                (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
-                (not self.needs_admin or is_admin) and
-                (not self.needs_auth or is_logged_in))
-
-    async def __call__(self, evt: CommandEvent):
-        error = await self.get_permission_error(evt)
-        if error is not None:
-            return await evt.reply(error)
-        return await self._handler(evt)
-
-    @property
-    def has_help(self) -> bool:
-        return bool(self.help_section) and bool(self._help_text)
-
-    @property
-    def help(self) -> str:
-        return f"**{self.name}** {self._help_args} - {self._help_text}"
+    def has_permission(self, key: HelpCacheKey) -> bool:
+        return (
+            super().has_permission(key)
+            and (not self.needs_puppeting or key.puppet_whitelisted)
+            and (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted)
+        )
 
 
-def command_handler(_func: Optional[Callable[[CommandEvent], None]] = None, *, needs_auth=True,
-                    needs_puppeting=True, needs_matrix_puppeting=False, needs_admin=False,
-                    management_only=False, name=None, help_text="", help_args="",
-                    help_section=None):
-    input_name = name
-
-    def decorator(func: Callable[[CommandEvent], None]):
-        name = input_name or func.__name__.replace("_", "-")
-        handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
-                                 needs_admin, management_only, name, help_text, help_args,
-                                 help_section)
-        command_handlers[handler.name] = handler
-        return handler
-
-    return decorator if _func is None else decorator(_func)
+def command_handler(
+    _func: CommandHandlerFunc | None = None,
+    *,
+    needs_auth: bool = True,
+    needs_puppeting: bool = True,
+    needs_matrix_puppeting: bool = False,
+    needs_admin: bool = False,
+    management_only: bool = False,
+    name: str | None = None,
+    help_text: str = "",
+    help_args: str = "",
+    help_section: HelpSection = None,
+) -> Callable[[CommandHandlerFunc], CommandHandler]:
+    return base_command_handler(
+        _func,
+        _handler_class=CommandHandler,
+        name=name,
+        help_text=help_text,
+        help_args=help_args,
+        help_section=help_section,
+        management_only=management_only,
+        needs_auth=needs_auth,
+        needs_admin=needs_admin,
+        needs_puppeting=needs_puppeting,
+        needs_matrix_puppeting=needs_matrix_puppeting,
+    )
 
 
-class CommandProcessor:
-    log = logging.getLogger("mau.commands")
+class CommandProcessor(BaseCommandProcessor):
+    def __init__(self, bridge: "TelegramBridge") -> None:
+        super().__init__(event_class=CommandEvent, bridge=bridge)
+        self.tgbot = bridge.bot
+        self.public_website = bridge.public_website
 
-    def __init__(self, context: c.Context):
-        self.az, self.db, self.config, self.loop, self.tgbot = context
-        self.public_website = context.public_website
-        self.command_prefix = self.config["bridge.command_prefix"]
-
-    async def handle(self, room: str, sender: u.User, command: str, args: List[str],
-                     is_management: bool, is_portal: bool):
-        evt = CommandEvent(self, room, sender, command, args, is_management, is_portal)
-        orig_command = command
-        command = command.lower()
+    @staticmethod
+    async def _run_handler(
+        handler: Callable[[CommandEvent], Awaitable[Any]], evt: CommandEvent
+    ) -> Any:
         try:
-            command = command_handlers[command]
-        except KeyError:
-            if sender.command_status and "next" in sender.command_status:
-                args.insert(0, orig_command)
-                evt.command = ""
-                command = sender.command_status["next"]
-            else:
-                command = command_handlers["unknown-command"]
-        try:
-            await command(evt)
+            return await handler(evt)
         except FloodWaitError as e:
             return await evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
-        except Exception:
-            self.log.exception("Unhandled error while handling command "
-                               f"{evt.command} {' '.join(args)} from {sender.mxid}")
-            return await evt.reply("Unhandled error while handling command. "
-                                   "Check logs for more details.")

mautrix_telegram/commands/matrix_auth.py

@@ -0,0 +1,85 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.bridge import InvalidAccessToken, OnlyLoginSelf
+from mautrix.types import EventID
+
+from .. import puppet as pu
+from . import SECTION_AUTH, CommandEvent, command_handler
+
+
+@command_handler(
+    needs_auth=True,
+    management_only=True,
+    needs_matrix_puppeting=True,
+    help_section=SECTION_AUTH,
+    help_text="Replace your Telegram account's Matrix puppet with your own Matrix account.",
+)
+async def login_matrix(evt: CommandEvent) -> EventID:
+    puppet = await pu.Puppet.get_by_tgid(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply(
+            "You have already logged in with your Matrix account. "
+            "Log out with `$cmdprefix+sp logout-matrix` first."
+        )
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        token = evt.public_website.make_token(evt.sender.mxid, "/matrix-login")
+        url = f"{prefix}/matrix-login?token={token}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history."
+            )
+        return await evt.reply(
+            "This bridge instance does not allow logging in inside Matrix.\n\n"
+            f"Please visit [the login page]({url}) to log in."
+        )
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in."
+        )
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+async def enter_matrix_token(evt: CommandEvent) -> EventID:
+    evt.sender.command_status = None
+
+    puppet = await pu.Puppet.get_by_tgid(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply(
+            "You have already logged in with your Matrix account. "
+            "Log out with `$cmdprefix+sp logout-matrix` first."
+        )
+    try:
+        await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    except OnlyLoginSelf:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    except InvalidAccessToken:
+        return await evt.reply("Failed to verify access token.")
+    return await evt.reply(
+        "Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}."
+    )

mautrix_telegram/commands/meta.py

@@ -1,69 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from . import command_handler, CommandEvent, _command_handlers, SECTION_GENERAL
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_GENERAL,
-                 help_text="Cancel an ongoing action (such as login)")
-def cancel(evt: CommandEvent):
-    if evt.sender.command_status:
-        action = evt.sender.command_status["action"]
-        evt.sender.command_status = None
-        return evt.reply(f"{action} cancelled.")
-    else:
-        return evt.reply("No ongoing command.")
-
-
-@command_handler(needs_auth=False, needs_puppeting=False)
-def unknown_command(evt: CommandEvent):
-    return evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
-
-
-help_cache = {}
-
-
-async def _get_help_text(evt: CommandEvent):
-    cache_key = (evt.is_management, evt.sender.puppet_whitelisted,
-                 evt.sender.matrix_puppet_whitelisted, evt.sender.is_admin,
-                 await evt.sender.is_logged_in())
-    if cache_key not in help_cache:
-        help = {}
-        for handler in _command_handlers.values():
-            if handler.has_help and handler.has_permission(*cache_key):
-                help.setdefault(handler.help_section, [])
-                help[handler.help_section].append(handler.help + "  ")
-        help = sorted(help.items(), key=lambda item: item[0].order)
-        help = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help]
-        help_cache[cache_key] = "\n".join(help)
-    return help_cache[cache_key]
-
-
-def _get_management_status(evt: CommandEvent):
-    if evt.is_management:
-        return "This is a management room: prefixing commands with `$cmdprefix` is not required."
-    elif evt.is_portal:
-        return ("**This is a portal room**: you must always prefix commands with `$cmdprefix`.\n"
-                "Management commands will not be sent to Telegram.")
-    return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_GENERAL,
-                 help_text="Show this help message.")
-async def help(evt: CommandEvent):
-    return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))

mautrix_telegram/commands/portal.py

@@ -1,490 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Callable
-import asyncio
-
-from telethon.errors import *
-from telethon.tl.types import ChatForbidden, ChannelForbidden
-from mautrix_appservice import MatrixRequestError, IntentAPI
-
-from .. import portal as po, user as u
-from . import (command_handler, CommandEvent,
-               SECTION_ADMIN, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT)
-
-
-@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
-                 help_section=SECTION_ADMIN,
-                 help_args="<_level_> [_mxid_]",
-                 help_text="Set a temporary power level without affecting Telegram.")
-async def set_power_level(evt: CommandEvent):
-    try:
-        level = int(evt.args[0])
-    except KeyError:
-        return await evt.reply("**Usage:** `$cmdprefix+sp set-power <level> [mxid]`")
-    except ValueError:
-        return await evt.reply("The level must be an integer.")
-    levels = await evt.az.intent.get_power_levels(evt.room_id)
-    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
-    levels["users"][mxid] = level
-    try:
-        await evt.az.intent.set_power_levels(evt.room_id, levels)
-    except MatrixRequestError:
-        evt.log.exception("Failed to set power level.")
-        return await evt.reply("Failed to set power level.")
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Get a Telegram invite link to the current chat.")
-async def invite_link(evt: CommandEvent):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
-        return await evt.reply("You can't invite users to private chats.")
-
-    try:
-        link = await portal.get_invite_link(evt.sender)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to create an invite link.")
-
-
-async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50):
-    if sender.is_admin:
-        return True
-    # Make sure the state store contains the power levels.
-    try:
-        await intent.get_power_levels(room)
-    except MatrixRequestError:
-        return False
-    return intent.state_store.has_power_level(room, sender.mxid,
-                                              event=f"net.maunium.telegram.{event}",
-                                              default=default)
-
-
-async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
-                                           action: Optional[str] = None):
-    room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if not portal:
-        that_this = "This" if room_id == evt.room_id else "That"
-        return await evt.reply(f"{that_this} is not a portal room."), False
-
-    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
-        action = action or f"{permission.replace('_', ' ')}s"
-        return await evt.reply(f"You do not have the permissions to {action} that portal."), False
-    return portal, True
-
-
-def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
-                                completed_message: str):
-    async def post_confirm(confirm):
-        confirm.sender.command_status = None
-        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
-            await function()
-            if confirm.room_id != room_id:
-                return await confirm.reply(completed_message)
-        else:
-            return await confirm.reply(f"{action} cancelled.")
-
-    return {
-        "next": post_confirm,
-        "action": action,
-    }
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Remove all users from the current portal room and forget the portal. "
-                           "Only works for group chats; to delete a private chat portal, simply "
-                           "leave the room.")
-async def delete_portal(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "unbridge")
-    if not ok:
-        return
-
-    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
-                                                            portal.cleanup_and_delete, "delete",
-                                                            "Portal successfully deleted.")
-    return await evt.reply("Please confirm deletion of portal "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           f"to Telegram chat \"{portal.title}\" "
-                           "by typing `$cmdprefix+sp confirm-delete`"
-                           "\n\n"
-                           "**WARNING:** If the bridge bot has the power level to do so, **this "
-                           "will kick ALL users** in the room. If you just want to remove the "
-                           "bridge, use `$cmdprefix+sp unbridge` instead.")
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Remove puppets from the current portal room and forget the portal.")
-async def unbridge(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "unbridge")
-    if not ok:
-        return
-
-    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
-                                                            portal.unbridge, "unbridge",
-                                                            "Room successfully unbridged.")
-    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           "by typing `$cmdprefix+sp confirm-unbridge`")
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_args="[_id_]",
-                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
-                           "ID. The ID must be the prefixed version that you get with the `/id` "
-                           "command of the Telegram-side bot.")
-async def bridge(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** "
-                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
-    room_id = evt.args[1] if len(evt.args) > 1 else evt.room_id
-    that_this = "This" if room_id == evt.room_id else "That"
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if portal:
-        return await evt.reply(f"{that_this} room is already a portal room.")
-
-    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
-
-    # The /id bot command provides the prefixed ID, so we assume
-    tgid = evt.args[0]
-    if tgid.startswith("-100"):
-        tgid = int(tgid[4:])
-        peer_type = "channel"
-    elif tgid.startswith("-"):
-        tgid = -int(tgid)
-        peer_type = "chat"
-    else:
-        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
-                               "If you did not get the ID using the `/id` bot command, please "
-                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
-                               "Bridging private chats to existing rooms is not allowed.")
-
-    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
-    if not portal.allow_bridging():
-        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
-                               "If you're the bridge admin, try "
-                               "`$cmdprefix+sp whitelist <Telegram chat ID>` first.")
-    if portal.mxid:
-        has_portal_message = (
-            "That Telegram chat already has a portal at "
-            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
-        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-            return await evt.reply(f"{has_portal_message}"
-                                   "Additionally, you do not have the permissions to unbridge "
-                                   "that room.")
-        evt.sender.command_status = {
-            "next": confirm_bridge,
-            "action": "Room bridging",
-            "mxid": portal.mxid,
-            "bridge_to_mxid": room_id,
-            "tgid": portal.tgid,
-            "peer_type": portal.peer_type,
-        }
-        return await evt.reply(f"{has_portal_message}"
-                               "However, you have the permissions to unbridge that room.\n\n"
-                               "To delete that portal completely and continue bridging, use "
-                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
-                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
-                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
-    evt.sender.command_status = {
-        "next": confirm_bridge,
-        "action": "Room bridging",
-        "bridge_to_mxid": room_id,
-        "tgid": portal.tgid,
-        "peer_type": portal.peer_type,
-    }
-    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
-                           "chat to this room, use `$cmdprefix+sp continue`")
-
-
-async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"):
-    if not portal.mxid:
-        await evt.reply("The portal seems to have lost its Matrix room between you"
-                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                        "Continuing without touching previous Matrix room...")
-        return True, None
-    elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Portal deleted (moving to another room)")
-    elif evt.args[0] == "unbridge-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Room unbridged (portal moving to another room)",
-                                         puppets_only=True)
-    else:
-        await evt.reply(
-            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
-            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
-            "continue` to either delete or unbridge the existing room (respectively) and "
-            "continue with the bridging.\n\n"
-            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
-        return False, None
-
-
-async def confirm_bridge(evt: CommandEvent):
-    status = evt.sender.command_status
-    try:
-        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
-        bridge_to_mxid = status["bridge_to_mxid"]
-    except KeyError:
-        evt.sender.command_status = None
-        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
-                               "This shouldn't happen unless you're messing with the command "
-                               "handler code.")
-    if "mxid" in status:
-        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
-        if not ok:
-            return
-        elif coro:
-            asyncio.ensure_future(coro, loop=evt.loop)
-            await evt.reply("Cleaning up previous portal room...")
-    elif portal.mxid:
-        evt.sender.command_status = None
-        return await evt.reply("The portal seems to have created a Matrix room between you "
-                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                               "Please start over by calling the bridge command again.")
-    elif evt.args[0] != "continue":
-        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
-                               "`$cmdprefix+sp cancel` to cancel.")
-
-    is_logged_in = await evt.sender.is_logged_in()
-    user = evt.sender if is_logged_in else evt.tgbot
-    try:
-        entity = await user.client.get_entity(portal.peer)
-    except Exception:
-        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
-        if is_logged_in:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You are logged in, are you in that chat?")
-        else:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You're not logged in, is the relay bot in the chat?")
-    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
-        if is_logged_in:
-            return await evt.reply("You don't seem to be in that chat.")
-        else:
-            return await evt.reply("The bot doesn't seem to be in that chat.")
-
-    direct = False
-
-    portal.mxid = bridge_to_mxid
-    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
-    portal.photo_id = ""
-    portal.save()
-
-    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
-                          loop=evt.loop)
-
-    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")
-
-
-async def get_initial_state(intent: IntentAPI, room_id: str):
-    state = await intent.get_room_state(room_id)
-    title = None
-    about = None
-    levels = None
-    for event in state:
-        try:
-            if event["type"] == "m.room.name":
-                title = event["content"]["name"]
-            elif event["type"] == "m.room.topic":
-                about = event["content"]["topic"]
-            elif event["type"] == "m.room.power_levels":
-                levels = event["content"]
-            elif event["type"] == "m.room.canonical_alias":
-                title = title or event["content"]["alias"]
-        except KeyError:
-            # Some state event probably has empty content
-            pass
-    return title, about, levels
-
-
-@command_handler(help_section=SECTION_CREATING_PORTALS,
-                 help_args="[_type_]",
-                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
-                           "The type is either `group`, `supergroup` or `channel` (defaults to "
-                           "`group`).")
-async def create(evt: CommandEvent):
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
-
-    if po.Portal.get_by_mxid(evt.room_id):
-        return await evt.reply("This is already a portal room.")
-
-    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply("You do not have the permissions to bridge this room.")
-
-    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
-    if not title:
-        return await evt.reply("Please set a title before creating a Telegram chat.")
-
-    supergroup = type == "supergroup"
-    type = {
-        "supergroup": "channel",
-        "channel": "channel",
-        "chat": "chat",
-        "group": "chat",
-    }[type]
-
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
-    try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
-    except ValueError as e:
-        portal.delete()
-        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Upgrade a normal Telegram group to a supergroup.")
-async def upgrade(evt: CommandEvent):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type == "channel":
-        return await evt.reply("This is already a supergroup or a channel.")
-    elif portal.peer_type == "user":
-        return await evt.reply("You can't upgrade private chats.")
-
-    try:
-        await portal.upgrade_telegram_chat(evt.sender)
-        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to upgrade this group.")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_args="<_name_|`-`>",
-                 help_text="Change the username of a supergroup/channel. "
-                           "To disable, use a dash (`-`) as the name.")
-async def group_name(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type != "channel":
-        return await evt.reply("Only channels and supergroups have usernames.")
-
-    try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
-        if portal.username:
-            return await evt.reply(f"Username of channel changed to {portal.username}.")
-        else:
-            return await evt.reply(f"Channel is now private.")
-    except ChatAdminRequiredError:
-        return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
-    except UsernameNotModifiedError:
-        if portal.username:
-            return await evt.reply("That is already the username of this channel.")
-        else:
-            return await evt.reply("This channel is already private")
-    except UsernameOccupiedError:
-        return await evt.reply("That username is already in use.")
-    except UsernameInvalidError:
-        return await evt.reply("Invalid username")
-
-
-@command_handler(needs_admin=True,
-                 help_section=SECTION_ADMIN,
-                 help_args="<`whitelist`|`blacklist`>",
-                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
-                           "default.")
-async def filter_mode(evt: CommandEvent):
-    try:
-        mode = evt.args[0]
-        if mode not in ("whitelist", "blacklist"):
-            raise ValueError()
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
-
-    evt.config["bridge.filter.mode"] = mode
-    evt.config.save()
-    po.Portal.filter_mode = mode
-    if mode == "whitelist":
-        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
-                               "To allow bridging a specific chat, use"
-                               "`!filter whitelist <chat ID>`.")
-    else:
-        return await evt.reply("The bridge will now allow bridging chats by default.\n"
-                               "To disallow bridging a specific chat, use"
-                               "`!filter blacklist <chat ID>`.")
-
-
-@command_handler(needs_admin=True,
-                 help_section=SECTION_ADMIN,
-                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
-                 help_text="Allow or disallow bridging a specific chat.")
-async def filter(evt: CommandEvent):
-    try:
-        action = evt.args[0]
-        if action not in ("whitelist", "blacklist", "add", "remove"):
-            raise ValueError()
-
-        id = evt.args[1]
-        if id.startswith("-100"):
-            id = int(id[4:])
-        elif id.startswith("-"):
-            id = int(id[1:])
-        else:
-            id = int(id)
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
-
-    mode = evt.config["bridge.filter.mode"]
-    if mode not in ("blacklist", "whitelist"):
-        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
-
-    list = evt.config["bridge.filter.list"]
-
-    if action in ("blacklist", "whitelist"):
-        action = "add" if mode == action else "remove"
-
-    def save():
-        evt.config["bridge.filter.list"] = list
-        evt.config.save()
-        po.Portal.filter_list = list
-
-    if action == "add":
-        if id in list:
-            return await evt.reply(f"That chat is already {mode}ed.")
-        list.append(id)
-        save()
-        return await evt.reply(f"Chat ID added to {mode}.")
-    elif action == "remove":
-        if id not in list:
-            return await evt.reply(f"That chat is not {mode}ed.")
-        list.remove(id)
-        save()
-        return await evt.reply(f"Chat ID removed from {mode}.")

mautrix_telegram/commands/portal/__init__.py

@@ -0,0 +1 @@
+from . import admin, bridge, config, create_chat, filter, misc, unbridge

mautrix_telegram/commands/portal/admin.py

@@ -0,0 +1,81 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import asyncio
+
+from mautrix.types import EventID
+
+from ... import portal as po, puppet as pu, user as u
+from .. import SECTION_ADMIN, CommandEvent, command_handler
+
+
+@command_handler(
+    needs_admin=True,
+    needs_auth=False,
+    help_section=SECTION_ADMIN,
+    help_args="<`portal`|`puppet`|`user`>",
+    help_text="Clear internal bridge caches",
+)
+async def clear_db_cache(evt: CommandEvent) -> EventID:
+    try:
+        section = evt.args[0].lower()
+    except IndexError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+    if section == "portal":
+        po.Portal.by_tgid = {}
+        po.Portal.by_mxid = {}
+        await evt.reply("Cleared portal cache")
+    elif section == "puppet":
+        pu.Puppet.by_tgid = {}
+        for puppet in pu.Puppet.by_custom_mxid.values():
+            puppet.stop()
+        pu.Puppet.by_custom_mxid = {}
+        await asyncio.gather(
+            *[puppet.try_start() async for puppet in pu.Puppet.all_with_custom_mxid()]
+        )
+        await evt.reply("Cleared puppet cache and restarted custom puppet syncers")
+    elif section == "user":
+        u.User.by_mxid = {user.mxid: user for user in u.User.by_tgid.values()}
+        await evt.reply("Cleared non-logged-in user cache")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+
+
+@command_handler(
+    needs_admin=True,
+    needs_auth=False,
+    help_section=SECTION_ADMIN,
+    help_args="[_mxid_]",
+    help_text="Reload and reconnect a user",
+)
+async def reload_user(evt: CommandEvent) -> EventID:
+    if len(evt.args) > 0:
+        mxid = evt.args[0]
+    else:
+        mxid = evt.sender.mxid
+    user = await u.User.get_by_mxid(mxid, create=False)
+    if not user:
+        return await evt.reply("User not found")
+    puppet = await pu.Puppet.get_by_custom_mxid(mxid)
+    if puppet:
+        puppet.stop()
+    await user.stop()
+    del u.User.by_tgid[user.tgid]
+    del u.User.by_mxid[user.mxid]
+    user = await u.User.get_by_mxid(mxid)
+    await user.ensure_started()
+    if puppet:
+        await puppet.start()
+    return await evt.reply(f"Reloaded and reconnected {user.mxid} (telegram: {user.human_tg_id})")

mautrix_telegram/commands/portal/bridge.py

@@ -0,0 +1,228 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Awaitable
+import asyncio
+
+from telethon.tl.types import ChannelForbidden, ChatForbidden
+
+from mautrix.types import EventID, RoomID
+
+from ... import portal as po
+from ...types import TelegramID
+from .. import SECTION_CREATING_PORTALS, CommandEvent, command_handler
+from .util import get_initial_state, user_has_power_level, warn_missing_power
+
+
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="[_id_]",
+    help_text=(
+        "Bridge the current Matrix room to the Telegram chat with the given ID. The ID must be "
+        "the prefixed version that you get with the `/id` command of the Telegram-side bot."
+    ),
+)
+async def bridge(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`"
+        )
+    force_use_bot = False
+    if evt.args[0] == "--usebot" and evt.sender.is_admin:
+        force_use_bot = True
+        evt.args = evt.args[1:]
+    room_id = RoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
+    that_this = "This" if room_id == evt.room_id else "That"
+
+    portal = await po.Portal.get_by_mxid(room_id)
+    if portal:
+        return await evt.reply(f"{that_this} room is already a portal room.")
+
+    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
+
+    # The /id bot command provides the prefixed ID, so we assume
+    tgid_str = evt.args[0]
+    if tgid_str.startswith("-100"):
+        tgid = TelegramID(int(tgid_str[4:]))
+        peer_type = "channel"
+    elif tgid_str.startswith("-"):
+        tgid = TelegramID(-int(tgid_str))
+        peer_type = "chat"
+    else:
+        return await evt.reply(
+            "That doesn't seem like a prefixed Telegram chat ID.\n\n"
+            "If you did not get the ID using the `/id` bot command, please "
+            "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+            "Bridging private chats to existing rooms is not allowed."
+        )
+
+    portal = await po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    if not portal.allow_bridging:
+        return await evt.reply(
+            "This bridge doesn't allow bridging that Telegram chat.\n"
+            "If you're the bridge admin, try "
+            "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first."
+        )
+    if portal.mxid:
+        has_portal_message = (
+            "That Telegram chat already has a portal at "
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). "
+        )
+        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+            return await evt.reply(
+                f"{has_portal_message}"
+                "Additionally, you do not have the permissions to unbridge that room."
+            )
+        evt.sender.command_status = {
+            "next": confirm_bridge,
+            "action": "Room bridging",
+            "mxid": portal.mxid,
+            "bridge_to_mxid": room_id,
+            "tgid": portal.tgid,
+            "peer_type": portal.peer_type,
+            "force_use_bot": force_use_bot,
+        }
+        return await evt.reply(
+            f"{has_portal_message}"
+            "However, you have the permissions to unbridge that room.\n\n"
+            "To delete that portal completely and continue bridging, use "
+            "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+            "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+            "continue`. To cancel, use `$cmdprefix+sp cancel`"
+        )
+    evt.sender.command_status = {
+        "next": confirm_bridge,
+        "action": "Room bridging",
+        "bridge_to_mxid": room_id,
+        "tgid": portal.tgid,
+        "peer_type": portal.peer_type,
+        "force_use_bot": force_use_bot,
+    }
+    return await evt.reply(
+        "That Telegram chat has no existing portal. To confirm bridging the "
+        "chat to this room, use `$cmdprefix+sp continue`"
+    )
+
+
+async def cleanup_old_portal_while_bridging(
+    evt: CommandEvent, portal: po.Portal
+) -> tuple[bool, Awaitable[None] | None]:
+    if not portal.mxid:
+        await evt.reply(
+            "The portal seems to have lost its Matrix room between you"
+            "calling `$cmdprefix+sp bridge` and this command.\n\n"
+            "Continuing without touching previous Matrix room..."
+        )
+        return True, None
+    elif evt.args[0] == "delete-and-continue":
+        return True, portal.cleanup_portal("Portal deleted (moving to another room)", delete=False)
+    elif evt.args[0] == "unbridge-and-continue":
+        return True, portal.cleanup_portal(
+            "Room unbridged (portal moving to another room)", puppets_only=True, delete=False
+        )
+    else:
+        await evt.reply(
+            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
+            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
+            "continue` to either delete or unbridge the existing room (respectively) and "
+            "continue with the bridging.\n\n"
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel."
+        )
+        return False, None
+
+
+async def confirm_bridge(evt: CommandEvent) -> EventID | None:
+    status = evt.sender.command_status
+    try:
+        portal = await po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        bridge_to_mxid = status["bridge_to_mxid"]
+    except KeyError:
+        evt.sender.command_status = None
+        return await evt.reply(
+            "Fatal error: tgid or peer_type missing from command_status. "
+            "This shouldn't happen unless you're messing with the command handler code."
+        )
+
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+
+    if "mxid" in status:
+        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
+        if not ok:
+            return None
+        elif coro:
+            asyncio.create_task(coro)
+            await evt.reply("Cleaning up previous portal room...")
+    elif portal.mxid:
+        evt.sender.command_status = None
+        return await evt.reply(
+            "The portal seems to have created a Matrix room between you "
+            "calling `$cmdprefix+sp bridge` and this command.\n\n"
+            "Please start over by calling the bridge command again."
+        )
+    elif evt.args[0] != "continue":
+        return await evt.reply(
+            "Please use `$cmdprefix+sp continue` to confirm the bridging or "
+            "`$cmdprefix+sp cancel` to cancel."
+        )
+
+    evt.sender.command_status = None
+    async with portal._room_create_lock:
+        await _locked_confirm_bridge(
+            evt, portal=portal, room_id=bridge_to_mxid, is_logged_in=is_logged_in
+        )
+
+
+async def _locked_confirm_bridge(
+    evt: CommandEvent, portal: po.Portal, room_id: RoomID, is_logged_in: bool
+) -> EventID | None:
+    user = evt.sender if is_logged_in else evt.tgbot
+    try:
+        entity = await user.client.get_entity(portal.peer)
+    except Exception:
+        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+        if is_logged_in:
+            return await evt.reply(
+                "Failed to get info of telegram chat. You are logged in, are you in that chat?"
+            )
+        else:
+            return await evt.reply(
+                "Failed to get info of telegram chat. "
+                "You're not logged in, is the relay bot in the chat?"
+            )
+    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
+        if is_logged_in:
+            return await evt.reply("You don't seem to be in that chat.")
+        else:
+            return await evt.reply("The bot doesn't seem to be in that chat.")
+
+    portal.mxid = room_id
+    portal.by_mxid[portal.mxid] = portal
+    (portal.title, portal.about, levels, portal.encrypted) = await get_initial_state(
+        evt.az.intent, evt.room_id
+    )
+    portal.photo_id = ""
+    await portal.save()
+    await portal.update_bridge_info()
+
+    asyncio.create_task(portal.update_matrix_room(user, entity, direct=False, levels=levels))
+
+    await warn_missing_power(levels, evt)
+
+    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -0,0 +1,161 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Any, Awaitable
+from io import StringIO
+
+from ruamel.yaml import YAMLError
+
+from mautrix.types import EventID
+from mautrix.util.config import yaml
+
+from ... import portal as po, util
+from .. import SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
+
+
+@command_handler(
+    needs_auth=False,
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="View or change per-portal settings.",
+    help_args="<`help`|_subcommand_> [...]",
+)
+async def config(evt: CommandEvent) -> None:
+    cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
+    if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
+        await config_help(evt)
+        return
+    elif cmd == "defaults":
+        await config_defaults(evt)
+        return
+
+    portal = await po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        await evt.reply("This is not a portal room.")
+        return
+    elif cmd == "view":
+        await config_view(evt, portal)
+        return
+
+    if not await portal.can_user_perform(evt.sender, "config"):
+        await evt.reply("You do not have the permissions to configure this room.")
+        return
+
+    key = evt.args[1] if len(evt.args) > 1 else None
+    try:
+        value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    except YAMLError as e:
+        await evt.reply(f"Invalid value provided. Values must be valid YAML.\n{e}")
+        return
+    if cmd == "set":
+        await config_set(evt, portal, key, value)
+    elif cmd == "unset":
+        await config_unset(evt, portal, key)
+    elif cmd == "add" or cmd == "del":
+        await config_add_del(evt, portal, key, value, cmd)
+    else:
+        return
+    await portal.save()
+
+
+def config_help(evt: CommandEvent) -> Awaitable[EventID]:
+    return evt.reply(
+        """**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
+
+* **help** - View this help text.
+* **view** - View the current config data.
+* **defaults** - View the default config values.
+* **set** <_key_> <_value_> - Set a config value.
+* **unset** <_key_> - Remove a config value.
+* **add** <_key_> <_value_> - Add a value to an array.
+* **del** <_key_> <_value_> - Remove a value from an array.
+"""
+    )
+
+
+def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[EventID]:
+    return evt.reply(f"Room-specific config:\n{_str_value(portal.local_config).rstrip()}")
+
+
+def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
+    value = _str_value(
+        {
+            "bridge_notices": {
+                "default": evt.config["bridge.bridge_notices.default"],
+                "exceptions": evt.config["bridge.bridge_notices.exceptions"],
+            },
+            "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
+            "inline_images": evt.config["bridge.inline_images"],
+            "message_formats": evt.config["bridge.message_formats"],
+            "emote_format": evt.config["bridge.emote_format"],
+            "state_event_formats": evt.config["bridge.state_event_formats"],
+            "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
+        }
+    )
+    return evt.reply(f"Bridge instance wide config:\n{value.rstrip()}")
+
+
+def _str_value(value: Any) -> str:
+    stream = StringIO()
+    yaml.dump(value, stream)
+    value_str = stream.getvalue()
+    if "\n" in value_str:
+        return f"\n```yaml\n{value_str}\n```\n"
+    else:
+        return f"`{value_str}`"
+
+
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: Any) -> Awaitable[EventID]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
+    elif util.recursive_set(portal.local_config, key, value):
+        return evt.reply(f"Successfully set the value of `{key}` to {_str_value(value)}".rstrip())
+    else:
+        return evt.reply(f"Failed to set value of `{key}`. Does the path contain non-map types?")
+
+
+def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[EventID]:
+    if not key:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config unset <key>`")
+    elif util.recursive_del(portal.local_config, key):
+        return evt.reply(f"Successfully deleted `{key}` from config.")
+    else:
+        return evt.reply(f"`{key}` not found in config.")
+
+
+def config_add_del(
+    evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
+) -> Awaitable[EventID]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
+
+    arr = util.recursive_get(portal.local_config, key)
+    if not arr:
+        return evt.reply(
+            f"`{key}` not found in config. Maybe do `$cmdprefix+sp config set {key} []` first?"
+        )
+    elif not isinstance(arr, list):
+        return evt.reply("`{key}` does not seem to be an array.")
+    elif cmd == "add":
+        if value in arr:
+            return evt.reply(f"The array at `{key}` already contains {_str_value(value)}".rstrip())
+        arr.append(value)
+        return evt.reply(f"Successfully added {_str_value(value)} to the array at `{key}`")
+    else:
+        if value not in arr:
+            return evt.reply(f"The array at `{key}` does not contain {_str_value(value)}")
+        arr.remove(value)
+        return evt.reply(f"Successfully removed {_str_value(value)} from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -0,0 +1,84 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from mautrix.types import EventID
+
+from ... import portal as po
+from ...types import TelegramID
+from .. import SECTION_CREATING_PORTALS, CommandEvent, command_handler
+from .util import get_initial_state, user_has_power_level, warn_missing_power
+
+
+@command_handler(
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="[_type_]",
+    help_text=(
+        "Create a Telegram chat of the given type for the current Matrix room. "
+        "The type is either `group`, `supergroup` or `channel` (defaults to `supergroup`)."
+    ),
+)
+async def create(evt: CommandEvent) -> EventID:
+    type = evt.args[0] if len(evt.args) > 0 else "supergroup"
+    if type not in ("chat", "group", "supergroup", "channel"):
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`"
+        )
+
+    if await po.Portal.get_by_mxid(evt.room_id):
+        return await evt.reply("This is already a portal room.")
+
+    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply("You do not have the permissions to bridge this room.")
+
+    title, about, levels, encrypted = await get_initial_state(evt.az.intent, evt.room_id)
+    if not title:
+        return await evt.reply("Please set a title before creating a Telegram chat.")
+
+    supergroup = type == "supergroup"
+    type = {
+        "supergroup": "channel",
+        "channel": "channel",
+        "chat": "chat",
+        "group": "chat",
+    }[type]
+
+    portal = po.Portal(
+        tgid=TelegramID(0),
+        tg_receiver=TelegramID(0),
+        peer_type=type,
+        mxid=evt.room_id,
+        title=title,
+        about=about,
+        encrypted=encrypted,
+    )
+    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender)
+    if len(errors) > 0:
+        error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
+        await evt.reply(
+            f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
+            "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
+            "those users."
+        )
+
+    await warn_missing_power(levels, evt)
+
+    try:
+        await portal.create_telegram_chat(evt.sender, invites=invites, supergroup=supergroup)
+    except ValueError as e:
+        await portal.delete()
+        return await evt.reply(e.args[0])
+    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/filter.py

@@ -0,0 +1,105 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from mautrix.types import EventID
+
+from ... import portal as po
+from .. import SECTION_ADMIN, CommandEvent, command_handler
+
+
+@command_handler(
+    needs_admin=True,
+    help_section=SECTION_ADMIN,
+    help_args="<`whitelist`|`blacklist`>",
+    help_text="Change whether the bridge will allow or disallow bridging rooms by default.",
+)
+async def filter_mode(evt: CommandEvent) -> EventID:
+    try:
+        mode = evt.args[0]
+        if mode not in ("whitelist", "blacklist"):
+            raise ValueError()
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
+
+    evt.config["bridge.filter.mode"] = mode
+    evt.config.save()
+    po.Portal.filter_mode = mode
+    if mode == "whitelist":
+        return await evt.reply(
+            "The bridge will now disallow bridging chats by default.\n"
+            "To allow bridging a specific chat, use"
+            "`!filter whitelist <chat ID>`."
+        )
+    else:
+        return await evt.reply(
+            "The bridge will now allow bridging chats by default.\n"
+            "To disallow bridging a specific chat, use"
+            "`!filter blacklist <chat ID>`."
+        )
+
+
+@command_handler(
+    name="filter",
+    needs_admin=True,
+    help_section=SECTION_ADMIN,
+    help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+    help_text="Allow or disallow bridging a specific chat.",
+)
+async def edit_filter(evt: CommandEvent) -> EventID:
+    try:
+        action = evt.args[0]
+        if action not in ("whitelist", "blacklist", "add", "remove"):
+            raise ValueError()
+
+        id_str = evt.args[1]
+        if id_str.startswith("-100"):
+            filter_id = int(id_str[4:])
+        elif id_str.startswith("-"):
+            filter_id = int(id_str[1:])
+        else:
+            filter_id = int(id_str)
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
+
+    mode = evt.config["bridge.filter.mode"]
+    if mode not in ("blacklist", "whitelist"):
+        return await evt.reply(f'Unknown filter mode "{mode}". Please fix the bridge config.')
+
+    filter_id_list = evt.config["bridge.filter.list"]
+
+    if action in ("blacklist", "whitelist"):
+        action = "add" if mode == action else "remove"
+
+    def save() -> None:
+        evt.config["bridge.filter.list"] = filter_id_list
+        evt.config.save()
+        po.Portal.filter_list = filter_id_list
+
+    if action == "add":
+        if filter_id in filter_id_list:
+            return await evt.reply(f"That chat is already {mode}ed.")
+        filter_id_list.append(filter_id)
+        save()
+        return await evt.reply(f"Chat ID added to {mode}.")
+    elif action == "remove":
+        if filter_id not in filter_id_list:
+            return await evt.reply(f"That chat is not {mode}ed.")
+        filter_id_list.remove(filter_id)
+        save()
+        return await evt.reply(f"Chat ID removed from {mode}.")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")

mautrix_telegram/commands/portal/misc.py

@@ -0,0 +1,257 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+import re
+
+from telethon.errors import (
+    ChatAdminRequiredError,
+    RPCError,
+    UsernameInvalidError,
+    UsernameNotModifiedError,
+    UsernameOccupiedError,
+)
+from telethon.tl.functions.channels import GetFullChannelRequest
+from telethon.tl.functions.messages import GetFullChatRequest
+
+from mautrix.types import EventID
+
+from ... import portal as po
+from .. import SECTION_MISC, SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
+from .util import user_has_power_level
+
+
+@command_handler(
+    needs_admin=False,
+    needs_puppeting=False,
+    needs_auth=False,
+    help_section=SECTION_MISC,
+    help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.",
+)
+async def sync_state(evt: CommandEvent) -> EventID:
+    portal = await po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to synchronize this room.")
+
+    await portal.main_intent.get_joined_members(portal.mxid)
+    await evt.reply("Synchronization complete")
+
+
+@command_handler(
+    needs_admin=False, needs_puppeting=False, needs_auth=False, help_section=SECTION_MISC
+)
+async def sync_full(evt: CommandEvent) -> EventID:
+    portal = await po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if len(evt.args) > 0 and evt.args[0] == "--usebot" and evt.sender.is_admin:
+        src = evt.tgbot
+    else:
+        src = evt.tgbot if await evt.sender.needs_relaybot(portal) else evt.sender
+
+    try:
+        if portal.peer_type == "channel":
+            res = await src.client(GetFullChannelRequest(portal.peer))
+        elif portal.peer_type == "chat":
+            res = await src.client(GetFullChatRequest(portal.tgid))
+        else:
+            return await evt.reply("This is not a channel or chat portal.")
+    except (ValueError, RPCError):
+        return await evt.reply("Failed to get portal info from Telegram.")
+
+    await portal.update_matrix_room(src, res.full_chat)
+    return await evt.reply("Portal synced successfully.")
+
+
+@command_handler(
+    name="id",
+    needs_admin=False,
+    needs_puppeting=False,
+    needs_auth=False,
+    help_section=SECTION_MISC,
+    help_text="Get the ID of the Telegram chat where this room is bridged.",
+)
+async def get_id(evt: CommandEvent) -> EventID:
+    portal = await po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    tgid = portal.tgid
+    if portal.peer_type == "chat":
+        tgid = -tgid
+    elif portal.peer_type == "channel":
+        tgid = f"-100{tgid}"
+    await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
+
+
+invite_link_usage = (
+    "**Usage:** `$cmdprefix+sp invite-link [--uses=<amount>] [--expire=<delta>]`"
+    "\n\n"
+    "* `--uses`: the number of times the invite link can be used."
+    "            Defaults to unlimited.\n"
+    "* `--expire`: the duration after which the link will expire."
+    "              A number suffixed with d(ay), h(our), m(inute) or s(econd)"
+)
+
+
+def _parse_flag(args: list[str]) -> tuple[str, str]:
+    arg = args.pop(0).lower()
+    if arg.startswith("--"):
+        value_start = arg.index("=")
+        if value_start:
+            flag = arg[2:value_start]
+            value = arg[value_start + 1 :]
+        else:
+            flag = arg[2:]
+            value = args.pop(0).lower()
+    elif arg.startswith("-"):
+        flag = arg[1]
+        if len(arg) > 3 and arg[2] == "=":
+            value = arg[3:]
+        else:
+            value = args.pop(0).lower()
+    else:
+        raise ValueError("invalid flag")
+    return flag, value
+
+
+delta_regex = re.compile(
+    "([0-9]+)(w(?:eek)?|d(?:ay)?|h(?:our)?|m(?:in(?:ute)?)?|s(?:ec(?:ond)?)?)"
+)
+
+
+def _parse_delta(value: str) -> timedelta | None:
+    match = delta_regex.fullmatch(value)
+    if not match:
+        return None
+    number = int(match.group(1))
+    unit = match.group(2)[0]
+    if unit == "w":
+        return timedelta(weeks=number)
+    elif unit == "d":
+        return timedelta(days=number)
+    elif unit == "h":
+        return timedelta(hours=number)
+    elif unit == "m":
+        return timedelta(minutes=number)
+    elif unit == "s":
+        return timedelta(seconds=number)
+    else:
+        return None
+
+
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="Get a Telegram invite link to the current chat.",
+    help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>]",
+)
+async def invite_link(evt: CommandEvent) -> EventID:
+    # TODO once we switch to Python 3.9 minimum, use argparse with exit_on_error=False
+    uses = None
+    expire = None
+    while evt.args:
+        try:
+            flag, value = _parse_flag(evt.args)
+        except (ValueError, IndexError):
+            return await evt.reply(invite_link_usage)
+        if flag in ("uses", "u"):
+            try:
+                uses = int(value)
+            except ValueError:
+                await evt.reply("The number of uses must be an integer")
+        elif flag in ("expire", "e"):
+            expire_delta = _parse_delta(value)
+            if not expire_delta:
+                await evt.reply("Invalid format for expiry time delta")
+            expire = datetime.now() + expire_delta
+
+    portal = await po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if portal.peer_type == "user":
+        return await evt.reply("You can't invite users to private chats.")
+
+    try:
+        link = await portal.get_invite_link(evt.sender, uses=uses, expire=expire)
+        return await evt.reply(f"Invite link to {portal.title}: {link}")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to create an invite link.")
+
+
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="Upgrade a normal Telegram group to a supergroup.",
+)
+async def upgrade(evt: CommandEvent) -> EventID:
+    portal = await po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type == "channel":
+        return await evt.reply("This is already a supergroup or a channel.")
+    elif portal.peer_type == "user":
+        return await evt.reply("You can't upgrade private chats.")
+
+    try:
+        await portal.upgrade_telegram_chat(evt.sender)
+        return await evt.reply(f"Group upgraded to supergroup. New ID: -100{portal.tgid}")
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to upgrade this group.")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+
+
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_args="<_name_|`-`>",
+    help_text=(
+        "Change the username of a supergroup/channel. To disable, use a dash (`-`) as the name."
+    ),
+)
+async def group_name(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
+
+    portal = await po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type != "channel":
+        return await evt.reply("Only channels and supergroups have usernames.")
+
+    try:
+        await portal.set_telegram_username(evt.sender, evt.args[0] if evt.args[0] != "-" else "")
+        if portal.username:
+            return await evt.reply(f"Username of channel changed to {portal.username}.")
+        else:
+            return await evt.reply(f"Channel is now private.")
+    except ChatAdminRequiredError:
+        return await evt.reply(
+            "You don't have the permission to set the username of this channel."
+        )
+    except UsernameNotModifiedError:
+        if portal.username:
+            return await evt.reply("That is already the username of this channel.")
+        else:
+            return await evt.reply("This channel is already private")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username")

mautrix_telegram/commands/portal/unbridge.py

@@ -0,0 +1,118 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Callable
+
+from mautrix.types import EventID, RoomID
+
+from ... import portal as po
+from .. import SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
+from .util import user_has_power_level
+
+
+async def _get_portal_and_check_permission(evt: CommandEvent) -> po.Portal | None:
+    room_id = RoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
+
+    portal = await po.Portal.get_by_mxid(room_id)
+    if not portal:
+        that_this = "This" if room_id == evt.room_id else "That"
+        await evt.reply(f"{that_this} is not a portal room.")
+        return None
+
+    if portal.peer_type == "user":
+        if portal.tg_receiver != evt.sender.tgid:
+            await evt.reply("You do not have the permissions to unbridge that portal.")
+            return None
+        return portal
+
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+        await evt.reply("You do not have the permissions to unbridge that portal.")
+        return None
+    return portal
+
+
+def _get_portal_murder_function(
+    action: str, room_id: str, function: Callable, command: str, completed_message: str
+) -> dict:
+    async def post_confirm(confirm) -> EventID | None:
+        confirm.sender.command_status = None
+        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
+            await function()
+            if confirm.room_id != room_id:
+                return await confirm.reply(completed_message)
+        else:
+            return await confirm.reply(f"{action} cancelled.")
+        return None
+
+    return {
+        "next": post_confirm,
+        "action": action,
+    }
+
+
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text=(
+        "Remove all users from the current portal room and forget the portal. "
+        "Only works for group chats; to delete a private chat portal, simply leave the room."
+    ),
+)
+async def delete_portal(evt: CommandEvent) -> EventID | None:
+    portal = await _get_portal_and_check_permission(evt)
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function(
+        "Portal deletion",
+        portal.mxid,
+        portal.cleanup_and_delete,
+        "delete",
+        "Portal successfully deleted.",
+    )
+    return await evt.reply(
+        "Please confirm deletion of portal "
+        f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+        f'to Telegram chat "{portal.title}" '
+        "by typing `$cmdprefix+sp confirm-delete`"
+        "\n\n"
+        "**WARNING:** If the bridge bot has the power level to do so, **this "
+        "will kick ALL users** in the room. If you just want to remove the "
+        "bridge, use `$cmdprefix+sp unbridge` instead."
+    )
+
+
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="Remove puppets from the current portal room and forget the portal.",
+)
+async def unbridge(evt: CommandEvent) -> EventID | None:
+    portal = await _get_portal_and_check_permission(evt)
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function(
+        "Room unbridging", portal.mxid, portal.unbridge, "unbridge", "Room successfully unbridged."
+    )
+    return await evt.reply(
+        f'Please confirm unbridging chat "{portal.title}" from room '
+        f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+        "by typing `$cmdprefix+sp confirm-unbridge`"
+    )

mautrix_telegram/commands/portal/util.py

@@ -0,0 +1,72 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from mautrix.appservice import IntentAPI
+from mautrix.errors import MatrixRequestError
+from mautrix.types import EventType, PowerLevelStateEventContent, RoomID
+
+from ... import user as u
+from .. import CommandEvent
+
+
+async def get_initial_state(
+    intent: IntentAPI, room_id: RoomID
+) -> tuple[str | None, str | None, PowerLevelStateEventContent | None, bool]:
+    state = await intent.get_state(room_id)
+    title: str | None = None
+    about: str | None = None
+    levels: PowerLevelStateEventContent | None = None
+    encrypted: bool = False
+    for event in state:
+        try:
+            if event.type == EventType.ROOM_NAME:
+                title = event.content.name
+            elif event.type == EventType.ROOM_TOPIC:
+                about = event.content.topic
+            elif event.type == EventType.ROOM_POWER_LEVELS:
+                levels = event.content
+            elif event.type == EventType.ROOM_CANONICAL_ALIAS:
+                title = title or event.content.canonical_alias
+            elif event.type == EventType.ROOM_ENCRYPTION:
+                encrypted = True
+        except KeyError:
+            # Some state event probably has empty content
+            pass
+    return title, about, levels, encrypted
+
+
+async def warn_missing_power(levels: PowerLevelStateEventContent, evt: CommandEvent) -> None:
+    if levels.get_user_level(evt.az.bot_mxid) < levels.redact:
+        await evt.reply(
+            "Warning: The bot does not have privileges to redact messages on Matrix. "
+            "Message deletions from Telegram will not be bridged unless you give "
+            f"redaction permissions to [{evt.az.bot_mxid}](https://matrix.to/#/{evt.az.bot_mxid})"
+        )
+
+
+async def user_has_power_level(
+    room_id: RoomID, intent: IntentAPI, sender: u.User, event: str
+) -> bool:
+    if sender.is_admin:
+        return True
+    # Make sure the state store contains the power levels.
+    try:
+        await intent.get_power_levels(room_id)
+    except MatrixRequestError:
+        return False
+    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
+    return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)

mautrix_telegram/commands/telegram.py

@@ -1,154 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from telethon.errors import *
-from telethon.tl.types import User as TLUser
-from telethon.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest
-from telethon.tl.functions.channels import JoinChannelRequest
-
-from .. import puppet as pu, portal as po
-from . import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
-
-
-@command_handler(help_section=SECTION_MISC,
-                 help_args="[_-r|--remote_] <_query_>",
-                 help_text="Search your contacts or the Telegram servers for users.")
-async def search(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
-
-    force_remote = False
-    if evt.args[0] in {"-r", "--remote"}:
-        force_remote = True
-        evt.args.pop(0)
-
-    query = " ".join(evt.args)
-    if force_remote and len(query) < 5:
-        return await evt.reply("Minimum length of query for remote search is 5 characters.")
-
-    results, remote = await evt.sender.search(query, force_remote)
-
-    if not results:
-        if len(query) < 5 and remote:
-            return await evt.reply("No local results. "
-                                   "Minimum length of remote query is 5 characters.")
-        return await evt.reply("No results 3:")
-
-    reply = []
-    if remote:
-        reply += ["**Results from Telegram server:**", ""]
-    else:
-        reply += ["**Results in contacts:**", ""]
-    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
-               f"{puppet.id} ({similarity}% match)")
-              for puppet, similarity in results]
-
-    # TODO somehow show remote channel results when joining by alias is possible?
-
-    return await evt.reply("\n".join(reply))
-
-
-@command_handler(name="pm",
-                 help_section=SECTION_CREATING_PORTALS,
-                 help_args="<_identifier_>",
-                 help_text="Open a private chat with the given Telegram user. The identifier is "
-                           "either the internal user ID, the username or the phone number. "
-                           "**N.B.** The phone numbers you start chats with must already be in "
-                           "your contacts.")
-async def private_message(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
-
-    try:
-        user = await evt.sender.client.get_entity(evt.args[0])
-    except ValueError:
-        return await evt.reply("Invalid user identifier or user not found.")
-
-    if not user:
-        return await evt.reply("User not found.")
-    elif not isinstance(user, TLUser):
-        return await evt.reply("That doesn't seem to be a user.")
-    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
-    await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply("Created private chat room with "
-                           f"{pu.Puppet.get_displayname(user, False)}")
-
-
-async def _join(evt: CommandEvent, arg: str):
-    if arg.startswith("joinchat/"):
-        invite_hash = arg[len("joinchat/"):]
-        try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
-        except InviteHashInvalidError:
-            return None, await evt.reply("Invalid invite link.")
-        except InviteHashExpiredError:
-            return None, await evt.reply("Invite link expired.")
-        try:
-            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
-        except UserAlreadyParticipantError:
-            return None, await evt.reply("You are already in that chat.")
-    else:
-        channel = await evt.sender.client.get_entity(arg)
-        if not channel:
-            return None, await evt.reply("Channel/supergroup not found.")
-        return await evt.sender.client(JoinChannelRequest(channel)), None
-
-
-@command_handler(help_section=SECTION_CREATING_PORTALS,
-                 help_args="<_link_>",
-                 help_text="Join a chat with an invite link.")
-async def join(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
-
-    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
-    arg = regex.match(evt.args[0])
-    if not arg:
-        return await evt.reply("That doesn't look like a Telegram invite link.")
-
-    updates, _ = await _join(evt, arg.group(1))
-    if not updates:
-        return
-
-    for chat in updates.chats:
-        portal = po.Portal.get_by_entity(chat)
-        if portal.mxid:
-            await portal.invite_to_matrix([evt.sender.mxid])
-            return await evt.reply(f"Invited you to portal of {portal.title}")
-        else:
-            await evt.reply(f"Creating room for {chat.title}... This might take a while.")
-            await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
-            return await evt.reply(f"Created room for {portal.title}")
-
-
-@command_handler(help_section=SECTION_MISC,
-                 help_args="[`chats`|`contacts`|`me`]",
-                 help_text="Synchronize your chat portals, contacts and/or own info.")
-async def sync(evt: CommandEvent):
-    if len(evt.args) > 0:
-        sync_only = evt.args[0]
-        if sync_only not in ("chats", "contacts", "me"):
-            return await evt.reply("**Usage:** `$cmdprefix+sp sync [chats|contacts|me]`")
-    else:
-        sync_only = None
-
-    if not sync_only or sync_only == "chats":
-        await evt.sender.sync_dialogs(synchronous_create=True)
-    if not sync_only or sync_only == "contacts":
-        await evt.sender.sync_contacts()
-    if not sync_only or sync_only == "me":
-        await evt.sender.update_info()
-    return await evt.reply("Synchronization complete.")

mautrix_telegram/commands/telegram/__init__.py

@@ -0,0 +1 @@
+from . import account, auth, misc

mautrix_telegram/commands/telegram/account.py

@@ -0,0 +1,173 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from telethon.errors import (
+    AboutTooLongError,
+    AuthKeyError,
+    FirstNameInvalidError,
+    HashInvalidError,
+    UsernameInvalidError,
+    UsernameNotModifiedError,
+    UsernameOccupiedError,
+)
+from telethon.tl.functions.account import (
+    GetAuthorizationsRequest,
+    ResetAuthorizationRequest,
+    UpdateProfileRequest,
+    UpdateUsernameRequest,
+)
+from telethon.tl.types import Authorization
+
+from mautrix.types import EventID
+
+from .. import SECTION_AUTH, CommandEvent, command_handler
+
+
+@command_handler(
+    needs_auth=True,
+    help_section=SECTION_AUTH,
+    help_args="<_new username_>",
+    help_text="Change your Telegram username.",
+)
+async def username(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp username <new username>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own username.")
+    new_name = evt.args[0]
+    if new_name == "-":
+        new_name = ""
+    try:
+        await evt.sender.client(UpdateUsernameRequest(username=new_name))
+    except UsernameInvalidError:
+        return await evt.reply(
+            "Invalid username. Usernames must be between 5 and 30 alphanumeric characters."
+        )
+    except UsernameNotModifiedError:
+        return await evt.reply("That is your current username.")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    await evt.sender.update_info()
+    if not evt.sender.tg_username:
+        await evt.reply("Username removed")
+    else:
+        await evt.reply(f"Username changed to {evt.sender.tg_username}")
+
+
+@command_handler(
+    needs_auth=True,
+    help_section=SECTION_AUTH,
+    help_args="<_new about_>",
+    help_text="Change your Telegram about section.",
+)
+async def about(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp about <new about>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own about section.")
+    new_about = " ".join(evt.args)
+    if new_about == "-":
+        new_about = ""
+    try:
+        await evt.sender.client(UpdateProfileRequest(about=new_about))
+    except AboutTooLongError:
+        return await evt.reply("The provided about section is too long")
+    return await evt.reply("About section updated")
+
+
+@command_handler(
+    needs_auth=True,
+    help_section=SECTION_AUTH,
+    help_args="<_new displayname_>",
+    help_text="Change your Telegram displayname.",
+)
+async def displayname(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own displayname.")
+
+    first_name, last_name = (
+        (evt.args[0], "") if len(evt.args) == 1 else (" ".join(evt.args[:-1]), evt.args[-1])
+    )
+    try:
+        await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid first name")
+    await evt.sender.update_info()
+    return await evt.reply("Displayname updated")
+
+
+def _format_session(sess: Authorization) -> str:
+    return (
+        f"**{sess.app_name} {sess.app_version}**  \n"
+        f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
+        f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
+        f"  **From:** {sess.ip} - {sess.region}, {sess.country}"
+    )
+
+
+@command_handler(
+    needs_auth=True,
+    help_section=SECTION_AUTH,
+    help_args="<`list`|`terminate`> [_hash_]",
+    help_text="View or delete other Telegram sessions.",
+)
+async def session(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't manage their sessions")
+    cmd = evt.args[0].lower()
+    if cmd == "list":
+        res = await evt.sender.client(GetAuthorizationsRequest())
+        session_list = res.authorizations
+        current = [s for s in session_list if s.current][0]
+        current_text = _format_session(current)
+        other_text = "\n".join(
+            f"* {_format_session(sess)}  \n  **Hash:** {sess.hash}"
+            for sess in session_list
+            if not sess.current
+        )
+        return await evt.reply(
+            f"### Current session\n"
+            f"{current_text}\n"
+            f"\n"
+            f"### Other active sessions\n"
+            f"{other_text}"
+        )
+    elif cmd == "terminate" and len(evt.args) > 1:
+        try:
+            session_hash = int(evt.args[1])
+        except ValueError:
+            return await evt.reply("Hash must be an integer")
+        try:
+            ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
+        except HashInvalidError:
+            return await evt.reply("Invalid session hash.")
+        except AuthKeyError as e:
+            if e.message == "FRESH_RESET_AUTHORISATION_FORBIDDEN":
+                return await evt.reply(
+                    "New sessions can't terminate other sessions. Please wait a while."
+                )
+            raise
+        if ok:
+            return await evt.reply("Session terminated successfully.")
+        else:
+            return await evt.reply("Session not found.")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")

mautrix_telegram/commands/telegram/auth.py

@@ -0,0 +1,433 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Any
+import asyncio
+import io
+
+from telethon.errors import (
+    AccessTokenExpiredError,
+    AccessTokenInvalidError,
+    FirstNameInvalidError,
+    FloodWaitError,
+    PasswordHashInvalidError,
+    PhoneCodeExpiredError,
+    PhoneCodeInvalidError,
+    PhoneNumberAppSignupForbiddenError,
+    PhoneNumberBannedError,
+    PhoneNumberFloodError,
+    PhoneNumberInvalidError,
+    PhoneNumberOccupiedError,
+    PhoneNumberUnoccupiedError,
+    SessionPasswordNeededError,
+)
+from telethon.tl.types import User
+
+from mautrix.types import (
+    EventID,
+    ImageInfo,
+    MediaMessageEventContent,
+    MessageType,
+    TextMessageEventContent,
+    UserID,
+)
+from mautrix.util.format_duration import format_duration as fmt_duration
+
+from ... import user as u
+from ...commands import SECTION_AUTH, CommandEvent, command_handler
+from ...types import TelegramID
+
+try:
+    from telethon.tl.custom import QRLogin
+    import PIL as _
+    import qrcode
+except ImportError:
+    qrcode = None
+    QRLogin = None
+
+
+@command_handler(
+    needs_auth=False, help_section=SECTION_AUTH, help_text="Check if you're logged into Telegram."
+)
+async def ping(evt: CommandEvent) -> EventID:
+    if await evt.sender.is_logged_in():
+        me = await evt.sender.get_me()
+        if me:
+            human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
+            return await evt.reply(f"You're logged in as {human_tg_id}")
+        else:
+            return await evt.reply("You were logged in, but there appears to have been an error.")
+    else:
+        return await evt.reply("You're not logged in.")
+
+
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_AUTH,
+    help_text="Get the info of the message relay Telegram bot.",
+)
+async def ping_bot(evt: CommandEvent) -> EventID:
+    if not evt.tgbot:
+        return await evt.reply("Telegram message relay bot not configured.")
+    info, mxid = await evt.tgbot.get_me(use_cache=False)
+    return await evt.reply(
+        "Telegram message relay bot is active: "
+        f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
+        "To use the bot, simply invite it to a portal room."
+    )
+
+
+@command_handler(
+    needs_auth=False,
+    management_only=True,
+    help_section=SECTION_AUTH,
+    help_args="<_phone_> <_full name_>",
+    help_text="Register to Telegram",
+)
+async def register(evt: CommandEvent) -> EventID:
+    if await evt.sender.is_logged_in():
+        return await evt.reply("You are already logged in.")
+    elif len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
+
+    phone_number = evt.args[0]
+    if len(evt.args) == 2:
+        full_name = evt.args[1], ""
+    else:
+        full_name = " ".join(evt.args[1:-1]), evt.args[-1]
+
+    await _request_code(
+        evt,
+        phone_number,
+        {
+            "next": enter_code_register,
+            "action": "Register",
+            "full_name": full_name,
+        },
+    )
+    return await evt.reply(
+        "By signing up for Telegram, you agree to "
+        "the terms of service: https://telegram.org/tos"
+    )
+
+
+async def enter_code_register(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        first_name, last_name = evt.sender.command_status["full_name"]
+        user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
+        asyncio.create_task(evt.sender.post_login(user, first_login=True))
+        evt.sender.command_status = None
+        return await evt.reply(f"Successfully registered to Telegram.")
+    except PhoneNumberOccupiedError:
+        return await evt.reply(
+            "That phone number has already been registered. "
+            "You can log in with `$cmdprefix+sp login`."
+        )
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
+    except PhoneCodeExpiredError:
+        return await evt.reply(
+            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`."
+        )
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
+    except Exception:
+        evt.log.exception("Error sending phone code")
+        return await evt.reply(
+            "Unhandled exception while sending code. Check console for more details."
+        )
+
+
+@command_handler(
+    needs_auth=False,
+    management_only=True,
+    help_section=SECTION_AUTH,
+    help_text="Log in by scanning a QR code.",
+)
+async def login_qr(evt: CommandEvent) -> EventID:
+    login_as = evt.sender
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        login_as = await u.User.get_by_mxid(UserID(evt.args[0]))
+    if not qrcode or not QRLogin:
+        return await evt.reply("This bridge instance does not support logging in with a QR code.")
+    if await login_as.is_logged_in():
+        return await evt.reply(f"You are already logged in as {login_as.human_tg_id}.")
+
+    await login_as.ensure_started(even_if_no_session=True)
+    qr_login = QRLogin(login_as.client, ignored_ids=[])
+    qr_event_id: EventID | None = None
+
+    async def upload_qr() -> None:
+        nonlocal qr_event_id
+        buffer = io.BytesIO()
+        image = qrcode.make(qr_login.url)
+        size = image.pixel_size
+        image.save(buffer, "PNG")
+        qr = buffer.getvalue()
+        mxc = await evt.az.intent.upload_media(qr, "image/png", "login-qr.png", len(qr))
+        content = MediaMessageEventContent(
+            body=qr_login.url,
+            url=mxc,
+            msgtype=MessageType.IMAGE,
+            info=ImageInfo(mimetype="image/png", size=len(qr), width=size, height=size),
+        )
+        if qr_event_id:
+            content.set_edit(qr_event_id)
+            await evt.az.intent.send_message(evt.room_id, content)
+        else:
+            content.set_reply(evt.event_id)
+            qr_event_id = await evt.az.intent.send_message(evt.room_id, content)
+
+    retries = 4
+    while retries > 0:
+        await qr_login.recreate()
+        await upload_qr()
+        try:
+            user = await qr_login.wait()
+            break
+        except asyncio.TimeoutError:
+            retries -= 1
+        except SessionPasswordNeededError:
+            evt.sender.command_status = {
+                "next": enter_password,
+                "login_as": login_as if login_as != evt.sender else None,
+                "action": "Login (password entry)",
+            }
+            return await evt.reply(
+                "Your account has two-factor authentication. Please send your password here."
+            )
+    else:
+        timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
+        timeout.set_edit(qr_event_id)
+        return await evt.az.intent.send_message(evt.room_id, timeout)
+
+    return await _finish_sign_in(evt, user, login_as=login_as)
+
+
+@command_handler(
+    needs_auth=False,
+    management_only=True,
+    help_section=SECTION_AUTH,
+    help_text="Get instructions on how to log in.",
+)
+async def login(evt: CommandEvent) -> EventID:
+    override_sender = False
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        evt.sender = await u.User.get_and_start_by_mxid(UserID(evt.args[0]))
+        override_sender = True
+    if await evt.sender.is_logged_in():
+        return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
+
+    allow_matrix_login = evt.config["bridge.allow_matrix_login"]
+    if allow_matrix_login and not override_sender:
+        evt.sender.command_status = {
+            "next": enter_phone_or_token,
+            "action": "Login",
+        }
+
+    nb = "**N.B. Logging in grants the bridge full access to your Telegram account.**"
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
+        if override_sender:
+            return await evt.reply(
+                f"[Click here to log in]({url}) as "
+                f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid})."
+            )
+        elif allow_matrix_login:
+            return await evt.reply(
+                f"[Click here to log in]({url}). Alternatively, send your phone"
+                f" number (or bot auth token) here to log in.\n\n{nb}"
+            )
+        return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
+    elif allow_matrix_login:
+        if override_sender:
+            return await evt.reply(
+                "This bridge instance does not allow you to log in outside of Matrix. "
+                "Logging in as another user inside Matrix is not currently possible."
+            )
+        return await evt.reply(
+            "Please send your phone number (or bot auth token) here to start "
+            f"the login process.\n\n{nb}"
+        )
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+async def _request_code(
+    evt: CommandEvent, phone_number: str, next_status: dict[str, Any]
+) -> EventID:
+    ok = False
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        await evt.sender.client.sign_in(phone_number)
+        ok = True
+        return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
+    except PhoneNumberAppSignupForbiddenError:
+        return await evt.reply("Your phone number does not allow 3rd party apps to sign in.")
+    except PhoneNumberFloodError:
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            "The ban is usually applied for around a day."
+        )
+    except FloodWaitError as e:
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            f"Please wait for {fmt_duration(e.seconds)} before trying again."
+        )
+    except PhoneNumberBannedError:
+        return await evt.reply("Your phone number has been banned from Telegram.")
+    except PhoneNumberUnoccupiedError:
+        return await evt.reply(
+            "That phone number has not been registered. "
+            "Please register with `$cmdprefix+sp register <phone>`."
+        )
+    except PhoneNumberInvalidError:
+        return await evt.reply("That phone number is not valid.")
+    except Exception:
+        evt.log.exception("Error requesting phone code")
+        return await evt.reply(
+            "Unhandled exception while requesting code. Check console for more details."
+        )
+    finally:
+        evt.sender.command_status = next_status if ok else None
+
+
+@command_handler(needs_auth=False)
+async def enter_phone_or_token(evt: CommandEvent) -> EventID | None:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply(
+            "This bridge instance does not allow in-Matrix login. "
+            "Please use `$cmdprefix+sp login` to get login instructions"
+        )
+
+    # phone numbers don't contain colons but telegram bot auth tokens do
+    if evt.args[0].find(":") > 0:
+        try:
+            await _sign_in(evt, bot_token=evt.args[0])
+        except Exception:
+            evt.log.exception("Error sending auth token")
+            return await evt.reply(
+                "Unhandled exception while sending auth token. Check console for more details."
+            )
+    else:
+        await _request_code(evt, evt.args[0], {"next": enter_code, "action": "Login"})
+    return None
+
+
+@command_handler(needs_auth=False)
+async def enter_code(evt: CommandEvent) -> EventID | None:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply(
+            "This bridge instance does not allow in-Matrix login. "
+            "Please use `$cmdprefix+sp login` to get login instructions"
+        )
+    try:
+        await _sign_in(evt, code=evt.args[0])
+    except Exception:
+        evt.log.exception("Error sending phone code")
+        return await evt.reply(
+            "Unhandled exception while sending code. Check console for more details."
+        )
+    return None
+
+
+@command_handler(needs_auth=False)
+async def enter_password(evt: CommandEvent) -> EventID | None:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply(
+            "This bridge instance does not allow in-Matrix login. "
+            "Please use `$cmdprefix+sp login` to get login instructions"
+        )
+    try:
+        await _sign_in(
+            evt,
+            login_as=evt.sender.command_status.get("login_as", None),
+            password=" ".join(evt.args),
+        )
+    except AccessTokenInvalidError:
+        return await evt.reply("That bot token is not valid.")
+    except AccessTokenExpiredError:
+        return await evt.reply("That bot token has expired.")
+    except Exception:
+        evt.log.exception("Error sending password")
+        return await evt.reply(
+            "Unhandled exception while sending password. Check console for more details."
+        )
+    return None
+
+
+async def _sign_in(evt: CommandEvent, login_as: u.User = None, **sign_in_info) -> EventID:
+    login_as = login_as or evt.sender
+    try:
+        await login_as.ensure_started(even_if_no_session=True)
+        user = await login_as.client.sign_in(**sign_in_info)
+        await _finish_sign_in(evt, user)
+    except PhoneCodeExpiredError:
+        return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
+    except PasswordHashInvalidError:
+        return await evt.reply("Incorrect password.")
+    except SessionPasswordNeededError:
+        evt.sender.command_status = {
+            "next": enter_password,
+            "action": "Login (password entry)",
+        }
+        return await evt.reply(
+            "Your account has two-factor authentication. Please send your password here."
+        )
+
+
+async def _finish_sign_in(evt: CommandEvent, user: User, login_as: u.User = None) -> EventID:
+    login_as = login_as or evt.sender
+    existing_user = await u.User.get_by_tgid(TelegramID(user.id))
+    if existing_user and existing_user != login_as:
+        await existing_user.log_out()
+        await evt.reply(
+            f"[{existing_user.displayname}] (https://matrix.to/#/{existing_user.mxid})"
+            " was logged out from the account."
+        )
+    asyncio.create_task(login_as.post_login(user, first_login=True))
+    evt.sender.command_status = None
+    name = f"@{user.username}" if user.username else f"+{user.phone}"
+    if login_as != evt.sender:
+        msg = (
+            f"Successfully logged in [{login_as.mxid}](https://matrix.to/#/{login_as.mxid})"
+            f" as {name}"
+        )
+    else:
+        msg = f"Successfully logged in as {name}"
+    return await evt.reply(msg)
+
+
+@command_handler(needs_auth=False, help_section=SECTION_AUTH, help_text="Log out from Telegram.")
+async def logout(evt: CommandEvent) -> EventID:
+    if not evt.sender.tgid:
+        return await evt.reply("You're not logged in")
+    if await evt.sender.log_out():
+        return await evt.reply("Logged out successfully.")
+    return await evt.reply("Failed to log out.")

mautrix_telegram/commands/telegram/misc.py

@@ -0,0 +1,440 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import cast
+import base64
+import codecs
+import re
+
+from aiohttp import ClientSession, InvalidURL
+from telethon.errors import (
+    ChatIdInvalidError,
+    EmoticonInvalidError,
+    InviteHashExpiredError,
+    InviteHashInvalidError,
+    OptionsTooMuchError,
+    TakeoutInitDelayError,
+    UserAlreadyParticipantError,
+)
+from telethon.tl.functions.channels import JoinChannelRequest
+from telethon.tl.functions.messages import (
+    CheckChatInviteRequest,
+    GetBotCallbackAnswerRequest,
+    ImportChatInviteRequest,
+    SendVoteRequest,
+)
+from telethon.tl.patched import Message
+from telethon.tl.types import (
+    InputMediaDice,
+    MessageMediaGame,
+    MessageMediaPoll,
+    TypeInputPeer,
+    TypeUpdates,
+    User as TLUser,
+)
+from telethon.tl.types.messages import BotCallbackAnswer
+
+from mautrix.types import EventID, Format
+
+from ... import portal as po, puppet as pu
+from ...abstract_user import AbstractUser
+from ...commands import (
+    SECTION_CREATING_PORTALS,
+    SECTION_MISC,
+    SECTION_PORTAL_MANAGEMENT,
+    CommandEvent,
+    command_handler,
+)
+from ...db import Message as DBMessage
+from ...types import TelegramID
+
+
+@command_handler(
+    needs_auth=False,
+    help_section=SECTION_MISC,
+    help_args="<_caption_>",
+    help_text="Set a caption for the next image you send",
+)
+async def caption(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp caption <caption>`")
+
+    prefix = f"{evt.command_prefix} caption "
+    if evt.content.format == Format.HTML:
+        evt.content.formatted_body = evt.content.formatted_body.replace(prefix, "", 1)
+    evt.content.body = evt.content.body.replace(prefix, "", 1)
+    evt.sender.command_status = {"caption": evt.content, "action": "Caption"}
+    return await evt.reply(
+        "Your next image or file will be sent with that caption. "
+        "Use `$cmdprefix+sp cancel` to cancel the caption."
+    )
+
+
+@command_handler(
+    help_section=SECTION_MISC,
+    help_args="[_-r|--remote_] <_query_>",
+    help_text="Search your contacts or the Telegram servers for users.",
+)
+async def search(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
+
+    force_remote = False
+    if evt.args[0] in {"-r", "--remote"}:
+        force_remote = True
+        evt.args.pop(0)
+
+    query = " ".join(evt.args)
+    if force_remote and len(query) < 5:
+        return await evt.reply("Minimum length of query for remote search is 5 characters.")
+
+    results, remote = await evt.sender.search(query, force_remote)
+
+    if not results:
+        if len(query) < 5 and remote:
+            return await evt.reply(
+                "No local results. Minimum length of remote query is 5 characters."
+            )
+        return await evt.reply("No results 3:")
+
+    reply: list[str] = []
+    if remote:
+        reply += ["**Results from Telegram server:**", ""]
+    else:
+        reply += ["**Results in contacts:**", ""]
+    reply += [
+        (
+            f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
+            f"{puppet.id} ({similarity}% match)"
+        )
+        for puppet, similarity in results
+    ]
+
+    # TODO somehow show remote channel results when joining by alias is possible?
+
+    return await evt.reply("\n".join(reply))
+
+
+@command_handler(
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="<_identifier_>",
+    help_text="Open a private chat with the given Telegram user. The identifier is "
+    "either the internal user ID, the username or the phone number. "
+    "**N.B.** The phone numbers you start chats with must already be in "
+    "your contacts.",
+)
+async def pm(evt: CommandEvent) -> EventID:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
+
+    try:
+        id = "".join(evt.args).translate({ord(c): None for c in "+()- "})
+        user = await evt.sender.client.get_entity(id)
+    except ValueError:
+        return await evt.reply("Invalid user identifier or user not found.")
+
+    if not user:
+        return await evt.reply("User not found.")
+    elif not isinstance(user, TLUser):
+        return await evt.reply("That doesn't seem to be a user.")
+    portal = await po.Portal.get_by_entity(user, tg_receiver=evt.sender.tgid)
+    await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
+    displayname, _ = pu.Puppet.get_displayname(user, False)
+    return await evt.reply(f"Created private chat room with {displayname}")
+
+
+async def _join(
+    evt: CommandEvent, identifier: str, link_type: str
+) -> tuple[TypeUpdates | None, EventID | None]:
+    if link_type == "joinchat":
+        try:
+            await evt.sender.client(CheckChatInviteRequest(identifier))
+        except InviteHashInvalidError:
+            return None, await evt.reply("Invalid invite link.")
+        except InviteHashExpiredError:
+            return None, await evt.reply("Invite link expired.")
+        try:
+            return (await evt.sender.client(ImportChatInviteRequest(identifier))), None
+        except UserAlreadyParticipantError:
+            return None, await evt.reply("You are already in that chat.")
+    else:
+        channel = await evt.sender.client.get_entity(identifier)
+        if not channel:
+            return None, await evt.reply("Channel/supergroup not found.")
+        return await evt.sender.client(JoinChannelRequest(channel)), None
+
+
+@command_handler(
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="<_link_>",
+    help_text="Join a chat with an invite link.",
+)
+async def join(evt: CommandEvent) -> EventID | None:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
+
+    url = evt.args[0]
+    if evt.config["bridge.invite_link_resolve"]:
+        try:
+            async with ClientSession() as sess, sess.get(url) as resp:
+                url = str(resp.url)
+        except InvalidURL:
+            return await evt.reply("That doesn't look like a Telegram invite link.")
+
+    regex = re.compile(
+        r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:/(?P<type>joinchat|s))?/(?P<id>[^/]+)/?",
+        flags=re.IGNORECASE,
+    )
+    arg = regex.match(url)
+    if not arg:
+        return await evt.reply("That doesn't look like a Telegram invite link.")
+
+    data = arg.groupdict()
+    identifier = data["id"]
+    link_type = data["type"]
+    if link_type:
+        link_type = link_type.lower()
+    updates, _ = await _join(evt, identifier, link_type)
+    if not updates:
+        return None
+
+    for chat in updates.chats:
+        portal = await po.Portal.get_by_entity(chat)
+        if portal.mxid:
+            await portal.invite_to_matrix([evt.sender.mxid])
+            return await evt.reply(f"Invited you to portal of {portal.title}")
+        else:
+            await evt.reply(f"Creating room for {chat.title}... This might take a while.")
+            try:
+                await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
+            except ChatIdInvalidError as e:
+                evt.log.trace(
+                    "ChatIdInvalidError while creating portal from !tg join command: %s",
+                    updates.stringify(),
+                )
+                raise e
+            return await evt.reply(f"Created room for {portal.title}")
+    return None
+
+
+@command_handler(
+    help_section=SECTION_MISC,
+    help_args="[`chats`|`contacts`|`me`]",
+    help_text="Synchronize your chat portals, contacts and/or own info.",
+)
+async def sync(evt: CommandEvent) -> EventID:
+    if len(evt.args) > 0:
+        sync_only = evt.args[0]
+        if sync_only not in ("chats", "contacts", "me"):
+            return await evt.reply("**Usage:** `$cmdprefix+sp sync [chats|contacts|me]`")
+    else:
+        sync_only = None
+
+    if not sync_only or sync_only == "chats":
+        await evt.reply("Synchronizing chats...")
+        await evt.sender.sync_dialogs()
+    if not sync_only or sync_only == "contacts":
+        await evt.reply("Synchronizing contacts...")
+        await evt.sender.sync_contacts()
+    if not sync_only or sync_only == "me":
+        await evt.sender.update_info()
+    return await evt.reply("Synchronization complete.")
+
+
+PEER_TYPE_CHAT = b"g"
+
+
+class MessageIDError(ValueError):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+
+async def _parse_encoded_msgid(
+    user: AbstractUser, enc_id: str, type_name: str
+) -> tuple[TypeInputPeer, Message]:
+    try:
+        enc_id += (4 - len(enc_id) % 4) * "="
+        enc_id = base64.b64decode(enc_id)
+        peer_type, enc_id = bytes([enc_id[0]]), enc_id[1:]
+        tgid = TelegramID(int(codecs.encode(enc_id[0:5], "hex_codec"), 16))
+        msg_id = TelegramID(int(codecs.encode(enc_id[5:10], "hex_codec"), 16))
+        space = None
+        if peer_type == PEER_TYPE_CHAT:
+            space = TelegramID(int(codecs.encode(enc_id[10:15], "hex_codec"), 16))
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (format)") from e
+
+    if peer_type == PEER_TYPE_CHAT:
+        orig_msg = await DBMessage.get_one_by_tgid(msg_id, space)
+        if not orig_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
+        new_msg = await DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
+        if not new_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (your copy of message not found in db)")
+        msg_id = new_msg.tgid
+    try:
+        peer = await user.client.get_input_entity(tgid)
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (chat not found)") from e
+
+    msg = await user.client.get_messages(entity=peer, ids=msg_id)
+    if not msg:
+        raise MessageIDError(f"Invalid {type_name} ID (message not found)")
+    return peer, cast(Message, msg)
+
+
+@command_handler(
+    help_section=SECTION_MISC, help_args="<_play ID_>", help_text="Play a Telegram game."
+)
+async def play(evt: CommandEvent) -> EventID:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp play <play ID>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to play games.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't play games :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="play")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaGame):
+        return await evt.reply("Invalid play ID (message doesn't look like a game)")
+
+    game = await evt.sender.client(
+        GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True)
+    )
+    if not isinstance(game, BotCallbackAnswer):
+        return await evt.reply("Game request response invalid")
+
+    return await evt.reply(
+        f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
+        f"{msg.media.game.description}"
+    )
+
+
+@command_handler(
+    help_section=SECTION_MISC,
+    help_args="<_poll ID_> <_choice number_>",
+    help_text="Vote in a Telegram poll.",
+)
+async def vote(evt: CommandEvent) -> EventID | None:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to vote in polls.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't vote in polls :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="poll")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaPoll):
+        return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
+
+    options = []
+    for option in evt.args[1:]:
+        try:
+            if len(option) > 10:
+                raise ValueError("option index too long")
+            option_index = int(option) - 1
+        except ValueError:
+            option_index = None
+        if option_index is None:
+            return await evt.reply(
+                f'Invalid option number "{option}"', render_markdown=False, allow_html=False
+            )
+        elif option_index < 0:
+            return await evt.reply(
+                f"Invalid option number {option}. Option numbers must be positive."
+            )
+        elif option_index >= len(msg.media.poll.answers):
+            return await evt.reply(
+                f"Invalid option number {option}. "
+                f"The poll only has {len(msg.media.poll.answers)} options."
+            )
+        options.append(msg.media.poll.answers[option_index].option)
+    options = [msg.media.poll.answers[int(option) - 1].option for option in evt.args[1:]]
+    try:
+        await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
+    except OptionsTooMuchError:
+        return await evt.reply("You passed too many options.")
+    # TODO use response
+    return await evt.mark_read()
+
+
+@command_handler(
+    help_section=SECTION_MISC,
+    help_args="<_emoji_>",
+    help_text="Roll a dice (\U0001F3B2), kick a football (\u26BD\uFE0F) or throw a "
+    "dart (\U0001F3AF) or basketball (\U0001F3C0) on the Telegram servers.",
+)
+async def random(evt: CommandEvent) -> EventID:
+    if not evt.is_portal:
+        return await evt.reply("You can only randomize values in portal rooms")
+    portal = await po.Portal.get_by_mxid(evt.room_id)
+    arg = evt.args[0] if len(evt.args) > 0 else "dice"
+    emoticon = {
+        "dart": "\U0001F3AF",
+        "dice": "\U0001F3B2",
+        "ball": "\U0001F3C0",
+        "basketball": "\U0001F3C0",
+        "football": "\u26BD",
+        "soccer": "\u26BD",
+    }.get(arg, arg)
+    try:
+        await evt.sender.client.send_media(
+            await portal.get_input_entity(evt.sender), InputMediaDice(emoticon)
+        )
+    except EmoticonInvalidError:
+        return await evt.reply("Invalid emoji for randomization")
+
+
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_args="[_limit_]",
+    help_text="Backfill messages from Telegram history.",
+)
+async def backfill(evt: CommandEvent) -> None:
+    if not evt.is_portal:
+        await evt.reply("You can only use backfill in portal rooms")
+        return
+    try:
+        limit = int(evt.args[0])
+    except (ValueError, IndexError):
+        limit = -1
+    portal = await po.Portal.get_by_mxid(evt.room_id)
+    if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
+        await evt.reply("Backfilling normal groups is disabled in the bridge config")
+        return
+    try:
+        await portal.backfill(evt.sender, limit=limit)
+    except TakeoutInitDelayError:
+        msg = (
+            "Please accept the data export request from a mobile device, "
+            "then re-run the backfill command."
+        )
+        if portal.peer_type == "user":
+            from mautrix.appservice import IntentAPI
+
+            await portal.main_intent.send_notice(evt.room_id, msg)
+        else:
+            await evt.reply(msg)

mautrix_telegram/config.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,148 +13,70 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Any, Optional
-from ruamel.yaml import YAML
+from typing import Any, List, NamedTuple
+import os
+
 from ruamel.yaml.comments import CommentedMap
-import random
-import string
 
-yaml = YAML()
-yaml.indent(4)
+from mautrix.bridge.config import BaseBridgeConfig
+from mautrix.client import Client
+from mautrix.types import UserID
+from mautrix.util.config import ConfigUpdateHelper, ForbiddenDefault, ForbiddenKey
+
+Permissions = NamedTuple(
+    "Permissions",
+    relaybot=bool,
+    user=bool,
+    puppeting=bool,
+    matrix_puppeting=bool,
+    admin=bool,
+    level=str,
+)
 
 
-class DictWithRecursion:
-    def __init__(self, data: CommentedMap = None):
-        self._data = data or CommentedMap()  # type: CommentedMap
-
-    def _recursive_get(self, data: CommentedMap, key: str, default_value: Any) -> Any:
-        if '.' in key:
-            key, next_key = key.split('.', 1)
-            next_data = data.get(key, CommentedMap())
-            return self._recursive_get(next_data, next_key, default_value)
-        return data.get(key, default_value)
-
-    def get(self, key: str, default_value: Any, allow_recursion: bool = True) -> Any:
-        if allow_recursion and '.' in key:
-            return self._recursive_get(self._data, key, default_value)
-        return self._data.get(key, default_value)
-
+class Config(BaseBridgeConfig):
     def __getitem__(self, key: str) -> Any:
-        return self.get(key, None)
-
-    def __contains__(self, key: str) -> bool:
-        return self[key] is not None
-
-    def _recursive_set(self, data: CommentedMap, key: str, value: Any):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
-            if key not in data:
-                data[key] = CommentedMap()
-            next_data = data.get(key, CommentedMap())
-            self._recursive_set(next_data, next_key, value)
-            return
-        data[key] = value
-
-    def set(self, key: str, value: Any, allow_recursion: bool = True):
-        if allow_recursion and '.' in key:
-            self._recursive_set(self._data, key, value)
-            return
-        self._data[key] = value
-
-    def __setitem__(self, key: str, value: Any):
-        self.set(key, value)
-
-    def _recursive_del(self, data: CommentedMap, key: str):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
-            if key not in data:
-                return
-            next_data = data[key]
-            self._recursive_del(next_data, next_key)
-            return
         try:
-            del data[key]
-            del data.ca.items[key]
+            return os.environ[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
         except KeyError:
-            pass
+            return super().__getitem__(key)
 
-    def delete(self, key: str, allow_recursion: bool = True):
-        if allow_recursion and '.' in key:
-            self._recursive_del(self._data, key)
-            return
-        try:
-            del self._data[key]
-            del self._data.ca.items[key]
-        except KeyError:
-            pass
+    @property
+    def forbidden_defaults(self) -> List[ForbiddenDefault]:
+        return [
+            *super().forbidden_defaults,
+            ForbiddenDefault(
+                "appservice.database",
+                "postgres://username:password@hostname/dbname",
+            ),
+            ForbiddenDefault(
+                "appservice.public.external",
+                "https://example.com/public",
+                condition="appservice.public.enabled",
+            ),
+            ForbiddenDefault("bridge.permissions", ForbiddenKey("example.com")),
+            ForbiddenDefault("telegram.api_id", 12345),
+            ForbiddenDefault("telegram.api_hash", "tjyd5yge35lbodk1xwzw2jstp90k55qz"),
+        ]
 
-    def __delitem__(self, key: str):
-        self.delete(key)
+    def do_update(self, helper: ConfigUpdateHelper) -> None:
+        super().do_update(helper)
+        copy, copy_dict, base = helper
 
-
-class Config(DictWithRecursion):
-    def __init__(self, path: str, registration_path: str, base_path: str):
-        super().__init__()
-        self.path = path  # type: str
-        self.registration_path = registration_path  # type: str
-        self.base_path = base_path  # type: str
-        self._registration = None  # type: dict
-
-    def load(self):
-        with open(self.path, 'r') as stream:
-            self._data = yaml.load(stream)
-
-    def load_base(self) -> Optional[DictWithRecursion]:
-        try:
-            with open(self.base_path, 'r') as stream:
-                return DictWithRecursion(yaml.load(stream))
-        except OSError:
-            pass
-        return None
-
-    def save(self):
-        with open(self.path, 'w') as stream:
-            yaml.dump(self._data, stream)
-        if self._registration and self.registration_path:
-            with open(self.registration_path, 'w') as stream:
-                yaml.dump(self._registration, stream)
-
-    @staticmethod
-    def _new_token() -> str:
-        return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
-
-    def update(self):
-        base = self.load_base()
-        if not base:
-            return
-
-        def copy(from_path, to_path=None):
-            if from_path in self:
-                base[to_path or from_path] = self[from_path]
-
-        def copy_dict(from_path, to_path=None, override_existing_map=True):
-            if from_path in self:
-                to_path = to_path or from_path
-                if override_existing_map or to_path not in base:
-                    base[to_path] = CommentedMap()
-                for key, value in self[from_path].items():
-                    base[to_path][key] = value
-
-        copy("homeserver.address")
-        copy("homeserver.verify_ssl")
-        copy("homeserver.domain")
+        copy("homeserver.asmux")
 
         if "appservice.protocol" in self and "appservice.address" not in self:
-            protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
-                                        self["appservice.port"])
+            protocol, hostname, port = (
+                self["appservice.protocol"],
+                self["appservice.hostname"],
+                self["appservice.port"],
+            )
             base["appservice.address"] = f"{protocol}://{hostname}:{port}"
-        else:
-            copy("appservice.address")
-        copy("appservice.hostname")
-        copy("appservice.port")
-        copy("appservice.max_body_size")
-
-        copy("appservice.database")
+        if "appservice.debug" in self and "logging" not in self:
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
+            base["logging.root.level"] = level
+            base["logging.loggers.mau.level"] = level
+            base["logging.loggers.telethon.level"] = level
 
         copy("appservice.public.enabled")
         copy("appservice.public.prefix")
@@ -167,34 +88,95 @@ class Config(DictWithRecursion):
         if base["appservice.provisioning.shared_secret"] == "generate":
             base["appservice.provisioning.shared_secret"] = self._new_token()
 
-        copy("appservice.id")
-        copy("appservice.bot_username")
-        copy("appservice.bot_displayname")
-        copy("appservice.bot_avatar")
+        if "pool_size" in base["appservice.database_opts"]:
+            pool_size = base["appservice.database_opts"].pop("pool_size")
+            base["appservice.database_opts.min_size"] = pool_size
+            base["appservice.database_opts.max_size"] = pool_size
+        if "pool_pre_ping" in base["appservice.database_opts"]:
+            del base["appservice.database_opts.pool_pre_ping"]
 
-        copy("appservice.as_token")
-        copy("appservice.hs_token")
+        copy("appservice.community_id")
+
+        copy("metrics.enabled")
+        copy("metrics.listen_port")
 
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
 
         copy("bridge.displayname_preference")
+        copy("bridge.displayname_max_length")
+        copy("bridge.allow_avatar_remove")
 
-        copy("bridge.edits_as_replies")
-        copy("bridge.highlight_edits")
-        copy("bridge.bridge_notices")
-        copy("bridge.bot_messages_as_notices")
         copy("bridge.max_initial_member_sync")
         copy("bridge.sync_channel_members")
+        copy("bridge.skip_deleted_members")
+        copy("bridge.startup_sync")
+        if "bridge.sync_dialog_limit" in self:
+            base["bridge.sync_create_limit"] = self["bridge.sync_dialog_limit"]
+            base["bridge.sync_update_limit"] = self["bridge.sync_dialog_limit"]
+        else:
+            copy("bridge.sync_update_limit")
+            copy("bridge.sync_create_limit")
+        copy("bridge.sync_direct_chats")
         copy("bridge.max_telegram_delete")
+        copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
-        copy("bridge.inline_images")
         copy("bridge.plaintext_highlights")
         copy("bridge.public_portals")
-        copy("bridge.native_stickers")
-        copy("bridge.catch_up")
         copy("bridge.sync_with_custom_puppets")
+        copy("bridge.sync_direct_chat_list")
+        copy("bridge.double_puppet_server_map")
+        copy("bridge.double_puppet_allow_discovery")
+        if "bridge.login_shared_secret" in self:
+            base["bridge.login_shared_secret_map"] = {
+                base["homeserver.domain"]: self["bridge.login_shared_secret"]
+            }
+        else:
+            copy("bridge.login_shared_secret_map")
+        copy("bridge.telegram_link_preview")
+        copy("bridge.invite_link_resolve")
+        copy("bridge.inline_images")
+        copy("bridge.image_as_file_size")
+        copy("bridge.max_document_size")
+        copy("bridge.parallel_file_transfer")
+        copy("bridge.federate_rooms")
+        copy("bridge.animated_sticker.target")
+        copy("bridge.animated_sticker.args")
+        copy("bridge.encryption.allow")
+        copy("bridge.encryption.default")
+        copy("bridge.encryption.database")
+        copy("bridge.encryption.key_sharing.allow")
+        copy("bridge.encryption.key_sharing.require_cross_signing")
+        copy("bridge.encryption.key_sharing.require_verification")
+        copy("bridge.private_chat_portal_meta")
+        copy("bridge.delivery_receipts")
+        copy("bridge.delivery_error_reports")
+        copy("bridge.resend_bridge_info")
+        copy("bridge.mute_bridging")
+        copy("bridge.pinned_tag")
+        copy("bridge.archive_tag")
+        copy("bridge.tag_only_on_create")
+        copy("bridge.bridge_matrix_leave")
+        copy("bridge.kick_on_logout")
+        copy("bridge.backfill.invite_own_puppet")
+        copy("bridge.backfill.takeout_limit")
+        copy("bridge.backfill.initial_limit")
+        copy("bridge.backfill.missed_limit")
+        copy("bridge.backfill.disable_notifications")
+        copy("bridge.backfill.normal_groups")
+
+        copy("bridge.initial_power_level_overrides.group")
+        copy("bridge.initial_power_level_overrides.user")
+
+        copy("bridge.bot_messages_as_notices")
+        if isinstance(self["bridge.bridge_notices"], bool):
+            base["bridge.bridge_notices"] = {
+                "default": self["bridge.bridge_notices"],
+                "exceptions": ["@importantbot:example.com"],
+            }
+        else:
+            copy("bridge.bridge_notices")
 
         copy("bridge.deduplication.pre_db_check")
         copy("bridge.deduplication.cache_queue_length")
@@ -202,6 +184,8 @@ class Config(DictWithRecursion):
         if "bridge.message_formats.m_text" in self:
             del self["bridge.message_formats"]
         copy_dict("bridge.message_formats", override_existing_map=False)
+        copy("bridge.emote_format")
+
         copy("bridge.state_event_formats.join")
         copy("bridge.state_event_formats.leave")
         copy("bridge.state_event_formats.name_change")
@@ -211,9 +195,11 @@ class Config(DictWithRecursion):
 
         copy("bridge.command_prefix")
 
-        migrate_permissions = ("bridge.permissions" not in self
-                               or "bridge.whitelist" in self
-                               or "bridge.admins" in self)
+        migrate_permissions = (
+            "bridge.permissions" not in self
+            or "bridge.whitelist" in self
+            or "bridge.admins" in self
+        )
         if migrate_permissions:
             permissions = self["bridge.permissions"] or CommentedMap()
             for entry in self["bridge.whitelist"] or []:
@@ -227,6 +213,11 @@ class Config(DictWithRecursion):
         if "bridge.relaybot" not in self:
             copy("bridge.authless_relaybot_portals", "bridge.relaybot.authless_portals")
         else:
+            copy("bridge.relaybot.private_chat.invite")
+            copy("bridge.relaybot.private_chat.state_changes")
+            copy("bridge.relaybot.private_chat.message")
+            copy("bridge.relaybot.group_chat_invite")
+            copy("bridge.relaybot.ignore_unbridged_group_chat")
             copy("bridge.relaybot.authless_portals")
             copy("bridge.relaybot.whitelist_group_admins")
             copy("bridge.relaybot.whitelist")
@@ -235,6 +226,24 @@ class Config(DictWithRecursion):
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
+
+        copy("telegram.connection.timeout")
+        copy("telegram.connection.retries")
+        copy("telegram.connection.retry_delay")
+        copy("telegram.connection.flood_sleep_threshold")
+        copy("telegram.connection.request_retries")
+
+        copy("telegram.device_info.device_model")
+        copy("telegram.device_info.system_version")
+        copy("telegram.device_info.app_version")
+        copy("telegram.device_info.lang_code")
+        copy("telegram.device_info.system_lang_code")
+
+        copy("telegram.server.enabled")
+        copy("telegram.server.dc")
+        copy("telegram.server.ip")
+        copy("telegram.server.port")
+
         copy("telegram.proxy.type")
         copy("telegram.proxy.address")
         copy("telegram.proxy.port")
@@ -242,63 +251,22 @@ class Config(DictWithRecursion):
         copy("telegram.proxy.username")
         copy("telegram.proxy.password")
 
-        if "appservice.debug" in self and "logging" not in self:
-            level = "DEBUG" if self["appservice.debug"] else "INFO"
-            base["logging.root.level"] = level
-            base["logging.loggers.mau.level"] = level
-            base["logging.loggers.telethon.level"] = level
-        else:
-            copy("logging")
-
-        self._data = base._data
-        self.save()
-
-    def _get_permissions(self, key: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
+    def _get_permissions(self, key: str) -> Permissions:
         level = self["bridge.permissions"].get(key, "")
         admin = level == "admin"
         matrix_puppeting = level == "full" or admin
         puppeting = level == "puppeting" or matrix_puppeting
         user = level == "user" or puppeting
         relaybot = level == "relaybot" or user
-        return relaybot, user, puppeting, matrix_puppeting, admin, level
+        return Permissions(relaybot, user, puppeting, matrix_puppeting, admin, level)
 
-    def get_permissions(self, mxid: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
-        permissions = self["bridge.permissions"] or {}
+    def get_permissions(self, mxid: UserID) -> Permissions:
+        permissions = self["bridge.permissions"]
         if mxid in permissions:
             return self._get_permissions(mxid)
 
-        homeserver = mxid[mxid.index(":") + 1:]
+        _, homeserver = Client.parse_user_id(mxid)
         if homeserver in permissions:
             return self._get_permissions(homeserver)
 
         return self._get_permissions("*")
-
-    def generate_registration(self):
-        homeserver = self["homeserver.domain"]
-
-        username_format = self.get("bridge.username_template", "telegram_{userid}") \
-            .format(userid=".+")
-        alias_format = self.get("bridge.alias_template", "telegram_{groupname}") \
-            .format(groupname=".+")
-
-        self.set("appservice.as_token", self._new_token())
-        self.set("appservice.hs_token", self._new_token())
-
-        self._registration = {
-            "id": self["appservice.id"] or "telegram",
-            "as_token": self["appservice.as_token"],
-            "hs_token": self["appservice.hs_token"],
-            "namespaces": {
-                "users": [{
-                    "exclusive": True,
-                    "regex": f"@{username_format}:{homeserver}"
-                }],
-                "aliases": [{
-                    "exclusive": True,
-                    "regex": f"#{alias_format}:{homeserver}"
-                }]
-            },
-            "url": self["appservice.address"],
-            "sender_localpart": self["appservice.bot_username"],
-            "rate_limited": False
-        }

mautrix_telegram/context.py

@@ -1,51 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import TYPE_CHECKING, Optional
-
-if TYPE_CHECKING:
-    import asyncio
-
-    from sqlalchemy.orm import scoped_session
-
-    from alchemysession import AlchemySessionContainer
-    from mautrix_appservice import AppService
-
-    from .web import PublicBridgeWebsite, ProvisioningAPI
-    from .config import Config
-    from .bot import Bot
-    from .matrix import MatrixHandler
-
-
-class Context:
-    def __init__(self, az: "AppService", db: "scoped_session", config: "Config",
-                 loop: "asyncio.AbstractEventLoop", session_container: "AlchemySessionContainer"):
-        self.az = az  # type: AppService
-        self.db = db  # type: scoped_session
-        self.config = config  # type: Config
-        self.loop = loop  # type: asyncio.AbstractEventLoop
-        self.bot = None  # type: Optional[Bot]
-        self.mx = None  # type: MatrixHandler
-        self.session_container = session_container  # type: AlchemySessionContainer
-        self.public_website = None  # type: PublicBridgeWebsite
-        self.provisioning_api = None  # type: ProvisioningAPI
-
-    def __iter__(self):
-        yield self.az
-        yield self.db
-        yield self.config
-        yield self.loop
-        yield self.bot

mautrix_telegram/db.py

@@ -1,183 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
-                        BigInteger, String, Boolean, Text)
-from sqlalchemy.sql import expression
-from sqlalchemy.orm import relationship, Query
-import json
-
-from .base import Base
-
-
-class Portal(Base):
-    query = None  # type: Query
-    __tablename__ = "portal"
-
-    # Telegram chat information
-    tgid = Column(Integer, primary_key=True)
-    tg_receiver = Column(Integer, primary_key=True)
-    peer_type = Column(String, nullable=False)
-    megagroup = Column(Boolean)
-
-    # Matrix portal information
-    mxid = Column(String, unique=True, nullable=True)
-
-    # Telegram chat metadata
-    username = Column(String, nullable=True)
-    title = Column(String, nullable=True)
-    about = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-
-
-class Message(Base):
-    query = None  # type: Query
-    __tablename__ = "message"
-
-    mxid = Column(String)
-    mx_room = Column(String)
-    tgid = Column(Integer, primary_key=True)
-    tg_space = Column(Integer, primary_key=True)
-
-    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
-
-
-class UserPortal(Base):
-    query = None  # type: Query
-    __tablename__ = "user_portal"
-
-    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
-                  primary_key=True)
-    portal = Column(Integer, primary_key=True)
-    portal_receiver = Column(Integer, primary_key=True)
-
-    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
-                                           ("portal.tgid", "portal.tg_receiver"),
-                                           onupdate="CASCADE", ondelete="CASCADE"),)
-
-
-class User(Base):
-    query = None  # type: Query
-    __tablename__ = "user"
-
-    mxid = Column(String, primary_key=True)
-    tgid = Column(Integer, nullable=True, unique=True)
-    tg_username = Column(String, nullable=True)
-    saved_contacts = Column(Integer, default=0, nullable=False)
-    contacts = relationship("Contact", uselist=True,
-                            cascade="save-update, merge, delete, delete-orphan")
-    portals = relationship("Portal", secondary="user_portal")
-
-
-class RoomState(Base):
-    query = None  # type: Query
-    __tablename__ = "mx_room_state"
-
-    room_id = Column(String, primary_key=True)
-    _power_levels_text = Column("power_levels", Text, nullable=True)
-    _power_levels_json = None
-
-    @property
-    def has_power_levels(self):
-        return bool(self._power_levels_text)
-
-    @property
-    def power_levels(self):
-        if not self._power_levels_json and self._power_levels_text:
-            self._power_levels_json = json.loads(self._power_levels_text)
-        return self._power_levels_json or {}
-
-    @power_levels.setter
-    def power_levels(self, val):
-        self._power_levels_json = val
-        self._power_levels_text = json.dumps(val)
-
-
-class UserProfile(Base):
-    query = None  # type: Query
-    __tablename__ = "mx_user_profile"
-
-    room_id = Column(String, primary_key=True)
-    user_id = Column(String, primary_key=True)
-    membership = Column(String, nullable=False, default="leave")
-    displayname = Column(String, nullable=True)
-    avatar_url = Column(String, nullable=True)
-
-    def dict(self):
-        return {
-            "membership": self.membership,
-            "displayname": self.displayname,
-            "avatar_url": self.avatar_url,
-        }
-
-
-class Contact(Base):
-    query = None  # type: Query
-    __tablename__ = "contact"
-
-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
-    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
-
-
-class Puppet(Base):
-    query = None  # type: Query
-    __tablename__ = "puppet"
-
-    id = Column(Integer, primary_key=True)
-    custom_mxid = Column(String, nullable=True)
-    access_token = Column(String, nullable=True)
-    displayname = Column(String, nullable=True)
-    displayname_source = Column(Integer, nullable=True)
-    username = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-    is_bot = Column(Boolean, nullable=True)
-    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
-
-
-# Fucking Telegram not telling bots what chats they are in 3:<
-class BotChat(Base):
-    query = None  # type: Query
-    __tablename__ = "bot_chat"
-    id = Column(Integer, primary_key=True)
-    type = Column(String, nullable=False)
-
-
-class TelegramFile(Base):
-    query = None  # type: Query
-    __tablename__ = "telegram_file"
-
-    id = Column(String, primary_key=True)
-    mxc = Column(String)
-    mime_type = Column(String)
-    was_converted = Column(Boolean)
-    timestamp = Column(BigInteger)
-    size = Column(Integer, nullable=True)
-    width = Column(Integer, nullable=True)
-    height = Column(Integer, nullable=True)
-    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
-    thumbnail = relationship("TelegramFile", uselist=False)
-
-
-def init(db_session):
-    Portal.query = db_session.query_property()
-    Message.query = db_session.query_property()
-    UserPortal.query = db_session.query_property()
-    User.query = db_session.query_property()
-    Puppet.query = db_session.query_property()
-    BotChat.query = db_session.query_property()
-    TelegramFile.query = db_session.query_property()
-    UserProfile.query = db_session.query_property()
-    RoomState.query = db_session.query_property()

mautrix_telegram/db/__init__.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,23 +13,31 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Database
+
+from .bot_chat import BotChat
+from .message import Message
+from .portal import Portal
+from .puppet import Puppet
+from .telegram_file import TelegramFile
+from .telethon_session import PgSession
+from .upgrade import upgrade_table
+from .user import User
 
 
-def format_duration(seconds: int) -> str:
-    def pluralize(count, singular):
-        return singular if count == 1 else singular + "s"
+def init(db: Database) -> None:
+    for table in (Portal, Message, User, Puppet, TelegramFile, BotChat, PgSession):
+        table.db = db
 
-    def include(count, word):
-        return f"{count} {pluralize(count, word)}" if count > 0 else ""
 
-    minutes, seconds = divmod(seconds, 60)
-    hours, minutes = divmod(minutes, 60)
-    days, hours = divmod(hours, 24)
-    parts = [a for a in [
-        include(days, "day"),
-        include(hours, "hour"),
-        include(minutes, "minute"),
-        include(seconds, "second")] if a]
-    if len(parts) > 2:
-        return "{} and {}".format(", ".join(parts[:-1]), parts[-1])
-    return " and ".join(parts)
+__all__ = [
+    "upgrade_table",
+    "init",
+    "Portal",
+    "Message",
+    "User",
+    "Puppet",
+    "TelegramFile",
+    "BotChat",
+    "PgSession",
+]

mautrix_telegram/db/bot_chat.py

@@ -0,0 +1,55 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.util.async_db import Database
+
+from ..types import TelegramID
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+# Fucking Telegram not telling bots what chats they are in 3:<
+@dataclass
+class BotChat:
+    db: ClassVar[Database] = fake_db
+
+    id: TelegramID
+    type: str
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> BotChat | None:
+        if row is None:
+            return None
+        return cls(**row)
+
+    @classmethod
+    async def delete_by_id(cls, chat_id: TelegramID) -> None:
+        await cls.db.execute("DELETE FROM bot_chat WHERE id=$1", chat_id)
+
+    @classmethod
+    async def all(cls) -> list[BotChat]:
+        rows = await cls.db.fetch("SELECT id, type FROM bot_chat")
+        return [cls._from_row(row) for row in rows]
+
+    async def insert(self) -> None:
+        q = "INSERT INTO bot_chat (id, type) VALUES ($1, $2)"
+        await self.db.execute(q, self.id, self.type)

mautrix_telegram/db/message.py

@@ -0,0 +1,161 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.types import EventID, RoomID
+from mautrix.util.async_db import Database
+
+from ..types import TelegramID
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+@dataclass
+class Message:
+    db: ClassVar[Database] = fake_db
+
+    mxid: EventID
+    mx_room: RoomID
+    tgid: TelegramID
+    tg_space: TelegramID
+    edit_index: int
+    redacted: bool = False
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> Message | None:
+        if row is None:
+            return None
+        return cls(**row)
+
+    columns: ClassVar[str] = "mxid, mx_room, tgid, tg_space, edit_index, redacted"
+
+    @classmethod
+    async def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> list[Message]:
+        q = f"SELECT {cls.columns} FROM message WHERE tgid=$1 AND tg_space=$2"
+        rows = await cls.db.fetch(q, tgid, tg_space)
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def get_one_by_tgid(
+        cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+    ) -> Message | None:
+        if edit_index < 0:
+            q = (
+                f"SELECT {cls.columns} FROM message WHERE tgid=$1 AND tg_space=$2 "
+                f"ORDER BY edit_index DESC LIMIT 1 OFFSET {-edit_index - 1}"
+            )
+            row = await cls.db.fetchrow(q, tgid, tg_space)
+        else:
+            q = (
+                f"SELECT {cls.columns} FROM message"
+                " WHERE tgid=$1 AND tg_space=$2 AND edit_index=$3"
+            )
+            row = await cls.db.fetchrow(q, tgid, tg_space, edit_index)
+        return cls._from_row(row)
+
+    @classmethod
+    async def get_first_by_tgids(
+        cls, tgids: list[TelegramID], tg_space: TelegramID
+    ) -> list[Message]:
+        if cls.db.scheme == "postgres":
+            q = (
+                f"SELECT {cls.columns} FROM message"
+                " WHERE tgid=ANY($1) AND tg_space=$2 AND edit_index=0"
+            )
+            rows = await cls.db.fetch(q, tgids, tg_space)
+        else:
+            tgid_placeholders = ("?," * len(tgids)).rstrip(",")
+            q = (
+                f"SELECT {cls.columns} FROM message "
+                f"WHERE tg_space=? AND edit_index=0 AND tgid IN ({tgid_placeholders})"
+            )
+            rows = await cls.db.fetch(q, tg_space, *tgids)
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
+        return (
+            await cls.db.fetchval(
+                "SELECT COUNT(tg_space) FROM message WHERE mxid=$1 AND mx_room=$2", mxid, mx_room
+            )
+            or 0
+        )
+
+    @classmethod
+    async def find_last(cls, mx_room: RoomID, tg_space: TelegramID) -> Message | None:
+        q = (
+            f"SELECT {cls.columns} FROM message WHERE mx_room=$1 AND tg_space=$2 "
+            f"ORDER BY tgid DESC LIMIT 1"
+        )
+        return cls._from_row(await cls.db.fetchrow(q, mx_room, tg_space))
+
+    @classmethod
+    async def delete_all(cls, mx_room: RoomID) -> None:
+        await cls.db.execute("DELETE FROM message WHERE mx_room=$1", mx_room)
+
+    @classmethod
+    async def get_by_mxid(
+        cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
+    ) -> Message | None:
+        q = f"SELECT {cls.columns} FROM message WHERE mxid=$1 AND mx_room=$2 AND tg_space=$3"
+        return cls._from_row(await cls.db.fetchrow(q, mxid, mx_room, tg_space))
+
+    @classmethod
+    async def get_by_mxids(
+        cls, mxids: list[EventID], mx_room: RoomID, tg_space: TelegramID
+    ) -> list[Message]:
+        if cls.db.scheme == "postgres":
+            q = (
+                f"SELECT {cls.columns} FROM message"
+                " WHERE mxid=ANY($1) AND mx_room=$2 AND tg_space=$3"
+            )
+            rows = await cls.db.fetch(q, mxids, mx_room, tg_space)
+        else:
+            mxid_placeholders = ("?," * len(mxids)).rstrip(",")
+            q = (
+                f"SELECT {cls.columns} FROM message "
+                f"WHERE mx_room=? AND tg_space=? AND mxid IN ({mxid_placeholders})"
+            )
+            rows = await cls.db.fetch(q, mx_room, tg_space, *mxids)
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def replace_temp_mxid(cls, temp_mxid: str, mx_room: RoomID, real_mxid: EventID) -> None:
+        q = "UPDATE message SET mxid=$1 WHERE mxid=$2 AND mx_room=$3"
+        await cls.db.execute(q, real_mxid, temp_mxid, mx_room)
+
+    async def insert(self) -> None:
+        q = (
+            "INSERT INTO message (mxid, mx_room, tgid, tg_space, edit_index, redacted) "
+            "VALUES ($1, $2, $3, $4, $5, $6)"
+        )
+        await self.db.execute(
+            q, self.mxid, self.mx_room, self.tgid, self.tg_space, self.edit_index, self.redacted
+        )
+
+    async def delete(self) -> None:
+        q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2 AND tg_space=$3"
+        await self.db.execute(q, self.mxid, self.mx_room, self.tg_space)
+
+    async def mark_redacted(self) -> None:
+        self.redacted = True
+        q = "UPDATE message SET redacted=true WHERE mxid=$1 AND mx_room=$2"
+        await self.db.execute(q, self.mxid, self.mx_room)

mautrix_telegram/db/portal.py

@@ -0,0 +1,148 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar
+import json
+
+from asyncpg import Record
+from attr import dataclass
+import attr
+
+from mautrix.types import ContentURI, EventID, RoomID
+from mautrix.util.async_db import Database
+
+from ..types import TelegramID
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+@dataclass
+class Portal:
+    db: ClassVar[Database] = fake_db
+
+    # Telegram chat information
+    tgid: TelegramID
+    tg_receiver: TelegramID
+    peer_type: str
+    megagroup: bool
+
+    # Matrix portal information
+    mxid: RoomID | None
+    avatar_url: ContentURI | None
+    encrypted: bool
+
+    sponsored_event_id: EventID | None
+    sponsored_event_ts: int | None
+    sponsored_msg_random_id: bytes | None
+
+    # Telegram chat metadata
+    username: str | None
+    title: str | None
+    about: str | None
+    photo_id: str | None
+
+    local_config: dict[str, Any] = attr.ib(factory=lambda: {})
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> Portal | None:
+        if row is None:
+            return None
+        data = {**row}
+        data["local_config"] = json.loads(data.pop("config", None) or "{}")
+        return cls(**data)
+
+    columns: ClassVar[str] = (
+        "tgid, tg_receiver, peer_type, megagroup, mxid, avatar_url, encrypted, sponsored_event_id,"
+        "sponsored_event_ts, sponsored_msg_random_id, username, title, about, photo_id, config"
+    )
+
+    @classmethod
+    async def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Portal | None:
+        q = f"SELECT {cls.columns} FROM portal WHERE tgid=$1 AND tg_receiver=$2"
+        return cls._from_row(await cls.db.fetchrow(q, tgid, tg_receiver))
+
+    @classmethod
+    async def get_by_mxid(cls, mxid: RoomID) -> Portal | None:
+        q = f"SELECT {cls.columns} FROM portal WHERE mxid=$1"
+        return cls._from_row(await cls.db.fetchrow(q, mxid))
+
+    @classmethod
+    async def find_by_username(cls, username: str) -> Portal | None:
+        q = f"SELECT {cls.columns} FROM portal WHERE lower(username)=$1"
+        return cls._from_row(await cls.db.fetchrow(q, username.lower()))
+
+    @classmethod
+    async def find_private_chats(cls, tg_receiver: TelegramID) -> list[Portal]:
+        q = f"SELECT {cls.columns} FROM portal WHERE tg_receiver=$1 AND peer_type='user'"
+        return [cls._from_row(row) for row in await cls.db.fetch(q, tg_receiver)]
+
+    @classmethod
+    async def all(cls) -> list[Portal]:
+        rows = await cls.db.fetch(f"SELECT {cls.columns} FROM portal")
+        return [cls._from_row(row) for row in rows]
+
+    @property
+    def _values(self):
+        return (
+            self.tgid,
+            self.tg_receiver,
+            self.peer_type,
+            self.mxid,
+            self.avatar_url,
+            self.encrypted,
+            self.sponsored_event_id,
+            self.sponsored_event_ts,
+            self.sponsored_msg_random_id,
+            self.username,
+            self.title,
+            self.about,
+            self.photo_id,
+            self.megagroup,
+            json.dumps(self.local_config) if self.local_config else None,
+        )
+
+    async def save(self) -> None:
+        q = (
+            "UPDATE portal SET mxid=$4, avatar_url=$5, encrypted=$6, sponsored_event_id=$7,"
+            "                  sponsored_event_ts=$8, sponsored_msg_random_id=$9, username=$10,"
+            "                  title=$11, about=$12, photo_id=$13, megagroup=$14, config=$15 "
+            "WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)"
+        )
+        await self.db.execute(q, *self._values)
+
+    async def update_id(self, id: TelegramID, peer_type: str) -> None:
+        q = (
+            "UPDATE portal SET tgid=$1, tg_receiver=$1, peer_type=$2 "
+            "WHERE tgid=$3 AND tg_receiver=$3"
+        )
+        await self.db.execute(q, id, peer_type, self.tgid)
+        self.tgid = id
+        self.tg_receiver = id
+        self.peer_type = peer_type
+
+    async def insert(self) -> None:
+        q = (
+            "INSERT INTO portal (tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,"
+            "                    sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,"
+            "                    username, title, about, photo_id, megagroup, config) "
+            "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"
+        )
+        await self.db.execute(q, *self._values)
+
+    async def delete(self) -> None:
+        q = "DELETE FROM portal WHERE tgid=$1 AND tg_receiver=$2"
+        await self.db.execute(q, self.tgid, self.tg_receiver)

mautrix_telegram/db/puppet.py

@@ -0,0 +1,130 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from asyncpg import Record
+from attr import dataclass
+from yarl import URL
+
+from mautrix.types import SyncToken, UserID
+from mautrix.util.async_db import Database
+
+from ..types import TelegramID
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+@dataclass
+class Puppet:
+    db: ClassVar[Database] = fake_db
+
+    id: TelegramID
+
+    is_registered: bool
+
+    displayname: str | None
+    displayname_source: TelegramID | None
+    displayname_contact: bool
+    displayname_quality: int
+    disable_updates: bool
+    username: str | None
+    photo_id: str | None
+    is_bot: bool | None
+
+    custom_mxid: UserID | None
+    access_token: str | None
+    next_batch: SyncToken | None
+    base_url: URL | None
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> Puppet | None:
+        if row is None:
+            return None
+        data = {**row}
+        base_url = data.pop("base_url", None)
+        return cls(**data, base_url=URL(base_url) if base_url else None)
+
+    columns: ClassVar[str] = (
+        "id, is_registered, displayname, displayname_source, displayname_contact, "
+        "displayname_quality, disable_updates, username, photo_id, is_bot, "
+        "custom_mxid, access_token, next_batch, base_url"
+    )
+
+    @classmethod
+    async def all_with_custom_mxid(cls) -> list[Puppet]:
+        q = f"SELECT {cls.columns} FROM puppet WHERE custom_mxid<>''"
+        return [cls._from_row(row) for row in await cls.db.fetch(q)]
+
+    @classmethod
+    async def get_by_tgid(cls, tgid: TelegramID) -> Puppet | None:
+        q = f"SELECT {cls.columns} FROM puppet WHERE id=$1"
+        return cls._from_row(await cls.db.fetchrow(q, tgid))
+
+    @classmethod
+    async def get_by_custom_mxid(cls, mxid: UserID) -> Puppet | None:
+        q = f"SELECT {cls.columns} FROM puppet WHERE custom_mxid=$1"
+        return cls._from_row(await cls.db.fetchrow(q, mxid))
+
+    @classmethod
+    async def find_by_username(cls, username: str) -> Puppet | None:
+        q = f"SELECT {cls.columns} FROM puppet WHERE lower(username)=$1"
+        return cls._from_row(await cls.db.fetchrow(q, username.lower()))
+
+    @classmethod
+    async def find_by_displayname(cls, displayname: str) -> Puppet | None:
+        q = f"SELECT {cls.columns} FROM puppet WHERE displayname=$1"
+        return cls._from_row(await cls.db.fetchrow(q, displayname))
+
+    @property
+    def _values(self):
+        return (
+            self.id,
+            self.is_registered,
+            self.displayname,
+            self.displayname_source,
+            self.displayname_contact,
+            self.displayname_quality,
+            self.disable_updates,
+            self.username,
+            self.photo_id,
+            self.is_bot,
+            self.custom_mxid,
+            self.access_token,
+            self.next_batch,
+            str(self.base_url) if self.base_url else None,
+        )
+
+    async def save(self) -> None:
+        q = (
+            "UPDATE puppet "
+            "SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,"
+            "    displayname_quality=$6, disable_updates=$7, username=$8, photo_id=$9, is_bot=$10,"
+            "    custom_mxid=$11, access_token=$12, next_batch=$13, base_url=$14 "
+            "WHERE id=$1"
+        )
+        await self.db.execute(q, *self._values)
+
+    async def insert(self) -> None:
+        q = (
+            "INSERT INTO puppet ("
+            "    id, is_registered, displayname, displayname_source, displayname_contact,"
+            "    displayname_quality, disable_updates, username, photo_id, is_bot,"
+            "    custom_mxid, access_token, next_batch, base_url"
+            ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)"
+        )
+        await self.db.execute(q, *self._values)

mautrix_telegram/db/telegram_file.py

@@ -0,0 +1,82 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from attr import dataclass
+
+from mautrix.types import ContentURI, EncryptedFile
+from mautrix.util.async_db import Database
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+@dataclass
+class TelegramFile:
+    db: ClassVar[Database] = fake_db
+
+    id: str
+    mxc: ContentURI
+    mime_type: str
+    was_converted: bool
+    timestamp: int
+    size: int | None
+    width: int | None
+    height: int | None
+    decryption_info: EncryptedFile | None
+    thumbnail: TelegramFile | None = None
+
+    @classmethod
+    async def get(cls, loc_id: str, *, _thumbnail: bool = False) -> TelegramFile | None:
+        q = (
+            "SELECT id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail,"
+            "       decryption_info "
+            "FROM telegram_file WHERE id=$1"
+        )
+        row = await cls.db.fetchrow(q, loc_id)
+        if row is None:
+            return None
+        data = {**row}
+        thumbnail_id = data.pop("thumbnail", None)
+        if _thumbnail:
+            # Don't allow more than one level of recursion
+            thumbnail_id = None
+        decryption_info = data.pop("decryption_info", None)
+        return cls(
+            **data,
+            thumbnail=(await cls.get(thumbnail_id, _thumbnail=True)) if thumbnail_id else None,
+            decryption_info=EncryptedFile.parse_json(decryption_info) if decryption_info else None,
+        )
+
+    async def insert(self) -> None:
+        q = (
+            "INSERT INTO telegram_file (id, mxc, mime_type, was_converted, size, width, height, "
+            "                           thumbnail, decryption_info) "
+            "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"
+        )
+        await self.db.execute(
+            q,
+            self.id,
+            self.mxc,
+            self.mime_type,
+            self.was_converted,
+            self.size,
+            self.width,
+            self.height,
+            self.thumbnail.id if self.thumbnail else None,
+            self.decryption_info.json() if self.decryption_info else None,
+        )

mautrix_telegram/db/telethon_session.py

@@ -0,0 +1,207 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+import asyncio
+import datetime
+
+from telethon import utils
+from telethon.crypto import AuthKey
+from telethon.sessions import MemorySession
+from telethon.tl.types import PeerChannel, PeerChat, PeerUser, updates
+
+from mautrix.util.async_db import Database
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+class PgSession(MemorySession):
+    db: ClassVar[Database] = fake_db
+
+    session_id: str
+    _dc_id: int
+    _server_address: str | None
+    _port: int | None
+    _auth_key: AuthKey | None
+    _takeout_id: int | None
+    _process_entities_lock: asyncio.Lock
+
+    def __init__(
+        self,
+        session_id: str,
+        dc_id: int = 0,
+        server_address: str | None = None,
+        port: int | None = None,
+        auth_key: AuthKey | None = None,
+        takeout_id: int | None = None,
+    ) -> None:
+        super().__init__()
+        self.session_id = session_id
+        self._dc_id = dc_id
+        self._server_address = server_address
+        self._port = port
+        self._auth_key = auth_key
+        self._takeout_id = takeout_id
+        self._process_entities_lock = asyncio.Lock()
+
+    def clone(self, to_instance=None) -> MemorySession:
+        # We don't want to store data of clones
+        # (which are used for temporarily connecting to different DCs)
+        return super().clone(MemorySession())
+
+    @property
+    def auth_key_bytes(self) -> bytes | None:
+        return self._auth_key.key if self._auth_key else None
+
+    @classmethod
+    async def get(cls, session_id: str) -> PgSession:
+        q = (
+            "SELECT session_id, dc_id, server_address, port, auth_key FROM telethon_sessions "
+            "WHERE session_id=$1"
+        )
+        row = await cls.db.fetchrow(q, session_id)
+        if row is None:
+            return cls(session_id)
+        data = {**row}
+        auth_key = AuthKey(data.pop("auth_key", None))
+        return cls(**data, auth_key=auth_key)
+
+    @classmethod
+    async def has(cls, session_id: str) -> bool:
+        q = "SELECT COUNT(*) FROM telethon_sessions WHERE session_id=$1"
+        count = await cls.db.fetchval(q, session_id)
+        return count > 0
+
+    async def save(self) -> None:
+        q = (
+            "INSERT INTO telethon_sessions (session_id, dc_id, server_address, port, auth_key) "
+            "VALUES ($1, $2, $3, $4, $5) ON CONFLICT (session_id) "
+            "DO UPDATE SET dc_id=$2, server_address=$3, port=$4, auth_key=$5"
+        )
+        await self.db.execute(
+            q, self.session_id, self.dc_id, self.server_address, self.port, self.auth_key_bytes
+        )
+
+    _tables: ClassVar[tuple[str, ...]] = (
+        "telethon_sessions",
+        "telethon_entities",
+        "telethon_sent_files",
+        "telethon_update_state",
+    )
+
+    async def delete(self) -> None:
+        async with self.db.acquire() as conn, conn.transaction():
+            for table in self._tables:
+                await conn.execute(f"DELETE FROM {table} WHERE session_id=$1", self.session_id)
+
+    async def close(self) -> None:
+        # Nothing to do here, DB connection is global
+        pass
+
+    async def get_update_state(self, entity_id: int) -> updates.State | None:
+        q = (
+            "SELECT pts, qts, date, seq, unread_count FROM telethon_update_state "
+            "WHERE session_id=$1 AND entity_id=$2"
+        )
+        row = await self.db.fetchrow(q, self.session_id, entity_id)
+        if row is None:
+            return None
+        date = datetime.datetime.utcfromtimestamp(row["date"])
+        return updates.State(row["pts"], row["qts"], date, row["seq"], row["unread_count"])
+
+    async def set_update_state(self, entity_id: int, row: updates.State) -> None:
+        q = (
+            "INSERT INTO telethon_update_state"
+            "    (session_id, entity_id, pts, qts, date, seq, unread_count) "
+            "VALUES ($1, $2, $3, $4, $5, $6, $7)"
+            "ON CONFLICT (session_id, entity_id) DO UPDATE"
+            "    SET pts=$3, qts=$4, date=$5, seq=$6, unread_count=$7"
+        )
+        ts = row.date.timestamp()
+        await self.db.execute(
+            q, self.session_id, entity_id, row.pts, row.qts, ts, row.seq, row.unread_count
+        )
+
+    def _entity_values_to_row(
+        self, id: int, hash: int, username: str | None, phone: str | int | None, name: str | None
+    ) -> tuple[str, int, int, str | None, str | None, str | None]:
+        return self.session_id, id, hash, username, str(phone) if phone else None, name
+
+    async def process_entities(self, tlo) -> None:
+        # Postgres likes to deadlock on simultaneous upserts, so just lock the whole thing here
+        # TODO: make sure postgres doesn't deadlock on upserts when session_id is different
+        async with self._process_entities_lock:
+            await self._locked_process_entities(tlo)
+
+    async def _locked_process_entities(self, tlo) -> None:
+        rows: list[
+            tuple[str, int, int, str | None, str | None, str | None]
+        ] = self._entities_to_rows(tlo)
+        if not rows:
+            return
+        if self.db.scheme == "postgres":
+            q = (
+                "INSERT INTO telethon_entities (session_id, id, hash, username, phone, name) "
+                "VALUES ($1, unnest($2::bigint[]), unnest($3::bigint[]), "
+                "        unnest($4::text[]), unnest($5::text[]), unnest($6::text[])) "
+                "ON CONFLICT (session_id, id) DO UPDATE"
+                "  SET hash=excluded.hash, username=excluded.username,"
+                "      phone=excluded.phone, name=excluded.name"
+            )
+            _, ids, hashes, usernames, phones, names = zip(*rows)
+            await self.db.execute(q, self.session_id, ids, hashes, usernames, phones, names)
+        else:
+            q = (
+                "INSERT INTO telethon_entities (session_id, id, hash, username, phone, name) "
+                "VALUES ($1, $2, $3, $4, $5, $6) "
+                "ON CONFLICT (session_id, id) DO UPDATE "
+                "    SET hash=$3, username=$4, phone=$5, name=$6"
+            )
+            await self.db.executemany(q, rows)
+
+    async def _select_entity(
+        self, constraint: str, *args: str | int | tuple[int, ...]
+    ) -> tuple[int, int] | None:
+        row = await self.db.fetchrow(
+            f"SELECT id, hash FROM telethon_entities WHERE {constraint}", *args
+        )
+        if row is None:
+            return None
+        return row["id"], row["hash"]
+
+    async def get_entity_rows_by_phone(self, key: str | int) -> tuple[int, int] | None:
+        return await self._select_entity("phone=$1", str(key))
+
+    async def get_entity_rows_by_username(self, key: str) -> tuple[int, int] | None:
+        return await self._select_entity("username=$1", key)
+
+    async def get_entity_rows_by_name(self, key: str) -> tuple[int, int] | None:
+        return await self._select_entity("name=$1", key)
+
+    async def get_entity_rows_by_id(self, key: int, exact: bool = True) -> tuple[int, int] | None:
+        if exact:
+            return await self._select_entity("id=$1", key)
+
+        ids = (
+            utils.get_peer_id(PeerUser(key)),
+            utils.get_peer_id(PeerChat(key)),
+            utils.get_peer_id(PeerChannel(key)),
+        )
+        if self.db.scheme == "postgres":
+            return await self._select_entity("id=ANY($1)", ids)
+        else:
+            return await self._select_entity(f"id IN ($1, $2, $3)", *ids)

mautrix_telegram/db/upgrade/__init__.py

@@ -0,0 +1,5 @@
+from mautrix.util.async_db import UpgradeTable
+
+upgrade_table = UpgradeTable()
+
+from . import v01_initial_revision, v02_sponsored_events

mautrix_telegram/db/upgrade/v01_initial_revision.py

@@ -0,0 +1,315 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from . import upgrade_table
+
+legacy_version_query = "SELECT version_num FROM alembic_version"
+last_legacy_version = "bfc0a39bfe02"
+
+
+def table_exists(scheme: str, name: str) -> str:
+    if scheme == "sqlite":
+        return f"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='{name}')"
+    elif scheme == "postgres":
+        return f"SELECT EXISTS(SELECT FROM information_schema.tables WHERE table_name='{name}')"
+    raise RuntimeError("unsupported database scheme")
+
+
+@upgrade_table.register(description="Initial asyncpg revision")
+async def upgrade_v1(conn: Connection, scheme: str) -> None:
+    is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
+    if is_legacy:
+        await migrate_legacy_to_v1(conn, scheme)
+    else:
+        await create_v1_tables(conn)
+
+
+async def migrate_legacy_to_v1(conn: Connection, scheme: str) -> None:
+    legacy_version = await conn.fetchval(legacy_version_query)
+    if legacy_version != last_legacy_version:
+        raise RuntimeError(
+            "Legacy database is not on last version. "
+            "Please upgrade the old database with alembic or drop it completely first."
+        )
+    if scheme != "sqlite":
+        await conn.execute(
+            """
+            ALTER TABLE contact
+              DROP CONSTRAINT contact_user_fkey,
+              DROP CONSTRAINT contact_contact_fkey,
+              ADD CONSTRAINT contact_user_fkey FOREIGN KEY (contact) REFERENCES puppet(id)
+                ON DELETE CASCADE ON UPDATE CASCADE,
+              ADD CONSTRAINT contact_contact_fkey FOREIGN KEY ("user") REFERENCES "user"(tgid)
+                ON DELETE CASCADE ON UPDATE CASCADE
+            """
+        )
+        await conn.execute(
+            """
+            ALTER TABLE telethon_sessions
+              DROP CONSTRAINT telethon_sessions_pkey,
+              ADD CONSTRAINT telethon_sessions_pkey PRIMARY KEY (session_id)
+            """
+        )
+        await conn.execute(
+            """
+            ALTER TABLE telegram_file
+              DROP CONSTRAINT fk_file_thumbnail,
+              ADD CONSTRAINT fk_file_thumbnail
+                FOREIGN KEY (thumbnail) REFERENCES telegram_file(id)
+                ON UPDATE CASCADE ON DELETE SET NULL
+            """
+        )
+        await conn.execute("ALTER TABLE puppet ALTER COLUMN id DROP DEFAULT")
+        await conn.execute("DROP SEQUENCE puppet_id_seq")
+        await conn.execute("ALTER TABLE bot_chat ALTER COLUMN id DROP DEFAULT")
+        await conn.execute("DROP SEQUENCE bot_chat_id_seq")
+        await conn.execute("ALTER TABLE portal ALTER COLUMN config TYPE jsonb USING config::jsonb")
+        await conn.execute(
+            "ALTER TABLE telegram_file ALTER COLUMN decryption_info TYPE jsonb "
+            "USING decryption_info::jsonb"
+        )
+        await varchar_to_text(conn)
+    else:
+        await conn.execute(
+            """CREATE TABLE telethon_sessions_new (
+                session_id     TEXT PRIMARY KEY,
+                dc_id          INTEGER,
+                server_address TEXT,
+                port           INTEGER,
+                auth_key       bytea
+            )"""
+        )
+        await conn.execute(
+            """
+            INSERT INTO telethon_sessions_new (session_id, dc_id, server_address, port, auth_key)
+            SELECT session_id, dc_id, server_address, port, auth_key FROM telethon_sessions
+            """
+        )
+        await conn.execute("DROP TABLE telethon_sessions")
+        await conn.execute("ALTER TABLE telethon_sessions_new RENAME TO telethon_sessions")
+
+    await update_state_store(conn, scheme)
+    await conn.execute('ALTER TABLE "user" ADD COLUMN is_bot BOOLEAN NOT NULL DEFAULT false')
+    await conn.execute("ALTER TABLE puppet RENAME COLUMN matrix_registered TO is_registered")
+    await conn.execute("DROP TABLE telethon_version")
+    await conn.execute("DROP TABLE alembic_version")
+
+
+async def update_state_store(conn: Connection, scheme: str) -> None:
+    # The Matrix state store already has more or less the correct schema, so set the version
+    await conn.execute("CREATE TABLE mx_version (version INTEGER PRIMARY KEY)")
+    await conn.execute("INSERT INTO mx_version (version) VALUES (2)")
+    await conn.execute("UPDATE mx_user_profile SET membership='LEAVE' WHERE membership='LEFT'")
+    if scheme != "sqlite":
+        # Also add the membership type on postgres
+        await conn.execute(
+            "CREATE TYPE membership AS ENUM ('join', 'leave', 'invite', 'ban', 'knock')"
+        )
+        await conn.execute(
+            "ALTER TABLE mx_user_profile ALTER COLUMN membership TYPE membership "
+            "USING LOWER(membership)::membership"
+        )
+    else:
+        # On SQLite there's no custom type, but we still want to lowercase everything
+        await conn.execute("UPDATE mx_user_profile SET membership=LOWER(membership)")
+
+
+async def varchar_to_text(conn: Connection) -> None:
+    columns_to_adjust = {
+        "user": ("mxid", "tg_username", "tg_phone"),
+        "portal": (
+            "peer_type",
+            "mxid",
+            "username",
+            "title",
+            "about",
+            "photo_id",
+            "avatar_url",
+            "config",
+        ),
+        "message": ("mxid", "mx_room"),
+        "puppet": (
+            "displayname",
+            "username",
+            "photo_id",
+            "access_token",
+            "custom_mxid",
+            "next_batch",
+            "base_url",
+        ),
+        "bot_chat": ("type",),
+        "telegram_file": ("id", "mxc", "mime_type", "thumbnail"),
+        # Phone is a bigint in the old schema, which is safe, but we don't do math on it,
+        # so let's change it to a string
+        "telethon_entities": ("session_id", "username", "name", "phone"),
+        "telethon_sent_files": ("session_id",),
+        "telethon_sessions": ("session_id", "server_address"),
+        "telethon_update_state": ("session_id",),
+        "mx_room_state": ("room_id",),
+        "mx_user_profile": ("room_id", "user_id", "displayname", "avatar_url"),
+    }
+    for table, columns in columns_to_adjust.items():
+        for column in columns:
+            await conn.execute(f'ALTER TABLE "{table}" ALTER COLUMN {column} TYPE TEXT')
+
+
+async def create_v1_tables(conn: Connection) -> None:
+    await conn.execute(
+        """CREATE TABLE "user" (
+            mxid TEXT   PRIMARY KEY,
+            tgid BIGINT UNIQUE,
+            tg_username    TEXT,
+            tg_phone       TEXT,
+            is_bot         BOOLEAN NOT NULL DEFAULT false,
+            saved_contacts INTEGER NOT NULL DEFAULT 0
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE portal (
+            tgid        BIGINT,
+            tg_receiver BIGINT,
+            peer_type   TEXT NOT NULL,
+            mxid        TEXT UNIQUE,
+            avatar_url  TEXT,
+            encrypted   BOOLEAN NOT NULL DEFAULT false,
+            username    TEXT,
+            title       TEXT,
+            about       TEXT,
+            photo_id    TEXT,
+            megagroup   BOOLEAN,
+            config      jsonb,
+            PRIMARY KEY (tgid, tg_receiver)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE message (
+            mxid       TEXT,
+            mx_room    TEXT,
+            tgid       BIGINT NOT NULL,
+            tg_space   BIGINT NOT NULL,
+            edit_index INTEGER NOT NULL,
+            redacted   BOOLEAN NOT NULL DEFAULT false,
+            PRIMARY KEY (tgid, tg_space, edit_index),
+            UNIQUE (mxid, mx_room, tg_space)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE puppet (
+            id BIGINT PRIMARY KEY,
+
+            is_registered BOOLEAN NOT NULL DEFAULT false,
+
+            displayname         TEXT,
+            displayname_source  BIGINT,
+            displayname_contact BOOLEAN NOT NULL DEFAULT true,
+            displayname_quality INTEGER NOT NULL DEFAULT 0,
+            disable_updates     BOOLEAN NOT NULL DEFAULT false,
+            username            TEXT,
+            photo_id            TEXT,
+            is_bot              BOOLEAN,
+
+            access_token TEXT,
+            custom_mxid  TEXT,
+            next_batch   TEXT,
+            base_url     TEXT
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telegram_file (
+            id              TEXT PRIMARY KEY,
+            mxc             TEXT NOT NULL,
+            mime_type       TEXT,
+            was_converted   BOOLEAN NOT NULL DEFAULT false,
+            timestamp       BIGINT  NOT NULL DEFAULT 0,
+            size            BIGINT,
+            width           INTEGER,
+            height          INTEGER,
+            thumbnail       TEXT,
+            decryption_info jsonb,
+            FOREIGN KEY (thumbnail) REFERENCES telegram_file(id)
+                ON UPDATE CASCADE ON DELETE SET NULL
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE bot_chat (
+            id   BIGINT PRIMARY KEY,
+            type TEXT NOT NULL
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE user_portal (
+            "user"          BIGINT,
+            portal          BIGINT,
+            portal_receiver BIGINT,
+            PRIMARY KEY ("user", portal, portal_receiver),
+            FOREIGN KEY ("user") REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (portal, portal_receiver) REFERENCES portal(tgid, tg_receiver)
+                 ON DELETE CASCADE ON UPDATE CASCADE
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE contact (
+            "user"  BIGINT,
+            contact BIGINT,
+            PRIMARY KEY ("user", contact),
+            FOREIGN KEY ("user")  REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (contact) REFERENCES puppet(id)   ON DELETE CASCADE ON UPDATE CASCADE
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_sessions (
+            session_id     TEXT PRIMARY KEY,
+            dc_id          INTEGER,
+            server_address TEXT,
+            port           INTEGER,
+            auth_key       bytea
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_entities (
+            session_id TEXT,
+            id         BIGINT,
+            hash       BIGINT NOT NULL,
+            username   TEXT,
+            phone      TEXT,
+            name       TEXT,
+            PRIMARY KEY (session_id, id)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_sent_files (
+            session_id TEXT,
+            md5_digest bytea,
+            file_size  INTEGER,
+            type       INTEGER,
+            id         BIGINT,
+            hash       BIGINT,
+            PRIMARY KEY (session_id, md5_digest, file_size, type)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_update_state (
+            session_id   TEXT,
+            entity_id    BIGINT,
+            pts          BIGINT,
+            qts          BIGINT,
+            date         BIGINT,
+            seq          BIGINT,
+            unread_count INTEGER,
+            PRIMARY KEY (session_id, entity_id)
+        )"""
+    )

mautrix_telegram/db/upgrade/v02_sponsored_events.py

@@ -0,0 +1,25 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from asyncpg import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add column to store sponsored message event ID in channels")
+async def upgrade_v2(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN sponsored_event_id TEXT")
+    await conn.execute("ALTER TABLE portal ADD COLUMN sponsored_event_ts BIGINT")
+    await conn.execute("ALTER TABLE portal ADD COLUMN sponsored_msg_random_id bytea")

mautrix_telegram/db/user.py

@@ -0,0 +1,138 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar, Iterable
+
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.types import UserID
+from mautrix.util.async_db import Database
+
+from ..types import TelegramID
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+@dataclass
+class User:
+    db: ClassVar[Database] = fake_db
+
+    mxid: UserID
+    tgid: TelegramID | None
+    tg_username: str | None
+    tg_phone: str | None
+    is_bot: bool
+    saved_contacts: int
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> User | None:
+        if row is None:
+            return None
+        return cls(**row)
+
+    columns: ClassVar[str] = "mxid, tgid, tg_username, tg_phone, is_bot, saved_contacts"
+
+    @classmethod
+    async def get_by_tgid(cls, tgid: TelegramID) -> User | None:
+        q = f'SELECT {cls.columns} FROM "user" WHERE tgid=$1'
+        return cls._from_row(await cls.db.fetchrow(q, tgid))
+
+    @classmethod
+    async def get_by_mxid(cls, mxid: UserID) -> User | None:
+        q = f'SELECT {cls.columns} FROM "user" WHERE mxid=$1'
+        return cls._from_row(await cls.db.fetchrow(q, mxid))
+
+    @classmethod
+    async def find_by_username(cls, username: str) -> User | None:
+        q = f'SELECT {cls.columns} FROM "user" WHERE lower(tg_username)=$1'
+        return cls._from_row(await cls.db.fetchrow(q, username.lower()))
+
+    @classmethod
+    async def all_with_tgid(cls) -> list[User]:
+        q = f'SELECT {cls.columns} FROM "user" WHERE tgid IS NOT NULL'
+        return [cls._from_row(row) for row in await cls.db.fetch(q)]
+
+    async def delete(self) -> None:
+        await self.db.execute('DELETE FROM "user" WHERE mxid=$1', self.mxid)
+
+    @property
+    def _values(self):
+        return (
+            self.mxid,
+            self.tgid,
+            self.tg_username,
+            self.tg_phone,
+            self.is_bot,
+            self.saved_contacts,
+        )
+
+    async def save(self) -> None:
+        q = (
+            'UPDATE "user" SET tgid=$2, tg_username=$3, tg_phone=$4, is_bot=$5, saved_contacts=$6 '
+            "WHERE mxid=$1"
+        )
+        await self.db.execute(q, *self._values)
+
+    async def insert(self) -> None:
+        q = (
+            'INSERT INTO "user" (mxid, tgid, tg_username, tg_phone, is_bot, saved_contacts) '
+            "VALUES ($1, $2, $3, $4, $5, $6)"
+        )
+        await self.db.execute(q, *self._values)
+
+    async def get_contacts(self) -> list[TelegramID]:
+        rows = await self.db.fetch('SELECT contact FROM contact WHERE "user"=$1', self.tgid)
+        return [TelegramID(row["contact"]) for row in rows]
+
+    async def set_contacts(self, puppets: Iterable[TelegramID]) -> None:
+        columns = ["user", "contact"]
+        records = [(self.tgid, puppet_id) for puppet_id in puppets]
+        async with self.db.acquire() as conn, conn.transaction():
+            await conn.execute('DELETE FROM contact WHERE "user"=$1', self.tgid)
+            if self.db.scheme == "postgres":
+                await conn.copy_records_to_table("contact", records=records, columns=columns)
+            else:
+                q = 'INSERT INTO contact ("user", contact) VALUES ($1, $2)'
+                await conn.executemany(q, records)
+
+    async def get_portals(self) -> list[tuple[TelegramID, TelegramID]]:
+        q = 'SELECT portal, portal_receiver FROM user_portal WHERE "user"=$1'
+        rows = await self.db.fetch(q, self.tgid)
+        return [(TelegramID(row["portal"]), TelegramID(row["portal_receiver"])) for row in rows]
+
+    async def set_portals(self, portals: Iterable[tuple[TelegramID, TelegramID]]) -> None:
+        columns = ["user", "portal", "portal_receiver"]
+        records = [(self.tgid, tgid, tg_receiver) for tgid, tg_receiver in portals]
+        async with self.db.acquire() as conn, conn.transaction():
+            await conn.execute('DELETE FROM user_portal WHERE "user"=$1', self.tgid)
+            if self.db.scheme == "postgres":
+                await conn.copy_records_to_table("user_portal", records=records, columns=columns)
+            else:
+                q = 'INSERT INTO user_portal ("user", portal, portal_receiver) VALUES ($1, $2, $3)'
+                await conn.executemany(q, records)
+
+    async def register_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
+        q = (
+            'INSERT INTO user_portal ("user", portal, portal_receiver) VALUES ($1, $2, $3) '
+            'ON CONFLICT ("user", portal, portal_receiver) DO NOTHING'
+        )
+        await self.db.execute(q, self.tgid, tgid, tg_receiver)
+
+    async def unregister_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
+        q = 'DELETE FROM user_portal WHERE "user"=$1 AND portal=$2 AND portal_receiver=$3'
+        await self.db.execute(q, self.tgid, tgid, tg_receiver)

mautrix_telegram/example-config.yaml

@@ -0,0 +1,555 @@
+# Homeserver details
+homeserver:
+    # The address that this appservice can use to connect to the homeserver.
+    address: https://example.com
+    # The domain of the homeserver (for MXIDs, etc).
+    domain: example.com
+    # Whether or not to verify the SSL certificate of the homeserver.
+    # Only applies if address starts with https://
+    verify_ssl: true
+    asmux: false
+    # Number of retries for all HTTP requests if the homeserver isn't reachable.
+    http_retry_count: 4
+    # The URL to push real-time bridge status to.
+    # If set, the bridge will make POST requests to this URL whenever a user's Telegram connection state changes.
+    # The bridge will use the appservice as_token to authorize requests.
+    status_endpoint: null
+    # Endpoint for reporting per-message status.
+    message_send_checkpoint_endpoint: null
+
+# Application service host/registration related details
+# Changing these values requires regeneration of the registration.
+appservice:
+    # The address that the homeserver can use to connect to this appservice.
+    address: http://localhost:29317
+    # When using https:// the TLS certificate and key files for the address.
+    tls_cert: false
+    tls_key: false
+
+    # The hostname and port where this appservice should listen.
+    hostname: 0.0.0.0
+    port: 29317
+    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+    max_body_size: 1
+
+    # The full URI to the database. SQLite and Postgres are supported.
+    # Format examples:
+    #   SQLite:   sqlite:///filename.db
+    #   Postgres: postgres://username:password@hostname/dbname
+    database: postgres://username:password@hostname/dbname
+    # Additional arguments for asyncpg.create_pool() or sqlite3.connect()
+    # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool
+    # https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
+    # For sqlite, min_size is used as the connection thread pool size and max_size is ignored.
+    database_opts:
+        min_size: 1
+        max_size: 10
+
+    # Public part of web server for out-of-Matrix interaction with the bridge.
+    # Used for things like login if the user wants to make sure the 2FA password isn't stored in
+    # the HS database.
+    public:
+        # Whether or not the public-facing endpoints should be enabled.
+        enabled: false
+        # The prefix to use in the public-facing endpoints.
+        prefix: /public
+        # The base URL where the public-facing endpoints are available. The prefix is not added
+        # implicitly.
+        external: https://example.com/public
+
+    # Provisioning API part of the web server for automated portal creation and fetching information.
+    # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager).
+    provisioning:
+        # Whether or not the provisioning API should be enabled.
+        enabled: true
+        # The prefix to use in the provisioning API endpoints.
+        prefix: /_matrix/provision/v1
+        # The shared secret to authorize users of the API.
+        # Set to "generate" to generate and save a new token.
+        shared_secret: generate
+
+    # The unique ID of this appservice.
+    id: telegram
+    # Username of the appservice bot.
+    bot_username: telegrambot
+    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+    # to leave display name/avatar as-is.
+    bot_displayname: Telegram bridge bot
+    bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
+
+    # Community ID for bridged users (changes registration file) and rooms.
+    # Must be created manually.
+    #
+    # Example: "+telegram:example.com". Set to false to disable.
+    community_id: false
+
+    # Whether or not to receive ephemeral events via appservice transactions.
+    # Requires MSC2409 support (i.e. Synapse 1.22+).
+    # You should disable bridge -> sync_with_custom_puppets when this is enabled.
+    ephemeral_events: false
+
+    # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
+    as_token: "This value is generated when generating the registration"
+    hs_token: "This value is generated when generating the registration"
+
+# Prometheus telemetry config. Requires prometheus-client to be installed.
+metrics:
+    enabled: false
+    listen_port: 8000
+
+# Manhole config.
+manhole:
+    # Whether or not opening the manhole is allowed.
+    enabled: false
+    # The path for the unix socket.
+    path: /var/tmp/mautrix-telegram.manhole
+    # The list of UIDs who can be added to the whitelist.
+    # If empty, any UIDs can be specified in the open-manhole command.
+    whitelist:
+    - 0
+
+# Bridge config
+bridge:
+    # Localpart template of MXIDs for Telegram users.
+    # {userid} is replaced with the user ID of the Telegram user.
+    username_template: "telegram_{userid}"
+    # Localpart template of room aliases for Telegram portal rooms.
+    # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} )
+    alias_template: "telegram_{groupname}"
+    # Displayname template for Telegram users.
+    # {displayname} is replaced with the display name of the Telegram user.
+    displayname_template: "{displayname} (Telegram)"
+
+    # Set the preferred order of user identifiers which to use in the Matrix puppet display name.
+    # In the (hopefully unlikely) scenario that none of the given keys are found, the numeric user
+    # ID is used.
+    #
+    # If the bridge is working properly, a phone number or an username should always be known, but
+    # the other one can very well be empty.
+    #
+    # Valid keys:
+    #   "full name"          (First and/or last name)
+    #   "full name reversed" (Last and/or first name)
+    #   "first name"
+    #   "last name"
+    #   "username"
+    #   "phone number"
+    displayname_preference:
+    - full name
+    - username
+    - phone number
+    # Maximum length of displayname
+    displayname_max_length: 100
+    # Remove avatars from Telegram ghost users when removed on Telegram. This is disabled by default
+    # as there's no way to determine whether an avatar is removed or just hidden from some users. If
+    # you're on a single-user instance, this should be safe to enable.
+    allow_avatar_remove: false
+
+    # Maximum number of members to sync per portal when starting up. Other members will be
+    # synced when they send messages. The maximum is 10000, after which the Telegram server
+    # will not send any more members.
+    # -1 means no limit (which means it's limited to 10000 by the server)
+    max_initial_member_sync: 100
+    # Whether or not to sync the member list in channels.
+    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
+    # list regardless of this setting.
+    sync_channel_members: true
+    # Whether or not to skip deleted members when syncing members.
+    skip_deleted_members: true
+    # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
+    # their Telegram account at startup.
+    startup_sync: true
+    # Number of most recently active dialogs to check when syncing chats.
+    # Set to 0 to remove limit.
+    sync_update_limit: 0
+    # Number of most recently active dialogs to create portals for when syncing chats.
+    # Set to 0 to remove limit.
+    sync_create_limit: 30
+    # Whether or not to sync and create portals for direct chats at startup.
+    sync_direct_chats: false
+    # The maximum number of simultaneous Telegram deletions to handle.
+    # A large number of simultaneous redactions could put strain on your homeserver.
+    max_telegram_delete: 10
+    # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
+    # at startup and when creating a bridge.
+    sync_matrix_state: true
+    # Allow logging in within Matrix. If false, users can only log in using login-qr or the
+    # out-of-Matrix login website (see appservice.public config section)
+    allow_matrix_login: true
+    # Whether or not to bridge plaintext highlights.
+    # Only enable this if your displayname_template has some static part that the bridge can use to
+    # reliably identify what is a plaintext highlight.
+    plaintext_highlights: false
+    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
+    public_portals: true
+    # Whether or not to use /sync to get presence, read receipts and typing notifications
+    # when double puppeting is enabled
+    sync_with_custom_puppets: true
+    # Whether or not to update the m.direct account data event when double puppeting is enabled.
+    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
+    # and is therefore prone to race conditions.
+    sync_direct_chat_list: false
+    # Servers to always allow double puppeting from
+    double_puppet_server_map:
+        example.com: https://example.com
+    # Allow using double puppeting from any server with a valid client .well-known file.
+    double_puppet_allow_discovery: false
+    # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth
+    #
+    # If set, custom puppets will be enabled automatically for local users
+    # instead of users having to find an access token and run `login-matrix`
+    # manually.
+    # If using this for other servers than the bridge's server,
+    # you must also set the URL in the double_puppet_server_map.
+    login_shared_secret_map:
+        example.com: foobar
+    # Set to false to disable link previews in messages sent to Telegram.
+    telegram_link_preview: true
+    # Whether or not the !tg join command should do a HTTP request
+    # to resolve redirects in invite links.
+    invite_link_resolve: false
+    # Use inline images instead of a separate message for the caption.
+    # N.B. Inline images are not supported on all clients (e.g. Element iOS/Android).
+    inline_images: false
+    # Maximum size of image in megabytes before sending to Telegram as a document.
+    image_as_file_size: 10
+    # Maximum size of Telegram documents in megabytes to bridge.
+    max_document_size: 100
+    # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
+    # streaming from/to Matrix and using many connections for Telegram.
+    # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
+    # This option uses internal Telethon implementation details and may break with minor updates.
+    parallel_file_transfer: false
+    # Whether or not created rooms should have federation enabled.
+    # If false, created portal rooms will never be federated.
+    federate_rooms: true
+    # Settings for converting animated stickers.
+    animated_sticker:
+        # Format to which animated stickers should be converted.
+        # disable - No conversion, send as-is (gzipped lottie)
+        # png - converts to non-animated png (fastest),
+        # gif - converts to animated gif
+        # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support
+        target: gif
+        # Arguments for converter. All converters take width and height.
+        args:
+            width: 256
+            height: 256
+            fps: 25 # only for webm and gif (2, 5, 10, 20 or 25 recommended)
+    # End-to-bridge encryption support options.
+    #
+    # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info.
+    encryption:
+        # Allow encryption, work in group chat rooms with e2ee enabled
+        allow: false
+        # Default to encryption, force-enable encryption in all portals the bridge creates
+        # This will cause the bridge bot to be in private chats for the encryption to work properly.
+        default: false
+        # Database for the encryption data. If set to `default`, will use the appservice database.
+        database: default
+        # Options for automatic key sharing.
+        key_sharing:
+            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
+            # You must use a client that supports requesting keys from other users to use this feature.
+            allow: false
+            # Require the requesting device to have a valid cross-signing signature?
+            # This doesn't require that the bridge has verified the device, only that the user has verified it.
+            # Not yet implemented.
+            require_cross_signing: false
+            # Require devices to be verified by the bridge?
+            # Verification by the bridge is not yet implemented.
+            require_verification: true
+    # Whether or not to explicitly set the avatar and room name for private
+    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
+    private_chat_portal_meta: false
+    # Whether or not the bridge should send a read receipt from the bridge bot when a message has
+    # been sent to Telegram.
+    delivery_receipts: false
+    # Whether or not delivery errors should be reported as messages in the Matrix room.
+    delivery_error_reports: false
+    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
+    # This field will automatically be changed back to false after it,
+    # except if the config file is not writable.
+    resend_bridge_info: false
+    # When using double puppeting, should muted chats be muted in Matrix?
+    mute_bridging: false
+    # When using double puppeting, should pinned chats be moved to a specific tag in Matrix?
+    # The favorites tag is `m.favourite`.
+    pinned_tag: null
+    # Same as above for archived chats, the low priority tag is `m.lowpriority`.
+    archive_tag: null
+    # Whether or not mute status and tags should only be bridged when the portal room is created.
+    tag_only_on_create: true
+    # Should leaving the room on Matrix make the user leave on Telegram?
+    bridge_matrix_leave: true
+    # Should the user be kicked out of all portals when logging out of the bridge?
+    kick_on_logout: true
+    # Settings for backfilling messages from Telegram.
+    backfill:
+        # Whether or not the Telegram ghosts of logged in Matrix users should be
+        # invited to private chats when backfilling history from Telegram. This is
+        # usually needed to prevent rate limits and to allow timestamp massaging.
+        invite_own_puppet: true
+        # Maximum number of messages to backfill without using a takeout.
+        # The first time a takeout is used, the user has to manually approve it from a different
+        # device. If initial_limit or missed_limit are higher than this value, the bridge will ask
+        # the user to accept the takeout after logging in before syncing any chats.
+        takeout_limit: 100
+        # Maximum number of messages to backfill initially.
+        # Set to 0 to disable backfilling when creating portal, or -1 to disable the limit.
+        #
+        # N.B. Initial backfill will only start after member sync. Make sure your
+        #      max_initial_member_sync is set to a low enough value so it doesn't take forever.
+        initial_limit: 0
+        # Maximum number of messages to backfill if messages were missed while the bridge was
+        # disconnected. Note that this only works for logged in users and only if the chat isn't
+        # older than sync_update_limit
+        # Set to 0 to disable backfilling missed messages.
+        missed_limit: 50
+        # If using double puppeting, should notifications be disabled
+        # while the initial backfill is in progress?
+        disable_notifications: false
+        # Whether or not to enable backfilling in normal groups.
+        # Normal groups have numerous technical problems in Telegram, and backfilling normal groups
+        # will likely cause problems if there are multiple Matrix users in the group.
+        normal_groups: false
+
+    # Overrides for base power levels.
+    initial_power_level_overrides:
+        user: {}
+        group: {}
+
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    bridge_notices:
+        # Whether or not Matrix bot messages (type m.notice) should be bridged.
+        default: false
+        # List of user IDs for whom the previous flag is flipped.
+        # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
+        #      notices from users listed here will be bridged.
+        exceptions:
+        - "@importantbot:example.com"
+
+    # Some config options related to Telegram message deduplication.
+    # The default values are usually fine, but some debug messages/warnings might recommend you
+    # change these.
+    deduplication:
+        # Whether or not to check the database if the message about to be sent is a duplicate.
+        pre_db_check: false
+        # The number of latest events to keep when checking for duplicates.
+        # You might need to increase this on high-traffic bridge instances.
+        cache_queue_length: 20
+
+    # The formats to use when sending messages to Telegram via the relay bot.
+    # Text msgtypes (m.text, m.notice and m.emote) support HTML, media msgtypes don't.
+    #
+    # Available variables:
+    #   $sender_displayname - The display name of the sender (e.g. Example User)
+    #   $sender_username    - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
+    #   $sender_mxid        - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $message            - The message content
+    message_formats:
+        m.text: "<b>$sender_displayname</b>: $message"
+        m.notice: "<b>$sender_displayname</b>: $message"
+        m.emote: "* <b>$sender_displayname</b> $message"
+        m.file: "<b>$sender_displayname</b> sent a file: $message"
+        m.image: "<b>$sender_displayname</b> sent an image: $message"
+        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "<b>$sender_displayname</b> sent a video: $message"
+        m.location: "<b>$sender_displayname</b> sent a location: $message"
+    # Telegram doesn't have built-in emotes, this field specifies how m.emote's from authenticated
+    # users are sent to telegram. All fields in message_formats are supported. Additionally, the
+    # Telegram user info is available in the following variables:
+    #    $displayname - Telegram displayname
+    #    $username    - Telegram username (may not exist)
+    #    $mention     - Telegram @username or displayname mention (depending on which exists)
+    emote_format: "* $mention $formatted_body"
+
+    # The formats to use when sending state events to Telegram via the relay bot.
+    #
+    # Variables from `message_formats` that have the `sender_` prefix are available without the prefix.
+    # In name_change events, `$prev_displayname` is the previous displayname.
+    #
+    # Set format to an empty string to disable the messages for that event.
+    state_event_formats:
+        join: "<b>$displayname</b> joined the room."
+        leave: "<b>$displayname</b> left the room."
+        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+
+    # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
+    # `filter-mode` management commands.
+    #
+    # Filters do not affect direct chats.
+    # An empty blacklist will essentially disable the filter.
+    filter:
+        # Filter mode to use. Either "blacklist" or "whitelist".
+        # If the mode is "blacklist", the listed chats will never be bridged.
+        # If the mode is "whitelist", only the listed chats can be bridged.
+        mode: blacklist
+        # The list of group/channel IDs to filter.
+        list: []
+
+    # The prefix for commands. Only required in non-management rooms.
+    command_prefix: "!tg"
+
+    # Messages sent upon joining a management room.
+    # Markdown is supported. The defaults are listed below.
+    management_room_text:
+        # Sent when joining a room.
+        welcome: "Hello, I'm a Telegram bridge bot."
+        # Sent when joining a management room and the user is already logged in.
+        welcome_connected: "Use `help` for help."
+        # Sent when joining a management room and the user is not logged in.
+        welcome_unconnected: "Use `help` for help or `login` to log in."
+        # Optional extra text sent when joining a management room.
+        additional_help: ""
+
+    # Send each message separately (for readability in some clients)
+    management_room_multiple_messages: false
+
+    # Permissions for using the bridge.
+    # Permitted values:
+    #   relaybot - Only use the bridge via the relaybot, no access to commands.
+    #       user - Relaybot level + access to commands to create bridges.
+    #  puppeting - User level + logging in with a Telegram account.
+    #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
+    #      admin - Full access to use the bridge and some extra administration commands.
+    # Permitted keys:
+    #        * - All Matrix users
+    #   domain - All users on that homeserver
+    #     mxid - Specific user
+    permissions:
+        "*": "relaybot"
+        "public.example.com": "user"
+        "example.com": "full"
+        "@admin:example.com": "admin"
+
+    # Options related to the message relay Telegram bot.
+    relaybot:
+        private_chat:
+            # List of users to invite to the portal when someone starts a private chat with the bot.
+            # If empty, private chats with the bot won't create a portal.
+            invite: []
+            # Whether or not to bridge state change messages in relaybot private chats.
+            state_changes: true
+            # When private_chat_invite is empty, this message is sent to users /starting the
+            # relaybot. Telegram's "markdown" is supported.
+            message: This is a Matrix bridge relaybot and does not support direct chats
+        # List of users to invite to all group chat portals created by the bridge.
+        group_chat_invite: []
+        # Whether or not the relaybot should not bridge events in unbridged group chats.
+        # If false, portals will be created when the relaybot receives messages, just like normal
+        # users. This behavior is usually not desirable, as it interferes with manually bridging
+        # the chat to another room.
+        ignore_unbridged_group_chat: true
+        # Whether or not to allow creating portals from Telegram.
+        authless_portals: true
+        # Whether or not to allow Telegram group admins to use the bot commands.
+        whitelist_group_admins: true
+        # Whether or not to ignore incoming events sent by the relay bot.
+        ignore_own_incoming_events: true
+        # List of usernames/user IDs who are also allowed to use the bot commands.
+        whitelist:
+        - myusername
+        - 12345678
+
+# Telegram config
+telegram:
+    # Get your own API keys at https://my.telegram.org/apps
+    api_id: 12345
+    api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
+    # (Optional) Create your own bot at https://t.me/BotFather
+    bot_token: disabled
+
+    # Telethon connection options.
+    connection:
+        # The timeout in seconds to be used when connecting.
+        timeout: 120
+        # How many times the reconnection should retry, either on the initial connection or when
+        # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
+        # this is not recommended, since the program can get stuck in an infinite loop.
+        retries: 5
+        # The delay in seconds to sleep between automatic reconnections.
+        retry_delay: 1
+        # The threshold below which the library should automatically sleep on flood wait errors
+        # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
+        # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
+        # the error instead. Values larger than a day (86400) will be changed to a day.
+        flood_sleep_threshold: 60
+        # How many times a request should be retried. Request are retried when Telegram is having
+        # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
+        # there's a migrate error. May take a negative or null value for infinite retries, but this
+        # is not recommended, since some requests can always trigger a call fail (such as searching
+        # for messages).
+        request_retries: 5
+
+    # Device info sent to Telegram.
+    device_info:
+        # "auto" = OS name+version.
+        device_model: auto
+        # "auto" = Telethon version.
+        system_version: auto
+        # "auto" = mautrix-telegram version.
+        app_version: auto
+        lang_code: en
+        system_lang_code: en
+
+    # Custom server to connect to.
+    server:
+        # Set to true to use these server settings. If false, will automatically
+        # use production server assigned by Telegram. Set to false in production.
+        enabled: false
+        # The DC ID to connect to.
+        dc: 2
+        # The IP to connect to.
+        ip: 149.154.167.40
+        # The port to connect to. 443 may not work, 80 is better and both are equally secure.
+        port: 80
+
+    # Telethon proxy configuration.
+    # You must install PySocks from pip for proxies to work.
+    proxy:
+        # Allowed types: disabled, socks4, socks5, http, mtproxy
+        type: disabled
+        # Proxy IP address and port.
+        address: 127.0.0.1
+        port: 1080
+        # Whether or not to perform DNS resolving remotely. Only for socks/http proxies.
+        rdns: true
+        # Proxy authentication (optional). Put MTProxy secret in password field.
+        username: ""
+        password: ""
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        colored:
+            (): mautrix_telegram.util.ColorFormatter
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+        normal:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: normal
+            filename: ./mautrix-telegram.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: colored
+    loggers:
+        mau:
+            level: DEBUG
+        telethon:
+            level: INFO
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,2 @@
-from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
-                          init_mx)
-from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
-from .. import context as c
-
-
-def init(context: c.Context):
-    init_mx(context)
-    init_tg(context)
+from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,143 +13,93 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, List, Tuple, Callable, Pattern, Match, TYPE_CHECKING
+from __future__ import annotations
+
 import re
-import logging
 
-from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
-                               TypeMessageEntity)
+from telethon import TelegramClient
+from telethon.helpers import add_surrogate, del_surrogate
+from telethon.tl.types import MessageEntityItalic, TypeMessageEntity
+
+from mautrix.types import MessageEventContent, RoomID
 
-from ... import puppet as pu
 from ...db import Message as DBMessage
-from ..util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                    trim_reply_fallback_text)
-from .parser_common import ParsedMessage
+from ...types import TelegramID
+from .parser import MatrixParser
 
-try:
-    from mautrix_telegram.formatter.from_matrix.parser_lxml import parse_html
-except ImportError:
-    from mautrix_telegram.formatter.from_matrix.parser_htmlparser import parse_html
+command_regex = re.compile(r"^!([A-Za-z0-9@]+)")
+not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")
 
-if TYPE_CHECKING:
-    from ...context import Context
-
-log = logging.getLogger("mau.fmt.mx")  # type: logging.Logger
-should_bridge_plaintext_highlights = False  # type: bool
-
-command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
-not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
-plain_mention_regex = None  # type: Pattern
-
-
-def plain_mention_to_html(match: Match) -> str:
-    puppet = pu.Puppet.find_by_displayname(match.group(2))
-    if puppet:
-        return (f"{match.group(1)}"
-                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                f"{puppet.displayname}"
-                "</a>")
-    return "".join(match.groups())
-
-
-def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
-    if len(message) > 4096:
-        message = message[0:4082] + " [message cut]"
-        new_entities = []
-        for entity in entities:
-            if entity.offset > 4082:
-                continue
-            if entity.offset + entity.length > 4082:
-                entity.length = 4082 - entity.offset
-            new_entities.append(entity)
-        new_entities.append(MessageEntityItalic(4082, len(" [message cut]")))
-        entities = new_entities
-    return message, entities
+MAX_LENGTH = 4096
+CUTOFF_TEXT = " [message cut]"
+CUT_MAX_LENGTH = MAX_LENGTH - len(CUTOFF_TEXT)
 
 
 class FormatError(Exception):
     pass
 
 
-def matrix_to_telegram(html: str) -> ParsedMessage:
+async def matrix_reply_to_telegram(
+    content: MessageEventContent, tg_space: TelegramID, room_id: RoomID | None = None
+) -> TelegramID | None:
+    event_id = content.get_reply_to()
+    if not event_id:
+        return
+    content.trim_reply_fallback()
+
+    message = await DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
+    return None
+
+
+async def matrix_to_telegram(
+    client: TelegramClient, *, text: str | None = None, html: str | None = None
+) -> tuple[str, list[TypeMessageEntity]]:
+    if html is not None:
+        return await _matrix_html_to_telegram(client, html)
+    elif text is not None:
+        return _matrix_text_to_telegram(text), []
+    else:
+        raise ValueError("text or html must be provided to convert formatting")
+
+
+async def _matrix_html_to_telegram(
+    client: TelegramClient, html: str
+) -> tuple[str, list[TypeMessageEntity]]:
     try:
         html = command_regex.sub(r"<command>\1</command>", html)
         html = html.replace("\t", " " * 4)
         html = not_command_regex.sub(r"\1", html)
-        if should_bridge_plaintext_highlights:
-            html = plain_mention_regex.sub(plain_mention_to_html, html)
 
-        html = add_surrogates(html)
-        text, entities = parse_html(add_surrogates(html))
-        text = remove_surrogates(text.strip())
-        text, entities = cut_long_message(text, entities)
+        parsed = await MatrixParser(client).parse(add_surrogate(html))
+        text = del_surrogate(parsed.text.strip())
+        text, entities = _cut_long_message(text, parsed.telegram_entities)
 
         return text, entities
     except Exception as e:
         raise FormatError(f"Failed to convert Matrix format: {html}") from e
 
 
-def matrix_reply_to_telegram(content: dict, tg_space: int, room_id: Optional[str] = None
-                             ) -> Optional[int]:
-    try:
-        reply = content["m.relates_to"]["m.in_reply_to"]
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.query.filter(DBMessage.mxid == event_id,
-                                         DBMessage.tg_space == tg_space,
-                                         DBMessage.mx_room == room_id).one_or_none()
-        if message:
-            return message.tgid
-    except KeyError:
-        pass
-    return None
+def _cut_long_message(
+    message: str, entities: list[TypeMessageEntity]
+) -> tuple[str, list[TypeMessageEntity]]:
+    if len(message) > MAX_LENGTH:
+        message = message[0:CUT_MAX_LENGTH] + CUTOFF_TEXT
+        new_entities = []
+        for entity in entities:
+            if entity.offset > CUT_MAX_LENGTH:
+                continue
+            if entity.offset + entity.length > CUT_MAX_LENGTH:
+                entity.length = CUT_MAX_LENGTH - entity.offset
+            new_entities.append(entity)
+        new_entities.append(MessageEntityItalic(CUT_MAX_LENGTH, len(CUTOFF_TEXT)))
+        entities = new_entities
+    return message, entities
 
 
-def matrix_text_to_telegram(text: str) -> ParsedMessage:
+def _matrix_text_to_telegram(text: str) -> str:
     text = command_regex.sub(r"/\1", text)
     text = text.replace("\t", " " * 4)
     text = not_command_regex.sub(r"\1", text)
-    if should_bridge_plaintext_highlights:
-        entities, pmr_replacer = plain_mention_to_text()
-        text = plain_mention_regex.sub(pmr_replacer, text)
-    else:
-        entities = []
-    return text, entities
-
-
-def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
-    entities = []
-
-    def replacer(match) -> str:
-        puppet = pu.Puppet.find_by_displayname(match.group(2))
-        if puppet:
-            offset = match.start()
-            length = match.end() - offset
-            if puppet.username:
-                entity = MessageEntityMention(offset, length)
-                text = f"@{puppet.username}"
-            else:
-                entity = MessageEntityMentionName(offset, length, user_id=puppet.tgid)
-                text = puppet.displayname
-            entities.append(entity)
-            return text
-        return "".join(match.groups())
-
-    return entities, replacer
-
-
-def init_mx(context: "Context"):
-    global plain_mention_regex, should_bridge_plaintext_highlights
-    config = context.config
-    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
-    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
-    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
-    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False
+    return text

mautrix_telegram/formatter/from_matrix/parser.py

@@ -0,0 +1,99 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import logging
+
+from telethon import TelegramClient
+
+from mautrix.types import RoomID, UserID
+from mautrix.util.formatter import MatrixParser as BaseMatrixParser, RecursionContext
+from mautrix.util.formatter.html_reader_htmlparser import HTMLNode, read_html
+from mautrix.util.logging import TraceLogger
+
+from ... import portal as po, puppet as pu, user as u
+from .telegram_message import TelegramEntityType, TelegramMessage
+
+log: TraceLogger = logging.getLogger("mau.fmt.mx")
+
+
+class MatrixParser(BaseMatrixParser[TelegramMessage]):
+    e = TelegramEntityType
+    fs = TelegramMessage
+    read_html = staticmethod(read_html)
+    client: TelegramClient
+
+    def __init__(self, client: TelegramClient) -> None:
+        self.client = client
+
+    async def custom_node_to_fstring(
+        self, node: HTMLNode, ctx: RecursionContext
+    ) -> TelegramMessage | None:
+        msg = await self.tag_aware_parse_node(node, ctx)
+        if node.tag == "command":
+            return msg.prepend("/").format(TelegramEntityType.COMMAND)
+        return None
+
+    async def user_pill_to_fstring(self, msg: TelegramMessage, user_id: UserID) -> TelegramMessage:
+        user = await pu.Puppet.get_by_mxid(user_id) or await u.User.get_by_mxid(
+            user_id, create=False
+        )
+        if not user:
+            return msg
+        if user.tg_username:
+            return TelegramMessage(f"@{user.tg_username}").format(TelegramEntityType.MENTION)
+        elif user.tgid:
+            displayname = user.plain_displayname or msg.text
+            msg = TelegramMessage(displayname)
+            try:
+                input_entity = self.client.get_input_entity(user.tgid)
+            except (ValueError, TypeError) as e:
+                log.trace(f"Dropping mention of {user.tgid}: {e}")
+            else:
+                msg = msg.format(TelegramEntityType.MENTION_NAME, user_id=input_entity)
+        return msg
+
+    async def url_to_fstring(self, msg: TelegramMessage, url: str) -> TelegramMessage:
+        if url == msg.text:
+            return msg.format(self.e.URL)
+        else:
+            return msg.format(self.e.INLINE_URL, url=url)
+
+    async def room_pill_to_fstring(self, msg: TelegramMessage, room_id: RoomID) -> TelegramMessage:
+        username = po.Portal.get_username_from_mx_alias(room_id)
+        portal = await po.Portal.find_by_username(username)
+        if portal and portal.username:
+            return TelegramMessage(f"@{portal.username}").format(TelegramEntityType.MENTION)
+
+    async def header_to_fstring(self, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        children = await self.node_to_fstrings(node, ctx)
+        length = int(node.tag[1])
+        prefix = "#" * length + " "
+        return TelegramMessage.join(children, "").prepend(prefix).format(TelegramEntityType.BOLD)
+
+    async def blockquote_to_fstring(
+        self, node: HTMLNode, ctx: RecursionContext
+    ) -> TelegramMessage:
+        msg = await self.tag_aware_parse_node(node, ctx)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")
+
+    async def color_to_fstring(self, msg: TelegramMessage, color: str) -> TelegramMessage:
+        return msg
+
+    async def spoiler_to_fstring(self, msg: TelegramMessage, spoiler: str) -> TelegramMessage:
+        return msg

mautrix_telegram/formatter/from_matrix/parser_htmlparser.py

@@ -1,236 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import (Optional, List, Tuple, Type, Dict, Any, Deque, Match)
-from html import unescape
-from html.parser import HTMLParser
-from collections import deque
-import math
-
-from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityEmail,
-                               MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
-                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
-                               MessageEntityBotCommand, TypeMessageEntity)
-
-from ... import user as u, puppet as pu, portal as po
-from ..util import html_to_unicode
-from .parser_common import MatrixParserCommon, ParsedMessage
-
-
-def parse_html(html: str) -> ParsedMessage:
-    parser = MatrixParser()
-    parser.feed(html)
-    return parser.text, parser.entities
-
-
-class MatrixParser(HTMLParser, MatrixParserCommon):
-    def __init__(self):
-        super(MatrixParser, self).__init__()
-        self.text = ""  # type: str
-        self.entities = []  # type: List[TypeMessageEntity]
-        self._building_entities = {}  # type: Dict[str, TypeMessageEntity]
-        self._list_counter = 0  # type: int
-        self._open_tags = deque()  # type: Deque[str]
-        self._open_tags_meta = deque()  # type: Deque[Any]
-        self._line_is_new = True  # type: bool
-        self._list_entry_is_new = False  # type: bool
-
-    def _parse_url(self, url: str, args: Dict[str, Any]
-                   ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]:
-        mention = self.mention_regex.match(url)  # type: Match
-        if mention:
-            mxid = mention.group(1)
-            user = (pu.Puppet.get_by_mxid(mxid)
-                    or u.User.get_by_mxid(mxid, create=False))
-            if not user:
-                return None, None
-            if user.username:
-                return MessageEntityMention, f"@{user.username}"
-            elif user.tgid:
-                args["user_id"] = user.tgid
-                return MessageEntityMentionName, user.displayname or None
-            else:
-                return None, None
-
-        room = self.room_regex.match(url)  # type: Match
-        if room:
-            username = po.Portal.get_username_from_mx_alias(room.group(1))
-            portal = po.Portal.find_by_username(username)
-            if portal and portal.username:
-                return MessageEntityMention, f"@{portal.username}"
-
-        if url.startswith("mailto:"):
-            return MessageEntityEmail, url[len("mailto:"):]
-        elif self.get_starttag_text() == url:
-            return MessageEntityUrl, url
-        else:
-            args["url"] = url
-            return MessageEntityTextUrl, None
-
-    def handle_starttag(self, tag: str, attrs: List[Tuple[str, str]]):
-        self._open_tags.appendleft(tag)
-        self._open_tags_meta.appendleft(0)
-
-        attrs = dict(attrs)
-        entity_type = None  # type: type(TypeMessageEntity)
-        args = {}  # type: Dict[str, Any]
-        if tag in ("strong", "b"):
-            entity_type = MessageEntityBold
-        elif tag in ("em", "i"):
-            entity_type = MessageEntityItalic
-        elif tag == "code":
-            try:
-                pre = self._building_entities["pre"]
-                try:
-                    # Pre tag and language found, add language to MessageEntityPre
-                    pre.language = attrs["class"][len("language-"):]
-                except KeyError:
-                    # Pre tag found, but language not found, keep pre as-is
-                    pass
-            except KeyError:
-                # No pre tag found, this is inline code
-                entity_type = MessageEntityCode
-        elif tag == "pre":
-            entity_type = MessageEntityPre
-            args["language"] = ""
-        elif tag == "command":
-            entity_type = MessageEntityBotCommand
-        elif tag == "li":
-            self._list_entry_is_new = True
-        elif tag == "a":
-            try:
-                url = attrs["href"]
-            except KeyError:
-                return
-            entity_type, url = self._parse_url(url, args)
-            self._open_tags_meta.popleft()
-            self._open_tags_meta.appendleft(url)
-
-        if tag in self.block_tags and ("blockquote" not in self._open_tags or tag == "br"):
-            self._newline()
-
-        if entity_type and tag not in self._building_entities:
-            offset = len(self.text)
-            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
-
-    @property
-    def _list_indent(self) -> int:
-        indent = 0
-        first_skipped = False
-        for index, tag in enumerate(self._open_tags):
-            if not first_skipped and tag in ("ol", "ul"):
-                # The first list level isn't indented, so skip it.
-                first_skipped = True
-                continue
-            if tag == "ol":
-                n = self._open_tags_meta[index]
-                extra_length_for_long_index = (int(math.log(n, 10)) - 1) * 3
-                indent += 4 + extra_length_for_long_index
-            elif tag == "ul":
-                indent += 3
-        return indent
-
-    def _newline(self, allow_multi: bool = False):
-        if self._line_is_new and not allow_multi:
-            return
-        self.text += "\n"
-        self._line_is_new = True
-        for entity in self._building_entities.values():
-            entity.length += 1
-
-    def _handle_special_previous_tags(self, text: str) -> str:
-        if "pre" not in self._open_tags and "code" not in self._open_tags:
-            text = text.replace("\n", "")
-        else:
-            text = text.strip()
-
-        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
-        if previous_tag == "a":
-            url = self._open_tags_meta[0]
-            if url:
-                text = url
-        elif previous_tag == "command":
-            text = f"/{text}"
-        return text
-
-    def _html_to_unicode(self, text: str) -> str:
-        strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags
-        if strikethrough and underline:
-            text = html_to_unicode(text, "\u0336\u0332")
-        elif strikethrough:
-            text = html_to_unicode(text, "\u0336")
-        elif underline:
-            text = html_to_unicode(text, "\u0332")
-        return text
-
-    def _handle_tags_for_data(self, text: str) -> Tuple[str, int]:
-        extra_offset = 0
-        list_entry_handled_once = False
-        # In order to maintain order of things like blockquotes in lists or lists in blockquotes,
-        # we can't just have ifs/elses and we need to actually loop through the open tags in order.
-        for index, tag in enumerate(self._open_tags):
-            if tag == "blockquote" and self._line_is_new:
-                text = f"> {text}"
-                extra_offset += 2
-            elif tag == "li" and not list_entry_handled_once:
-                list_type_index = index + 1
-                list_type = self._open_tags[list_type_index]
-                indent = self._list_indent * " " if self._line_is_new else ""
-                if list_type == "ol":
-                    n = self._open_tags_meta[list_type_index]
-                    if self._list_entry_is_new:
-                        n += 1
-                        self._open_tags_meta[list_type_index] = n
-                        prefix = f"{n}. "
-                    else:
-                        prefix = int(math.log(n, 10)) * 3 * " " + 4 * " "
-                else:
-                    prefix = "* " if self._list_entry_is_new else 3 * " "
-                if not self._list_entry_is_new and not self._line_is_new:
-                    prefix = ""
-                extra_offset += len(indent) + len(prefix)
-                text = indent + prefix + text
-                self._list_entry_is_new = False
-                list_entry_handled_once = True
-        return text, extra_offset
-
-    def _extend_entities_in_construction(self, text: str, extra_offset: int):
-        for tag, entity in self._building_entities.items():
-            entity.length += len(text) - extra_offset
-            entity.offset += extra_offset
-
-    def handle_data(self, text: str):
-        text = unescape(text)
-        text = self._handle_special_previous_tags(text)
-        text = self._html_to_unicode(text)
-        text, extra_offset = self._handle_tags_for_data(text)
-        self._extend_entities_in_construction(text, extra_offset)
-        self._line_is_new = False
-        self.text += text
-
-    def handle_endtag(self, tag: str):
-        try:
-            self._open_tags.popleft()
-            self._open_tags_meta.popleft()
-        except IndexError:
-            pass
-
-        entity = self._building_entities.pop(tag, None)
-        if entity:
-            self.entities.append(entity)
-
-        if tag in self.block_tags and tag != "br" and "blockquote" not in self._open_tags:
-            self._newline(allow_multi=tag == "br")

mautrix_telegram/formatter/from_matrix/parser_lxml.py

@@ -1,343 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, List, Tuple, Union, Callable
-from lxml import html
-
-from telethon.tl.types import (MessageEntityMention as Mention,
-                               MessageEntityMentionName as MentionName, MessageEntityEmail as Email,
-                               MessageEntityUrl as URL, MessageEntityTextUrl as TextURL,
-                               MessageEntityBold as Bold, MessageEntityItalic as Italic,
-                               MessageEntityCode as Code, MessageEntityPre as Pre,
-                               MessageEntityBotCommand as Command, TypeMessageEntity,
-                               InputMessageEntityMentionName as InputMentionName)
-
-from ... import user as u, puppet as pu, portal as po
-from ..util import html_to_unicode
-from .parser_common import MatrixParserCommon, ParsedMessage
-
-
-def parse_html(html: str) -> ParsedMessage:
-    return MatrixParser.parse(html)
-
-
-class Entity:
-    @staticmethod
-    def copy(entity: TypeMessageEntity) -> Optional[TypeMessageEntity]:
-        if not entity:
-            return None
-        kwargs = {
-            "offset": entity.offset,
-            "length": entity.length,
-        }
-        if isinstance(entity, Pre):
-            kwargs["language"] = entity.language
-        elif isinstance(entity, TextURL):
-            kwargs["url"] = entity.url
-        elif isinstance(entity, (MentionName, InputMentionName)):
-            kwargs["user_id"] = entity.user_id
-        return entity.__class__(**kwargs)
-
-    @classmethod
-    def adjust(cls, entity: Union[TypeMessageEntity, List[TypeMessageEntity]],
-               func: Callable[[TypeMessageEntity], None]
-               ) -> Union[Optional[TypeMessageEntity], List[TypeMessageEntity]]:
-        if isinstance(entity, list):
-            return [Entity.adjust(element, func) for element in entity if entity]
-        elif not entity:
-            return None
-        entity = cls.copy(entity)
-        func(entity)
-        if entity.offset < 0:
-            entity.length += entity.offset
-            entity.offset = 0
-        return entity
-
-
-def offset_diff(amount: int):
-    def func(entity: TypeMessageEntity):
-        entity.offset += amount
-
-    return func
-
-
-def offset_length_multiply(amount: int):
-    def func(entity: TypeMessageEntity):
-        entity.offset *= amount
-        entity.length *= amount
-
-    return func
-
-
-class TelegramMessage:
-    def __init__(self, text: str = "", entities: Optional[List[TypeMessageEntity]] = None):
-        self.text = text  # type: str
-        self.entities = entities or []  # type: List[TypeMessageEntity]
-
-    def offset_entities(self, offset: int) -> "TelegramMessage":
-        def apply_offset(entity: TypeMessageEntity, inner_offset: int
-                         ) -> Optional[TypeMessageEntity]:
-            entity = Entity.copy(entity)
-            entity.offset += inner_offset
-            if entity.offset < 0:
-                entity.offset = 0
-            elif entity.offset > len(self.text):
-                return None
-            elif entity.offset + entity.length > len(self.text):
-                entity.length = len(self.text) - entity.offset
-            return entity
-
-        self.entities = [apply_offset(entity, offset) for entity in self.entities if entity]
-        self.entities = [x for x in self.entities if x is not None]
-        return self
-
-    def append(self, *args: Union[str, "TelegramMessage"]) -> "TelegramMessage":
-        for msg in args:
-            if isinstance(msg, str):
-                msg = TelegramMessage(text=msg)
-            self.entities += Entity.adjust(msg.entities, offset_diff(len(self.text)))
-            self.text += msg.text
-        return self
-
-    def prepend(self, *args: Union[str, "TelegramMessage"]) -> "TelegramMessage":
-        for msg in args:
-            if isinstance(msg, str):
-                msg = TelegramMessage(text=msg)
-            self.entities = msg.entities + Entity.adjust(self.entities, offset_diff(len(msg.text)))
-            self.text = msg.text + self.text
-        return self
-
-    def format(self, entity_type: type(TypeMessageEntity), offset: int = None, length: int = None,
-               **kwargs) -> "TelegramMessage":
-        self.entities.append(entity_type(offset=offset or 0,
-                                         length=length if length is not None else len(self.text),
-                                         **kwargs))
-        return self
-
-    def concat(self, *args: Union[str, "TelegramMessage"]) -> "TelegramMessage":
-        return TelegramMessage().append(self, *args)
-
-    def trim(self) -> "TelegramMessage":
-        orig_len = len(self.text)
-        self.text = self.text.lstrip()
-        diff = orig_len - len(self.text)
-        self.text = self.text.rstrip()
-        self.offset_entities(-diff)
-        return self
-
-    def split(self, separator, max_items: int = 0) -> List["TelegramMessage"]:
-        text_parts = self.text.split(separator, max_items - 1)
-        output = []  # type: List[TelegramMessage]
-
-        offset = 0
-        for part in text_parts:
-            msg = TelegramMessage(part)
-            for entity in self.entities:
-                start_in_range = len(part) > entity.offset - offset >= 0
-                end_in_range = len(part) >= entity.offset - offset + entity.length > 0
-                if start_in_range and end_in_range:
-                    msg.entities.append(Entity.adjust(entity, offset_diff(-offset)))
-            output.append(msg)
-
-            offset += len(part)
-            offset += len(separator)
-
-        return output
-
-    @staticmethod
-    def join(items: List[Union[str, "TelegramMessage"]], separator: str = " ") -> "TelegramMessage":
-        main = TelegramMessage()
-        for msg in items:
-            if isinstance(msg, str):
-                msg = TelegramMessage(text=msg)
-            main.entities += Entity.adjust(msg.entities, offset_diff(len(main.text)))
-            main.text += msg.text + separator
-        main.text = main.text[:-len(separator)]
-        return main
-
-
-class MatrixParser(MatrixParserCommon):
-    @classmethod
-    def list_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
-        ordered = node.tag == "ol"
-        tagged_children = cls.node_to_tagged_tmessages(node, strip_linebreaks)
-        counter = 1
-        indent_length = 0
-        if ordered:
-            try:
-                counter = int(node.attrib.get("start", "1"))
-            except ValueError:
-                counter = 1
-
-            longest_index = counter - 1 + len(tagged_children)
-            indent_length = len(str(longest_index))
-        indent = (indent_length + 4) * " "
-        children = []  # type: List[TelegramMessage]
-        for child, tag in tagged_children:
-            if tag != "li":
-                continue
-
-            if ordered:
-                prefix = f"{counter}. "
-                counter += 1
-            else:
-                prefix = "● "
-            child = child.prepend(prefix)
-            parts = child.split("\n")
-            parts = parts[:1] + [part.prepend(indent) for part in parts[1:]]
-            child = TelegramMessage.join(parts, "\n")
-            children.append(child)
-        return TelegramMessage.join(children, "\n")
-
-    @classmethod
-    def blockquote_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
-        msg = cls.tag_aware_parse_node(node, strip_linebreaks)
-        children = msg.trim().split("\n")
-        children = [child.prepend("> ") for child in children]
-        return TelegramMessage.join(children, "\n")
-
-    @classmethod
-    def header_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
-        children = cls.node_to_tmessages(node, strip_linebreaks)
-        length = int(node.tag[1])
-        prefix = "#" * length + " "
-        return TelegramMessage.join(children, "").prepend(prefix)
-
-    @classmethod
-    def basic_format_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
-        msg = cls.tag_aware_parse_node(node, strip_linebreaks)
-        if node.tag in ("b", "strong"):
-            msg.format(Bold)
-        elif node.tag in ("i", "em"):
-            msg.format(Italic)
-        elif node.tag == "command":
-            msg.format(Command)
-        elif node.tag in ("s", "del"):
-            msg.text = html_to_unicode(msg.text, "\u0336")
-        elif node.tag in ("u", "ins"):
-            msg.text = html_to_unicode(msg.text, "\u0332")
-
-        if node.tag in ("s", "del", "u", "ins"):
-            msg.entities = Entity.adjust(msg.entities, offset_length_multiply(2))
-
-        return msg
-
-    @classmethod
-    def link_to_tstring(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
-        msg = cls.tag_aware_parse_node(node, strip_linebreaks)
-        href = node.attrib.get("href", "")
-        if not href:
-            return msg
-
-        if href.startswith("mailto:"):
-            return TelegramMessage(href[len("mailto:"):]).format(Email)
-
-        mention = cls.mention_regex.match(href)
-        if mention:
-            mxid = mention.group(1)
-            user = (pu.Puppet.get_by_mxid(mxid)
-                    or u.User.get_by_mxid(mxid, create=False))
-            if not user:
-                return msg
-            if user.username:
-                return TelegramMessage(f"@{user.username}").format(Mention)
-            elif user.tgid:
-                return TelegramMessage(user.displayname or msg.text).format(MentionName,
-                                                                            user_id=user.tgid)
-            return msg
-
-        room = cls.room_regex.match(href)
-        if room:
-            username = po.Portal.get_username_from_mx_alias(room.group(1))
-            portal = po.Portal.find_by_username(username)
-            if portal and portal.username:
-                return TelegramMessage(f"@{portal.username}").format(Mention)
-
-        return (msg.format(URL)
-                if msg.text == href
-                else msg.format(TextURL, url=href))
-
-    @classmethod
-    def node_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
-        if node.tag == "blockquote":
-            return cls.blockquote_to_tmessage(node, strip_linebreaks)
-        elif node.tag in ("ol", "ul"):
-            return cls.list_to_tmessage(node, strip_linebreaks)
-        elif node.tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
-            return cls.header_to_tmessage(node, strip_linebreaks)
-        elif node.tag == "br":
-            return TelegramMessage("\n")
-        elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
-            return cls.basic_format_to_tmessage(node, strip_linebreaks)
-        elif node.tag == "a":
-            return cls.link_to_tstring(node, strip_linebreaks)
-        elif node.tag == "p":
-            return cls.tag_aware_parse_node(node, strip_linebreaks).append("\n")
-        elif node.tag == "pre":
-            lang = ""
-            try:
-                if node[0].tag == "code":
-                    lang = node[0].attrib["class"][len("language-"):]
-                    node = node[0]
-            except (IndexError, KeyError):
-                pass
-            return cls.parse_node(node, strip_linebreaks=False).format(Pre, language=lang)
-        elif node.tag == "code":
-            return cls.parse_node(node, strip_linebreaks=False).format(Code)
-        return cls.tag_aware_parse_node(node, strip_linebreaks)
-
-    @staticmethod
-    def text_to_tmessage(text: str, strip_linebreaks: bool = True) -> TelegramMessage:
-        if strip_linebreaks:
-            text = text.replace("\n", "")
-        return TelegramMessage(text)
-
-    @classmethod
-    def node_to_tagged_tmessages(cls, node: html.HtmlElement, strip_linebreaks: bool = True
-                                 ) -> List[Tuple[TelegramMessage, str]]:
-        output = []
-
-        if node.text:
-            output.append((cls.text_to_tmessage(node.text, strip_linebreaks), "text"))
-        for child in node:
-            output.append((cls.node_to_tmessage(child, strip_linebreaks), child.tag))
-            if child.tail:
-                output.append((cls.text_to_tmessage(child.tail, strip_linebreaks), "text"))
-        return output
-
-    @classmethod
-    def node_to_tmessages(cls, node: html.HtmlElement, strip_linebreaks) -> List[TelegramMessage]:
-        return [msg for (msg, tag) in cls.node_to_tagged_tmessages(node, strip_linebreaks)]
-
-    @classmethod
-    def tag_aware_parse_node(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
-        msgs = cls.node_to_tagged_tmessages(node, strip_linebreaks)
-        output = TelegramMessage()
-        for msg, tag in msgs:
-            if tag in cls.block_tags:
-                msg = msg.append("\n").prepend("\n")
-            output = output.append(msg)
-        return output.trim()
-
-    @classmethod
-    def parse_node(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
-        return TelegramMessage.join(cls.node_to_tmessages(node, strip_linebreaks))
-
-    @classmethod
-    def parse(cls, data: str) -> ParsedMessage:
-        document = html.fromstring(f"<html>{data}</html>")
-        msg = cls.parse_node(document, strip_linebreaks=True)
-        return msg.text, msg.entities

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -0,0 +1,120 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Any, Type
+from enum import Enum
+
+from telethon.tl.types import (
+    InputMessageEntityMentionName as InputMentionName,
+    MessageEntityBlockquote as Blockquote,
+    MessageEntityBold as Bold,
+    MessageEntityBotCommand as Command,
+    MessageEntityCode as Code,
+    MessageEntityEmail as Email,
+    MessageEntityItalic as Italic,
+    MessageEntityMention as Mention,
+    MessageEntityMentionName as MentionName,
+    MessageEntityPre as Pre,
+    MessageEntityStrike as Strike,
+    MessageEntityTextUrl as TextURL,
+    MessageEntityUnderline as Underline,
+    MessageEntityUrl as URL,
+    TypeMessageEntity,
+)
+
+from mautrix.util.formatter import EntityString, SemiAbstractEntity
+
+
+class TelegramEntityType(Enum):
+    """EntityType is a Matrix formatting entity type."""
+
+    BOLD = Bold
+    ITALIC = Italic
+    STRIKETHROUGH = Strike
+    UNDERLINE = Underline
+    URL = URL
+    INLINE_URL = TextURL
+    EMAIL = Email
+    PREFORMATTED = Pre
+    INLINE_CODE = Code
+    BLOCKQUOTE = Blockquote
+    MENTION = Mention
+    MENTION_NAME = InputMentionName
+    COMMAND = Command
+
+    USER_MENTION = 1
+    ROOM_MENTION = 2
+    HEADER = 3
+
+
+class TelegramEntity(SemiAbstractEntity):
+    internal: TypeMessageEntity
+
+    def __init__(
+        self,
+        type: TelegramEntityType | Type[TypeMessageEntity],
+        offset: int,
+        length: int,
+        extra_info: dict[str, Any],
+    ) -> None:
+        if isinstance(type, TelegramEntityType):
+            if isinstance(type.value, int):
+                raise ValueError(f"Can't create Entity with non-Telegram EntityType {type}")
+            type = type.value
+        self.internal = type(offset=offset, length=length, **extra_info)
+
+    def copy(self) -> TelegramEntity:
+        extra_info = {}
+        if isinstance(self.internal, Pre):
+            extra_info["language"] = self.internal.language
+        elif isinstance(self.internal, TextURL):
+            extra_info["url"] = self.internal.url
+        elif isinstance(self.internal, (MentionName, InputMentionName)):
+            extra_info["user_id"] = self.internal.user_id
+        return TelegramEntity(
+            type(self.internal),
+            offset=self.internal.offset,
+            length=self.internal.length,
+            extra_info=extra_info,
+        )
+
+    def __repr__(self) -> str:
+        return str(self.internal)
+
+    @property
+    def offset(self) -> int:
+        return self.internal.offset
+
+    @offset.setter
+    def offset(self, value: int) -> None:
+        self.internal.offset = value
+
+    @property
+    def length(self) -> int:
+        return self.internal.length
+
+    @length.setter
+    def length(self, value: int) -> None:
+        self.internal.length = value
+
+
+class TelegramMessage(EntityString[TelegramEntity, TelegramEntityType]):
+    entity_class = TelegramEntity
+
+    @property
+    def telegram_entities(self) -> list[TypeMessageEntity]:
+        return [entity.internal for entity in self.entities]

mautrix_telegram/formatter/from_telegram.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,273 +13,310 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, List, Tuple, TYPE_CHECKING
+from __future__ import annotations
+
 from html import escape
 import logging
 import re
 
-from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName,
-                               MessageEntityEmail, MessageEntityUrl, MessageEntityTextUrl,
-                               MessageEntityBold, MessageEntityItalic, MessageEntityCode,
-                               MessageEntityPre, MessageEntityBotCommand, Message, PeerChannel,
-                               MessageEntityHashtag, TypeMessageEntity, MessageFwdHeader, PeerUser)
+from telethon.errors import RPCError
+from telethon.helpers import add_surrogate, del_surrogate
+from telethon.tl.custom import Message
+from telethon.tl.types import (
+    MessageEntityBlockquote,
+    MessageEntityBold,
+    MessageEntityBotCommand,
+    MessageEntityCashtag,
+    MessageEntityCode,
+    MessageEntityEmail,
+    MessageEntityHashtag,
+    MessageEntityItalic,
+    MessageEntityMention,
+    MessageEntityMentionName,
+    MessageEntityPhone,
+    MessageEntityPre,
+    MessageEntityStrike,
+    MessageEntityTextUrl,
+    MessageEntityUnderline,
+    MessageEntityUrl,
+    MessageFwdHeader,
+    PeerChannel,
+    PeerChat,
+    PeerUser,
+    SponsoredMessage,
+    TypeMessageEntity,
+)
 
-from mautrix_appservice import MatrixRequestError
-from mautrix_appservice.intent_api import IntentAPI
+from mautrix.appservice import IntentAPI
+from mautrix.types import (
+    EventType,
+    Format,
+    MessageType,
+    RelatesTo,
+    RelationType,
+    TextMessageEventContent,
+)
 
-from .. import user as u, puppet as pu, portal as po
+from .. import abstract_user as au, portal as po, puppet as pu, user as u
 from ..db import Message as DBMessage
-from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text, unicode_to_html)
+from ..types import TelegramID
 
-if TYPE_CHECKING:
-    from ..abstract_user import AbstractUser
-    from ..context import Context
-
-try:
-    from lxml.html.diff import htmldiff
-except ImportError:
-    htmldiff = None  # type: function
+log: logging.Logger = logging.getLogger("mau.fmt.tg")
 
 
-log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
-should_highlight_edits = False  # type: bool
-
-
-def telegram_reply_to_matrix(evt: Message, source: "AbstractUser") -> dict:
-    if evt.reply_to_msg_id:
-        space = (evt.to_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
-                 else source.tgid)
-        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+async def telegram_reply_to_matrix(evt: Message, source: au.AbstractUser) -> RelatesTo | None:
+    if evt.reply_to:
+        space = (
+            evt.peer_id.channel_id
+            if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
+            else source.tgid
+        )
+        msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
         if msg:
-            return {
-                "m.in_reply_to": {
-                    "event_id": msg.mxid,
-                    "room_id": msg.mx_room,
-                }
-            }
-    return {}
+            return RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
+    return None
 
 
-async def _add_forward_header(source, text: str, html: Optional[str],
-                              fwd_from: MessageFwdHeader) -> Tuple[str, str]:
-    if not html:
-        html = escape(text)
+async def _add_forward_header(
+    source: au.AbstractUser, content: TextMessageEventContent, fwd_from: MessageFwdHeader
+) -> None:
+    if not content.formatted_body or content.format != Format.HTML:
+        content.format = Format.HTML
+        content.formatted_body = escape(content.body)
     fwd_from_html, fwd_from_text = None, None
-    if fwd_from.from_id:
-        user = u.User.get_by_tgid(fwd_from.from_id)
+    if isinstance(fwd_from.from_id, PeerUser):
+        user = await u.User.get_by_tgid(TelegramID(fwd_from.from_id.user_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
-            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
+            fwd_from_html = (
+                f"<a href='https://matrix.to/#/{user.mxid}'>{escape(fwd_from_text)}</a>"
+            )
 
         if not fwd_from_text:
-            puppet = pu.Puppet.get(fwd_from.from_id, create=False)
+            puppet = await pu.Puppet.get_by_tgid(
+                TelegramID(fwd_from.from_id.user_id), create=False
+            )
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
-                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
+                fwd_from_html = (
+                    f"<a href='https://matrix.to/#/{puppet.mxid}'>{escape(fwd_from_text)}</a>"
+                )
 
         if not fwd_from_text:
-            user = await source.client.get_entity(PeerUser(fwd_from.from_id))
-            if user:
-                fwd_from_text = pu.Puppet.get_displayname(user, False)
-                fwd_from_html = f"<b>{fwd_from_text}</b>"
-
-    if not fwd_from_text:
-        if fwd_from.from_id:
-            fwd_from_text = "Unknown user"
+            try:
+                user = await source.client.get_entity(fwd_from.from_id)
+                if user:
+                    fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
+                    fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+            except (ValueError, RPCError):
+                fwd_from_text = fwd_from_html = "unknown user"
+    elif isinstance(fwd_from.from_id, (PeerChannel, PeerChat)):
+        from_id = (
+            fwd_from.from_id.chat_id
+            if isinstance(fwd_from.from_id, PeerChat)
+            else fwd_from.from_id.channel_id
+        )
+        portal = await po.Portal.get_by_tgid(TelegramID(from_id))
+        if portal and portal.title:
+            fwd_from_text = portal.title
+            if portal.alias:
+                fwd_from_html = (
+                    f"<a href='https://matrix.to/#/{portal.alias}'>{escape(fwd_from_text)}</a>"
+                )
+            else:
+                fwd_from_html = f"channel <b>{escape(fwd_from_text)}</b>"
         else:
-            fwd_from_text = "Unknown source"
-        fwd_from_html = f"<b>{fwd_from_text}</b>"
+            try:
+                channel = await source.client.get_entity(fwd_from.from_id)
+                if channel:
+                    fwd_from_text = f"channel {channel.title}"
+                    fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
+            except (ValueError, RPCError):
+                fwd_from_text = fwd_from_html = "unknown channel"
+    elif fwd_from.from_name:
+        fwd_from_text = fwd_from.from_name
+        fwd_from_html = f"<b>{escape(fwd_from.from_name)}</b>"
+    else:
+        fwd_from_text = "unknown source"
+        fwd_from_html = f"unknown source"
 
-    text = "\n".join([f"> {line}" for line in text.split("\n")])
-    text = f"Forwarded from {fwd_from_text}:\n{text}"
-    html = (f"Forwarded message from {fwd_from_html}<br/>"
-            f"<tg-forward><blockquote>{html}</blockquote></tg-forward>")
-    return text, html
+    content.body = "\n".join([f"> {line}" for line in content.body.split("\n")])
+    content.body = f"Forwarded from {fwd_from_text}:\n{content.body}"
+    content.formatted_body = (
+        f"Forwarded message from {fwd_from_html}<br/>"
+        f"<tg-forward><blockquote>{content.formatted_body}</blockquote></tg-forward>"
+    )
 
 
-def highlight_edits(new_html: str, old_html: str) -> str:
-    # Don't include `Edit:` text in diff.
-    if old_html.startswith("<u>Edit:</u> "):
-        old_html = old_html[len("<u>Edit:</u> "):]
+async def _add_reply_header(
+    source: au.AbstractUser, content: TextMessageEventContent, evt: Message, main_intent: IntentAPI
+) -> None:
+    space = (
+        evt.peer_id.channel_id
+        if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
+        else source.tgid
+    )
 
-    # Generate diff with lxml
-    new_html = htmldiff(old_html, new_html)
-
-    # Replace <ins> with <u> since Riot doesn't allow <ins>
-    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
-    # Remove <del>s since we just want to hide deletions.
-    new_html = re.sub("<del>.+?</del>", "", new_html)
-    return new_html
-
-
-async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
-                            relates_to: dict, main_intent: IntentAPI, is_edit: bool
-                            ) -> Tuple[str, str]:
-    space = (evt.to_id.channel_id
-             if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
-             else source.tgid)
-
-    msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+    msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
     if not msg:
-        return text, html
+        return
 
-    relates_to["m.in_reply_to"] = {
-        "event_id": msg.mxid,
-        "room_id": msg.mx_room,
-    }
+    content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
 
     try:
         event = await main_intent.get_event(msg.mx_room, msg.mxid)
-
-        content = event["content"]
-        r_sender = event["sender"]
-
-        r_text_body = trim_reply_fallback_text(content["body"])
-        r_html_body = trim_reply_fallback_html(content["formatted_body"]
-                                               if "formatted_body" in content
-                                               else escape(content["body"]))
-
-        puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
-        r_displayname = puppet.displayname if puppet else r_sender
-        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
-
-        if is_edit and should_highlight_edits:
-            html = highlight_edits(html or escape(text), r_html_body)
-    except (ValueError, KeyError, MatrixRequestError):
-        r_sender_link = "unknown user"
-        r_displayname = "unknown user"
-        r_text_body = "Failed to fetch message"
-        r_html_body = "<em>Failed to fetch message</em>"
-
-    if is_edit:
-        html = f"<u>Edit:</u> {html or escape(text)}"
-        text = f"Edit: {text}"
-
-    r_keyword = "In reply to" if not is_edit else "Edit to"
-    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
-    html = (
-        f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
-        + (html or escape(text)))
-
-    lines = r_text_body.strip().split("\n")
-    text_with_quote = f"> <{r_displayname}> {lines.pop(0)}"
-    for line in lines:
-        if line:
-            text_with_quote += f"\n> {line}"
-    text_with_quote += "\n\n"
-    text_with_quote += text
-    return text_with_quote, html
+        if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
+            event = await source.bridge.matrix.e2ee.decrypt(event)
+        if isinstance(event.content, TextMessageEventContent):
+            event.content.trim_reply_fallback()
+        puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
+        content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
+    except Exception:
+        log.exception("Failed to get event to add reply fallback")
 
 
-async def telegram_to_matrix(evt: Message, source: "AbstractUser",
-                             main_intent: Optional[IntentAPI] = None,
-                             is_edit: bool = False, prefix_text: Optional[str] = None,
-                             prefix_html: Optional[str] = None) -> Tuple[str, str, dict]:
-    text = add_surrogates(evt.message)
-    html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None
-    relates_to = {}
+async def telegram_to_matrix(
+    evt: Message | SponsoredMessage,
+    source: au.AbstractUser,
+    main_intent: IntentAPI | None = None,
+    prefix_text: str | None = None,
+    prefix_html: str | None = None,
+    override_text: str = None,
+    override_entities: list[TypeMessageEntity] = None,
+    no_reply_fallback: bool = False,
+    require_html: bool = False,
+) -> TextMessageEventContent:
+    content = TextMessageEventContent(
+        msgtype=MessageType.TEXT,
+        body=add_surrogate(override_text or evt.message),
+    )
+    entities = override_entities or evt.entities
+    if entities:
+        content.format = Format.HTML
+        html = await _telegram_entities_to_matrix_catch(add_surrogate(content.body), entities)
+        content.formatted_body = del_surrogate(html).replace("\n", "<br/>")
+
+    def force_html():
+        if not content.formatted_body:
+            content.format = Format.HTML
+            content.formatted_body = escape(content.body)
+
+    if require_html:
+        force_html()
 
     if prefix_html:
-        html = prefix_html + (html or escape(text))
+        force_html()
+        content.formatted_body = prefix_html + content.formatted_body
     if prefix_text:
-        text = prefix_text + text
+        content.body = prefix_text + content.body
 
-    if evt.fwd_from:
-        text, html = await _add_forward_header(source, text, html, evt.fwd_from)
+    if getattr(evt, "fwd_from", None):
+        await _add_forward_header(source, content, evt.fwd_from)
 
-    if evt.reply_to_msg_id:
-        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
-                                             is_edit)
+    if getattr(evt, "reply_to", None) and not no_reply_fallback:
+        await _add_reply_header(source, content, evt, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
-        if not html:
-            html = escape(text)
-        text += f"\n- {evt.post_author}"
-        html += f"<br/><i>- <u>{evt.post_author}</u></i>"
+        force_html()
+        content.body += f"\n- {evt.post_author}"
+        content.formatted_body += f"<br/><i>- <u>{evt.post_author}</u></i>"
 
-    html = unicode_to_html(text, html, "\u0336", "del")
-    html = unicode_to_html(text, html, "\u0332", "u")
-
-    if html:
-        html = html.replace("\n", "<br/>")
-
-    return remove_surrogates(text), remove_surrogates(html), relates_to
+    return content
 
 
-def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEntity]) -> str:
+async def _telegram_entities_to_matrix_catch(text: str, entities: list[TypeMessageEntity]) -> str:
     try:
-        return _telegram_entities_to_matrix(text, entities)
+        return await _telegram_entities_to_matrix(text, entities)
     except Exception:
-        log.exception("Failed to convert Telegram format:\n"
-                      "message=%s\n"
-                      "entities=%s",
-                      text, entities)
+        log.exception(
+            "Failed to convert Telegram format:\nmessage=%s\nentities=%s", text, entities
+        )
+    return "[failed conversion in _telegram_entities_to_matrix]"
 
 
-def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
+async def _telegram_entities_to_matrix(
+    text: str, entities: list[TypeMessageEntity], offset: int = 0, length: int = None
+) -> str:
     if not entities:
-        return text
+        return escape(text)
+    if length is None:
+        length = len(text)
     html = []
     last_offset = 0
-    for entity in entities:
-        if entity.offset > last_offset:
-            html.append(escape(text[last_offset:entity.offset]))
-        elif entity.offset < last_offset:
+    for i, entity in enumerate(entities):
+        if entity.offset > offset + length:
+            break
+        relative_offset = entity.offset - offset
+        if relative_offset > last_offset:
+            html.append(escape(text[last_offset:relative_offset]))
+        elif relative_offset < last_offset:
             continue
 
         skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length])
+        entity_text = await _telegram_entities_to_matrix(
+            text=text[relative_offset : relative_offset + entity.length],
+            entities=entities[i + 1 :],
+            offset=entity.offset,
+            length=entity.length,
+        )
         entity_type = type(entity)
 
         if entity_type == MessageEntityBold:
             html.append(f"<strong>{entity_text}</strong>")
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
+        elif entity_type == MessageEntityUnderline:
+            html.append(f"<u>{entity_text}</u>")
+        elif entity_type == MessageEntityStrike:
+            html.append(f"<del>{entity_text}</del>")
+        elif entity_type == MessageEntityBlockquote:
+            html.append(f"<blockquote>{entity_text}</blockquote>")
         elif entity_type == MessageEntityCode:
-            html.append(f"<code>{entity_text}</code>")
+            html.append(
+                f"<pre><code>{entity_text}</code></pre>"
+                if "\n" in entity_text
+                else f"<code>{entity_text}</code>"
+            )
         elif entity_type == MessageEntityPre:
             skip_entity = _parse_pre(html, entity_text, entity.language)
         elif entity_type == MessageEntityMention:
-            skip_entity = _parse_mention(html, entity_text)
+            skip_entity = await _parse_mention(html, entity_text)
         elif entity_type == MessageEntityMentionName:
-            skip_entity = _parse_name_mention(html, entity_text, entity.user_id)
+            skip_entity = await _parse_name_mention(html, entity_text, TelegramID(entity.user_id))
         elif entity_type == MessageEntityEmail:
             html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
-        elif entity_type in {MessageEntityTextUrl, MessageEntityUrl}:
-            skip_entity = _parse_url(html, entity_text,
-                                     entity.url if entity_type == MessageEntityTextUrl else None)
+        elif entity_type in (MessageEntityTextUrl, MessageEntityUrl):
+            skip_entity = await _parse_url(
+                html, entity_text, entity.url if entity_type == MessageEntityTextUrl else None
+            )
         elif entity_type == MessageEntityBotCommand:
-            html.append(f"<font color='blue'>!{entity_text[1:]}</font>")
-        elif entity_type == MessageEntityHashtag:
+            html.append(f"<font color='blue'>{entity_text}</font>")
+        elif entity_type in (MessageEntityHashtag, MessageEntityCashtag, MessageEntityPhone):
             html.append(f"<font color='blue'>{entity_text}</font>")
         else:
             skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:])
+        last_offset = relative_offset + (0 if skip_entity else entity.length)
+    html.append(escape(text[last_offset:]))
 
     return "".join(html)
 
 
-def _parse_pre(html: List[str], entity_text: str, language: str) -> bool:
+def _parse_pre(html: list[str], entity_text: str, language: str) -> bool:
     if language:
-        html.append("<pre>"
-                    f"<code class='language-{language}'>{entity_text}</code>"
-                    "</pre>")
+        html.append(f"<pre><code class='language-{language}'>{entity_text}</code></pre>")
     else:
         html.append(f"<pre><code>{entity_text}</code></pre>")
     return False
 
 
-def _parse_mention(html: List[str], entity_text: str) -> bool:
+async def _parse_mention(html: list[str], entity_text: str) -> bool:
     username = entity_text[1:]
 
-    user = u.User.find_by_username(username) or pu.Puppet.find_by_username(username)
+    user = await u.User.find_by_username(username) or await pu.Puppet.find_by_username(username)
     if user:
         mxid = user.mxid
     else:
-        portal = po.Portal.find_by_username(username)
+        portal = await po.Portal.find_by_username(username)
         mxid = portal.alias or portal.mxid if portal else None
 
     if mxid:
@@ -290,12 +326,12 @@ def _parse_mention(html: List[str], entity_text: str) -> bool:
     return False
 
 
-def _parse_name_mention(html: List[str], entity_text: str, user_id: int) -> bool:
-    user = u.User.get_by_tgid(user_id)
+async def _parse_name_mention(html: list[str], entity_text: str, user_id: TelegramID) -> bool:
+    user = await u.User.get_by_tgid(user_id)
     if user:
         mxid = user.mxid
     else:
-        puppet = pu.Puppet.get(user_id, create=False)
+        puppet = await pu.Puppet.get_by_tgid(user_id, create=False)
         mxid = puppet.mxid if puppet else None
     if mxid:
         html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
@@ -305,29 +341,25 @@ def _parse_name_mention(html: List[str], entity_text: str, user_id: int) -> bool
 
 
 message_link_regex = re.compile(
-    r"https?://t(?:elegram)?\.(?:me|dog)/([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})")
+    r"https?://t(?:elegram)?\.(?:me|dog)/([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})"
+)
 
 
-def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
+async def _parse_url(html: list[str], entity_text: str, url: str) -> bool:
     url = escape(url) if url else entity_text
     if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
         url = "http://" + url
 
     message_link_match = message_link_regex.match(url)
     if message_link_match:
-        group, msgid = message_link_match.groups()
-        msgid = int(msgid)
+        group, msgid_str = message_link_match.groups()
+        msgid = int(msgid_str)
 
-        portal = po.Portal.find_by_username(group)
+        portal = await po.Portal.find_by_username(group)
         if portal:
-            message = DBMessage.query.get((msgid, portal.tgid))
+            message = await DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 
     html.append(f"<a href='{url}'>{entity_text}</a>")
     return False
-
-
-def init_tg(context: "Context"):
-    global should_highlight_edits
-    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/formatter/util.py

@@ -1,86 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Pattern
-from html import escape
-import struct
-import re
-
-
-# add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
-# Licensed under the MIT license.
-# https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py
-def add_surrogates(text: Optional[str]) -> Optional[str]:
-    if text is None:
-        return None
-    return "".join("".join(chr(y) for y in struct.unpack("<HH", x.encode("utf-16-le")))
-                   if (0x10000 <= ord(x) <= 0x10FFFF) else x for x in text)
-
-
-def remove_surrogates(text: Optional[str]) -> Optional[str]:
-    if text is None:
-        return None
-    return text.encode("utf-16", "surrogatepass").decode("utf-16")
-
-
-def trim_reply_fallback_text(text: str) -> str:
-    if not text.startswith("> ") or "\n" not in text:
-        return text
-    lines = text.split("\n")
-    while len(lines) > 0 and lines[0].startswith("> "):
-        lines.pop(0)
-    return "\n".join(lines)
-
-
-html_reply_fallback_regex = re.compile("^<mx-reply>"
-                                       r"[\s\S]+?"
-                                       "</mx-reply>")  # type: Pattern
-
-
-def trim_reply_fallback_html(html: str) -> str:
-    return html_reply_fallback_regex.sub("", html)
-
-
-def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
-    if ctrl not in text:
-        return html
-    if not html:
-        html = escape(text)
-    tag_start = f"<{tag}>"
-    tag_end = f"</{tag}>"
-    characters = html.split(ctrl)
-    html = ""
-    in_tag = False
-    for char in characters:
-        if not in_tag:
-            if len(char) > 1:
-                html += char[0:-1]
-                char = char[-1]
-            html += tag_start
-            in_tag = True
-            html += char
-        else:
-            if len(char) > 1:
-                html += tag_end
-                in_tag = False
-            html += char
-    if in_tag:
-        html += tag_end
-    return html
-
-
-def html_to_unicode(text: str, ctrl: str) -> str:
-    return ctrl.join(text) + ctrl

mautrix_telegram/get_version.py

@@ -0,0 +1,49 @@
+import os
+import shutil
+import subprocess
+
+from . import __version__
+
+cmd_env = {
+    "PATH": os.environ["PATH"],
+    "HOME": os.environ["HOME"],
+    "LANG": "C",
+    "LC_ALL": "C",
+}
+
+
+def run(cmd):
+    return subprocess.check_output(cmd, stderr=subprocess.DEVNULL, env=cmd_env)
+
+
+if os.path.exists(".git") and shutil.which("git"):
+    try:
+        git_revision = run(["git", "rev-parse", "HEAD"]).strip().decode("ascii")
+        git_revision_url = f"https://github.com/mautrix/telegram/commit/{git_revision}"
+        git_revision = git_revision[:8]
+    except (subprocess.SubprocessError, OSError):
+        git_revision = "unknown"
+        git_revision_url = None
+
+    try:
+        git_tag = run(["git", "describe", "--exact-match", "--tags"]).strip().decode("ascii")
+    except (subprocess.SubprocessError, OSError):
+        git_tag = None
+else:
+    git_revision = "unknown"
+    git_revision_url = None
+    git_tag = None
+
+git_tag_url = f"https://github.com/mautrix/telegram/releases/tag/{git_tag}" if git_tag else None
+
+if git_tag and __version__ == git_tag[1:].replace("-", ""):
+    version = __version__
+    linkified_version = f"[{version}]({git_tag_url})"
+else:
+    if not __version__.endswith("+dev"):
+        __version__ += "+dev"
+    version = f"{__version__}.{git_revision}"
+    if git_revision_url:
+        linkified_version = f"{__version__}.[{git_revision}]({git_revision_url})"
+    else:
+        linkified_version = version

mautrix_telegram/matrix.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,392 +13,411 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Dict, Tuple, Set, Match
-import logging
-import asyncio
-import re
+from __future__ import annotations
 
-from mautrix_appservice import MatrixRequestError, IntentError
+from typing import TYPE_CHECKING, Iterable
 
-from . import user as u, portal as po, puppet as pu, commands as com
+from mautrix.appservice import DOUBLE_PUPPET_SOURCE_KEY
+from mautrix.bridge import BaseMatrixHandler
+from mautrix.errors import MatrixError
+from mautrix.types import (
+    Event,
+    EventID,
+    EventType,
+    MemberStateEventContent,
+    MessageType,
+    PresenceEvent,
+    PresenceState,
+    ReceiptEvent,
+    ReceiptEventContent,
+    ReceiptType,
+    RedactionEvent,
+    RoomAvatarStateEventContent as AvatarContent,
+    RoomID,
+    RoomNameStateEventContent as NameContent,
+    RoomTopicStateEventContent as TopicContent,
+    SingleReceiptEventContent,
+    StateEvent,
+    TextMessageEventContent,
+    TypingEvent,
+    UserID,
+)
+
+from . import commands as com, portal as po, puppet as pu, user as u
+
+if TYPE_CHECKING:
+    from .__main__ import TelegramBridge
 
 
-class MatrixHandler:
-    log = logging.getLogger("mau.mx")  # type: logging.Logger
+class MatrixHandler(BaseMatrixHandler):
+    commands: com.CommandProcessor
+    _previously_typing: dict[RoomID, set[UserID]]
 
-    def __init__(self, context):
-        self.az, self.db, self.config, _, self.tgbot = context
-        self.commands = com.CommandProcessor(context)  # type: com.CommandProcessor
-        self.previously_typing = []  # type: List[str]
+    def __init__(self, bridge: "TelegramBridge") -> None:
+        prefix, suffix = bridge.config["bridge.username_template"].format(userid=":").split(":")
+        homeserver = bridge.config["homeserver.domain"]
+        self.user_id_prefix = f"@{prefix}"
+        self.user_id_suffix = f"{suffix}:{homeserver}"
 
-        self.az.matrix_event_handler(self.handle_event)
+        super().__init__(command_processor=com.CommandProcessor(bridge), bridge=bridge)
 
-    async def init_as_bot(self):
-        displayname = self.config["appservice.bot_displayname"]
-        if displayname:
-            try:
-                await self.az.intent.set_display_name(
-                    displayname if displayname != "remove" else "")
-            except asyncio.TimeoutError:
-                self.log.exception("TimeoutError when trying to set displayname")
+        self._previously_typing = {}
 
-        avatar = self.config["appservice.bot_avatar"]
-        if avatar:
-            try:
-                await self.az.intent.set_avatar(avatar if avatar != "remove" else "")
-            except asyncio.TimeoutError:
-                self.log.exception("TimeoutError when trying to set avatar")
-
-    async def handle_puppet_invite(self, room_id, puppet: pu.Puppet, inviter: u.User):
+    async def handle_puppet_invite(
+        self, room_id: RoomID, puppet: pu.Puppet, inviter: u.User, event_id: EventID
+    ) -> None:
         intent = puppet.default_mxid_intent
-        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}")
+        self.log.debug(f"{inviter.mxid} invited puppet for {puppet.tgid} to {room_id}")
         if not await inviter.is_logged_in():
             await intent.error_and_leave(
-                room_id, text="Please log in before inviting Telegram puppets.")
+                room_id, text="Please log in before inviting Telegram puppets."
+            )
             return
-        portal = po.Portal.get_by_mxid(room_id)
+        portal = await po.Portal.get_by_mxid(room_id)
         if portal:
             if portal.peer_type == "user":
                 await intent.error_and_leave(
-                    room_id, text="You can not invite additional users to private chats.")
+                    room_id, text="You can not invite additional users to private chats."
+                )
                 return
             await portal.invite_telegram(inviter, puppet)
             await intent.join_room(room_id)
             return
         try:
-            members = await self.az.intent.get_room_members(room_id)
-        except MatrixRequestError:
-            members = []
+            members = await intent.get_room_members(room_id)
+        except MatrixError:
+            self.log.exception(f"Failed to get members after joining {room_id} as {intent.mxid}")
+            return
         if self.az.bot_mxid not in members:
-            if len(members) > 1:
-                await intent.error_and_leave(room_id, text=None, html=(
-                    f"Please invite "
-                    f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
-                    f"first if you want to create a Telegram chat."))
+            if len(members) > 2:
+                await intent.error_and_leave(
+                    room_id,
+                    text=None,
+                    html=(
+                        f"Please invite "
+                        f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
+                        f"first if you want to create a Telegram chat."
+                    ),
+                )
                 return
 
             await intent.join_room(room_id)
-            portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
+            portal = await po.Portal.get_by_tgid(
+                puppet.tgid, tg_receiver=inviter.tgid, peer_type="user"
+            )
             if portal.mxid:
                 try:
-                    await intent.invite(portal.mxid, inviter.mxid)
-                    await intent.send_notice(room_id, text=None, html=(
-                        "You already have a private chat with me: "
-                        f"<a href='https://matrix.to/#/{portal.mxid}'>"
-                        "Link to room"
-                        "</a>"))
+                    await portal.invite_to_matrix(inviter.mxid)
+                    await intent.send_notice(
+                        room_id,
+                        text=f"You already have a private chat with me: {portal.mxid}",
+                        html=(
+                            "You already have a private chat with me: "
+                            f"<a href='https://matrix.to/#/{portal.mxid}'>Link to room</a>"
+                        ),
+                    )
                     await intent.leave_room(room_id)
                     return
-                except MatrixRequestError:
+                except MatrixError:
                     pass
             portal.mxid = room_id
-            portal.save()
-            inviter.register_portal(portal)
-            await intent.send_notice(room_id, "po.Portal to private chat created.")
+            e2be_ok = await portal.check_dm_encryption()
+            await portal.save()
+            await inviter.register_portal(portal)
+            if e2be_ok is True:
+                evt_type, content = await self.e2ee.encrypt(
+                    room_id,
+                    EventType.ROOM_MESSAGE,
+                    TextMessageEventContent(
+                        msgtype=MessageType.NOTICE,
+                        body=(
+                            "Portal to private chat created and end-to-bridge encryption enabled."
+                        ),
+                    ),
+                )
+                await intent.send_message_event(room_id, evt_type, content)
+            else:
+                message = "Portal to private chat created."
+                if e2be_ok is False:
+                    message += "\n\nWarning: Failed to enable end-to-bridge encryption"
+                await intent.send_notice(room_id, message)
+            await portal.update_bridge_info()
         else:
             await intent.join_room(room_id)
-            await intent.send_notice(room_id, "This puppet will remain inactive until a "
-                                              "Telegram chat is created for this room.")
+            await intent.send_notice(
+                room_id,
+                "This puppet will remain inactive until a Telegram chat is created for this room.",
+            )
 
-    async def accept_bot_invite(self, room_id: str, inviter: u.User):
-        tries = 0
-        while tries < 5:
-            try:
-                await self.az.intent.join_room(room_id)
-                break
-            except (IntentError, MatrixRequestError):
-                tries += 1
-                wait_for_seconds = (tries + 1) * 10
-                if tries < 5:
-                    self.log.exception(f"Failed to join room {room_id} with bridge bot, "
-                                       f"retrying in {wait_for_seconds} seconds...")
-                    await asyncio.sleep(wait_for_seconds)
-                else:
-                    self.log.exception("Failed to join room {room}, giving up.")
-                    return
-
-        if not inviter.whitelisted:
-            await self.az.intent.send_notice(
-                room_id, text=None,
-                html="You are not whitelisted to use this bridge.<br/><br/>"
-                     "If you are the owner of this bridge, see the "
-                     "<code>bridge.permissions</code> section in your config file.")
-            await self.az.intent.leave_room(room_id)
-
-    async def handle_invite(self, room_id: str, user_id: str, inviter_mxid: str):
-        self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
-        inviter = await u.User.get_by_mxid(inviter_mxid).ensure_started()
-        if user_id == self.az.bot_mxid:
-            return await self.accept_bot_invite(room_id, inviter)
-        elif not inviter.whitelisted:
-            return
-
-        puppet = pu.Puppet.get_by_mxid(user_id)
-        if puppet:
-            await self.handle_puppet_invite(room_id, puppet, inviter)
-            return
-
-        user = u.User.get_by_mxid(user_id, create=False)
+    async def handle_invite(
+        self, room_id: RoomID, user_id: UserID, inviter: u.User, event_id: EventID
+    ) -> None:
+        user = await u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
-        portal = po.Portal.get_by_mxid(room_id)
-        if user and await user.has_full_access(allow_bot=True) and portal:
-            await portal.invite_telegram(inviter, user)
-            return
+        portal = await po.Portal.get_by_mxid(room_id)
+        if user and await user.has_full_access(allow_bot=True):
+            if portal and portal.allow_bridging:
+                await portal.invite_telegram(inviter, user)
 
-        # The rest can probably be ignored
+    async def handle_join(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None:
+        user = await u.User.get_and_start_by_mxid(user_id)
 
-    async def handle_join(self, room_id: str, user_id: str, event_id: str):
-        user = await u.User.get_by_mxid(user_id).ensure_started()
-
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
+        portal = await po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.allow_bridging:
             return
 
         if not user.relaybot_whitelisted:
-            await portal.main_intent.kick(room_id, user.mxid,
-                                          "You are not whitelisted on this Telegram bridge.")
+            await portal.main_intent.kick_user(
+                room_id, user.mxid, "You are not whitelisted on this Telegram bridge."
+            )
             return
         elif not await user.is_logged_in() and not portal.has_bot:
-            await portal.main_intent.kick(room_id, user.mxid,
-                                          "This chat does not have a bot relaying "
-                                          "messages for unauthenticated users.")
+            await portal.main_intent.kick_user(
+                room_id,
+                user.mxid,
+                "This chat does not have a bot relaying messages for unauthenticated users.",
+            )
             return
 
-        self.log.debug(f"{user} joined {room_id}")
+        self.log.debug(f"{user.mxid} joined {room_id}")
         if await user.is_logged_in() or portal.has_bot:
             await portal.join_matrix(user, event_id)
 
-    async def handle_part(self, room_id: str, user_id, sender_mxid: str, event_id: str):
+    async def handle_leave(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None:
         self.log.debug(f"{user_id} left {room_id}")
+        portal = await po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.allow_bridging:
+            return
 
-        sender = u.User.get_by_mxid(sender_mxid, create=False)
+        user = await u.User.get_by_mxid(user_id, create=False)
+        if not user:
+            return
+        await user.ensure_started()
+        await portal.leave_matrix(user, event_id)
+
+    async def handle_kick_ban(
+        self,
+        ban: bool,
+        room_id: RoomID,
+        user_id: UserID,
+        sender: UserID,
+        reason: str,
+        event_id: EventID,
+    ) -> None:
+        action = "banned" if ban else "kicked"
+        self.log.debug(f"{user_id} was {action} from {room_id} by {sender} for {reason}")
+        portal = await po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.allow_bridging:
+            return
+
+        if user_id == self.az.bot_mxid:
+            # Direct chat portal unbridging is handled in portal.kick_matrix
+            if portal.peer_type != "user":
+                await portal.unbridge()
+            return
+
+        sender = await u.User.get_by_mxid(sender, create=False)
         if not sender:
             return
         await sender.ensure_started()
 
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
+        puppet = await pu.Puppet.get_by_mxid(user_id)
+        if puppet:
+            if ban:
+                await portal.ban_matrix(puppet, sender)
+            else:
+                await portal.kick_matrix(puppet, sender)
             return
 
-        puppet = pu.Puppet.get_by_mxid(user_id)
-        if sender and puppet:
-            await portal.leave_matrix(puppet, sender, event_id)
-
-        user = u.User.get_by_mxid(user_id, create=False)
+        user = await u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
-        if await user.is_logged_in() or portal.has_bot:
-            await portal.leave_matrix(user, sender, event_id)
+        if ban:
+            await portal.ban_matrix(user, sender)
+        else:
+            await portal.kick_matrix(user, sender)
 
-    def is_command(self, message: dict) -> Tuple[bool, str]:
-        text = message.get("body", "")
-        prefix = self.config["bridge.command_prefix"]
-        is_command = text.startswith(prefix)
-        if is_command:
-            text = text[len(prefix) + 1:]
-        return is_command, text
+    async def handle_kick(
+        self, room_id: RoomID, user_id: UserID, kicked_by: UserID, reason: str, event_id: EventID
+    ) -> None:
+        await self.handle_kick_ban(False, room_id, user_id, kicked_by, reason, event_id)
 
-    async def handle_message(self, room, sender, message, event_id):
-        is_command, text = self.is_command(message)
-        sender = await u.User.get_by_mxid(sender).ensure_started()
-        if not sender.relaybot_whitelisted:
-            self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:"
-                           " u.User is not whitelisted.")
-            return
-        self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}")
+    async def handle_unban(
+        self, room_id: RoomID, user_id: UserID, unbanned_by: UserID, reason: str, event_id: EventID
+    ) -> None:
+        # TODO handle unbans properly instead of handling it as a kick
+        await self.handle_kick_ban(False, room_id, user_id, unbanned_by, reason, event_id)
 
-        portal = po.Portal.get_by_mxid(room)
-        if not is_command and portal and (await sender.is_logged_in() or portal.has_bot):
-            await portal.handle_matrix_message(sender, message, event_id)
-            return
+    async def handle_ban(
+        self, room_id: RoomID, user_id: UserID, banned_by: UserID, reason: str, event_id: EventID
+    ) -> None:
+        await self.handle_kick_ban(True, room_id, user_id, banned_by, reason, event_id)
 
-        if not sender.whitelisted or message.get("msgtype", "m.unknown") != "m.text":
-            return
+    async def allow_message(self, user: u.User) -> bool:
+        return user.relaybot_whitelisted
 
-        try:
-            is_management = len(await self.az.intent.get_room_members(room)) == 2
-        except MatrixRequestError:
-            # The AS bot is not in the room.
-            return
-
-        if is_command or is_management:
-            try:
-                command, arguments = text.split(" ", 1)
-                args = arguments.split(" ")
-            except ValueError:
-                # Not enough values to unpack, i.e. no arguments
-                command = text
-                args = []
-            await self.commands.handle(room, sender, command, args, is_management,
-                                       is_portal=portal is not None)
+    async def allow_command(self, user: u.User) -> bool:
+        return user.whitelisted
 
     @staticmethod
-    async def handle_redaction(room_id: str, sender_mxid: str, event_id: str):
-        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+    async def allow_bridging_message(user: u.User, portal: po.Portal) -> bool:
+        return await user.is_logged_in() or portal.has_bot
+
+    @staticmethod
+    async def handle_redaction(evt: RedactionEvent) -> None:
+        sender = await u.User.get_and_start_by_mxid(evt.sender)
         if not sender.relaybot_whitelisted:
             return
 
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
+        portal = await po.Portal.get_by_mxid(evt.room_id)
+        if not portal or not portal.allow_bridging:
             return
 
-        await portal.handle_matrix_deletion(sender, event_id)
+        await portal.handle_matrix_deletion(sender, evt.redacts, evt.event_id)
 
     @staticmethod
-    async def handle_power_levels(room_id: str, sender_mxid: str, new: dict, old: dict):
-        portal = po.Portal.get_by_mxid(room_id)
-        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
-        if await sender.has_full_access(allow_bot=True) and portal:
-            await portal.handle_matrix_power_levels(sender, new["users"], old["users"])
+    async def handle_power_levels(evt: StateEvent) -> None:
+        portal = await po.Portal.get_by_mxid(evt.room_id)
+        sender = await u.User.get_and_start_by_mxid(evt.sender)
+        if await sender.has_full_access(allow_bot=True) and portal and portal.allow_bridging:
+            await portal.handle_matrix_power_levels(
+                sender, evt.content.users, evt.unsigned.prev_content.users, evt.event_id
+            )
 
     @staticmethod
-    async def handle_room_meta(evt_type: str, room_id: str, sender_mxid: str, content: dict):
-        portal = po.Portal.get_by_mxid(room_id)
-        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
-        if await sender.has_full_access(allow_bot=True) and portal:
-            handler, content_key = {
-                "m.room.name": (portal.handle_matrix_title, "name"),
-                "m.room.topic": (portal.handle_matrix_about, "topic"),
-                "m.room.avatar": (portal.handle_matrix_avatar, "url"),
+    async def handle_room_meta(
+        evt_type: EventType,
+        room_id: RoomID,
+        sender_mxid: UserID,
+        content: NameContent | AvatarContent | TopicContent,
+        event_id: EventID,
+    ) -> None:
+        portal = await po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_and_start_by_mxid(sender_mxid)
+        if await sender.has_full_access(allow_bot=True) and portal and portal.allow_bridging:
+            handler, content_type, content_key = {
+                EventType.ROOM_NAME: (portal.handle_matrix_title, NameContent, "name"),
+                EventType.ROOM_TOPIC: (portal.handle_matrix_about, TopicContent, "topic"),
+                EventType.ROOM_AVATAR: (portal.handle_matrix_avatar, AvatarContent, "url"),
             }[evt_type]
-            if content_key not in content:
+            if not isinstance(content, content_type):
                 return
-            await handler(sender, content[content_key])
+            await handler(sender, content[content_key], event_id)
 
     @staticmethod
-    async def handle_room_pin(room_id: str, sender_mxid: str, new_events: Set[str],
-                              old_events: Set[str]):
-        portal = po.Portal.get_by_mxid(room_id)
-        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
-        if await sender.has_full_access(allow_bot=True) and portal:
-            events = new_events - old_events
-            if len(events) > 0:
-                # New event pinned, set that as pinned in Telegram.
-                await portal.handle_matrix_pin(sender, events.pop())
-            elif len(new_events) == 0:
-                # All pinned events removed, remove pinned event in Telegram.
-                await portal.handle_matrix_pin(sender, None)
+    async def handle_room_pin(
+        room_id: RoomID,
+        sender_mxid: UserID,
+        new_events: set[str],
+        old_events: set[str],
+        event_id: EventID,
+    ) -> None:
+        portal = await po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_and_start_by_mxid(sender_mxid)
+        if await sender.has_full_access(allow_bot=True) and portal and portal.allow_bridging:
+            if not new_events:
+                await portal.handle_matrix_unpin_all(sender, event_id)
+            else:
+                changes = {
+                    event_id: event_id in new_events for event_id in new_events ^ old_events
+                }
+                await portal.handle_matrix_pin(sender, changes, event_id)
 
     @staticmethod
-    async def handle_name_change(room_id: str, user_id: str, displayname: str,
-                                 prev_displayname: str, event_id: str):
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal or not portal.has_bot:
+    async def handle_room_upgrade(
+        room_id: RoomID, sender: UserID, new_room_id: RoomID, event_id: EventID
+    ) -> None:
+        portal = await po.Portal.get_by_mxid(room_id)
+        if portal and portal.allow_bridging:
+            await portal.handle_matrix_upgrade(sender, new_room_id, event_id)
+
+    async def handle_member_info_change(
+        self,
+        room_id: RoomID,
+        user_id: UserID,
+        profile: MemberStateEventContent,
+        prev_profile: MemberStateEventContent,
+        event_id: EventID,
+    ) -> None:
+        if profile.displayname == prev_profile.displayname:
             return
 
-        user = await u.User.get_by_mxid(user_id).ensure_started()
+        portal = await po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.has_bot or not portal.allow_bridging:
+            return
+
+        user = await u.User.get_and_start_by_mxid(user_id)
         if await user.needs_relaybot(portal):
-            await portal.name_change_matrix(user, displayname, prev_displayname, event_id)
+            await portal.name_change_matrix(
+                user, profile.displayname, prev_profile.displayname, event_id
+            )
+
+    async def handle_read_receipt(
+        self, user: u.User, portal: po.Portal, event_id: EventID, data: SingleReceiptEventContent
+    ) -> None:
+        if not portal.allow_bridging:
+            return
+        await portal.mark_read(user, event_id, data.get("ts", 0))
 
     @staticmethod
-    def parse_read_receipts(content: dict) -> Dict[str, str]:
-        return {user_id: event_id
-                for event_id, receipts in content.items()
-                for user_id in receipts.get("m.read", {})}
+    async def handle_presence(user_id: UserID, presence: PresenceState) -> None:
+        user = await u.User.get_by_mxid(user_id, check_db=False, create=False)
+        if user and await user.is_logged_in():
+            await user.set_presence(presence == PresenceState.ONLINE)
 
-    @staticmethod
-    async def handle_read_receipts(room_id: str, receipts: Dict[str, str]):
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
+    async def handle_typing(self, room_id: RoomID, now_typing: set[UserID]) -> None:
+        portal = await po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.allow_bridging:
             return
 
-        for user_id, event_id in receipts.items():
-            user = await u.User.get_by_mxid(user_id).ensure_started()
-            if not await user.is_logged_in():
-                continue
-            await portal.mark_read(user, event_id)
+        previously_typing = self._previously_typing.get(room_id, set())
 
-    @staticmethod
-    async def handle_presence(user_id: str, presence: str):
-        user = await u.User.get_by_mxid(user_id).ensure_started()
-        if not await user.is_logged_in():
-            return
-        await user.set_presence(presence == "online")
-
-    async def handle_typing(self, room_id: str, now_typing: List[str]):
-        portal = po.Portal.get_by_mxid(room_id)
-        if not portal:
-            return
-
-        for user_id in set(self.previously_typing + now_typing):
+        for user_id in set(previously_typing | now_typing):
             is_typing = user_id in now_typing
-            was_typing = user_id in self.previously_typing
+            was_typing = user_id in previously_typing
             if is_typing and was_typing:
                 continue
 
-            user = await u.User.get_by_mxid(user_id).ensure_started()
-            if not await user.is_logged_in():
-                continue
+            user = await u.User.get_by_mxid(user_id, check_db=False, create=False)
+            if user and await user.is_logged_in():
+                await portal.set_typing(user, is_typing)
 
-            await portal.set_typing(user, is_typing)
+        self._previously_typing[room_id] = now_typing
 
-        self.previously_typing = now_typing
+    async def handle_ephemeral_event(
+        self, evt: ReceiptEvent | PresenceEvent | TypingEvent
+    ) -> None:
+        if evt.type == EventType.RECEIPT:
+            await self.handle_receipt(evt)
+        elif evt.type == EventType.PRESENCE:
+            await self.handle_presence(evt.sender, evt.content.presence)
+        elif evt.type == EventType.TYPING:
+            await self.handle_typing(evt.room_id, set(evt.content.user_ids))
 
-    def filter_matrix_event(self, event: dict):
-        sender = event.get("sender", None)
-        if not sender:
-            return False
-        return (sender == self.az.bot_mxid
-                or pu.Puppet.get_id_from_mxid(sender) is not None)
+    async def handle_event(self, evt: Event) -> None:
+        if evt.type == EventType.ROOM_REDACTION:
+            await self.handle_redaction(evt)
 
-    async def try_handle_event(self, evt: dict):
-        try:
-            await self.handle_event(evt)
-        except Exception:
-            self.log.exception("Error handling manually received Matrix event")
-
-    async def handle_event(self, evt: dict):
-        if self.filter_matrix_event(evt):
-            return
-        self.log.debug("Received event: %s", evt)
-        evt_type = evt.get("type", "m.unknown")  # type: str
-        room_id = evt.get("room_id", None)  # type: str
-        event_id = evt.get("event_id", None)  # type: str
-        sender = evt.get("sender", None)  # type: str
-        content = evt.get("content", {})  # type: dict
-        if evt_type == "m.room.member":
-            state_key = evt["state_key"]  # type: str
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: dict
-            membership = content.get("membership", "")  # type: str
-            prev_membership = prev_content.get("membership", "leave")  # type: str
-            if membership == prev_membership:
-                match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
-                localpart = match.group(1)  # type: str
-                displayname = content.get("displayname", localpart)  # type: str
-                prev_displayname = prev_content.get("displayname", localpart)  # type: str
-                if displayname != prev_displayname:
-                    await self.handle_name_change(room_id, state_key, displayname,
-                                                  prev_displayname, event_id)
-            elif membership == "invite":
-                await self.handle_invite(room_id, state_key, sender)
-            elif prev_membership == "join" and membership == "leave":
-                await self.handle_part(room_id, state_key, sender, event_id)
-            elif membership == "join":
-                await self.handle_join(room_id, state_key, event_id)
-        elif evt_type in ("m.room.message", "m.sticker"):
-            if evt_type != "m.room.message":
-                content["msgtype"] = evt_type
-            await self.handle_message(room_id, sender, content, event_id)
-        elif evt_type == "m.room.redaction":
-            await self.handle_redaction(room_id, sender, evt["redacts"])
-        elif evt_type == "m.room.power_levels":
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: dict
-            await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
-        elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
-            await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
-        elif evt_type == "m.room.pinned_events":
-            new_events = set(evt["content"]["pinned"])
+    async def handle_state_event(self, evt: StateEvent) -> None:
+        if evt.type == EventType.ROOM_POWER_LEVELS:
+            await self.handle_power_levels(evt)
+        elif evt.type in (EventType.ROOM_NAME, EventType.ROOM_AVATAR, EventType.ROOM_TOPIC):
+            await self.handle_room_meta(
+                evt.type, evt.room_id, evt.sender, evt.content, evt.event_id
+            )
+        elif evt.type == EventType.ROOM_PINNED_EVENTS:
+            new_events = set(evt.content.pinned)
             try:
-                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
-            except KeyError:
+                old_events = set(evt.unsigned.prev_content.pinned)
+            except (KeyError, ValueError, TypeError, AttributeError):
                 old_events = set()
-            await self.handle_room_pin(room_id, sender, new_events, old_events)
-        elif evt_type == "m.receipt":
-            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
-        elif evt_type == "m.presence":
-            await self.handle_presence(sender, content.get("presence", "offline"))
-        elif evt_type == "m.typing":
-            await self.handle_typing(room_id, content.get("user_ids", []))
+            await self.handle_room_pin(
+                evt.room_id, evt.sender, new_events, old_events, evt.event_id
+            )
+        elif evt.type == EventType.ROOM_TOMBSTONE:
+            await self.handle_room_upgrade(
+                evt.room_id, evt.sender, evt.content.replacement_room, evt.event_id
+            )

mautrix_telegram/portal_util/__init__.py

@@ -0,0 +1,6 @@
+from .deduplication import PortalDedup
+from .media_fallback import make_contact_event_content, make_dice_event_content
+from .participants import get_users
+from .power_levels import get_base_power_levels, participants_to_power_levels
+from .send_lock import PortalSendLock
+from .sponsored_message import get_sponsored_message, make_sponsored_message_content

mautrix_telegram/portal_util/deduplication.py

@@ -0,0 +1,134 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Tuple
+from collections import deque
+import hashlib
+
+from telethon.tl.patched import Message, MessageService
+from telethon.tl.types import (
+    MessageMediaContact,
+    MessageMediaDocument,
+    MessageMediaGeo,
+    MessageMediaPhoto,
+    TypeMessage,
+    TypeUpdates,
+    UpdateNewChannelMessage,
+    UpdateNewMessage,
+)
+
+from mautrix.types import EventID
+
+from .. import portal as po
+from ..types import TelegramID
+
+DedupMXID = Tuple[EventID, TelegramID]
+
+
+class PortalDedup:
+    pre_db_check: bool = False
+    cache_queue_length: int = 20
+
+    _dedup: deque[str]
+    _dedup_mxid: dict[str, DedupMXID]
+    _dedup_action: deque[str]
+    _portal: po.Portal
+
+    def __init__(self, portal: po.Portal) -> None:
+        self._dedup = deque()
+        self._dedup_mxid = {}
+        self._dedup_action = deque()
+        self._portal = portal
+
+    @property
+    def _always_force_hash(self) -> bool:
+        return self._portal.peer_type == "chat"
+
+    @staticmethod
+    def _hash_event(event: TypeMessage) -> str:
+        # Non-channel messages are unique per-user (wtf telegram), so we have no other choice than
+        # to deduplicate based on a hash of the message content.
+
+        # The timestamp is only accurate to the second, so we can't rely solely on that either.
+        if isinstance(event, MessageService):
+            hash_content = [event.date.timestamp(), event.from_id, event.action]
+        else:
+            hash_content = [event.date.timestamp(), event.message.strip()]
+            if event.fwd_from:
+                hash_content += [event.fwd_from.from_id]
+            elif isinstance(event, Message) and event.media:
+                try:
+                    hash_content += {
+                        MessageMediaContact: lambda media: [media.user_id],
+                        MessageMediaDocument: lambda media: [media.document.id],
+                        MessageMediaPhoto: lambda media: [media.photo.id if media.photo else 0],
+                        MessageMediaGeo: lambda media: [media.geo.long, media.geo.lat],
+                    }[type(event.media)](event.media)
+                except KeyError:
+                    pass
+        return hashlib.md5("-".join(str(a) for a in hash_content).encode("utf-8")).hexdigest()
+
+    def check_action(self, event: TypeMessage) -> bool:
+        evt_hash = self._hash_event(event) if self._always_force_hash else event.id
+        if evt_hash in self._dedup_action:
+            return True
+
+        self._dedup_action.append(evt_hash)
+
+        if len(self._dedup_action) > self.cache_queue_length:
+            self._dedup_action.popleft()
+        return False
+
+    def update(
+        self,
+        event: TypeMessage,
+        mxid: DedupMXID = None,
+        expected_mxid: DedupMXID | None = None,
+        force_hash: bool = False,
+    ) -> DedupMXID | None:
+        evt_hash = self._hash_event(event) if self._always_force_hash or force_hash else event.id
+        try:
+            found_mxid = self._dedup_mxid[evt_hash]
+        except KeyError:
+            return EventID("None"), TelegramID(0)
+
+        if found_mxid != expected_mxid:
+            return found_mxid
+        self._dedup_mxid[evt_hash] = mxid
+        return None
+
+    def check(
+        self, event: TypeMessage, mxid: DedupMXID = None, force_hash: bool = False
+    ) -> DedupMXID | None:
+        evt_hash = self._hash_event(event) if self._always_force_hash or force_hash else event.id
+        if evt_hash in self._dedup:
+            return self._dedup_mxid[evt_hash]
+
+        self._dedup_mxid[evt_hash] = mxid
+        self._dedup.append(evt_hash)
+
+        if len(self._dedup) > self.cache_queue_length:
+            del self._dedup_mxid[self._dedup.popleft()]
+        return None
+
+    def register_outgoing_actions(self, response: TypeUpdates) -> None:
+        for update in response.updates:
+            check_dedup = isinstance(
+                update, (UpdateNewMessage, UpdateNewChannelMessage)
+            ) and isinstance(update.message, MessageService)
+            if check_dedup:
+                self.check(update.message)

mautrix_telegram/portal_util/media_fallback.py

@@ -0,0 +1,133 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import html
+
+from telethon.tl.types import MessageMediaContact, MessageMediaDice, PeerUser
+
+from mautrix.types import Format, MessageType, TextMessageEventContent
+
+from .. import abstract_user as au, puppet as pu
+from ..types import TelegramID
+
+try:
+    import phonenumbers
+except ImportError:
+    phonenumbers = None
+
+
+def _format_dice(roll: MessageMediaDice) -> str:
+    if roll.emoticon == "\U0001F3B0":
+        emojis = {
+            0: "\U0001F36B",  # "🍫",
+            1: "\U0001F352",  # "🍒",
+            2: "\U0001F34B",  # "🍋",
+            3: "7\ufe0f\u20e3",  # "7️⃣",
+        }
+        res = roll.value - 1
+        slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
+        return f"{slot1} {slot2} {slot3} ({roll.value})"
+    elif roll.emoticon == "\u26BD":
+        results = {
+            1: "miss",
+            2: "hit the woodwork",
+            3: "goal",  # seems to go in through the center
+            4: "goal",
+            5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
+        }
+    elif roll.emoticon == "\U0001F3B3":
+        results = {
+            1: "miss",
+            2: "1 pin down",
+            3: "3 pins down, split",
+            4: "4 pins down, split",
+            5: "5 pins down",
+            6: "strike 🎉",
+        }
+    # elif roll.emoticon == "\U0001F3C0":
+    #     results = {
+    #         2: "rolled off",
+    #         3: "stuck",
+    #     }
+    # elif roll.emoticon == "\U0001F3AF":
+    #     results = {
+    #         1: "bounced off",
+    #         2: "outer rim",
+    #
+    #         6: "bullseye",
+    #     }
+    else:
+        return str(roll.value)
+    return f"{results[roll.value]} ({roll.value})"
+
+
+def make_dice_event_content(roll: MessageMediaDice) -> TextMessageEventContent:
+    emoji_text = {
+        "\U0001F3AF": " Dart throw",
+        "\U0001F3B2": " Dice roll",
+        "\U0001F3C0": " Basketball throw",
+        "\U0001F3B0": " Slot machine",
+        "\U0001F3B3": " Bowling",
+        "\u26BD": " Football kick",
+    }
+    text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {_format_dice(roll)}"
+    content = TextMessageEventContent(
+        msgtype=MessageType.TEXT, format=Format.HTML, body=text, formatted_body=f"<h4>{text}</h4>"
+    )
+    content["net.maunium.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
+    return content
+
+
+async def make_contact_event_content(
+    source: au.AbstractUser, contact: MessageMediaContact
+) -> TextMessageEventContent:
+    name = " ".join(x for x in [contact.first_name, contact.last_name] if x)
+    formatted_phone = f"+{contact.phone_number}"
+    if phonenumbers is not None:
+        try:
+            parsed = phonenumbers.parse(formatted_phone)
+            fmt = phonenumbers.PhoneNumberFormat.INTERNATIONAL
+            formatted_phone = phonenumbers.format_number(parsed, fmt)
+        except phonenumbers.NumberParseException:
+            pass
+    content = TextMessageEventContent(
+        msgtype=MessageType.TEXT,
+        body=f"Shared contact info for {name}: {formatted_phone}",
+    )
+    content["net.maunium.telegram.contact"] = {
+        "user_id": contact.user_id,
+        "first_name": contact.first_name,
+        "last_name": contact.last_name,
+        "phone_number": contact.phone_number,
+        "vcard": contact.vcard,
+    }
+
+    puppet = await pu.Puppet.get_by_tgid(TelegramID(contact.user_id))
+    if not puppet.displayname:
+        try:
+            entity = await source.client.get_entity(PeerUser(contact.user_id))
+            await puppet.update_info(source, entity)
+        except Exception as e:
+            source.log.warning(f"Failed to sync puppet info of received contact: {e}")
+    else:
+        content.format = Format.HTML
+        content.formatted_body = (
+            f"Shared contact info for "
+            f"<a href='https://matrix.to/#/{puppet.mxid}'>{html.escape(name)}</a>: "
+            f"{html.escape(formatted_phone)}"
+        )
+    return content

mautrix_telegram/portal_util/participants.py

@@ -0,0 +1,106 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Iterable
+
+from telethon.errors import ChatAdminRequiredError
+from telethon.tl.functions.channels import GetParticipantsRequest
+from telethon.tl.functions.messages import GetFullChatRequest
+from telethon.tl.types import (
+    ChannelParticipantBanned,
+    ChannelParticipantsRecent,
+    ChannelParticipantsSearch,
+    InputChannel,
+    InputUser,
+    TypeChannelParticipant,
+    TypeChat,
+    TypeChatParticipant,
+    TypeInputPeer,
+    TypeUser,
+)
+
+from ..tgclient import MautrixTelegramClient
+
+
+def _filter_participants(
+    users: list[TypeUser], participants: list[TypeChatParticipant | TypeChannelParticipant]
+) -> Iterable[TypeUser]:
+    participant_map = {
+        part.user_id: part
+        for part in participants
+        if not isinstance(part, ChannelParticipantBanned)
+    }
+    for user in users:
+        try:
+            user.participant = participant_map[user.id]
+        except KeyError:
+            pass
+        else:
+            yield user
+
+
+async def _get_channel_users(
+    client: MautrixTelegramClient, entity: InputChannel, limit: int
+) -> list[TypeUser]:
+    if 0 < limit <= 200:
+        response = await client(
+            GetParticipantsRequest(
+                entity, ChannelParticipantsRecent(), offset=0, limit=limit, hash=0
+            )
+        )
+        return list(_filter_participants(response.users, response.participants))
+    elif limit > 200 or limit == -1:
+        users: list[TypeUser] = []
+        offset = 0
+        remaining_quota = limit if limit > 0 else 1000000
+        query = ChannelParticipantsSearch("") if limit == -1 else ChannelParticipantsRecent()
+        while True:
+            if remaining_quota <= 0:
+                break
+            response = await client(
+                GetParticipantsRequest(
+                    entity, query, offset=offset, limit=min(remaining_quota, 200), hash=0
+                )
+            )
+            if not response.users:
+                break
+            users += _filter_participants(response.users, response.participants)
+            offset += len(response.participants)
+            remaining_quota -= len(response.participants)
+        return users
+
+
+async def get_users(
+    client: MautrixTelegramClient,
+    tgid: int,
+    entity: TypeInputPeer | InputUser | TypeChat | TypeUser | InputChannel,
+    limit: int,
+    peer_type: str,
+) -> list[TypeUser]:
+    if peer_type == "chat":
+        chat = await client(GetFullChatRequest(chat_id=tgid))
+        users = list(_filter_participants(chat.users, chat.full_chat.participants.participants))
+        return users[:limit] if limit > 0 else users
+    elif peer_type == "channel":
+        try:
+            return await _get_channel_users(client, entity, limit)
+        except ChatAdminRequiredError:
+            return []
+    elif peer_type == "user":
+        return [entity]
+    else:
+        raise RuntimeError(f"Unexpected peer type {peer_type}")

mautrix_telegram/portal_util/power_levels.py

@@ -0,0 +1,154 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from telethon.tl.types import (
+    ChannelParticipantAdmin,
+    ChannelParticipantCreator,
+    ChatBannedRights,
+    ChatParticipantAdmin,
+    ChatParticipantCreator,
+    TypeChannelParticipant,
+    TypeChat,
+    TypeChatParticipant,
+    TypeUser,
+)
+
+from mautrix.types import EventType, PowerLevelStateEventContent as PowerLevelContent, UserID
+
+from .. import portal as po, puppet as pu, user as u
+from ..types import TelegramID
+
+
+def get_base_power_levels(
+    portal: po.Portal, levels: PowerLevelContent = None, entity: TypeChat = None
+) -> PowerLevelContent:
+    levels = levels or PowerLevelContent()
+    if portal.peer_type == "user":
+        overrides = portal.config["bridge.initial_power_level_overrides.user"]
+        levels.ban = overrides.get("ban", 100)
+        levels.kick = overrides.get("kick", 100)
+        levels.invite = overrides.get("invite", 100)
+        levels.redact = overrides.get("redact", 0)
+        levels.events[EventType.ROOM_NAME] = 0
+        levels.events[EventType.ROOM_AVATAR] = 0
+        levels.events[EventType.ROOM_TOPIC] = 0
+        levels.state_default = overrides.get("state_default", 0)
+        levels.users_default = overrides.get("users_default", 0)
+        levels.events_default = overrides.get("events_default", 0)
+    else:
+        overrides = portal.config["bridge.initial_power_level_overrides.group"]
+        dbr = entity.default_banned_rights
+        if not dbr:
+            portal.log.debug(f"default_banned_rights is None in {entity}")
+            dbr = ChatBannedRights(
+                invite_users=True,
+                change_info=True,
+                pin_messages=True,
+                send_stickers=False,
+                send_messages=False,
+                until_date=None,
+            )
+        levels.ban = overrides.get("ban", 50)
+        levels.kick = overrides.get("kick", 50)
+        levels.redact = overrides.get("redact", 50)
+        levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
+        levels.events[EventType.ROOM_ENCRYPTION] = 50 if portal.matrix.e2ee else 99
+        levels.events[EventType.ROOM_TOMBSTONE] = 99
+        levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
+        levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0
+        levels.events[EventType.ROOM_TOPIC] = 50 if dbr.change_info else 0
+        levels.events[EventType.ROOM_PINNED_EVENTS] = 50 if dbr.pin_messages else 0
+        levels.events[EventType.ROOM_POWER_LEVELS] = 75
+        levels.events[EventType.ROOM_HISTORY_VISIBILITY] = 75
+        levels.events[EventType.STICKER] = 50 if dbr.send_stickers else levels.events_default
+        levels.state_default = overrides.get("state_default", 50)
+        levels.users_default = overrides.get("users_default", 0)
+        levels.events_default = overrides.get(
+            "events_default",
+            50
+            if (
+                portal.peer_type == "channel"
+                and not entity.megagroup
+                or entity.default_banned_rights.send_messages
+            )
+            else 0,
+        )
+    for evt_type, value in overrides.get("events", {}).items():
+        levels.events[EventType.find(evt_type)] = value
+    levels.users = overrides.get("users", {})
+    if portal.main_intent.mxid not in levels.users:
+        levels.users[portal.main_intent.mxid] = 100
+    return levels
+
+
+async def participants_to_power_levels(
+    portal: po.Portal,
+    users: list[TypeUser | TypeChatParticipant | TypeChannelParticipant],
+    levels: PowerLevelContent,
+) -> bool:
+    bot_level = levels.get_user_level(portal.main_intent.mxid)
+    if bot_level < levels.get_event_level(EventType.ROOM_POWER_LEVELS):
+        return False
+    changed = False
+    admin_power_level = min(75 if portal.peer_type == "channel" else 50, bot_level)
+    if levels.get_event_level(EventType.ROOM_POWER_LEVELS) != admin_power_level:
+        changed = True
+        levels.events[EventType.ROOM_POWER_LEVELS] = admin_power_level
+
+    for user in users:
+        # The User objects we get from TelegramClient.get_participants have a custom
+        # participant property
+        participant = getattr(user, "participant", user)
+
+        puppet = await pu.Puppet.get_by_tgid(TelegramID(participant.user_id))
+        user = await u.User.get_by_tgid(TelegramID(participant.user_id))
+        new_level = _get_level_from_participant(portal.az.bot_mxid, participant, levels)
+
+        if user:
+            await user.register_portal(portal)
+            changed = _participant_to_power_levels(levels, user, new_level, bot_level) or changed
+
+        if puppet:
+            changed = _participant_to_power_levels(levels, puppet, new_level, bot_level) or changed
+    return changed
+
+
+def _get_level_from_participant(
+    bot_mxid: UserID,
+    participant: TypeUser | TypeChatParticipant | TypeChannelParticipant,
+    levels: PowerLevelContent,
+) -> int:
+    # TODO use the power level requirements to get better precision in channels
+    if isinstance(participant, (ChatParticipantAdmin, ChannelParticipantAdmin)):
+        return levels.state_default or 50
+    elif isinstance(participant, (ChatParticipantCreator, ChannelParticipantCreator)):
+        return levels.get_user_level(bot_mxid) - 5
+    return levels.users_default or 0
+
+
+def _participant_to_power_levels(
+    levels: PowerLevelContent,
+    user: u.User | pu.Puppet,
+    new_level: int,
+    bot_level: int,
+) -> bool:
+    new_level = min(new_level, bot_level)
+    user_level = levels.get_user_level(user.mxid)
+    if user_level != new_level and user_level < bot_level:
+        levels.users[user.mxid] = new_level
+        return True
+    return False

mautrix_telegram/portal_util/send_lock.py

@@ -0,0 +1,44 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from asyncio import Lock
+
+from ..types import TelegramID
+
+
+class FakeLock:
+    async def __aenter__(self) -> None:
+        pass
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        pass
+
+
+class PortalSendLock:
+    _send_locks: dict[int, Lock]
+    _noop_lock: Lock = FakeLock()
+
+    def __init__(self) -> None:
+        self._send_locks = {}
+
+    def __call__(self, user_id: TelegramID, required: bool = True) -> Lock:
+        if user_id is None and required:
+            raise ValueError("Required send lock for none id")
+        try:
+            return self._send_locks[user_id]
+        except KeyError:
+            return self._send_locks.setdefault(user_id, Lock()) if required else self._noop_lock

mautrix_telegram/portal_util/sponsored_message.py

@@ -0,0 +1,95 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import base64
+import html
+
+from telethon.tl.functions.channels import GetSponsoredMessagesRequest
+from telethon.tl.types import Channel, InputChannel, PeerChannel, PeerUser, SponsoredMessage, User
+
+from mautrix.types import MessageType, TextMessageEventContent
+
+from .. import user as u
+from ..formatter import telegram_to_matrix
+
+
+async def get_sponsored_message(
+    user: u.User,
+    entity: InputChannel,
+) -> tuple[SponsoredMessage | None, int | None, Channel | User | None]:
+    resp = await user.client(GetSponsoredMessagesRequest(entity))
+    if len(resp.messages) == 0:
+        return None, None, None
+    msg = resp.messages[0]
+    if isinstance(msg.from_id, PeerUser):
+        entities = resp.users
+        target_id = msg.from_id.user_id
+    else:
+        entities = resp.chats
+        target_id = msg.from_id.channel_id
+    try:
+        entity = next(ent for ent in entities if ent.id == target_id)
+    except StopIteration:
+        entity = None
+    return msg, target_id, entity
+
+
+async def make_sponsored_message_content(
+    source: u.User, msg: SponsoredMessage, entity: Channel | User
+) -> TextMessageEventContent | None:
+    content = await telegram_to_matrix(msg, source, require_html=True)
+    content.external_url = f"https://t.me/{entity.username}"
+    content.msgtype = MessageType.NOTICE
+    sponsored_meta = {
+        "random_id": base64.b64encode(msg.random_id).decode("utf-8"),
+    }
+    if isinstance(msg.from_id, PeerChannel):
+        sponsored_meta["channel_id"] = msg.from_id.channel_id
+        if getattr(msg, "channel_post", None) is not None:
+            sponsored_meta["channel_post"] = msg.channel_post
+            content.external_url += f"/{msg.channel_post}"
+            action = "View Post"
+        else:
+            action = "View Channel"
+    elif isinstance(msg.from_id, PeerUser):
+        sponsored_meta["bot_id"] = msg.from_id.user_id
+        if msg.start_param:
+            content.external_url += f"?start={msg.start_param}"
+        action = "View Bot"
+    else:
+        return None
+
+    if isinstance(entity, User):
+        name_parts = [entity.first_name, entity.last_name]
+        sponsor_name = " ".join(x for x in name_parts if x)
+        sponsor_name_html = f"<strong>{html.escape(sponsor_name)}</strong>"
+    elif isinstance(entity, Channel):
+        sponsor_name = entity.title
+        sponsor_name_html = f"<strong>{html.escape(sponsor_name)}</strong>"
+    else:
+        sponsor_name = sponsor_name_html = "unknown entity"
+
+    content["net.maunium.telegram.sponsored"] = sponsored_meta
+    content.formatted_body += (
+        f"<br/><br/>Sponsored message from {sponsor_name_html} "
+        f"- <a href='{content.external_url}'>{action}</a>"
+    )
+    content.body += (
+        f"\n\nSponsored message from {sponsor_name} - {action} at {content.external_url}"
+    )
+
+    return content

mautrix_telegram/puppet.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,443 +13,436 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Awaitable, Pattern, Dict, List, TYPE_CHECKING
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, AsyncGenerator, AsyncIterable, Awaitable, cast
 from difflib import SequenceMatcher
-import re
-import logging
-import asyncio
+import unicodedata
 
-from sqlalchemy import orm
+from telethon.tl.types import (
+    InputPeerPhotoFileLocation,
+    PeerUser,
+    TypeInputPeer,
+    TypeInputUser,
+    UpdateUserName,
+    User,
+    UserProfilePhoto,
+    UserProfilePhotoEmpty,
+)
+from yarl import URL
 
-from telethon.tl.types import UserProfilePhoto
-from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
+from mautrix.appservice import IntentAPI
+from mautrix.bridge import BasePuppet, async_getter_lock
+from mautrix.errors import MatrixError
+from mautrix.types import ContentURI, RoomID, SyncToken, UserID
+from mautrix.util.simple_template import SimpleTemplate
 
+from . import abstract_user as au, portal as p, util
+from .config import Config
 from .db import Puppet as DBPuppet
-from . import util
+from .types import TelegramID
 
 if TYPE_CHECKING:
-    from .matrix import MatrixHandler
-    from .config import Config
-    from .context import Context
-
-config = None  # type: Config
+    from .__main__ import TelegramBridge
 
 
-class Puppet:
-    log = logging.getLogger("mau.puppet")  # type: logging.Logger
-    db = None  # type: orm.Session
-    az = None  # type: AppService
-    mx = None  # type: MatrixHandler
-    loop = None  # type: asyncio.AbstractEventLoop
-    mxid_regex = None  # type: Pattern
-    username_template = None  # type: str
-    hs_domain = None  # type: str
-    cache = {}  # type: Dict[str, Puppet]
-    by_custom_mxid = {}  # type: Dict[str, Puppet]
+class Puppet(DBPuppet, BasePuppet):
+    config: Config
+    hs_domain: str
+    mxid_template: SimpleTemplate[TelegramID]
+    displayname_template: SimpleTemplate[str]
+
+    by_tgid: dict[TelegramID, Puppet] = {}
+    by_custom_mxid: dict[UserID, Puppet] = {}
+
+    def __init__(
+        self,
+        id: TelegramID,
+        is_registered: bool = False,
+        displayname: str | None = None,
+        displayname_source: TelegramID | None = None,
+        displayname_contact: bool = True,
+        displayname_quality: int = 0,
+        disable_updates: bool = False,
+        username: str | None = None,
+        photo_id: str | None = None,
+        is_bot: bool = False,
+        custom_mxid: UserID | None = None,
+        access_token: str | None = None,
+        next_batch: SyncToken | None = None,
+        base_url: str | None = None,
+    ) -> None:
+        super().__init__(
+            id=id,
+            is_registered=is_registered,
+            displayname=displayname,
+            displayname_source=displayname_source,
+            displayname_contact=displayname_contact,
+            displayname_quality=displayname_quality,
+            disable_updates=disable_updates,
+            username=username,
+            photo_id=photo_id,
+            is_bot=is_bot,
+            custom_mxid=custom_mxid,
+            access_token=access_token,
+            next_batch=next_batch,
+            base_url=base_url,
+        )
 
-    def __init__(self, id=None, access_token=None, custom_mxid=None, username=None,
-                 displayname=None, displayname_source=None, photo_id=None, is_bot=None,
-                 is_registered=False, db_instance=None):
-        self.id = id
-        self.access_token = access_token
-        self.custom_mxid = custom_mxid
-        self.is_real_user = self.custom_mxid and self.access_token
         self.default_mxid = self.get_mxid_from_id(self.id)
-        self.mxid = self.custom_mxid or self.default_mxid
-
-        self.username = username
-        self.displayname = displayname
-        self.displayname_source = displayname_source
-        self.photo_id = photo_id
-        self.is_bot = is_bot
-        self.is_registered = is_registered
-        self._db_instance = db_instance
-
         self.default_mxid_intent = self.az.intent.user(self.default_mxid)
-        self.intent = None  # type: IntentAPI
-        self.refresh_intents()
+        self.intent = self._fresh_intent()
 
-        self.cache[id] = self
+        self.by_tgid[id] = self
         if self.custom_mxid:
             self.by_custom_mxid[self.custom_mxid] = self
 
+        self.log = self.log.getChild(str(self.id))
+
     @property
-    def tgid(self):
+    def tgid(self) -> TelegramID:
         return self.id
 
-    @staticmethod
-    async def is_logged_in():
-        return True
-
-    # region Custom puppet management
-    def refresh_intents(self):
-        self.is_real_user = self.custom_mxid and self.access_token
-        self.intent = (self.az.intent.user(self.custom_mxid, self.access_token)
-                       if self.is_real_user else self.default_mxid_intent)
-
-    async def switch_mxid(self, access_token, mxid):
-        prev_mxid = self.custom_mxid
-        self.custom_mxid = mxid
-        self.access_token = access_token
-        self.refresh_intents()
-
-        err = await self.init_custom_mxid()
-        if err != 0:
-            return err
-
-        try:
-            del self.by_custom_mxid[prev_mxid]
-        except KeyError:
-            pass
-        self.mxid = self.custom_mxid or self.default_mxid
-        if self.mxid != self.default_mxid:
-            self.by_custom_mxid[self.mxid] = self
-            await self.leave_rooms_with_default_user()
-        self.save()
-        return 0
-
-    async def init_custom_mxid(self):
-        if not self.is_real_user:
-            return 0
-
-        mxid = await self.intent.whoami()
-        if not mxid or mxid != self.custom_mxid:
-            self.custom_mxid = None
-            self.access_token = None
-            self.refresh_intents()
-            if mxid != self.custom_mxid:
-                return 2
-            return 1
-        if config["bridge.sync_with_custom_puppets"]:
-            asyncio.ensure_future(self.sync(), loop=self.loop)
-        return 0
-
-    async def leave_rooms_with_default_user(self):
-        for room_id in await self.default_mxid_intent.get_joined_rooms():
-            try:
-                await self.default_mxid_intent.leave_room(room_id)
-                await self.intent.ensure_joined(room_id)
-            except (IntentError, MatrixRequestError):
-                pass
-
-    def create_sync_filter(self) -> Awaitable[str]:
-        return self.intent.client.create_filter(self.custom_mxid, {
-            "room": {
-                "include_leave": False,
-                "state": {
-                    "types": []
-                },
-                "timeline": {
-                    "types": [],
-                },
-                "ephemeral": {
-                    "types": ["m.typing", "m.receipt"],
-                },
-                "account_data": {
-                    "types": []
-                }
-            },
-            "account_data": {
-                "types": [],
-            },
-            "presence": {
-                "types": ["m.presence"],
-                "senders": [self.custom_mxid],
-            },
-        })
-
-    def filter_events(self, events):
-        new_events = []
-        for event in events:
-            evt_type = event.get("type", None)
-            event.setdefault("content", {})
-            if evt_type == "m.typing":
-                is_typing = self.custom_mxid in event["content"].get("user_ids", [])
-                event["content"]["user_ids"] = [self.custom_mxid] if is_typing else []
-            elif evt_type == "m.receipt":
-                val = None
-                evt = None
-                for event_id in event["content"]:
-                    try:
-                        val = event["content"][event_id]["m.read"][self.custom_mxid]
-                        evt = event_id
-                        break
-                    except KeyError:
-                        pass
-                if val and evt:
-                    event["content"] = {evt: {"m.read": {
-                        self.custom_mxid: val
-                    }}}
-                else:
-                    continue
-            new_events.append(event)
-        return new_events
-
-    def handle_sync(self, presence, ephemeral):
-        presence = [self.mx.try_handle_event(event) for event in presence]
-
-        for room_id, events in ephemeral.items():
-            for event in events:
-                event["room_id"] = room_id
-
-        ephemeral = [self.mx.try_handle_event(event)
-                     for events in ephemeral.values()
-                     for event in self.filter_events(events)]
-
-        events = ephemeral + presence
-        coro = asyncio.gather(*events, loop=self.loop)
-        asyncio.ensure_future(coro, loop=self.loop)
-
-    async def sync(self):
-        try:
-            await self._sync()
-        except Exception:
-            self.log.exception("Fatal error syncing")
-
-    async def _sync(self):
-        if not self.is_real_user:
-            self.log.warning("Called sync() for non-custom puppet.")
-            return
-        custom_mxid = self.custom_mxid
-        access_token_at_start = self.access_token
-        errors = 0
-        next_batch = None
-        filter_id = await self.create_sync_filter()
-        self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.")
-        while access_token_at_start == self.access_token:
-            try:
-                sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch,
-                                                          set_presence="offline")
-                errors = 0
-                if next_batch is not None:
-                    presence = sync_resp.get("presence", {}).get("events", [])
-                    ephemeral = {room: data.get("ephemeral", {}).get("events", [])
-                                 for room, data
-                                 in sync_resp.get("rooms", {}).get("join", {}).items()}
-                    self.handle_sync(presence, ephemeral)
-                next_batch = sync_resp.get("next_batch", None)
-            except MatrixRequestError as e:
-                wait = min(errors, 11) ** 2
-                self.log.warning(f"Syncer for {custom_mxid} errored: {e}. "
-                                 f"Waiting for {wait} seconds...")
-                errors += 1
-                await asyncio.sleep(wait)
-        self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.")
-
-    # endregion
-    # region DB conversion
+    @property
+    def tg_username(self) -> str | None:
+        return self.username
 
     @property
-    def db_instance(self):
-        if not self._db_instance:
-            self._db_instance = self.new_db_instance()
-        return self._db_instance
+    def peer(self) -> PeerUser:
+        return PeerUser(user_id=self.tgid)
 
-    def new_db_instance(self):
-        return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
-                        username=self.username, displayname=self.displayname,
-                        displayname_source=self.displayname_source, photo_id=self.photo_id,
-                        is_bot=self.is_bot, matrix_registered=self.is_registered)
+    @property
+    def plain_displayname(self) -> str:
+        return self.displayname_template.parse(self.displayname) or self.displayname
+
+    def get_input_entity(self, user: au.AbstractUser) -> Awaitable[TypeInputPeer | TypeInputUser]:
+        return user.client.get_input_entity(self.peer)
+
+    def intent_for(self, portal: p.Portal) -> IntentAPI:
+        if portal.tgid == self.tgid:
+            return self.default_mxid_intent
+        return self.intent
 
     @classmethod
-    def from_db(cls, db_puppet):
-        return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
-                      db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
-                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
-                      db_instance=db_puppet)
+    def init_cls(cls, bridge: "TelegramBridge") -> AsyncIterable[Awaitable[None]]:
+        cls.config = bridge.config
+        cls.loop = bridge.loop
+        cls.mx = bridge.matrix
+        cls.az = bridge.az
+        cls.hs_domain = cls.config["homeserver.domain"]
+        mxid_tpl = SimpleTemplate(
+            cls.config["bridge.username_template"],
+            "userid",
+            prefix="@",
+            suffix=f":{Puppet.hs_domain}",
+            type=int,
+        )
+        cls.mxid_template = cast(SimpleTemplate[TelegramID], mxid_tpl)
+        cls.displayname_template = SimpleTemplate(
+            cls.config["bridge.displayname_template"], "displayname"
+        )
+        cls.sync_with_custom_puppets = cls.config["bridge.sync_with_custom_puppets"]
+        cls.homeserver_url_map = {
+            server: URL(url)
+            for server, url in cls.config["bridge.double_puppet_server_map"].items()
+        }
+        cls.allow_discover_url = cls.config["bridge.double_puppet_allow_discovery"]
+        cls.login_shared_secret_map = {
+            server: secret.encode("utf-8")
+            for server, secret in cls.config["bridge.login_shared_secret_map"].items()
+        }
+        cls.login_device_name = "Telegram Bridge"
 
-    def save(self):
-        self.db_instance.access_token = self.access_token
-        self.db_instance.custom_mxid = self.custom_mxid
-        self.db_instance.username = self.username
-        self.db_instance.displayname = self.displayname
-        self.db_instance.displayname_source = self.displayname_source
-        self.db_instance.photo_id = self.photo_id
-        self.db_instance.is_bot = self.is_bot
-        self.db_instance.matrix_registered = self.is_registered
-        self.db.commit()
+        return (puppet.try_start() async for puppet in cls.all_with_custom_mxid())
 
-    # endregion
     # region Info updating
-    def similarity(self, query):
-        username_similarity = (SequenceMatcher(None, self.username, query).ratio()
-                               if self.username else 0)
-        displayname_similarity = (SequenceMatcher(None, self.displayname, query).ratio()
-                                  if self.displayname else 0)
+
+    def similarity(self, query: str) -> int:
+        username_similarity = (
+            SequenceMatcher(None, self.username, query).ratio() if self.username else 0
+        )
+        displayname_similarity = (
+            SequenceMatcher(None, self.plain_displayname, query).ratio() if self.displayname else 0
+        )
         similarity = max(username_similarity, displayname_similarity)
-        return round(similarity * 1000) / 10
+        return int(round(similarity * 100))
 
     @staticmethod
-    def get_displayname(info, enable_format=True):
+    def _filter_name(name: str) -> str:
+        if not name:
+            return ""
+        whitespace = (
+            "\t\n\r\v\f \u00a0\u034f\u180e\u2063\u202f\u205f\u2800\u3000\u3164\ufeff\u2000\u2001"
+            "\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u200c\u200d\u200e\u200f"
+            "\ufe0f"
+        )
+        allowed_other_format = ("\u200d", "\u200c")
+        name = "".join(
+            c
+            for c in name.strip(whitespace)
+            if unicodedata.category(c) != "Cf" or c in allowed_other_format
+        )
+        return name
+
+    @classmethod
+    def get_displayname(cls, info: User, enable_format: bool = True) -> tuple[str, int]:
+        fn = cls._filter_name(info.first_name)
+        ln = cls._filter_name(info.last_name)
         data = {
             "phone number": info.phone if hasattr(info, "phone") else None,
             "username": info.username,
-            "full name": " ".join([info.first_name or "", info.last_name or ""]).strip(),
-            "full name reversed": " ".join([info.first_name or "", info.last_name or ""]).strip(),
-            "first name": info.first_name,
-            "last name": info.last_name,
+            "full name": " ".join([fn, ln]).strip(),
+            "full name reversed": " ".join([ln, fn]).strip(),
+            "first name": fn,
+            "last name": ln,
         }
-        preferences = config.get("bridge.displayname_preference",
-                                 ["full name", "username", "phone"])
+        preferences = cls.config["bridge.displayname_preference"]
         name = None
+        quality = 99
         for preference in preferences:
             name = data[preference]
             if name:
                 break
+            quality -= 1
 
-        if info.deleted:
+        if isinstance(info, User) and info.deleted:
             name = f"Deleted account {info.id}"
+            quality = 99
         elif not name:
-            name = info.id
+            name = str(info.id)
+            quality = 0
 
-        if not enable_format:
-            return name
-        return config.get("bridge.displayname_template", "{displayname} (Telegram)").format(
-            displayname=name)
+        return (cls.displayname_template.format_full(name) if enable_format else name), quality
 
-    async def update_info(self, source, info):
+    async def try_update_info(self, source: au.AbstractUser, info: User) -> None:
+        try:
+            await self.update_info(source, info)
+        except Exception:
+            source.log.exception(f"Failed to update info of {self.tgid}")
+
+    async def update_info(self, source: au.AbstractUser, info: User) -> None:
         changed = False
         if self.username != info.username:
             self.username = info.username
             changed = True
 
-        changed = await self.update_displayname(source, info) or changed
-        if isinstance(info.photo, UserProfilePhoto):
-            changed = await self.update_avatar(source, info.photo.photo_big) or changed
+        if not self.disable_updates:
+            try:
+                changed = await self.update_displayname(source, info) or changed
+                changed = await self.update_avatar(source, info.photo) or changed
+            except Exception:
+                self.log.exception(f"Failed to update info from source {source.tgid}")
 
         self.is_bot = info.bot
 
         if changed:
-            self.save()
+            await self.save()
 
-    async def update_displayname(self, source, info):
-        ignore_source = (not source.is_relaybot
-                         and self.displayname_source is not None
-                         and self.displayname_source != source.tgid)
-        if ignore_source:
-            return
+    async def update_displayname(
+        self, source: au.AbstractUser, info: User | UpdateUserName
+    ) -> bool:
+        if self.disable_updates:
+            return False
+        if source.is_relaybot or source.is_bot:
+            allow_because = "user is bot"
+        elif self.displayname_source == source.tgid:
+            allow_because = "user is the primary source"
+        elif not isinstance(info, UpdateUserName) and not info.contact:
+            allow_because = "user is not a contact"
+        elif not self.displayname_source:
+            allow_because = "no primary source set"
+        elif not self.displayname:
+            allow_because = "user has no name"
+        else:
+            return False
 
-        displayname = self.get_displayname(info)
-        if displayname != self.displayname:
-            await self.default_mxid_intent.set_display_name(displayname)
+        if isinstance(info, UpdateUserName):
+            info = await source.client.get_entity(PeerUser(self.tgid))
+        if not info.contact:
+            self.displayname_contact = False
+        elif not self.displayname_contact:
+            if not self.displayname:
+                self.displayname_contact = True
+            else:
+                return False
+
+        displayname, quality = self.get_displayname(info)
+        if displayname != self.displayname and quality >= self.displayname_quality:
+            allow_because = f"{allow_because} and quality {quality} >= {self.displayname_quality}"
+            self.log.debug(
+                f"Updating displayname of {self.id} (src: {source.tgid}, allowed "
+                f"because {allow_because}) from {self.displayname} to {displayname}"
+            )
+            self.log.trace("Displayname source data: %s", info)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            self.displayname_quality = quality
+            try:
+                await self.default_mxid_intent.set_displayname(
+                    displayname[: self.config["bridge.displayname_max_length"]]
+                )
+            except MatrixError:
+                self.log.exception("Failed to set displayname")
+                self.displayname = ""
+                self.displayname_source = None
+                self.displayname_quality = 0
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
             return True
+        return False
 
-    async def update_avatar(self, source, photo):
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
+    async def update_avatar(
+        self, source: au.AbstractUser, photo: UserProfilePhoto | UserProfilePhotoEmpty
+    ) -> bool:
+        if self.disable_updates:
+            return False
+
+        if photo is None or isinstance(photo, UserProfilePhotoEmpty):
+            photo_id = ""
+        elif isinstance(photo, UserProfilePhoto):
+            photo_id = str(photo.photo_id)
+        else:
+            self.log.warning(f"Unknown user profile photo type: {type(photo)}")
+            return False
+        if not photo_id and not self.config["bridge.allow_avatar_remove"]:
+            return False
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(self.db, source.client,
-                                                      self.default_mxid_intent, photo)
+            if not photo_id:
+                self.photo_id = ""
+                try:
+                    await self.default_mxid_intent.set_avatar_url(ContentURI(""))
+                except MatrixError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
+                return True
+
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(source), photo_id=photo.photo_id, big=True
+            )
+            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
             if file:
-                await self.default_mxid_intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
+                try:
+                    await self.default_mxid_intent.set_avatar_url(file.mxc)
+                except MatrixError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
                 return True
         return False
 
+    async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool:
+        portal: p.Portal = await p.Portal.get_by_mxid(room_id)
+        return portal and not portal.backfill_lock.locked and portal.peer_type != "user"
+
     # endregion
     # region Getters
 
+    def _add_to_cache(self) -> None:
+        self.by_tgid[self.id] = self
+        if self.custom_mxid:
+            self.by_custom_mxid[self.custom_mxid] = self
+
     @classmethod
-    def get(cls, tgid, create=True) -> "Optional[Puppet]":
+    @async_getter_lock
+    async def get_by_tgid(cls, tgid: TelegramID, *, create: bool = True) -> Puppet | None:
+        if tgid is None:
+            return None
+
         try:
-            return cls.cache[tgid]
+            return cls.by_tgid[tgid]
         except KeyError:
             pass
 
-        puppet = DBPuppet.query.get(tgid)
+        puppet = cast(cls, await super().get_by_tgid(tgid))
         if puppet:
-            return cls.from_db(puppet)
+            puppet._add_to_cache()
+            return puppet
 
         if create:
             puppet = cls(tgid)
-            cls.db.add(puppet.db_instance)
-            cls.db.commit()
+            await puppet.insert()
+            puppet._add_to_cache()
             return puppet
 
         return None
 
     @classmethod
-    def get_by_mxid(cls, mxid, create=True) -> "Optional[Puppet]":
-        tgid = cls.get_id_from_mxid(mxid)
-        return cls.get(tgid, create) if tgid else None
+    def get_by_mxid(cls, mxid: UserID, create: bool = True) -> Awaitable[Puppet | None]:
+        return cls.get_by_tgid(cls.get_id_from_mxid(mxid), create=create)
 
     @classmethod
-    def get_by_custom_mxid(cls, mxid):
-        if not mxid:
-            raise ValueError("Matrix ID can't be empty")
-
+    @async_getter_lock
+    async def get_by_custom_mxid(cls, mxid: UserID) -> Puppet | None:
         try:
             return cls.by_custom_mxid[mxid]
         except KeyError:
             pass
 
-        puppet = DBPuppet.query.filter(DBPuppet.custom_mxid == mxid).one_or_none()
+        puppet = cast(cls, await super().get_by_custom_mxid(mxid))
         if puppet:
-            puppet = cls.from_db(puppet)
+            puppet._add_to_cache()
             return puppet
 
         return None
 
     @classmethod
-    def get_all_with_custom_mxid(cls):
-        return [cls.by_custom_mxid[puppet.mxid]
-                if puppet.custom_mxid in cls.by_custom_mxid
-                else cls.from_db(puppet)
-                for puppet in DBPuppet.query.filter(DBPuppet.custom_mxid is not None).all()]
+    async def all_with_custom_mxid(cls) -> AsyncGenerator[Puppet, None]:
+        puppets = await super().all_with_custom_mxid()
+        puppet: cls
+        for puppet in puppets:
+            try:
+                yield cls.by_tgid[puppet.tgid]
+            except KeyError:
+                puppet._add_to_cache()
+                yield puppet
 
     @classmethod
-    def get_id_from_mxid(cls, mxid):
-        match = cls.mxid_regex.match(mxid)
-        if match:
-            return int(match.group(1))
-        return None
+    def get_id_from_mxid(cls, mxid: UserID) -> TelegramID | None:
+        return cls.mxid_template.parse(mxid)
 
     @classmethod
-    def get_mxid_from_id(cls, tgid):
-        return f"@{cls.username_template.format(userid=tgid)}:{cls.hs_domain}"
+    def get_mxid_from_id(cls, tgid: TelegramID) -> UserID:
+        return UserID(cls.mxid_template.format_full(tgid))
 
     @classmethod
-    def find_by_username(cls, username) -> "Optional[Puppet]":
+    async def find_by_username(cls, username: str) -> Puppet | None:
         if not username:
             return None
 
-        for _, puppet in cls.cache.items():
-            if puppet.username and puppet.username.lower() == username.lower():
+        username = username.lower()
+
+        for _, puppet in cls.by_tgid.items():
+            if puppet.username and puppet.username.lower() == username:
                 return puppet
 
-        puppet = DBPuppet.query.filter(DBPuppet.username == username).one_or_none()
+        puppet = cast(cls, await super().find_by_username(username))
         if puppet:
-            return cls.from_db(puppet)
+            try:
+                return cls.by_tgid[puppet.tgid]
+            except KeyError:
+                puppet._add_to_cache()
+                return puppet
 
         return None
 
     @classmethod
-    def find_by_displayname(cls, displayname) -> "Optional[Puppet]":
+    async def find_by_displayname(cls, displayname: str) -> Puppet | None:
         if not displayname:
             return None
 
-        for _, puppet in cls.cache.items():
+        for _, puppet in cls.by_tgid.items():
             if puppet.displayname and puppet.displayname == displayname:
                 return puppet
 
-        puppet = DBPuppet.query.filter(DBPuppet.displayname == displayname).one_or_none()
+        puppet = cast(cls, await super().find_by_displayname(displayname))
         if puppet:
-            return cls.from_db(puppet)
+            try:
+                return cls.by_tgid[puppet.tgid]
+            except KeyError:
+                puppet._add_to_cache()
+                return puppet
 
         return None
+
     # endregion
-
-
-def init(context: "Context") -> List[Awaitable[int]]:
-    global config
-    Puppet.az, Puppet.db, config, Puppet.loop, _ = context
-    Puppet.mx = context.mx
-    Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
-    Puppet.hs_domain = config["homeserver"]["domain"]
-    Puppet.mxid_regex = re.compile(
-        f"@{Puppet.username_template.format(userid='(.+)')}:{Puppet.hs_domain}")
-    return [puppet.init_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()]

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -1,59 +0,0 @@
-import argparse
-import sqlalchemy as sql
-from sqlalchemy import orm
-from sqlalchemy.ext.declarative import declarative_base
-
-from alchemysession import AlchemySessionContainer
-
-parser = argparse.ArgumentParser(description="mautrix-telegram dbms migration script",
-                                 prog="python -m mautrix_telegram.scripts.dbms_migrate")
-parser.add_argument("-f", "--from-url", type=str, required=True, metavar="<url>",
-                    help="the old database path")
-parser.add_argument("-t", "--to-url", type=str, required=True, metavar="<url>",
-                    help="the new database path")
-args = parser.parse_args()
-
-
-def connect(to):
-    import mautrix_telegram.base as base
-    base.Base = declarative_base()
-    from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile,
-                                     Contact, Puppet, BotChat, TelegramFile)
-    db_engine = sql.create_engine(to)
-    db_factory = orm.sessionmaker(bind=db_engine)
-    db_session = orm.scoped_session(db_factory)  # type: orm.Session
-    base.Base.metadata.bind = db_engine
-    session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
-                                                table_base=base.Base, table_prefix="telethon_",
-                                                manage_tables=False)
-
-    return db_session, {
-        "Version": session_container.Version,
-        "Session": session_container.Session,
-        "Entity": session_container.Entity,
-        "SentFile": session_container.SentFile,
-        "UpdateState": session_container.UpdateState,
-        "Portal": Portal,
-        "Message": Message,
-        "Puppet": Puppet,
-        "User": User,
-        "UserPortal": UserPortal,
-        "RoomState": RoomState,
-        "UserProfile": UserProfile,
-        "Contact": Contact,
-        "BotChat": BotChat,
-        "TelegramFile": TelegramFile,
-    }
-
-
-session, tables = connect(args.from_url)
-
-data = {}
-for name, table in tables.items():
-    data[name] = session.query(table).all()
-
-session, tables = connect(args.to_url)
-for name, table in tables.items():
-    for row in data[name]:
-        session.merge(row)
-session.commit()

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -1,91 +0,0 @@
-import argparse
-import sqlalchemy as sql
-from sqlalchemy import orm
-
-from mautrix_telegram.base import Base
-from mautrix_telegram.config import Config
-from mautrix_telegram.db import Portal, Message, Puppet, BotChat
-from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as TelematrixBase
-
-parser = argparse.ArgumentParser(
-    description="mautrix-telegram telematrix import script",
-    prog="python -m mautrix_telegram.scripts.telematrix_import")
-parser.add_argument("-c", "--config", type=str, default="config.yaml",
-                    metavar="<path>", help="the path to your mautrix-telegram config file")
-parser.add_argument("-b", "--bot-id", type=int, required=True,
-                    metavar="<id>", help="the telegram user ID of your relay bot")
-parser.add_argument("-t", "--telematrix-database", type=str, default="sqlite:///database.db",
-                    metavar="<url>", help="your telematrix database URL")
-args = parser.parse_args()
-
-config = Config(args.config, None, None)
-config.load()
-
-mxtg_db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
-mxtg = orm.sessionmaker(bind=mxtg_db_engine)()
-Base.metadata.bind = mxtg_db_engine
-
-telematrix_db_engine = sql.create_engine(args.telematrix_database)
-telematrix = orm.sessionmaker(bind=telematrix_db_engine)()
-TelematrixBase.metadata.bind = telematrix_db_engine
-
-chat_links = telematrix.query(ChatLink).all()
-tg_users = telematrix.query(TgUser).all()
-mx_users = telematrix.query(MatrixUser).all()
-messages = telematrix.query(TMMessage).all()
-
-telematrix.close()
-telematrix_db_engine.dispose()
-
-portals = {}
-chats = {}
-messages = {}
-puppets = {}
-
-for chat_link in chat_links:
-    if type(chat_link.tg_room) is str:
-        print("Expected tg_room to be a number, got a string. Ignoring %s" % chat_link.tg_room)
-        continue
-    if chat_link.tg_room >= 0:
-        print("Unexpected unprefixed telegram chat ID: %s, ignoring..." % chat_link.tg_room)
-        continue
-    tgid = str(chat_link.tg_room)
-    if tgid.startswith("-100"):
-        tgid = int(tgid[4:])
-        peer_type = "channel"
-        megagroup = True
-    else:
-        tgid = -chat_link.tg_room
-        peer_type = "chat"
-        megagroup = False
-
-    portal = Portal(tgid=tgid, tg_receiver=tgid, peer_type=peer_type, megagroup=megagroup,
-                    mxid=chat_link.matrix_room)
-    bot_chat = BotChat(id=tgid, type=peer_type)
-    portals[chat_link.tg_room] = portal
-    chats[tgid] = bot_chat
-
-for tm_msg in messages:
-    try:
-        portal = portals[tm_msg.tg_group_id]
-    except KeyError:
-        print("Found message entry %d in unlinked chat %d, ignoring..." % (tm_msg.tg_message_id, tm_msg.tg_group_id))
-        continue
-    tg_space = portal.tgid if portal.peer_type == "channel" else args.bot_id
-    message = Message(mxid=tm_msg.matrix_event_id, mx_room=tm_msg.matrix_room_id,
-                      tgid=tm_msg.tg_message_id, tg_space=tg_space)
-    messages[tm_msg.matrix_event_id] = message
-
-for user in tg_users:
-    puppets[user.tg_id] = Puppet(id=user.tg_id, displayname=user.name, displayname_source=args.bot_id)
-
-for k, v in portals.items():
-    mxtg.add(v)
-for k, v in chats.items():
-    mxtg.add(v)
-for k, v in messages.items():
-    mxtg.add(v)
-for k, v in puppets.items():
-    mxtg.add(v)
-
-mxtg.commit()

mautrix_telegram/scripts/telematrix_import/models.py

@@ -1,44 +0,0 @@
-import sqlalchemy as sa
-from sqlalchemy.ext.declarative import declarative_base
-
-Base = declarative_base()
-
-
-class ChatLink(Base):
-    __tablename__ = 'chat_link'
-
-    id = sa.Column(sa.Integer, primary_key=True)
-    matrix_room = sa.Column(sa.String)
-    tg_room = sa.Column(sa.BigInteger)
-    active = sa.Column(sa.Boolean)
-
-
-class TgUser(Base):
-    __tablename__ = 'tg_user'
-
-    id = sa.Column(sa.Integer, primary_key=True)
-    tg_id = sa.Column(sa.BigInteger)
-    name = sa.Column(sa.String)
-    profile_pic_id = sa.Column(sa.String, nullable=True)
-
-
-class MatrixUser(Base):
-    __tablename__ = 'matrix_user'
-
-    id = sa.Column(sa.Integer, primary_key=True)
-    matrix_id = sa.Column(sa.String)
-    name = sa.Column(sa.String)
-
-
-class Message(Base):
-    """Describes a message in a room bridged between Telegram and Matrix"""
-    __tablename__ = "message"
-
-    id = sa.Column(sa.Integer, primary_key=True)
-    tg_group_id = sa.Column(sa.BigInteger)
-    tg_message_id = sa.Column(sa.BigInteger)
-
-    matrix_room_id = sa.Column(sa.String)
-    matrix_event_id = sa.Column(sa.String)
-
-    displayname = sa.Column(sa.String)

mautrix_telegram/sqlstatestore.py

@@ -1,120 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, Tuple
-
-from sqlalchemy import orm
-
-from mautrix_appservice import StateStore
-
-from . import puppet as pu
-from .db import RoomState, UserProfile
-
-
-class SQLStateStore(StateStore):
-    def __init__(self, db):
-        super().__init__()
-        self.db = db  # type: orm.Session
-        self.profile_cache = {}  # type: Dict[Tuple[str, str], UserProfile]
-        self.room_state_cache = {}  # type: Dict[str, RoomState]
-
-    @staticmethod
-    def is_registered(user: str) -> bool:
-        puppet = pu.Puppet.get_by_mxid(user)
-        return puppet.is_registered if puppet else False
-
-    @staticmethod
-    def registered(user: str):
-        puppet = pu.Puppet.get_by_mxid(user)
-        if puppet:
-            puppet.is_registered = True
-            puppet.save()
-
-    def update_state(self, event: dict):
-        event_type = event["type"]
-        if event_type == "m.room.power_levels":
-            self.set_power_levels(event["room_id"], event["content"])
-        elif event_type == "m.room.member":
-            self.set_member(event["room_id"], event["state_key"], event["content"])
-
-    def _get_user_profile(self, room_id: str, user_id: str, create: bool = True) -> UserProfile:
-        key = (room_id, user_id)
-        try:
-            return self.profile_cache[key]
-        except KeyError:
-            pass
-
-        profile = UserProfile.query.get(key)
-        if profile:
-            self.profile_cache[key] = profile
-        elif create:
-            profile = UserProfile(room_id=room_id, user_id=user_id)
-            self.db.add(profile)
-            self.db.commit()
-            self.profile_cache[key] = profile
-        return profile
-
-    def get_member(self, room: str, user: str) -> dict:
-        return self._get_user_profile(room, user).dict()
-
-    def set_member(self, room: str, user: str, member: dict):
-        profile = self._get_user_profile(room, user)
-        profile.membership = member.get("membership", profile.membership or "leave")
-        profile.displayname = member.get("displayname", profile.displayname)
-        profile.avatar_url = member.get("avatar_url", profile.avatar_url)
-        self.db.commit()
-
-    def set_membership(self, room: str, user: str, membership: str):
-        self.set_member(room, user, {
-            "membership": membership,
-        })
-
-    def _get_room_state(self, room_id: str, create: bool = True) -> RoomState:
-        try:
-            return self.room_state_cache[room_id]
-        except KeyError:
-            pass
-
-        room = RoomState.query.get(room_id)
-        if room:
-            self.room_state_cache[room_id] = room
-        elif create:
-            room = RoomState(room_id=room_id)
-            self.room_state_cache[room_id] = room
-        return room
-
-    def has_power_levels(self, room: str) -> bool:
-        return self._get_room_state(room).has_power_levels
-
-    def get_power_levels(self, room: str) -> dict:
-        return self._get_room_state(room).power_levels
-
-    def set_power_level(self, room: str, user: str, level: int):
-        room_state = self._get_room_state(room)
-        power_levels = room_state.power_levels
-        if not power_levels:
-            power_levels = {
-                "users": {},
-                "events": {},
-            }
-        power_levels[room]["users"][user] = level
-        room_state.power_levels = power_levels
-        self.db.commit()
-
-    def set_power_levels(self, room: str, content: dict):
-        state = self._get_room_state(room)
-        state.power_levels = content
-        self.db.commit()

mautrix_telegram/tgclient.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,20 +13,38 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Optional, Union
+
 from telethon import TelegramClient, utils
+from telethon.sessions.abstract import Session
 from telethon.tl.functions.messages import SendMediaRequest
-from telethon.tl.types import *
-from telethon.tl import custom
+from telethon.tl.patched import Message
+from telethon.tl.types import (
+    InputMediaUploadedDocument,
+    InputMediaUploadedPhoto,
+    TypeDocumentAttribute,
+    TypeInputMedia,
+    TypeInputPeer,
+    TypeMessageEntity,
+    TypeMessageMedia,
+    TypePeer,
+)
 
 
 class MautrixTelegramClient(TelegramClient):
-    async def upload_file_direct(self, file: bytes, mime_type: str = None,
-                                 attributes: List[TypeDocumentAttribute] = None,
-                                 file_name: str = None
-                                 ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
-        file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)
+    session: Session
 
-        if mime_type == "image/png" or mime_type == "image/jpeg":
+    async def upload_file_direct(
+        self,
+        file: bytes,
+        mime_type: str = None,
+        attributes: List[TypeDocumentAttribute] = None,
+        file_name: str = None,
+        max_image_size: float = 10 * 1000 ** 2,
+    ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
+        file_handle = await super().upload_file(file, file_name=file_name)
+
+        if (mime_type == "image/png" or mime_type == "image/jpeg") and len(file) < max_image_size:
             return InputMediaUploadedPhoto(file_handle)
         else:
             attributes = attributes or []
@@ -36,14 +53,20 @@ class MautrixTelegramClient(TelegramClient):
             return InputMediaUploadedDocument(
                 file=file_handle,
                 mime_type=mime_type or "application/octet-stream",
-                attributes=list(attr_dict.values()))
+                attributes=list(attr_dict.values()),
+            )
 
-    async def send_media(self, entity: Union[TypeInputPeer, TypePeer],
-                         media: Union[TypeInputMedia, TypeMessageMedia],
-                         caption: str = None, entities: List[TypeMessageEntity] = None,
-                         reply_to: int = None) -> Optional[custom.Message]:
+    async def send_media(
+        self,
+        entity: Union[TypeInputPeer, TypePeer],
+        media: Union[TypeInputMedia, TypeMessageMedia],
+        caption: str = None,
+        entities: List[TypeMessageEntity] = None,
+        reply_to: int = None,
+    ) -> Optional[Message]:
         entity = await self.get_input_entity(entity)
         reply_to = utils.get_message_id(reply_to)
-        request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],
-                                   reply_to_msg_id=reply_to)
+        request = SendMediaRequest(
+            entity, media, message=caption or "", entities=entities or [], reply_to_msg_id=reply_to
+        )
         return self._get_response_message(request, await self(request), entity)

mautrix_telegram/types.py

@@ -0,0 +1,3 @@
+from typing import NewType
+
+TelegramID = NewType("TelegramID", int)

mautrix_telegram/user.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,64 +13,108 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, Awaitable, Optional, Match, Tuple, TYPE_CHECKING
-import logging
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, AsyncGenerator, AsyncIterable, Awaitable, NamedTuple, cast
+from datetime import datetime, timezone
 import asyncio
-import re
 
-from telethon.tl.types import *
-from telethon.tl.types import User as TLUser
-from telethon.tl.types.contacts import ContactsNotModified
-from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
+from telethon.errors import AuthKeyDuplicatedError, RPCError, UnauthorizedError
+from telethon.tl.custom import Dialog
 from telethon.tl.functions.account import UpdateStatusRequest
-from mautrix_appservice import MatrixRequestError
+from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
+from telethon.tl.functions.updates import GetStateRequest
+from telethon.tl.functions.users import GetUsersRequest
+from telethon.tl.types import (
+    Chat,
+    ChatForbidden,
+    InputUserSelf,
+    NotifyPeer,
+    TypeUpdate,
+    UpdateFolderPeers,
+    UpdateNewChannelMessage,
+    UpdateNewMessage,
+    UpdateNotifySettings,
+    UpdatePinnedDialogs,
+    UpdateShortChatMessage,
+    UpdateShortMessage,
+    User as TLUser,
+)
+from telethon.tl.types.contacts import ContactsNotModified
+
+from mautrix.appservice import DOUBLE_PUPPET_SOURCE_KEY
+from mautrix.bridge import BaseUser, async_getter_lock
+from mautrix.client import Client
+from mautrix.errors import MatrixRequestError, MNotFound
+from mautrix.types import PushActionType, PushRuleKind, PushRuleScope, RoomID, RoomTagInfo, UserID
+from mautrix.util.bridge_state import BridgeState, BridgeStateEvent
+from mautrix.util.opt_prometheus import Gauge
 
-from .db import User as DBUser, Contact as DBContact, Portal as DBPortal
-from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
+from .abstract_user import AbstractUser
+from .db import Message as DBMessage, PgSession, User as DBUser
+from .types import TelegramID
 
 if TYPE_CHECKING:
-    from .config import Config
-    from .context import Context
+    from .__main__ import TelegramBridge
 
-config = None  # type: Config
+SearchResult = NamedTuple("SearchResult", puppet="pu.Puppet", similarity=int)
 
-SearchResults = List[Tuple["pu.Puppet", int]]
+METRIC_LOGGED_IN = Gauge("bridge_logged_in", "Users logged into bridge")
+METRIC_CONNECTED = Gauge("bridge_connected", "Users connected to Telegram")
+
+BridgeState.human_readable_errors.update(
+    {
+        "tg-not-connected": "Your Telegram connection failed",
+        "tg-auth-key-duplicated": "The bridge accidentally logged you out",
+        "tg-not-authenticated": "The stored auth token did not work",
+        "tg-no-auth": "You're not logged in",
+    }
+)
 
 
-class User(AbstractUser):
-    log = logging.getLogger("mau.user")  # type: logging.Logger
-    by_mxid = {}  # type: Dict[str, User]
-    by_tgid = {}  # type: Dict[int, User]
+class User(DBUser, AbstractUser, BaseUser):
+    by_mxid: dict[str, User] = {}
+    by_tgid: dict[int, User] = {}
 
-    def __init__(self, mxid: str, tgid: Optional[int] = None, username: Optional[str] = None,
-                 db_contacts: Optional[List[DBContact]] = None, saved_contacts: int = 0,
-                 is_bot: bool = False, db_portals: Optional[List[DBPortal]] = None,
-                 db_instance: Optional[DBUser] = None):
-        super().__init__()
-        self.mxid = mxid  # type: str
-        self.tgid = tgid  # type: int
-        self.is_bot = is_bot  # type: bool
-        self.username = username  # type: str
-        self.contacts = []  # type: List[pu.Puppet]
-        self.saved_contacts = saved_contacts  # type: int
-        self.db_contacts = db_contacts  # type: List[DBContact]
-        self.portals = {}  # type: Dict[Tuple[int, int], po.Portal]
-        self.db_portals = db_portals  # type: List[DBPortal]
-        self._db_instance = db_instance  # type: DBUser
+    _portals_cache: dict[tuple[TelegramID, TelegramID], po.Portal] | None
 
-        self.command_status = None  # type: dict
+    _ensure_started_lock: asyncio.Lock
+    _track_connection_task: asyncio.Task | None
+    _is_backfilling: bool
 
-        (self.relaybot_whitelisted,
-         self.whitelisted,
-         self.puppet_whitelisted,
-         self.matrix_puppet_whitelisted,
-         self.is_admin,
-         self.permissions) = config.get_permissions(self.mxid)
+    def __init__(
+        self,
+        mxid: UserID,
+        tgid: TelegramID | None = None,
+        tg_username: str | None = None,
+        tg_phone: str | None = None,
+        is_bot: bool = False,
+        saved_contacts: int = 0,
+    ) -> None:
+        super().__init__(
+            mxid=mxid,
+            tgid=tgid,
+            tg_username=tg_username,
+            tg_phone=tg_phone,
+            is_bot=is_bot,
+            saved_contacts=saved_contacts,
+        )
+        AbstractUser.__init__(self)
+        BaseUser.__init__(self)
+        self._ensure_started_lock = asyncio.Lock()
+        self._track_connection_task = None
+        self._is_backfilling = False
+        self._portals_cache = None
 
-        self.by_mxid[mxid] = self
-        if tgid:
-            self.by_tgid[tgid] = self
+        (
+            self.relaybot_whitelisted,
+            self.whitelisted,
+            self.puppet_whitelisted,
+            self.matrix_puppet_whitelisted,
+            self.is_admin,
+            self.permissions,
+        ) = self.config.get_permissions(self.mxid)
 
     @property
     def name(self) -> str:
@@ -79,8 +122,12 @@ class User(AbstractUser):
 
     @property
     def mxid_localpart(self) -> str:
-        match = re.compile("@(.+):(.+)").match(self.mxid)  # type: Match
-        return match.group(1)
+        localpart, server = Client.parse_user_id(self.mxid)
+        return localpart
+
+    @property
+    def human_tg_id(self) -> str:
+        return f"@{self.tg_username}" if self.tg_username else f"+{self.tg_phone}" or None
 
     # TODO replace with proper displayname getting everywhere
     @property
@@ -88,300 +135,619 @@ class User(AbstractUser):
         return self.mxid_localpart
 
     @property
-    def db_contacts(self) -> List[DBContact]:
-        return [self.db.merge(DBContact(user=self.tgid, contact=puppet.id))
-                for puppet in self.contacts]
-
-    @db_contacts.setter
-    def db_contacts(self, contacts: List[DBContact]):
-        self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts] if contacts else []
-
-    @property
-    def db_portals(self) -> List[DBPortal]:
-        return [portal.db_instance for portal in self.portals.values() if not portal.deleted]
-
-    @db_portals.setter
-    def db_portals(self, portals: List[DBPortal]):
-        self.portals = {(portal.tgid, portal.tg_receiver):
-                            po.Portal.get_by_tgid(portal.tgid, portal.tg_receiver)
-                        for portal in portals} if portals else {}
-
-    # region Database conversion
-
-    @property
-    def db_instance(self) -> DBUser:
-        if not self._db_instance:
-            self._db_instance = self.new_db_instance()
-        return self._db_instance
-
-    def new_db_instance(self) -> DBUser:
-        return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                      contacts=self.db_contacts, saved_contacts=self.saved_contacts or 0,
-                      portals=self.db_portals)
-
-    def save(self):
-        self.db_instance.tgid = self.tgid
-        self.db_instance.username = self.username
-        self.db_instance.contacts = self.db_contacts
-        self.db_instance.saved_contacts = self.saved_contacts or 0
-        self.db_instance.portals = self.db_portals
-        self.db.commit()
-
-    def delete(self):
-        try:
-            del self.by_mxid[self.mxid]
-            del self.by_tgid[self.tgid]
-        except KeyError:
-            pass
-        if self._db_instance:
-            self.db.delete(self._db_instance)
-            self.db.commit()
+    def plain_displayname(self) -> str:
+        return self.displayname
 
     @classmethod
-    def from_db(cls, db_user: DBUser) -> "User":
-        return User(db_user.mxid, db_user.tgid, db_user.tg_username, db_user.contacts,
-                    False, db_user.saved_contacts, db_user.portals, db_instance=db_user)
+    def init_cls(cls, bridge: "TelegramBridge") -> AsyncIterable[Awaitable[User]]:
+        cls.config = bridge.config
+        cls.bridge = bridge
+        cls.az = bridge.az
+        cls.loop = bridge.loop
+
+        return (user.try_ensure_started() async for user in cls.all_with_tgid())
 
-    # endregion
     # region Telegram connection management
 
-    async def start(self, delete_unless_authenticated: bool = False) -> "User":
-        await super().start()
-        if await self.is_logged_in():
+    async def try_ensure_started(self) -> None:
+        try:
+            await self.ensure_started()
+        except Exception:
+            self.log.exception("Exception in ensure_started")
+        else:
+            if not self.client and not await PgSession.has(self.mxid):
+                self.log.warning("Didn't start user: no session stored")
+                if self.tgid:
+                    await self.push_bridge_state(
+                        BridgeStateEvent.BAD_CREDENTIALS, error="tg-no-auth"
+                    )
+
+    async def ensure_started(self, even_if_no_session=False) -> User:
+        if not self.puppet_whitelisted or self.connected:
+            return self
+        async with self._ensure_started_lock:
+            return cast(User, await super().ensure_started(even_if_no_session))
+
+    async def start(self, delete_unless_authenticated: bool = False) -> User:
+        try:
+            await super().start()
+        except AuthKeyDuplicatedError:
+            self.log.warning("Got AuthKeyDuplicatedError in start()")
+            await self.push_bridge_state(
+                BridgeStateEvent.BAD_CREDENTIALS, error="tg-auth-key-duplicated"
+            )
+            await self.client.disconnect()
+            await self.client.session.delete()
+            self.client = None
+            if not delete_unless_authenticated:
+                # The caller wants the client to be connected, so restart the connection.
+                await super().start()
+            return self
+        except Exception:
+            await self.push_bridge_state(BridgeStateEvent.UNKNOWN_ERROR)
+            raise
+        try:
+            assert self.client, "client is undefined"
+            assert self.client.is_connected(), "client is not connected"
+            await self.client(GetStateRequest())
+        except AssertionError as e:
+            self.log.error(f"Client in bad state after start(): {e}")
+            if self.tgid:
+                await self.push_bridge_state(BridgeStateEvent.UNKNOWN_ERROR, message=str(e))
+        except UnauthorizedError as e:
+            self.log.error(f"Authorization error in start(): {type(e)}: {e}")
+            if self.tgid:
+                await self.push_bridge_state(
+                    BridgeStateEvent.BAD_CREDENTIALS,
+                    error="tg-auth-error",
+                    message=str(e),
+                    ttl=3600,
+                )
+        except RPCError as e:
+            self.log.error(f"Unknown RPC error in start(): {type(e)}: {e}")
+            if self.tgid:
+                await self.push_bridge_state(BridgeStateEvent.UNKNOWN_ERROR, message=str(e))
+        else:
+            # Authenticated, run post login
             self.log.debug(f"Ensuring post_login() for {self.name}")
-            asyncio.ensure_future(self.post_login(), loop=self.loop)
-        elif delete_unless_authenticated:
+            asyncio.create_task(self.post_login())
+            return self
+        # Not authenticated, delete data if necessary
+        if delete_unless_authenticated:
             self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
             await self.client.disconnect()
-            self.client.session.delete()
+            await self.client.session.delete()
         return self
 
-    async def post_login(self, info: TLUser = None):
+    @property
+    def _is_connected(self) -> bool:
+        return bool(
+            self.client and self.client._sender and self.client._sender._transport_connected()
+        )
+
+    async def _track_connection(self) -> None:
+        self.log.debug("Starting loop to track connection state")
+        while True:
+            await asyncio.sleep(3)
+            connected = self._is_connected
+            self._track_metric(METRIC_CONNECTED, connected)
+            if connected:
+                await self.push_bridge_state(
+                    BridgeStateEvent.BACKFILLING
+                    if self._is_backfilling
+                    else BridgeStateEvent.CONNECTED,
+                    ttl=3600,
+                )
+            else:
+                await self.push_bridge_state(
+                    BridgeStateEvent.UNKNOWN_ERROR, ttl=240, error="tg-not-connected"
+                )
+
+    async def fill_bridge_state(self, state: BridgeState) -> None:
+        await super().fill_bridge_state(state)
+        state.remote_id = str(self.tgid)
+        state.remote_name = self.human_tg_id
+
+    async def get_bridge_states(self) -> list[BridgeState]:
+        if not self.tgid:
+            return []
+        if self._is_connected and await self.is_logged_in():
+            state_event = (
+                BridgeStateEvent.BACKFILLING
+                if self._is_backfilling
+                else BridgeStateEvent.CONNECTED
+            )
+            ttl = 3600
+        else:
+            state_event = BridgeStateEvent.UNKNOWN_ERROR
+            ttl = 240
+        return [BridgeState(state_event=state_event, ttl=ttl)]
+
+    async def get_puppet(self) -> pu.Puppet | None:
+        if not self.tgid:
+            return None
+        return await pu.Puppet.get_by_tgid(self.tgid)
+
+    async def stop(self) -> None:
+        if self._track_connection_task:
+            self._track_connection_task.cancel()
+            self._track_connection_task = None
+        await super().stop()
+        self._track_metric(METRIC_CONNECTED, False)
+
+    async def post_login(self, info: TLUser = None, first_login: bool = False) -> None:
+        if self.config["metrics.enabled"] and not self._track_connection_task:
+            self._track_connection_task = asyncio.create_task(self._track_connection())
+
         try:
             await self.update_info(info)
-            if not self.is_bot:
+        except Exception:
+            self.log.exception("Failed to update telegram account info")
+            return
+
+        self._track_metric(METRIC_LOGGED_IN, True)
+
+        try:
+            puppet = await pu.Puppet.get_by_tgid(self.tgid)
+            if puppet.custom_mxid != self.mxid and puppet.can_auto_login(self.mxid):
+                self.log.info(f"Automatically enabling custom puppet")
+                await puppet.switch_mxid(access_token="auto", mxid=self.mxid)
+        except Exception:
+            self.log.exception("Failed to automatically enable custom puppet")
+
+        if not self.is_bot and self.config["bridge.startup_sync"]:
+            try:
+                self._is_backfilling = True
                 await self.sync_dialogs()
                 await self.sync_contacts()
-            if config["bridge.catch_up"]:
-                await self.client.catch_up()
-        except Exception:
-            self.log.exception("Failed to run post-login functions for %s", self.mxid)
+            except Exception:
+                self.log.exception("Failed to run post-login sync")
+            finally:
+                self._is_backfilling = False
 
-    async def update(self, update: TypeUpdate):
+    async def update(self, update: TypeUpdate) -> bool:
         if not self.is_bot:
-            return
+            return False
 
         if isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
-            message = update.message
-            if isinstance(message.to_id, PeerUser) and not message.out:
-                portal = po.Portal.get_by_tgid(message.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
-            else:
-                portal = po.Portal.get_by_entity(message.to_id, receiver_id=self.tgid)
+            portal = await po.Portal.get_by_entity(update.message.peer_id, tg_receiver=self.tgid)
         elif isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
         elif isinstance(update, UpdateShortMessage):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+            portal = await po.Portal.get_by_tgid(
+                TelegramID(update.user_id), tg_receiver=self.tgid, peer_type="user"
+            )
         else:
-            return
+            return False
 
-        self.register_portal(portal)
+        if portal:
+            await self.register_portal(portal)
+            return False
+
+        # Don't bother handling the update
+        return True
 
     # endregion
     # region Telegram actions that need custom methods
 
-    def ensure_started(self, even_if_no_session: bool = False) -> "Awaitable[User]":
-        return super().ensure_started(even_if_no_session)
+    async def set_presence(self, online: bool = True) -> None:
+        if not self.is_bot:
+            await self.client(UpdateStatusRequest(offline=not online))
 
-    def set_presence(self, online: bool = True):
-        if self.is_bot:
-            return
-        return self.client(UpdateStatusRequest(offline=not online))
+    async def get_me(self) -> TLUser | None:
+        try:
+            return (await self.client(GetUsersRequest([InputUserSelf()])))[0]
+        except UnauthorizedError as e:
+            self.log.error(f"Authorization error in get_me(): {type(e)}: {e}")
+            await self.push_bridge_state(
+                BridgeStateEvent.BAD_CREDENTIALS, error="tg-auth-error", message=str(e), ttl=3600
+            )
+            await self.stop()
+            return None
 
-    async def update_info(self, info: TLUser = None):
-        info = info or await self.client.get_me()
+    async def update_info(self, info: TLUser = None) -> None:
+        if not info:
+            info = await self.get_me()
+            if not info:
+                self.log.warning("get_me() returned None, aborting update_info()")
+                return
         changed = False
         if self.is_bot != info.bot:
             self.is_bot = info.bot
             changed = True
-        if self.username != info.username:
-            self.username = info.username
+        if self.tg_username != info.username:
+            self.tg_username = info.username
+            changed = True
+        if self.tg_phone != info.phone:
+            self.tg_phone = info.phone
             changed = True
         if self.tgid != info.id:
-            self.tgid = info.id
+            self.tgid = TelegramID(info.id)
             self.by_tgid[self.tgid] = self
         if changed:
-            self.save()
+            await self.save()
 
-    async def log_out(self):
-        puppet = pu.Puppet.get(self.tgid)
+    async def kick_from_portals(self) -> None:
+        if not self.config["bridge.kick_on_logout"]:
+            return
+        portals = await self.get_cached_portals()
+        for _, portal in portals.values():
+            if not portal or portal.deleted or not portal.mxid or portal.has_bot:
+                continue
+            if portal.peer_type == "user":
+                await portal.cleanup_portal("Logged out of Telegram")
+            else:
+                try:
+                    await portal.main_intent.kick_user(
+                        portal.mxid, self.mxid, "Logged out of Telegram."
+                    )
+                except MatrixRequestError:
+                    pass
+
+    async def log_out(self) -> bool:
+        puppet = await pu.Puppet.get_by_tgid(self.tgid)
         if puppet.is_real_user:
             await puppet.switch_mxid(None, None)
-        for _, portal in self.portals.items():
-            if not portal.mxid or portal.has_bot:
-                continue
-            try:
-                await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
-            except MatrixRequestError:
-                pass
-        self.portals = {}
-        self.contacts = []
-        self.save()
+        try:
+            await self.kick_from_portals()
+        except Exception:
+            self.log.exception("Failed to kick user from portals on logout")
+        await self.push_bridge_state(BridgeStateEvent.LOGGED_OUT)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
             except KeyError:
                 pass
             self.tgid = None
-            self.save()
         ok = await self.client.log_out()
-        if not ok:
-            return False
-        self.delete()
-        return True
+        await self.client.session.delete()
+        await self.delete()
+        self.by_mxid.pop(self.mxid, None)
+        await self.stop()
+        self._track_metric(METRIC_LOGGED_IN, False)
+        return ok
 
-    def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
-                      ) -> SearchResults:
-        results = []  # type: SearchResults
-        for contact in self.contacts:
+    async def _search_local(
+        self, query: str, max_results: int = 5, min_similarity: int = 45
+    ) -> list[SearchResult]:
+        results: list[SearchResult] = []
+        for contact_id in await self.get_contacts():
+            contact = await pu.Puppet.get_by_tgid(contact_id, create=False)
+            if not contact:
+                continue
             similarity = contact.similarity(query)
             if similarity >= min_similarity:
-                results.append((contact, similarity))
+                results.append(SearchResult(contact, similarity))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
-    async def _search_remote(self, query: str, max_results: int = 5) -> SearchResults:
+    async def _search_remote(self, query: str, max_results: int = 5) -> list[SearchResult]:
         if len(query) < 5:
             return []
         server_results = await self.client(SearchRequest(q=query, limit=max_results))
-        results = []  # type: SearchResults
+        results: list[SearchResult] = []
         for user in server_results.users:
-            puppet = pu.Puppet.get(user.id)
+            puppet = await pu.Puppet.get_by_tgid(user.id)
             await puppet.update_info(self, user)
-            results.append((puppet, puppet.similarity(query)))
+            results.append(SearchResult(puppet, puppet.similarity(query)))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
-    async def search(self, query: str, force_remote: bool = False) -> Tuple[SearchResults, bool]:
+    async def search(
+        self, query: str, force_remote: bool = False
+    ) -> tuple[list[SearchResult], bool]:
         if force_remote:
             return await self._search_remote(query), True
 
-        results = self._search_local(query)
+        results = await self._search_local(query)
         if results:
             return results, False
 
         return await self._search_remote(query), True
 
-    async def sync_dialogs(self, synchronous_create: bool = False):
+    async def get_direct_chats(self) -> dict[UserID, list[RoomID]]:
+        return {
+            pu.Puppet.get_mxid_from_id(portal.tgid): [portal.mxid]
+            async for portal in po.Portal.find_private_chats(self.tgid)
+            if portal.mxid
+        }
+
+    async def _tag_room(
+        self, puppet: pu.Puppet, portal: po.Portal, tag: str, active: bool
+    ) -> None:
+        if not tag or not portal or not portal.mxid:
+            return
+        tag_info = await puppet.intent.get_room_tag(portal.mxid, tag)
+        if active and tag_info is None:
+            tag_info = RoomTagInfo(order=0.5)
+            tag_info[DOUBLE_PUPPET_SOURCE_KEY] = self.bridge.name
+            await puppet.intent.set_room_tag(portal.mxid, tag, tag_info)
+        elif (
+            not active and tag_info and tag_info.get(DOUBLE_PUPPET_SOURCE_KEY) == self.bridge.name
+        ):
+            await puppet.intent.remove_room_tag(portal.mxid, tag)
+
+    async def _mute_room(cls, puppet: pu.Puppet, portal: po.Portal, mute_until: datetime) -> None:
+        if not cls.config["bridge.mute_bridging"] or not portal or not portal.mxid:
+            return
+        now = datetime.utcnow().replace(tzinfo=timezone.utc)
+        if mute_until is not None and mute_until > now:
+            await puppet.intent.set_push_rule(
+                PushRuleScope.GLOBAL,
+                PushRuleKind.ROOM,
+                portal.mxid,
+                actions=[PushActionType.DONT_NOTIFY],
+            )
+        else:
+            try:
+                await puppet.intent.remove_push_rule(
+                    PushRuleScope.GLOBAL, PushRuleKind.ROOM, portal.mxid
+                )
+            except MNotFound:
+                pass
+
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
+        if self.config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        for peer in update.folder_peers:
+            portal = await po.Portal.get_by_entity(peer.peer, tg_receiver=self.tgid, create=False)
+            await self._tag_room(
+                puppet, portal, self.config["bridge.archive_tag"], peer.folder_id == 1
+            )
+
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        if self.config["bridge.tag_only_on_create"]:
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        # TODO bridge unpinning properly
+        for pinned in update.order:
+            portal = await po.Portal.get_by_entity(
+                pinned.peer, tg_receiver=self.tgid, create=False
+            )
+            await self._tag_room(puppet, portal, self.config["bridge.pinned_tag"], True)
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        if self.config["bridge.tag_only_on_create"]:
+            return
+        elif not isinstance(update.peer, NotifyPeer):
+            # TODO handle global notification setting changes?
+            return
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        if not puppet or not puppet.is_real_user:
+            return
+        portal = await po.Portal.get_by_entity(
+            update.peer.peer, tg_receiver=self.tgid, create=False
+        )
+        await self._mute_room(puppet, portal, update.notify_settings.mute_until)
+
+    async def _sync_dialog(
+        self, portal: po.Portal, dialog: Dialog, should_create: bool, puppet: pu.Puppet | None
+    ) -> None:
+        was_created = False
+        if portal.mxid:
+            try:
+                await portal.backfill(self, last_id=dialog.message.id)
+            except Exception:
+                self.log.exception(f"Error while backfilling {portal.tgid_log}")
+            try:
+                await portal.update_matrix_room(self, dialog.entity)
+            except Exception:
+                self.log.exception(f"Error while updating {portal.tgid_log}")
+        elif should_create:
+            try:
+                await portal.create_matrix_room(self, dialog.entity, invites=[self.mxid])
+                was_created = True
+            except Exception:
+                self.log.exception(f"Error while creating {portal.tgid_log}")
+        if portal.mxid and puppet and puppet.is_real_user:
+            tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+            if dialog.unread_count == 0:
+                # This is usually more reliable than finding a specific message
+                # e.g. if the last read message is a service message that isn't in the message db
+                last_read = await DBMessage.find_last(portal.mxid, tg_space)
+            else:
+                last_read = await DBMessage.get_one_by_tgid(
+                    portal.tgid, tg_space, dialog.dialog.read_inbox_max_id
+                )
+            if last_read:
+                await puppet.intent.mark_read(last_read.mx_room, last_read.mxid)
+            if was_created or not self.config["bridge.tag_only_on_create"]:
+                await self._mute_room(puppet, portal, dialog.dialog.notify_settings.mute_until)
+                await self._tag_room(
+                    puppet, portal, self.config["bridge.pinned_tag"], dialog.pinned
+                )
+                await self._tag_room(
+                    puppet, portal, self.config["bridge.archive_tag"], dialog.archived
+                )
+
+    async def get_cached_portals(self) -> dict[tuple[TelegramID, TelegramID], po.Portal]:
+        if self._portals_cache is None:
+            self._portals_cache = {
+                (tgid, tg_receiver): await po.Portal.get_by_tgid(tgid, tg_receiver=tg_receiver)
+                for tgid, tg_receiver in await self.get_portals()
+            }
+        return self._portals_cache
+
+    async def sync_dialogs(self) -> None:
+        if self.is_bot:
+            return
         creators = []
-        for entity in await self.get_dialogs(limit=30):
-            portal = po.Portal.get_by_entity(entity)
-            self.portals[portal.tgid_full] = portal
-            creators.append(
-                portal.create_matrix_room(self, entity, invites=[self.mxid],
-                                          synchronous=synchronous_create))
-        self.save()
-        await asyncio.gather(*creators, loop=self.loop)
+        update_limit = self.config["bridge.sync_update_limit"] or None
+        create_limit = self.config["bridge.sync_create_limit"]
+        index = 0
+        self.log.debug(
+            f"Syncing dialogs (update_limit={update_limit}, create_limit={create_limit})"
+        )
+        await self.push_bridge_state(BridgeStateEvent.BACKFILLING)
+        puppet = await pu.Puppet.get_by_custom_mxid(self.mxid)
+        dialog: Dialog
+        old_portal_cache = await self.get_cached_portals()
+        new_portal_cache = old_portal_cache.copy()
+        async for dialog in self.client.iter_dialogs(
+            limit=update_limit, ignore_migrated=True, archived=False
+        ):
+            entity = dialog.entity
+            if isinstance(entity, ChatForbidden):
+                self.log.warning(f"Ignoring forbidden chat {entity} while syncing")
+                continue
+            elif isinstance(entity, Chat) and (entity.deactivated or entity.left):
+                self.log.warning(f"Ignoring deactivated or left chat {entity} while syncing")
+                continue
+            elif isinstance(entity, TLUser) and not self.config["bridge.sync_direct_chats"]:
+                self.log.trace(f"Ignoring user {entity.id} while syncing")
+                continue
+            portal = await po.Portal.get_by_entity(entity, tg_receiver=self.tgid)
+            new_portal_cache[portal.tgid_full] = portal
+            coro = self._sync_dialog(
+                portal=portal,
+                dialog=dialog,
+                puppet=puppet,
+                should_create=not create_limit or index < create_limit,
+            )
+            creators.append(asyncio.create_task(coro))
+            index += 1
+        if new_portal_cache.keys() != old_portal_cache.keys():
+            await self.set_portals(new_portal_cache.keys())
+            self._portals_cache = new_portal_cache
+        await asyncio.gather(*creators)
+        await self.update_direct_chats()
+        self.log.debug("Dialog syncing complete")
 
-    def register_portal(self, portal: po.Portal):
-        try:
-            if self.portals[portal.tgid_full] == portal:
+    async def register_portal(self, portal: po.Portal) -> None:
+        self.log.trace(f"Registering portal {portal.tgid_full}")
+        if self._portals_cache is not None:
+            if self._portals_cache.get(portal.tgid_full) == portal:
                 return
-        except KeyError:
-            pass
-        self.portals[portal.tgid_full] = portal
-        self.save()
+            self._portals_cache[portal.tgid_full] = portal
+        await super().register_portal(portal.tgid, portal.tg_receiver)
 
-    def unregister_portal(self, portal: po.Portal):
-        try:
-            del self.portals[portal.tgid_full]
-            self.save()
-        except KeyError:
-            pass
+    async def unregister_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
+        self.log.trace(f"Unregistering portal {(tgid, tg_receiver)}")
+        if self._portals_cache is not None:
+            self._portals_cache.pop((tgid, tg_receiver), None)
+        await super().unregister_portal(tgid, tg_receiver)
 
     async def needs_relaybot(self, portal: po.Portal) -> bool:
         return not await self.is_logged_in() or (
-            self.is_bot and portal.tgid_full not in self.portals)
+            (portal.has_bot or self.is_bot)
+            and portal.tgid_full not in await self.get_cached_portals()
+        )
 
-    def _hash_contacts(self) -> int:
+    @staticmethod
+    def _hash_contacts(count: int, ids: list[TelegramID]) -> int:
         acc = 0
-        for id in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
-            acc = (acc * 20261 + id) & 0xffffffff
-        return acc & 0x7fffffff
+        for contact in sorted([count] + ids):
+            acc = (acc * 20261 + contact) & 0xFFFFFFFF
+        return acc & 0x7FFFFFFF
 
-    async def sync_contacts(self):
-        response = await self.client(GetContactsRequest(hash=self._hash_contacts()))
+    async def sync_contacts(self) -> None:
+        existing_contacts = await self.get_contacts()
+        contact_hash = self._hash_contacts(self.saved_contacts, existing_contacts)
+        response = await self.client(GetContactsRequest(hash=contact_hash))
         if isinstance(response, ContactsNotModified):
             return
-        self.log.debug("Updating contacts...")
-        self.contacts = []
-        self.saved_contacts = response.saved_count
+        self.log.debug(f"Updating contacts of {self.name}...")
+        if self.saved_contacts != response.saved_count:
+            self.saved_contacts = response.saved_count
+            await self.save()
+        await self.set_contacts(user.id for user in response.users)
         for user in response.users:
-            puppet = pu.Puppet.get(user.id)
+            puppet = await pu.Puppet.get_by_tgid(user.id)
             await puppet.update_info(self, user)
-            self.contacts.append(puppet)
-        self.save()
 
     # endregion
     # region Class instance lookup
 
-    @classmethod
-    def get_by_mxid(cls, mxid: str, create: bool=True) -> "Optional[User]":
-        if not mxid:
-            raise ValueError("Matrix ID can't be empty")
+    def _add_to_cache(self) -> None:
+        self.by_mxid[self.mxid] = self
+        if self.tgid:
+            self.by_tgid[self.tgid] = self
 
+    @classmethod
+    async def get_and_start_by_mxid(cls, mxid: UserID, even_if_no_session: bool = False) -> User:
+        user = await cls.get_by_mxid(mxid, create=True)
+        await user.ensure_started(even_if_no_session=even_if_no_session)
+        return user
+
+    @classmethod
+    async def all_with_tgid(cls) -> AsyncGenerator[User, None]:
+        users = await super().all_with_tgid()
+        user: cls
+        for user in users:
+            try:
+                yield cls.by_mxid[user.mxid]
+            except KeyError:
+                user._add_to_cache()
+                yield user
+
+    @classmethod
+    @async_getter_lock
+    async def get_by_mxid(
+        cls, mxid: UserID, *, check_db: bool = True, create: bool = True
+    ) -> User | None:
+        if not mxid or pu.Puppet.get_id_from_mxid(mxid) or mxid == cls.az.bot_mxid:
+            return None
         try:
             return cls.by_mxid[mxid]
         except KeyError:
             pass
 
-        user = DBUser.query.get(mxid)
-        if user:
-            user = cls.from_db(user)
+        if not check_db:
+            return None
+
+        user = cast(cls, await super().get_by_mxid(mxid))
+        if user is not None:
+            user._add_to_cache()
             return user
 
         if create:
+            cls.log.debug(f"Creating user instance for {mxid}")
             user = cls(mxid)
-            cls.db.add(user.db_instance)
-            cls.db.commit()
+            await user.insert()
+            user._add_to_cache()
             return user
 
         return None
 
     @classmethod
-    def get_by_tgid(cls, tgid: int) -> "Optional[User]":
+    @async_getter_lock
+    async def get_by_tgid(cls, tgid: TelegramID) -> User | None:
         try:
             return cls.by_tgid[tgid]
         except KeyError:
             pass
 
-        user = DBUser.query.filter(DBUser.tgid == tgid).one_or_none()
-        if user:
-            user = cls.from_db(user)
+        user = cast(cls, await super().get_by_tgid(tgid))
+        if user is not None:
+            user._add_to_cache()
             return user
 
         return None
 
     @classmethod
-    def find_by_username(cls, username: str) -> "Optional[User]":
+    async def find_by_username(cls, username: str) -> User | None:
         if not username:
             return None
 
+        username = username.lower()
+
         for _, user in cls.by_tgid.items():
-            if user.username and user.username.lower() == username.lower():
+            if user.tg_username and user.tg_username.lower() == username:
                 return user
 
-        puppet = DBUser.query.filter(DBUser.tg_username == username).one_or_none()
-        if puppet:
-            return cls.from_db(puppet)
+        user = cast(cls, await super().find_by_username(username))
+        if user:
+            try:
+                return cls.by_mxid[user.mxid]
+            except KeyError:
+                user._add_to_cache()
+                return user
 
         return None
+
     # endregion
-
-
-def init(context: "Context") -> List[Awaitable[User]]:
-    global config
-    config = context.config
-
-    users = [User.from_db(user) for user in DBUser.query.all()]
-    return [user.ensure_started() for user in users]

mautrix_telegram/util/__init__.py

@@ -1,3 +1,4 @@
-from .file_transfer import transfer_file_to_matrix, convert_image
-from .format_duration import format_duration
-from .signed_token import sign_token, verify_token
+from .color_log import ColorFormatter
+from .file_transfer import convert_image, transfer_file_to_matrix
+from .parallel_file_transfer import parallel_transfer_to_telegram
+from .recursive_dict import recursive_del, recursive_get, recursive_set

mautrix_telegram/util/color_log.py

@@ -0,0 +1,36 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.logging.color import (
+    MXID_COLOR,
+    PREFIX,
+    RESET,
+    ColorFormatter as BaseColorFormatter,
+)
+
+TELETHON_COLOR = PREFIX + "35;1m"  # magenta
+TELETHON_MODULE_COLOR = PREFIX + "35m"
+
+
+class ColorFormatter(BaseColorFormatter):
+    def _color_name(self, module: str) -> str:
+        if module.startswith("telethon"):
+            prefix, user_id, module = module.split(".", 2)
+            return (
+                f"{TELETHON_COLOR}{prefix}{RESET}."
+                f"{MXID_COLOR}{user_id}{RESET}."
+                f"{TELETHON_MODULE_COLOR}{module}{RESET}"
+            )
+        return super()._color_name(module)

mautrix_telegram/util/file_transfer.py

@@ -1,6 +1,5 @@
-# -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,50 +13,80 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Tuple, Union, Dict
+from __future__ import annotations
+
+from typing import Optional, Union
 from io import BytesIO
-import time
-import logging
+from sqlite3 import IntegrityError
 import asyncio
+import logging
+import tempfile
+import time
 
+from asyncpg import UniqueViolationError
+from telethon.errors import (
+    AuthBytesInvalidError,
+    AuthKeyInvalidError,
+    FileIdInvalidError,
+    LocationInvalidError,
+    SecurityError,
+)
+from telethon.tl.types import (
+    Document,
+    InputDocumentFileLocation,
+    InputFileLocation,
+    InputPeerPhotoFileLocation,
+    InputPhotoFileLocation,
+    PhotoCachedSize,
+    PhotoSize,
+    TypePhotoSize,
+)
 import magic
-from sqlalchemy import orm
-from sqlalchemy.exc import IntegrityError, InvalidRequestError
-from sqlalchemy.orm.exc import FlushError
 
-from telethon.tl.types import (Document, FileLocation, InputFileLocation,
-                               InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
-from telethon.errors import *
-from mautrix_appservice import IntentAPI
+from mautrix.appservice import IntentAPI
 
-from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
+from ..tgclient import MautrixTelegramClient
+from ..util import sane_mimetypes
+from .parallel_file_transfer import parallel_transfer_to_matrix
+from .tgs_converter import convert_tgs_to
 
 try:
     from PIL import Image
 except ImportError:
     Image = None
+
 try:
     from moviepy.editor import VideoFileClip
-    import random
-    import string
-    import os
-    import mimetypes
 except ImportError:
-    VideoFileClip = random = string = os = mimetypes = None
+    VideoFileClip = None
 
-log = logging.getLogger("mau.util")  # type: logging.Logger
+try:
+    from mautrix.crypto.attachments import encrypt_attachment
+except ImportError:
+    encrypt_attachment = None
 
-TypeLocation = Union[Document, InputDocumentFileLocation, FileLocation, InputFileLocation]
+log: logging.Logger = logging.getLogger("mau.util")
+
+TypeLocation = Union[
+    Document,
+    InputDocumentFileLocation,
+    InputPeerPhotoFileLocation,
+    InputFileLocation,
+    InputPhotoFileLocation,
+]
 
 
-def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
-                  thumbnail_to: Optional[Tuple[int, int]] = None
-                  ) -> Tuple[str, bytes, Optional[int], Optional[int]]:
+def convert_image(
+    file: bytes,
+    source_mime: str = "image/webp",
+    target_type: str = "png",
+    thumbnail_to: tuple[int, int] | None = None,
+) -> tuple[str, bytes, int | None, int | None]:
     if not Image:
         return source_mime, file, None, None
     try:
-        image = Image.open(BytesIO(file)).convert("RGBA")  # type: Image.Image
+        image: Image.Image = Image.open(BytesIO(file)).convert("RGBA")
         if thumbnail_to:
             image.thumbnail(thumbnail_to, Image.ANTIALIAS)
         new_file = BytesIO()
@@ -69,46 +98,53 @@ def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str
         return source_mime, file, None, None
 
 
-def _temp_file_name(ext: str) -> str:
-    return ("/tmp/mxtg-video-"
-            + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
-            + ext)
-
-
-def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str = "png",
-                          max_size: Tuple[int, int] = (1024, 720)) -> Tuple[bytes, int, int]:
-    # We don't have any way to read the video from memory, so save it to disk.
-    temp_file = _temp_file_name(video_ext)
-    with open(temp_file, "wb") as file:
+def _read_video_thumbnail(
+    data: bytes,
+    video_ext: str = "mp4",
+    frame_ext: str = "png",
+    max_size: tuple[int, int] = (1024, 720),
+) -> tuple[bytes, int, int]:
+    with tempfile.NamedTemporaryFile(prefix="mxtg_video_", suffix=f".{video_ext}") as file:
+        # We don't have any way to read the video from memory, so save it to disk.
         file.write(data)
 
-    # Read temp file and get frame
-    clip = VideoFileClip(temp_file)
-    frame = clip.get_frame(0)
+        # Read temp file and get frame
+        frame = VideoFileClip(file.name).get_frame(0)
 
     # Convert to png and save to BytesIO
     image = Image.fromarray(frame).convert("RGBA")
+
     thumbnail_file = BytesIO()
     if max_size:
         image.thumbnail(max_size, Image.ANTIALIAS)
     image.save(thumbnail_file, frame_ext)
 
-    os.remove(temp_file)
-
     w, h = image.size
     return thumbnail_file.getvalue(), w, h
 
 
 def _location_to_id(location: TypeLocation) -> str:
-    if isinstance(location, (Document, InputDocumentFileLocation)):
-        return f"{location.id}-{location.version}"
-    elif isinstance(location, (FileLocation, InputFileLocation)):
+    if isinstance(location, Document):
+        return f"{location.id}-{location.access_hash}"
+    elif isinstance(location, (InputDocumentFileLocation, InputPhotoFileLocation)):
+        return f"{location.id}-{location.access_hash}-{location.thumb_size}"
+    elif isinstance(location, InputFileLocation):
         return f"{location.volume_id}-{location.local_id}"
+    elif isinstance(location, InputPeerPhotoFileLocation):
+        return str(location.photo_id)
 
 
-async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                       thumbnail_loc: TypeLocation, video: bytes,
-                                       mime: str) -> Optional[DBTelegramFile]:
+async def transfer_thumbnail_to_matrix(
+    client: MautrixTelegramClient,
+    intent: IntentAPI,
+    thumbnail_loc: TypeLocation,
+    mime_type: str,
+    encrypt: bool,
+    video: bytes | None,
+    custom_data: bytes | None = None,
+    width: int | None = None,
+    height: int | None = None,
+) -> DBTelegramFile | None:
     if not Image or not VideoFileClip:
         return None
 
@@ -116,8 +152,17 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if not loc_id:
         return None
 
-    video_ext = mimetypes.guess_extension(mime)
-    if VideoFileClip and video_ext:
+    if custom_data:
+        loc_id += "-mau_custom_thumbnail"
+
+    db_file = await DBTelegramFile.get(loc_id)
+    if db_file:
+        return db_file
+
+    video_ext = sane_mimetypes.guess_extension(mime_type)
+    if custom_data:
+        file = custom_data
+    elif VideoFileClip and video_ext and video:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
         except OSError:
@@ -128,24 +173,59 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
         width, height = None, None
         mime_type = magic.from_buffer(file, mime=True)
 
-    content_uri = await intent.upload_file(file, mime_type)
+    decryption_info = None
+    upload_mime_type = mime_type
+    if encrypt:
+        file, decryption_info = encrypt_attachment(file)
+        upload_mime_type = "application/octet-stream"
+    content_uri = await intent.upload_media(file, upload_mime_type)
+    if decryption_info:
+        decryption_info.url = content_uri
 
-    return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
-                          was_converted=False, timestamp=int(time.time()), size=len(file),
-                          width=width, height=height)
+    db_file = DBTelegramFile(
+        id=loc_id,
+        mxc=content_uri,
+        mime_type=mime_type,
+        was_converted=False,
+        timestamp=int(time.time()),
+        size=len(file),
+        width=width,
+        height=height,
+        decryption_info=decryption_info,
+    )
+    try:
+        await db_file.insert()
+    except (UniqueViolationError, IntegrityError) as e:
+        log.exception(
+            f"{e.__class__.__name__} while saving transferred file thumbnail data. "
+            "This was probably caused by two simultaneous transfers of the same file, "
+            "and might (but probably won't) cause problems with thumbnails or something."
+        )
+    return db_file
 
 
-transfer_locks = {}  # type: Dict[str, asyncio.Lock]
+transfer_locks: dict[str, asyncio.Lock] = {}
+
+TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
 
 
-async def transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: Optional[TypeLocation] = None,
-                                  is_sticker: bool = False) -> Optional[DBTelegramFile]:
+async def transfer_file_to_matrix(
+    client: MautrixTelegramClient,
+    intent: IntentAPI,
+    location: TypeLocation,
+    thumbnail: TypeThumbnail = None,
+    *,
+    is_sticker: bool = False,
+    tgs_convert: dict | None = None,
+    filename: str | None = None,
+    encrypt: bool = False,
+    parallel_id: int | None = None,
+) -> DBTelegramFile | None:
     location_id = _location_to_id(location)
     if not location_id:
         return None
 
-    db_file = DBTelegramFile.query.get(location_id)
+    db_file = await DBTelegramFile.get(location_id)
     if db_file:
         return db_file
 
@@ -155,61 +235,119 @@ async def transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient
         lock = asyncio.Lock()
         transfer_locks[location_id] = lock
     async with lock:
-        return await _unlocked_transfer_file_to_matrix(db, client, intent, location_id, location,
-                                                       thumbnail, is_sticker)
+        return await _unlocked_transfer_file_to_matrix(
+            client,
+            intent,
+            location_id,
+            location,
+            thumbnail,
+            is_sticker,
+            tgs_convert,
+            filename,
+            encrypt,
+            parallel_id,
+        )
 
 
-async def _unlocked_transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient,
-                                            intent: IntentAPI, loc_id: str, location: TypeLocation,
-                                            thumbnail: Optional[TypeLocation],
-                                            is_sticker: bool) -> Optional[DBTelegramFile]:
-    db_file = DBTelegramFile.query.get(loc_id)
+async def _unlocked_transfer_file_to_matrix(
+    client: MautrixTelegramClient,
+    intent: IntentAPI,
+    loc_id: str,
+    location: TypeLocation,
+    thumbnail: TypeThumbnail,
+    is_sticker: bool,
+    tgs_convert: dict | None,
+    filename: str | None,
+    encrypt: bool,
+    parallel_id: int | None,
+) -> DBTelegramFile | None:
+    db_file = await DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
-    try:
-        file = await client.download_file(location)
-    except LocationInvalidError:
-        return None
-    except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
-        log.exception(f"{e.__class__.__name__} while downloading a file.")
-        return None
+    converted_anim = None
 
-    width, height = None, None
-    mime_type = magic.from_buffer(file, mime=True)
+    if parallel_id and isinstance(location, Document) and (not is_sticker or not tgs_convert):
+        db_file = await parallel_transfer_to_matrix(
+            client, intent, loc_id, location, filename, encrypt, parallel_id
+        )
+        mime_type = location.mime_type
+        file = None
+    else:
+        try:
+            file = await client.download_file(location)
+        except (LocationInvalidError, FileIdInvalidError):
+            return None
+        except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
+            log.exception(f"{e.__class__.__name__} while downloading a file.")
+            return None
 
-    image_converted = False
-    if mime_type == "image/webp":
-        new_mime_type, file, width, height = convert_image(
-            file, source_mime="image/webp", target_type="png",
-            thumbnail_to=(256, 256) if is_sticker else None)
-        image_converted = new_mime_type != mime_type
-        mime_type = new_mime_type
-        thumbnail = None
+        width, height = None, None
+        mime_type = magic.from_buffer(file, mime=True)
 
-    content_uri = await intent.upload_file(file, mime_type)
+        image_converted = False
+        # A weird bug in alpine/magic makes it return application/octet-stream for gzips...
+        is_tgs = mime_type == "application/gzip" or (
+            mime_type == "application/octet-stream" and magic.from_buffer(file).startswith("gzip")
+        )
+        if is_sticker and tgs_convert and is_tgs:
+            converted_anim = await convert_tgs_to(
+                file, tgs_convert["target"], **tgs_convert["args"]
+            )
+            mime_type = converted_anim.mime
+            file = converted_anim.data
+            width, height = converted_anim.width, converted_anim.height
+            image_converted = mime_type != "application/gzip"
+            thumbnail = None
 
-    db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
-                             mime_type=mime_type, was_converted=image_converted,
-                             timestamp=int(time.time()), size=len(file),
-                             width=width, height=height)
+        decryption_info = None
+        upload_mime_type = mime_type
+        if encrypt and encrypt_attachment:
+            file, decryption_info = encrypt_attachment(file)
+            upload_mime_type = "application/octet-stream"
+        content_uri = await intent.upload_media(file, upload_mime_type)
+        if decryption_info:
+            decryption_info.url = content_uri
+
+        db_file = DBTelegramFile(
+            id=loc_id,
+            mxc=content_uri,
+            decryption_info=decryption_info,
+            mime_type=mime_type,
+            was_converted=image_converted,
+            timestamp=int(time.time()),
+            size=len(file),
+            width=width,
+            height=height,
+        )
     if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
         if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
             thumbnail = thumbnail.location
-        db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
-                                                               mime_type)
+        try:
+            db_file.thumbnail = await transfer_thumbnail_to_matrix(
+                client, intent, thumbnail, video=file, mime_type=mime_type, encrypt=encrypt
+            )
+        except FileIdInvalidError:
+            log.warning(f"Failed to transfer thumbnail for {thumbnail!s}", exc_info=True)
+    elif converted_anim and converted_anim.thumbnail_data:
+        db_file.thumbnail = await transfer_thumbnail_to_matrix(
+            client,
+            intent,
+            location,
+            video=None,
+            encrypt=encrypt,
+            custom_data=converted_anim.thumbnail_data,
+            mime_type=converted_anim.thumbnail_mime,
+            width=converted_anim.width,
+            height=converted_anim.height,
+        )
 
     try:
-        db.add(db_file)
-        db.commit()
-    except FlushError as e:
-        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
-                      "This was probably caused by two simultaneous transfers of the same file, "
-                      "and should not cause any problems.")
-    except (IntegrityError, InvalidRequestError) as e:
-        db.rollback()
-        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
-                      "This was probably caused by two simultaneous transfers of the same file, "
-                      "and should not cause any problems.")
-
+        await db_file.insert()
+    except (UniqueViolationError, IntegrityError) as e:
+        log.exception(
+            f"{e.__class__.__name__} while saving transferred file data. "
+            "This was probably caused by two simultaneous transfers of the same file, "
+            "and should not cause any problems."
+        )
     return db_file