Bump version to 0.12.1

* Custom emojis in reactions
* Premium users can react 3 times to a single message
* Reactions to recent messages are now polled on read receipt

.codeclimate.yml

@@ -1,8 +0,0 @@
-engines:
-  sonar-python:
-    enabled: true
-    checks:
-      python:S107:
-        enabled: false
-exclude_patterns:
-- "alembic/"

.editorconfig

@@ -8,11 +8,14 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 
+[*.md]
+trim_trailing_whitespace = false
+
 [*.py]
 max_line_length = 99
 
 [*.{yaml,yml,py}]
 indent_style = space
 
-[.gitlab-ci.yml]
+[{.gitlab-ci.yml,.pre-commit-config.yaml,mautrix_telegram/web/provisioning/spec.yaml}]
 indent_size = 2

.github/ISSUE_TEMPLATE/bug.md

@@ -0,0 +1,7 @@
+---
+name: Bug report
+about: If something is definitely wrong in the bridge (rather than just a setup issue),
+  file a bug report. Remember to include relevant logs.
+labels: bug
+
+---

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,7 @@
+contact_links:
+  - name: Troubleshooting docs & FAQ
+    url: https://docs.mau.fi/bridges/general/troubleshooting.html
+    about: Check this first if you're having problems setting up the bridge.
+  - name: Support room
+    url: https://matrix.to/#/#telegram:maunium.net
+    about: For setup issues not answered by the troubleshooting docs, ask in the Matrix room.

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,6 @@
+---
+name: Enhancement request
+about: Submit a feature request or other suggestion
+labels: enhancement
+
+---

.github/workflows/python-lint.yml

@@ -0,0 +1,26 @@
+name: Python lint
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: "3.10"
+    - uses: isort/isort-action@master
+      with:
+        sortPaths: "./mautrix_telegram"
+    - uses: psf/black@stable
+      with:
+        src: "./mautrix_telegram"
+        version: "22.3.0"
+    - name: pre-commit
+      run: |
+        pip install pre-commit
+        pre-commit run -av trailing-whitespace
+        pre-commit run -av end-of-file-fixer
+        pre-commit run -av check-yaml
+        pre-commit run -av check-added-large-files

.gitlab-ci.yml

@@ -1,48 +1,3 @@
-image: docker:stable
-
-stages:
-- build
-- manifest
-
-default:
-  before_script:
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-
-build amd64:
-  stage: build
-  tags:
-  - amd64
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  after_script:
-  - |
-    if [ "$CI_COMMIT_BRANCH" = "master" ]; then
-      apk add --update curl
-      rm -rf /var/cache/apk/*
-      curl "$NOVA_ADMIN_API_URL" -H "Content-Type: application/json" -d '{"password":"'"$NOVA_ADMIN_NIGHTLY_PASS"'","bridge":"'$NOVA_BRIDGE_TYPE'","image":"'$CI_REGISTRY_IMAGE':'$CI_COMMIT_SHA'-amd64"}'
-    fi
-
-build arm64:
-  stage: build
-  tags:
-  - arm64
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-
-manifest:
-  stage: manifest
-  before_script:
-  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-  script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
-  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+include:
+- project: 'mautrix/ci'
+  file: '/python.yml'

.isort.cfg

@@ -1,16 +0,0 @@
-[settings]
-line_length=99
-indent=4
-
-multi_line_output=5
-
-sections=FUTURE,STDLIB,THIRDPARTY,TELETHON,MAUTRIX,FIRSTPARTY,LOCALFOLDER
-no_lines_before=LOCALFOLDER
-default_section=FIRSTPARTY
-
-known_thirdparty=aiohttp,sqlalchemy,alembic,commonmark,ruamel.yaml,PIL,moviepy,prometheus_client,yarl,mako,pkg_resources
-known_telethon=telethon,alchemysession,cryptg
-known_mautrix=mautrix
-
-balanced_wrapping=True
-length_sort=True

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+      - id: trailing-whitespace
+        exclude_types: [markdown]
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        language_version: python3
+        files: ^mautrix_telegram/.*\.pyi?$
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        files: ^mautrix_telegram/.*\.pyi?$

CHANGELOG.md

@@ -0,0 +1,878 @@
+# v0.12.1 (2022-09-26)
+
+### Added
+* Support for custom emojis in reactions.
+  * Like other bridges with custom emoji reactions, they're bridged as `mxc://`
+    URIs, so client support is required to render them properly.
+
+### Improved
+* The bridge will now poll for reactions to 20 most recent messages when
+  receiving a read receipt. This works around Telegram's bad protocol that
+  doesn't notify clients on reactions to other users' messages.
+* The docker image now has an option to bypass the startup script by setting
+  the `MAUTRIX_DIRECT_STARTUP` environment variable. Additionally, it will
+  refuse to run as a non-root user if that variable is not set (and print an
+  error message suggesting to either set the variable or use a custom command).
+* Moved environment variable overrides for config fields to mautrix-python.
+  The new system also allows loading JSON values to enable overriding maps like
+  `login_shared_secret_map`.
+
+### Fixed
+* `ChatParticipantsForbidden` is handled properly when syncing non-supergroup
+  info.
+* Fixed some bugs with file transfers when using SQLite.
+* Fixed error when attempting to log in again after logging out.
+* Fixed QR login not working.
+* Fixed error syncing chats if bridging a message had previously been
+  interrupted.
+
+# v0.12.0 (2022-08-26)
+
+**N.B.** This release requires a homeserver with Matrix v1.1 support, which
+bumps up the minimum homeserver versions to Synapse 1.54 and Dendrite 0.8.7.
+Minimum Conduit version remains at 0.4.0.
+
+### Added
+* Added provisioning API for resolving Telegram identifiers (like usernames).
+* Added support for bridging Telegram custom emojis to Matrix.
+* Added option to not bridge chats with lots of members.
+* Added option to include captions in the same message as the media to
+  implement [MSC2530]. Sending captions the same way is also supported and
+  enabled by default.
+* Added commands to kick or ban relaybot users from Telegram.
+* Added support for Telegram's disappearing messages.
+* Added support for bridging forwarded messages as forwards on Telegram.
+  * Forwarding is not allowed in relay mode as the bot wouldn't be able to
+    specify who sent the message.
+  * Matrix doesn't have real forwarding (there's no forwarding metadata), so
+    only messages bridged from Telegram can be forwarded.
+  * Double puppeted messages from Telegram currently can't be forwarded without
+    removing the `fi.mau.double_puppet_source` key from the content.
+  * If forwarding fails (e.g. due to it being blocked in the source chat), the
+    bridge will automatically fall back to sending it as a normal new message.
+* Added options to make encryption more secure.
+  * The `encryption` -> `verification_levels` config options can be used to
+    make the bridge require encrypted messages to come from cross-signed
+    devices, with trust-on-first-use validation of the cross-signing master
+    key.
+  * The `encryption` -> `require` option can be used to make the bridge ignore
+    any unencrypted messages.
+  * Key rotation settings can be configured with the `encryption` -> `rotation`
+    config.
+
+### Improved
+* Improved handling the bridge user leaving chats on Telegram, and new users
+  being added on Telegram.
+* Improved animated sticker conversion options: added support for animated webp
+  and added option to convert video stickers (webm) to the specified image
+  format.
+* Audio and video metadata is now bridged properly to Telegram.
+* Added database index on Telegram usernames (used when bridging username
+  @-mentions in messages).
+* Changed `/login/send_code` provisioning API to return a proper error when the
+  phone number is not registered on Telegram.
+  * The same login code can be used for registering an account, but registering
+    is not currently supported in the provisioning API.
+* Removed `plaintext_highlights` config option (the code using it was already
+  removed in v0.11.0).
+* Enabled appservice ephemeral events by default for new installations.
+  * Existing bridges can turn it on by enabling `ephemeral_events` and disabling
+    `sync_with_custom_puppets` in the config, then regenerating the registration
+    file.
+* Updated to API layer 144 so that Telegram would send new message types like
+  premium stickers to the bridge.
+* Updated Docker image to Alpine 3.16 and made it smaller.
+
+### Fixed
+* Fixed command prefix in game and poll messages (thanks to [@cynhr] in [#804]).
+
+[MSC2530]: https://github.com/matrix-org/matrix-spec-proposals/pull/2530
+[@cynhr]: https://github.com/cynhr
+[#804]: https://github.com/mautrix/telegram/pull/804
+
+# v0.11.3 (2022-04-17)
+
+**N.B.** This release drops support for old homeservers which don't support the
+new `/v3` API endpoints. Synapse 1.48+, Dendrite 0.6.5+ and Conduit 0.4.0+ are
+supported. Legacy `r0` API support can be temporarily re-enabled with `pip install mautrix==0.16.0`.
+However, this option will not be available in future releases.
+
+### Added
+* Added `list-invite-links` command to list invite links in a chat.
+* Added option to use [MSC2246] async media uploads.
+* Provisioning API for listing contacts and starting private chats.
+
+### Improved
+* Dropped Python 3.7 support.
+* Telegram->Matrix message formatter will now replace `t.me/c/chatid/messageid`
+  style links with a link to the bridged Matrix event (in addition to the
+  previously supported `t.me/username/messageid` links).
+* Updated formatting converter to keep newlines in code blocks as `\n` instead
+  of converting them to `<br/>`.
+* Removed `max_document_size` option. The bridge will now fetch the max size
+  automatically using the media repo config endpoint.
+* Removed redundant `msgtype` field in sticker events sent to Matrix.
+* Disabled file logging in Docker image by default.
+  * If you want to enable it, set the `filename` in the file log handler to a
+    path that is writable, then add `"file"` back to `logging.root.handlers`.
+* Reactions are now marked as read when bridging read receipts from Matrix.
+
+### Fixed
+* Fixed `!tg bridge` throwing error if the parameter is not an integer
+* Fixed `!tg bridge` failing if the command had been previously run with an
+  incorrectly prefixed chat ID (e.g. `!tg bridge -1234567` followed by
+  `!tg bridge -1001234567`).
+* Fixed `bridge_matrix_leave` config option not actually being used correctly.
+* Fixed public channel mentions always bridging into a user mention on Matrix
+  rather than a room mention.
+  * The bridge will now make room mentions if the portal exists and fall back
+    to user mentions otherwise.
+* Fixed newlines being lost in unformatted forwarded messages.
+
+[MSC2246]: https://github.com/matrix-org/matrix-spec-proposals/pull/2246
+
+# v0.11.2 (2022-02-14)
+
+**N.B.** This will be the last release to support Python 3.7. Future versions
+will require Python 3.8 or higher. In general, the mautrix bridges will only
+support the lowest Python version in the latest Debian or Ubuntu LTS.
+
+### Added
+* Added simple fallback message for live location and venue messages from Telegram.
+* Added support for `t.me/+code` style invite links in `!tg join`.
+* Added support for showing channel profile when users send messages as a channel.
+* Added "user joined Telegram" message when Telegram auto-creates a DM chat for
+  a new user.
+
+### Improved
+* Added option for adding a random prefix to relayed user displaynames to help
+  distinguish them on the Telegram side.
+* Improved syncing profile info to room info when using encryption and/or the
+  `private_chat_profile_meta` config option.
+* Removed legacy `community_id` config option.
+
+### Fixed
+* Fixed newlines disappearing when bridging channel messages with signatures.
+* Fixed login throwing an error if a previous login code expired.
+* Fixed bug in v0.11.0 that broke `!tg create`.
+
+# v0.11.1 (2022-01-10)
+
+### Added
+* Added support for message reactions.
+* Added support for spoiler text.
+
+### Improved
+* Support for voice messages.
+* Changed color of blue text from Telegram to be more readable on dark themes.
+
+### Fixed
+* Fixed syncing contacts throwing an error for new accounts.
+* Fixed migrating pre-v0.11 legacy databases if the database schema had been
+  corrupted (e.g. by using 3rd party tools for SQLite -> Postgres migration).
+* Fixed converting animated stickers to webm with >33 FPS.
+* Fixed a bug in v0.11.0 that broke mentioning users in groups
+  (thanks to [@dfuchss] in [#724]).
+
+[@dfuchss]: https://github.com/dfuchss
+[#724]: https://github.com/mautrix/telegram/pull/724
+
+# v0.11.0 (2021-12-28)
+
+* Switched from SQLAlchemy to asyncpg/aiosqlite.
+  * The default database is now Postgres. If using SQLite, make sure you install
+    the `sqlite` [optional dependency](https://docs.mau.fi/bridges/python/optional-dependencies.html).
+  * **Alembic is no longer used**, schema migrations happen automatically on startup.
+  * **The automatic database migration requires you to be on the latest legacy
+    database version.** If you were running any v0.10.x version, you should be on
+    the latest version already. Otherwise, update to v0.10.2 first, upgrade the
+    database with `alembic`, then upgrade to v0.11.0 (or higher).
+* Added support for contact messages.
+* Added support for Telegram sponsored messages in channels.
+  * Only applies to broadcast channels with 1000+ members
+    (as per <https://t.me/durov/172>).
+  * Only applies if you're using puppeting with a normal user account,
+    because bots can't get sponsored messages.
+* Fixed non-supergroup member sync incorrectly kicking one user from the Matrix
+  side if there was no limit on the number of members to sync (broke in v0.10.2).
+* Updated animated sticker conversion to support [lottieconverter r0.2]
+  (thanks to [@sot-tech] in [#694]).
+* Updated Docker image to Alpine 3.15.
+* Formatted all code using [black](https://github.com/psf/black)
+  and [isort](https://github.com/PyCQA/isort).
+
+[lottieconverter r0.2]: https://github.com/sot-tech/LottieConverter/releases/tag/r0.2
+[#694]: https://github.com/mautrix/telegram/pull/694
+
+# v0.10.2 (2021-11-13)
+
+### Added
+* Added extensions when bridging unnamed files from Telegram.
+* Added support for custom bridge bot welcome messages
+  (thanks to [@justinbot] in [#676]).
+
+### Improved
+* Improved handling authorization errors if the bridge was logged out remotely.
+* Updated room syncer to use existing power levels to find appropriate levels
+  for admins and normal users instead of hardcoding 50 and 0.
+* Updated to Telegram API layer 133 to handle 64-bit user/chat/channel IDs.
+* Stopped logging message contents when message handling failed
+  (thanks to [@justinbot] in [#681]).
+* Removed Element iOS compatibility hack from non-sticker files.
+* Made `max_initial_member_sync` work for non-supergroups too
+  (thanks to [@tadzik] in [#680]).
+* SQLite is now supported for the crypto database. Pickle is no longer supported.
+  If you were using pickle, the bridge will create a new e2ee session and store
+  the data in SQLite this time.
+
+### Fixed
+* Fixed generating reply fallbacks to encrypted messages.
+* Fixed chat sync failing if the member list contained banned users.
+
+[@justinbot]: https://github.com/justinbot
+[@tadzik]: https://github.com/tadzik
+[#676]: https://github.com/mautrix/telegram/pull/676
+[#680]: https://github.com/mautrix/telegram/pull/680
+[#681]: https://github.com/mautrix/telegram/pull/681
+
+# v0.10.1 (2021-08-19)
+
+**N.B.** Docker images have moved from `dock.mau.dev/tulir/mautrix-telegram`
+to `dock.mau.dev/mautrix/telegram`. New versions are only available at the new
+path.
+
+### Added
+
+* Warning when bridging existing room if bridge bot doesn't have redaction
+  permissions.
+* Custom flag to invite events that will be auto-accepted using double puppeting.
+* Custom flags for animated stickers (same as what gifs already had).
+
+### Improved
+* Updated to Telethon 1.22.
+* Updated Docker image to Alpine 3.14.
+
+### Fixed
+* Fixed Bridging Matrix location messages with additional flags in `geo_uri`.
+* Editing encrypted messages will no longer add an asterisk on Telegram.
+* Matrix typing notifications won't be echoed back for double puppeted users anymore.
+* `AuthKeyDuplicatedError` is now handled properly instead of making the user
+  get stuck.
+* Fixed `public_portals` setting not being respected on room creation.
+
+# v0.10.0 (2021-06-14)
+
+* Added options to bridge archive, pin and mute status from Telegram to Matrix.
+* Added custom fields in Matrix events indicating Telegram gifs.
+* Allowed zero-width joiners in displaynames so things like multi-part emoji
+  would work correctly.
+* Fixed Telegram->Matrix typing notifications.
+
+## rc1 (2021-04-05)
+
+### Added
+* Support for multiple pins from/to Telegram.
+* Option to resolve redirects when joining invite links, for people who use
+  custom URLs as invite links.
+* Command to update about section in Telegram profile info
+  (thanks to [@MadhuranS] in [#599]).
+* Own read marker/unread status from Telegram is now synced to Matrix after backfilling.
+* Support for showing the individual slots in 🎰 dice rolls from Telegram.
+
+### Improved
+* Improved invite link regex to allow joining with less precise invite links.
+* Invite links can be customized with the `--uses=<amount>` and
+  `--expire=<delta>` flags for `!tg invite-link`.
+* Read receipts where the target message is unknown will now cause the chat to
+  be marked as fully read instead of the read receipt event being ignored.
+* WebP stickers are now sent as-is without converting to png.
+* Default power levels in rooms now allow enabling encryption with PL 50 if
+  e2be is enabled in config (thanks to [@Rafaeltheraven] in [#550]).
+* Updated Docker image to Alpine 3.13 and removed all edge repo stuff.
+
+### Fixed
+* Matrix->Telegram location message bridging no longer flips the coordinates.
+* Fixed some user displaynames constantly changing between contact/non-contact
+  names and other similar cases.
+
+[@Rafaeltheraven]: https://github.com/Rafaeltheraven
+[@MadhuranS]: https://github.com/MadhuranS
+[#550]: https://github.com/mautrix/telegram/pull/550
+[#599]: https://github.com/mautrix/telegram/pull/599
+
+# v0.9.0 (2020-11-17)
+
+* Fixed cleaning unidentified rooms.
+
+## rc3 (2020-11-12)
+
+### Added
+* Added retrying message sending if server returns 502.
+
+### Fixed
+* Fixed Matrix → Telegram name mentions.
+* Fixed some bugs with replies.
+
+## rc2 (2020-11-06)
+
+### Improved
+* Ephemeral event handling should be faster by not checking the database for
+  user existence.
+* Using the register command now sends a link to the Telegram terms of service.
+* The `bridge_connected` metric is now only set for users who are logged in.
+
+### Fixed
+* Fixed bug where syncing members sometimes kicked ghosts of users who were
+  actually still in the chat.
+* Fixed sending captions to Telegram with `!tg caption` (broken in rc1).
+* Logging out will now delete private chat portals, instead of only kicking the
+  user and leaving the portal in a broken state.
+* Unbridging direct chat portals is now possible.
+
+## rc1 (2020-10-24)
+
+### Breaking changes
+* Prometheus metric names are now prefixed with `bridge_`.
+* An entrypoint script is no longer automatically generated. This won't affect
+  most users, as `python -m mautrix_telegram` has been the official way to start
+  the bridge for a long time.
+
+### Added
+* Support for logging in by scanning a QR code from another Telegram client.
+* Automatic backfilling of old messages when creating portals.
+* Automatic backfilling of missed messages when starting bridge.
+* Option to update `m.direct` list when using double puppeting.
+* PNG thumbnails for animated stickers when converted to webm.
+* Support for receiving ephemeral events pushed directly with [MSC2409]
+  (requires Synapse 1.22 or higher).
+
+### Improved
+* Switched end-to-bridge encryption to mautrix-python instead of a hacky
+  matrix-nio solution.
+* End-to-bridge encryption no longer requires `login_shared_secret`, it uses
+  [MSC2778] instead (requires Synapse 1.21 or higher).
+* The bridge info state event is now updated whenever the chat name or avatar changes.
+* Double puppeting is no longer limited to users on the same homeserver as the bridge.
+* Delivery receipts are no longer sent in unencrypted private chat portals, as
+  the bridge bot is usually not present in them.
+
+### Fixed
+* File captions are now sent as a separate message like photo captions.
+* The relaybot no longer drops Telegram messages with commands.
+* Bridging events of a user whose power level is malformed (i.e. a string
+  instead of an integer) now works.
+
+[MSC2409]: https://github.com/matrix-org/matrix-spec-proposals/pull/2409
+[MSC2778]: https://github.com/matrix-org/matrix-spec-proposals/pull/2778
+
+# v0.8.2 (2020-07-27)
+
+* Fixed deleting messages from Matrix.
+* Fixed Alpine edge dependencies in Docker image.
+
+Note: this release is not on PyPI, as the only changes were a mautrix-python
+update (v0.5.8) and a fix to the Docker image.
+
+# v0.8.1 (2020-06-08)
+
+* Fixed starting bridge for the first time failing due to not registering the bridge bot.
+* Updated Docker image to Alpine 3.12.
+
+# v0.8.0 (2020-06-03)
+
+* Updated to mautrix-python 0.5.0 and matrix-nio 0.12.0.
+
+## rc5 (2020-05-30)
+
+* Added option to disable removing avatars from Telegram ghosts.
+* Added option to send delivery error notices.
+* Added option to send delivery receipts.
+* Bumped maximum Telethon version to 1.14.
+* Possibly fixed infinite loop of avatar changes when using double puppeting.
+
+## rc3 (2020-05-22)
+
+* Moved private information to trace log level.
+* Added `private_chat_portal_meta` option. This is implicitly enabled when
+  encryption is enabled, it was only added as an option for instances with
+  encryption disabled.
+* Removed avatars are now synced properly from Telegram, instead of leaving the
+  last known avatar forever.
+* Fixed admin detection on Telegram-side relaybot commands
+  (thanks to [@davidmehren] in [#468]).
+* Fixed bug handling `ChatForbidden` when syncing chats.
+
+[@davidmehren]: https://github.com/davidmehren
+[#468]: https://github.com/mautrix/telegram/pull/468
+
+## rc2 (2020-05-20)
+
+* Implemented [MSC2346]: Bridge information state event for newly created rooms.
+* Fixed `sync_direct_chats` option creating non-working portals.
+* Fixed video thumbnailing sometimes leaving behind downloaded videos in `/tmp`.
+
+[MSC2346]: https://github.com/matrix-org/matrix-spec-proposals/pull/2346
+
+## rc1 (2020-04-25)
+
+### Added
+* Command for backfilling room history from Telegram.
+* arm64 support in docker images.
+* Optional end-to-bridge encryption support.
+  See [docs](https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html) for more info.
+* Bridging for Telegram dice roll messages.
+
+### Fixed
+* Riot iOS not showing stickers properly.
+* Updated to Telethon 1.13 to fix bugs like [#443].
+
+[#443]: https://github.com/mautrix/telegram/issues/443
+
+# v0.7.2 (2020-04-04)
+
+* No changes since rc1.
+
+## rc1 (2020-02-08)
+
+* Fixed enabling double puppeting causing saved messages to become unusable.
+* Fixed receiving channel messages when `ignore_own_incoming_events` was enabled.
+
+# v0.7.1 (2020-02-04)
+
+* Fixed missing responses in logout provisioning API.
+
+## rc2 (2020-01-25)
+
+* Fixed import in database migration script (thanks to [@cubesky] in [#409]).
+* Fixed relaybot messages being allowed through to Matrix even when
+  `ignore_own_incoming_events` was set to `true`.
+
+[@cubesky]: https://github.com/cubesky
+[#409]: https://github.com/mautrix/telegram/pull/409
+
+## rc1 (2020-01-11)
+
+* Fixed incorrect parameter name causing `!tg config set` to throw an error.
+* Fixed potential dictionary size changed during iteration crash.
+
+# v0.7.0 (2019-12-28)
+
+* No changes since rc4.
+
+## rc4 (2019-12-25)
+
+* Fixed handling of Matrix `m.emote` events.
+
+## rc3 (2019-12-25)
+
+* Added option to log in to custom puppet with shared secret
+  (<https://github.com/devture/matrix-synapse-shared-secret-auth>).
+* Updated Docker image to Alpine 3.11.
+* Improved displayname syncing by trusting any displayname if user is not a contact.
+* Fixed error when cleaning up rooms.
+* Fixed stack traces being printed to non-admin users.
+* Fixed invite rejections being handles as leaves.
+* Fixed `version` command output in CI docker builds not showing the correct
+  git commit hash.
+
+## rc2 (2019-12-01)
+
+* Added command to get bridge version.
+* Made bridge refuse to start if config contains example values.
+* Removed some debug stack traces.
+* Ignored `ChatForbidden` when syncing chats that was causing the sync to fail.
+* Fixed DB migration causing some incorrect values to be left behind.
+
+## rc1 (2019-11-30)
+
+### Important changes
+* Dropped Python 3.5 compatibility.
+* Moved docker registry to [dock.mau.dev](https://mau.dev/tulir/mautrix-telegram/container_registry).
+
+### Added
+* Support for bridging animated stickers (thanks to [@sot-tech] in [#366]).
+  * Requires [LottieConverter](https://github.com/sot-tech/LottieConverter),
+    which is included in the docker image.
+  * Can be [configured](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L174-L187).
+* Support for MTProxy (thanks to [@sot-tech] in [#344]).
+* [Config option](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L117-L118)
+  for max length of displayname, with the default being 100.
+* Separate [config option](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L232-L238)
+  for `m.emote` formatting of logged in users.
+* Streamed file transfers and parallel telegram file download/upload.
+  * Files are streamed from telegram servers to the media repo rather than
+    downloading the whole file into memory.
+  * File transfers use multiple connections to telegram servers to transfer faster.
+  * Parallel and streamed file transfers can be enabled in the
+    [config](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L166-L169).
+* Command to set caption for files and images when sending to telegram.
+* Bridging bans to telegram.
+* Helm chart.
+
+### Improved
+* Switched from mautrix-appservice-python to [mautrix-python](https://github.com/mautrix/python).
+* Users with Matrix puppeting can now bridge their "Saved Messages" chat.
+* The bridge will refuse to start without access to the example config file.
+* Changed default port to 29317.
+* Mentions are now marked as read on Telegram when bridging read receipts using
+  double puppeting.
+* Kicking or banning the bridge bot will now unbridge the room.
+* Shrinked Docker image from 151mb to 77mb.
+
+### Fixed
+* The bridge will no longer crash if one user's startup fails.
+* (hopefully) Incorrect peer type being saved into database in some cases.
+* File names when bridging to Telegram.
+* Alembic config interpolating passwords with `%`.
+* A single chat failing to sync preventing any chat from syncing.
+* Users logged in as a bot not receiving any messages.
+* Username matching being case-sensitive in the database and preventing
+  telegram->matrix pillifying.
+* IndexError if running `!tg set-pl` with no parameters.
+
+[@sot-tech]: https://github.com/sot-tech
+[#344]: https://github.com/mautrix/telegram/pull/344
+[#366]: https://github.com/mautrix/telegram/pull/366
+
+# v0.6.0 (2019-07-09)
+
+* Fixed vulnerability in event handling.
+
+## rc2 (2019-07-06)
+
+* Nested formatting is now supported by Telegram, so the bridge also supports it.
+* Strikethrough and underline are now bridged into native Telegram formatting
+  rather than unicode hacks.
+* Fixed displayname not updating for users who the bridge only saw via a logged
+  in user who had the problematic user in their contacts.
+* Fixed handling unsupported media.
+* Added handling for `FileIdInvalidError` in file transfers that could disrupt
+  `sync`s.
+
+## rc1 (2019-06-22)
+
+### Added
+* Native Matrix edit support and new fallback format.
+* Config options for `retry_delay` and other TelegramClient constructor fields.
+* Config option for maximum document size to let through the bridge.
+* External URL field for chat and private channel messages.
+* Telegram user info (puppet displayname & avatar) is now updated every time
+  the user sends a message.
+* Command to change Telegram displayname.
+* Possibility to override config fields with environment variables
+  (thanks to [@pacien] in [#332]).
+
+### Improved
+* Simplified bridged poll message.
+* Telegram user info updates are now accepted from any logged in user as long
+  as the logged in user doesn't see a phone number for the Telegram user.
+* Some image errors are now handled by resending the image as a document.
+* Made getting started more user-friendly.
+* Updated to Telethon 1.8.
+
+### Fixed
+* Portal peer type not being saved in database after Telegram chat upgrade.
+* Newlines in unformatted messages not being bridged when using relaybot.
+* Mime type info field for stickers converted to PNG.
+* Content after newlines being stripped in messages sent by some clients.
+* Potential `NoneType is not iterable` exception when logging out
+  (thanks to [@turt2live] in [#315]).
+* Handling of Matrix messages where `m.relates_to` is null.
+* Internal server error when logging in with an account on another DC.
+* Spaces between command and arguments are now trimmed.
+* Changed migrations to use `batch_alter_table` for adding columns to have less
+  warnings with SQLite.
+* Error when `ping`ing without being logged in.
+* Terminating sessions with negative hashes.
+* State cache not being updated when sending events, causing invalid cache if
+  the server doesn't echo the sent events.
+
+[@pacien]: https://github.com/pacien
+[#315]: https://github.com/mautrix/telegram/pull/315
+[#332]: https://github.com/mautrix/telegram/pull/332
+
+# v0.5.2 (2019-05-25)
+
+* Fixed null `m.relates_to`'s that break Synapse 0.99.5.
+
+# v0.5.1 (2019-03-21)
+
+* Fixed Python 3.5 compatibility.
+* Fixed DBMS migration script.
+
+# v0.5.0 (2019-03-19)
+
+* Replaced rawgit with cdnjs in public website as rawgit is deprecated.
+* Fixed login command throwing error when web login is enabled.
+* Updated telethon-session-sqlalchemy to fix logging into an account on another DC.
+* Stopped adding reply fallback to caption when sending caption and image as
+  separate messages.
+
+## rc4 (2019-03-16)
+
+* Added verbose flag to migration script.
+* Added pytest setup and some tests (thanks to [@V02460] in [#290]).
+* Fixed scripts (DBMS migration and Telematrix import) not being included in builds.
+* Fixed some database problems.
+* Removed remaining traces of ORM that might have been the causes of some other
+  database problems.
+* Removed option to use lxml in HTML parsing as it was messing up emoji offset
+  handling. The new HTML parser supports using the default python HTMLParser
+  class since 0.5.0rc1, so lxml wasn't really useful anway.
+
+[#290]: https://github.com/mautrix/telegram/pull/290
+
+## rc3 (2019-02-16)
+
+* Fixed bridging documents without thumbnails to Matrix.
+* Added option to set maximum size of image to send to Telegram. Images above
+  the size limit will be sent as documents without the compression Telegram
+  applies to images.
+* Fixed saving user portals and contacts.
+* Added Telegram -> Matrix poll bridging and a command to vote in polls.
+
+## rc2 (2019-02-15)
+
+* Added missing future-fstrings comments that caused the bridge to not start on
+  Python 3.5.
+* Fixed handling of document thumbnails.
+* Fixed private chat portals failing to be created.
+* Made relaybot handle Telegram chat upgrade events.
+
+## rc1 (2019-02-14)
+
+### Added
+* More config options
+  * Option to to use Telegram test servers.
+  * Option to disable link previews on Telegram.
+  * Option to disable startup sync.
+  * Option to skip deleted members when syncing member lists.
+  * Option to change number of dialogs to handle in startup sync.
+* More commands
+  * `username` for setting Telegram username.
+  * `sync-state` for updating Matrix room state cache.
+  * `matrix-ping` for checking Matrix login status (thanks to [@krombel] in [#271]).
+  * `clear-db-cache` for clearing internal database caches.
+  * `reload-user` for reloading and reconnecting a Telegram user.
+  * `session` for listing and terminating other Telegram sessions.
+  * Added argument to `login` to allow admins to log in for other users.
+* Added warning when logging in that it grants the bridge full access to the
+  telegram account.
+* Telegram->Matrix bridging:
+  * Telegram games
+  * Message pins in normal groups
+  * Custom message for unsupported media like polls
+* Added client ID in logs when making requests to telegram.
+* Added handling for Matrix room upgrades.
+
+### Improved
+* Removed lxml dependency from the new HTML parser and removed the old parser
+  completely.
+* Switched mautrix-appservice-python state store and most mautrix-telegram
+  tables to SQLAlchemy core. This should speed things up and reduce problems
+  with the ORM getting stuck.
+* `ensure_started` is now only called for logged in users, which should improve
+  performance for large instances.
+* Displayname template extras (e.g. the `(Telegram)` suffix) are now stripped
+  when mentioning Telegram users with no username.
+* Updated Telethon.
+* Switched Dockerfile to use setup.py for dependencies to avoid dependency
+  updates breaking stuff.
+* The telematrix import script will now warn about and skip over duplicate portals.
+* Relaybot will now be used for users who have logged in, but are not in the chat.
+
+### Fixed
+* Bug where stickers with an unidentified emoji failed to bridge.
+* Invalid letter prefixes in clean-rooms output.
+* Messages forwarded from channels showing up as "Unknown source".
+* Matrix->Telegram room avatar bridging.
+
+[@krombel]: https://github.com/krombel
+[#271]: https://github.com/mautrix/telegram/pull/271
+
+# v0.4.0 (2018-11-28)
+
+* No changes since rc2.
+
+## rc2 (2018-11-15)
+
+* Fixed kicking Telegram puppets from Matrix.
+
+## rc1 (2018-11-15)
+
+### Added
+* Flag to indicate if user can unbridge portal in provisioning API
+  (thanks to [@turt2live] in [#225]).
+* Option to send captions as second message (replaces option to send caption
+  in `body`.
+* Room-specific settings.
+
+### Improved
+* (internal) Added type hints everywhere (mostly thanks to [@V02460] in [#206]).
+* Telegram->Matrix formatter now uses `<pre>` tags for multiline code even if
+  said code was in the telegram equivalent of inline code tags.
+* Better bullets and linebreak handling in Matrix->Telegram formatter.
+* Logging in will now show your phone number instead of `@None` if you don't
+  have a username.
+* Significantly improved performance on high-load instances (t2bot.io) by
+  moving most used database tables to SQLAlchemy Core.
+
+### Fixed
+* Bugs that caused database migrations to fail in some cases.
+* Editing the config (e.g. whitelisting chats) corrupting the config.
+* Negative numbers (chat IDs) in `/connect` of the provisioning API
+  (thanks to [@turt2live] in [#223]).
+* Relaybot creating portals automatically when receiving message.
+* Not being able to use a bridge bot localpart that would also match the puppet
+  localpart format.
+* Matrix login sync failing completely if the homeserver stopped during a sync
+  response.
+* Errors when cleaning rooms.
+* Bridging code blocks without a language.
+* Error and lost messages when trying to bridge PM from new users in some cases.
+* Logging in with an account that someone has already logged in failing
+  silently and then breaking the bridge.
+* Relaybot message when adding/removing Matrix displaynames.
+
+[@V02460]: https://github.com/V02460
+[#206]: https://github.com/mautrix/telegram/pull/206
+[#223]: https://github.com/mautrix/telegram/pull/223
+[#225]: https://github.com/mautrix/telegram/pull/225
+
+# v0.3.0 (2018-08-15)
+
+* Added database URI format examples.
+* Bumped maximum Telethon version to 1.2, possibly fixing the catch_up option.
+
+## rc3 (2018-08-08)
+
+* Improved Telegram message deduplication options.
+  * Added pre-send message database check for deduplication.
+  * Made dedup cache queue length configurable.
+
+## rc2 (2018-08-06)
+
+* Added option to change max body size for AS API.
+* Fixed a minor error regarding power level changes (thanks to [@turt2live] in [#203]).
+* Updated minimum mautrix-appservice version to include some recent bugfixes.
+
+[@turt2live]: https://github.com/turt2live
+[#203]: https://github.com/mautrix/telegram/pull/203
+
+## rc1 (2018-08-05)
+
+### Added
+* Logging in with a bot
+  (see [docs](https://docs.mau.fi/bridges/python/telegram/authentication.html#bot-token) for usage).
+  * You can log in with a personal Telegram bot to appear almost like a real
+    user without logging in with a real Telegram account.
+* Replacing your Telegram account's Matrix puppet with your Matrix account
+  (see [docs](https://docs.mau.fi/bridges/general/double-puppeting.html) for usage).
+* Formatting options for relaybot messages.
+  * Real displaynames are now supported and enabled by default.
+  * State events (join/leave/name change) can be independently disabled by
+    setting the format to a blank string.
+* New config sections
+  * Proper log config, including logging to file (by default)
+  * Proxy support (requires installing PySocks)
+  * Separate field for appservice address for homeserver
+    (useful if using a reverse proxy).
+* New permission levels to allow initiating bridges without allowing puppeting
+  and to allow Telegram puppeting without allowing Matrix puppeting.
+* Telematrix import script (see [docs](https://docs.mau.fi/bridges/python/telegram/migrating-from-telematrix.html) for usage).
+* Provisioning API (see [docs](https://docs.mau.fi/bridges/python/telegram/provisioning-api.html) for more info).
+* DBMS migration script (see [docs](https://docs.mau.fi/bridges/python/telegram/dbms-migration.html) for usage).
+
+### Improved
+* Tabs are now replaced with 4 spaces so that Telegram servers wouldn't change
+  the message.
+* Help page now detects your permissions and only shows commands you can use.
+* Moved Matrix state cache to the main database. This means that the
+  `mx-state.json` file is no longer needed and all non-config data is
+  stored in the main database.
+* Better lxml-based HTML parser for Matrix->Telegram formatting bridging.
+  lxml is still optional, so the old parser is used as fallback if lxml is not
+  installed.
+* Disabled Telegram->Matrix bridging of messages sent by the relaybot.
+  Can be re-enabled in config if necessary.
+
+### Fixed
+* A `ValueError` in some cases when syncing power levels.
+* Telegram connections being created for unauthenticated users possibly
+  triggering spam protection connection delays in the Telegram servers.
+* Logging out if a portal had been deleted/unbridged.
+
+# v0.2.0 (2018-06-08)
+
+* No changes since rc6.
+
+## rc6 (2018-06-06)
+
+* Added warning about `delete-portal` kicking all room members.
+* Fixed error when upgrading/creating SQLite database.
+
+## rc5 (2018-06-01)
+
+* Fixed relaybot automatically creating portal rooms when invited to Telegram chat ([#145]).
+* Fixed kicking Telegram puppets and fix error message when bridging chats you've left.
+* Fixed integrity error deleting portals from database.
+
+[#145]: https://github.com/mautrix/telegram/issues/145
+
+## rc4 (2018-05-29)
+
+* ~~Fixed~~ Added Postgres compatibility.
+* Fixed manual bridging (`!tg bridge`) for unauthenticated users.
+* Fixed inviting unauthenticated Matrix users from Telegram (via `/invite <mxid>`).
+* Changed Alembic to read database path from the config, so editing `alembic.ini`
+  is no longer necessary. Use `alembic -x config=/path/to/config.yaml ...` to
+  specify the config path.
+
+## rc3 (2018-05-25)
+
+* Reworked Dockerfile to remove virtualenv and use Alpine packages (thanks to
+  [@jcgruenhage] in [#142]). This fixes webp->png conversion for stickers.
+
+[#142]: https://github.com/mautrix/telegram/pull/142
+
+## rc2 (2018-05-21)
+
+* Added Dockerfile (thanks to [@jcgruenhage] in [#136]).
+
+[#136]: https://github.com/mautrix/telegram/pull/136
+[@jcgruenhage]: https://github.com/jcgruenhage
+
+## rc1 (2018-05-19)
+
+* Added
+  * Option to exclude telegram chats from being bridged.
+  * Support for using a relay bot to relay messages for unauthenticated users
+  * Bridging for message pinning and room mentions/pills.
+  * Matrix->Telegram sticker bridging.
+  * `!command` to `/command` conversion at the start of Matrix message text.
+  * Conversion of t.me message links to matrix.to message links
+  * Timestamp massaging (bridge Telegram timestamps to Matrix)
+  * Support for out-of-Matrix login (useful if you don't want your 2FA password to be stored in the homeserver)
+  * Optional HQ gif/video thumbnails using moviepy.
+  * Option to send bot messages as `m.notice`
+* Improved deduplication
+  * Matrix file uploads are now reused if the same Telegram file (e.g. a sticker) is sent multiple times
+  * Room metadata changes and other non-message actions are now deduplicated
+* Improved formatting bridging
+* Improved Telegram user display name handling in cases where one or more users have set custom display names for other users.
+* Fixed Alembic setup and removed automatic database generation.
+* Fixed outgoing message deduplication in cases where message is sent to other clients before responding to the sender.
+* Moved mautrix-appservice-python to separate repository.
+* Switched to telethon-session-sqlalchemy to have the session databases in the main database.
+* Switched license from GPLv3 to AGPLv3
+* Probably a bunch of other stuff I forgot
+
+# v0.1.1 (2018-02-18)
+
+Fixed bridging formatted messages from Matrix to Telegram
+
+# v0.1.0 (2018-02-17)
+
+First release.
+
+Things work.

Dockerfile

@@ -1,38 +1,34 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.13
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.16
 
 ARG TARGETARCH=amd64
 
-#RUN echo $'\
-#@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\n\
-#@edge http://dl-cdn.alpinelinux.org/alpine/edge/testing\n\
-#@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories
-
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
-      py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-magic \
-      py3-sqlalchemy \
-      py3-telethon-session-sqlalchemy \
-      py3-alembic \
-      py3-psycopg2 \
       py3-ruamel.yaml \
       py3-commonmark \
+      py3-phonenumbers \
+      py3-mako \
+      #py3-prometheus-client \ (pulls in twisted unnecessarily)
       # Indirect dependencies
       py3-idna \
+      py3-rsa \
       #moviepy
         py3-decorator \
         py3-tqdm \
         py3-requests \
+        #py3-proglog \
         #imageio
           py3-numpy \
-      #py3-telethon@edge \ (outdated)
+      #py3-telethon \ (outdated)
         # Optional for socks proxies
         py3-pysocks \
+        py3-pyaes \
         # cryptg
           py3-cffi \
-	  py3-qrcode \
+          py3-qrcode \
       py3-brotli \
       # Other dependencies
       ffmpeg \
@@ -40,7 +36,7 @@ RUN apk add --no-cache \
       su-exec \
       netcat-openbsd \
       # encryption
-      olm-dev \
+      py3-olm \
       py3-pycryptodome \
       py3-unpaddedbase64 \
       py3-future \
@@ -52,18 +48,16 @@ RUN apk add --no-cache \
 COPY requirements.txt /opt/mautrix-telegram/requirements.txt
 COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
 WORKDIR /opt/mautrix-telegram
-RUN apk add --virtual .build-deps \
-      python3-dev \
-      libffi-dev \
-      build-base \
- && sed -Ei 's/psycopg2-binary.+//' optional-requirements.txt \
- && pip3 install -r requirements.txt -r optional-requirements.txt \
- && apk del .build-deps
+RUN apk add --virtual .build-deps python3-dev libffi-dev build-base \
+ && pip3 install /cryptg-*.whl \
+ && pip3 install --no-cache-dir -r requirements.txt -r optional-requirements.txt \
+ && apk del .build-deps \
+ && rm -f /cryptg-*.whl
 
 COPY . /opt/mautrix-telegram
-RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics,e2be] && apk del git \
+RUN apk add git && pip3 install --no-cache-dir .[all] && apk del git \
   # This doesn't make the image smaller, but it's needed so that the `version` command works properly
-  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
+  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram .git build
 
 VOLUME /data
 ENV UID=1337 GID=1337 \

MANIFEST.in

@@ -1,4 +1,5 @@
 include README.md
+include CHANGELOG.md
 include LICENSE
 include requirements.txt
 include optional-requirements.txt

README.md

@@ -1,9 +1,10 @@
 # mautrix-telegram
-![Languages](https://img.shields.io/github/languages/top/tulir/mautrix-telegram.svg)
-[![License](https://img.shields.io/github/license/tulir/mautrix-telegram.svg)](LICENSE)
-[![Release](https://img.shields.io/github/release/tulir/mautrix-telegram/all.svg)](https://github.com/tulir/mautrix-telegram/releases)
-[![GitLab CI](https://mau.dev/tulir/mautrix-telegram/badges/master/pipeline.svg)](https://mau.dev/tulir/mautrix-telegram/container_registry)
-[![Maintainability](https://img.shields.io/codeclimate/maintainability/tulir/mautrix-telegram.svg)](https://codeclimate.com/github/tulir/mautrix-telegram)
+![Languages](https://img.shields.io/github/languages/top/mautrix/telegram.svg)
+[![License](https://img.shields.io/github/license/mautrix/telegram.svg)](LICENSE)
+[![Release](https://img.shields.io/github/release/mautrix/telegram/all.svg)](https://github.com/mautrix/telegram/releases)
+[![GitLab CI](https://mau.dev/mautrix/telegram/badges/master/pipeline.svg)](https://mau.dev/mautrix/telegram/container_registry)
+[![Code style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Imports](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
 
 A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
@@ -15,14 +16,14 @@ All setup and usage instructions are located on
 [docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
 Some quick links:
 
-* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
-  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
+* [Bridge setup](https://docs.mau.fi/bridges/python/setup.html?bridge=telegram)
+  (or [with Docker](https://docs.mau.fi/bridges/general/docker-setup.html?bridge=telegram))
 * Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
   [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
   [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)
 
 ### Features & Roadmap
-[ROADMAP.md](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
+[ROADMAP.md](https://github.com/mautrix/telegram/blob/master/ROADMAP.md)
 contains a general overview of what is supported by the bridge.
 
 ## Discussion

ROADMAP.md

@@ -3,6 +3,7 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message reactions
   * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
@@ -12,8 +13,8 @@
   * [x] Power level
     * [x] Normal chats
       * [ ] Non-hardcoded PL requirements
-	* [x] Supergroups/channels
-	  * [ ] Precise bridging (non-hardcoded PL requirements, bridge specific permissions, etc..)
+    * [x] Supergroups/channels
+      * [ ] Precise bridging (non-hardcoded PL requirements, bridge specific permissions, etc..)
   * [x] Membership actions (invite/kick/join/leave)
   * [x] Room metadata changes (name, topic, avatar)
   * [x] Initial room metadata
@@ -23,10 +24,12 @@
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
   * [ ] Advanced message content/media
+    * [x] Custom emojis
     * [x] Polls
 	* [x] Games
 	* [ ] Buttons
   * [x] Message deletions
+  * [x] Message reactions
   * [x] Message edits
   * [x] Message history
     * [x] Manually (`!tg backfill`)
@@ -52,12 +55,12 @@
   * [x] Automatic portal creation
     * [x] At startup
     * [x] When receiving invite or message
-  * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
+  * [x] Portal creation by inviting Matrix puppet of Telegram user to new room
   * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
   * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
   * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
-  * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
+  * [x] End-to-bridge encryption in Matrix rooms (see [docs](https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html))
 
-† Information not automatically sent from source, i.e. implementation may not be possible
+† Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic.ini

@@ -1,71 +0,0 @@
-# A generic, single database configuration.
-
-[alembic]
-# path to migration scripts
-script_location = alembic
-
-# template used to generate migration files
-# file_template = %%(rev)s_%%(slug)s
-
-# timezone to use when rendering the date
-# within the migration file as well as the filename.
-# string value is passed to dateutil.tz.gettz()
-# leave blank for localtime
-# timezone =
-
-# max length of characters to apply to the
-# "slug" field
-#truncate_slug_length = 40
-
-# set to 'true' to run the environment during
-# the 'revision' command, regardless of autogenerate
-# revision_environment = false
-
-# set to 'true' to allow .pyc and .pyo files without
-# a source .py file to be detected as revisions in the
-# versions/ directory
-# sourceless = false
-
-# version location specification; this defaults
-# to alembic/versions.  When using multiple version
-# directories, initial revisions must be specified with --version-path
-# version_locations = %(here)s/bar %(here)s/bat alembic/versions
-
-# the output encoding used when revision files
-# are written from script.py.mako
-# output_encoding = utf-8
-
-# Logging configuration
-[loggers]
-keys = root,sqlalchemy,alembic
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = WARN
-handlers = console
-qualname =
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-
-[logger_alembic]
-level = INFO
-handlers =
-qualname = alembic
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = NOTSET
-formatter = generic
-
-[formatter_generic]
-format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S

alembic/README

@@ -1 +0,0 @@
-Generic single-database configuration.

alembic/env.py

@@ -1,90 +0,0 @@
-from alembic import context
-from sqlalchemy import engine_from_config, pool
-from logging.config import fileConfig
-
-import sys
-from os.path import abspath, dirname
-
-sys.path.insert(0, dirname(dirname(abspath(__file__))))
-
-from mautrix.util.db import Base
-import mautrix_telegram.db
-from mautrix_telegram.config import Config
-from alchemysession import AlchemySessionContainer
-
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
-mxtg_config = Config(mxtg_config_path, None, None)
-mxtg_config.load()
-config.set_main_option("sqlalchemy.url", mxtg_config["appservice.database"].replace("%", "%%"))
-
-AlchemySessionContainer.create_table_classes(None, "telethon_", Base)
-
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-fileConfig(config.config_file_name)
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-target_metadata = Base.metadata
-
-
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-
-def run_migrations_offline():
-    """Run migrations in 'offline' mode.
-
-    This configures the context with just a URL
-    and not an Engine, though an Engine is acceptable
-    here as well.  By skipping the Engine creation
-    we don't even need a DBAPI to be available.
-
-    Calls to context.execute() here emit the given string to the
-    script output.
-
-    """
-    url = config.get_main_option("sqlalchemy.url")
-    context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True,
-        render_as_batch=True)
-
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-def run_migrations_online():
-    """Run migrations in 'online' mode.
-
-    In this scenario we need to create an Engine
-    and associate a connection with the context.
-
-    """
-    connectable = engine_from_config(
-        config.get_section(config.config_ini_section),
-        prefix='sqlalchemy.',
-        poolclass=pool.NullPool)
-
-    with connectable.connect() as connection:
-        context.configure(
-            connection=connection,
-            target_metadata=target_metadata,
-            render_as_batch=True
-        )
-
-        with context.begin_transaction():
-            context.run_migrations()
-
-
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()

alembic/script.py.mako

@@ -1,24 +0,0 @@
-"""${message}
-
-Revision ID: ${up_revision}
-Revises: ${down_revision | comma,n}
-Create Date: ${create_date}
-
-"""
-from alembic import op
-import sqlalchemy as sa
-${imports if imports else ""}
-
-# revision identifiers, used by Alembic.
-revision = ${repr(up_revision)}
-down_revision = ${repr(down_revision)}
-branch_labels = ${repr(branch_labels)}
-depends_on = ${repr(depends_on)}
-
-
-def upgrade():
-    ${upgrades if upgrades else "pass"}
-
-
-def downgrade():
-    ${downgrades if downgrades else "pass"}

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -1,27 +0,0 @@
-"""Add disable_updates field for puppets
-
-Revision ID: 17574c57f3f8
-Revises: a9119be92164
-Create Date: 2019-05-15 00:24:46.967529
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '17574c57f3f8'
-down_revision = 'a9119be92164'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
-                                      server_default=sa.sql.expression.false()))
-
-
-def downgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column("disable_updates")

alembic/versions/1b241f7e8530_add_telegramfile_table.py

@@ -1,28 +0,0 @@
-"""Add TelegramFile table
-
-Revision ID: 1b241f7e8530
-Revises: 97d2a942bcf8
-Create Date: 2018-02-19 23:52:06.605741
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '1b241f7e8530'
-down_revision = '97d2a942bcf8'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table('telegram_file',
-                    sa.Column('id', sa.String(), nullable=False),
-                    sa.Column('mxc', sa.String(), nullable=True),
-                    sa.Column('mime_type', sa.String(), nullable=True),
-                    sa.Column('was_converted', sa.Boolean(), nullable=True),
-                    sa.PrimaryKeyConstraint('id'))
-
-
-def downgrade():
-    op.drop_table('telegram_file')

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -1,26 +0,0 @@
-"""Add is_bot field to puppets
-
-Revision ID: 1fa46383a9d3
-Revises: 30eca60587f1
-Create Date: 2018-04-29 23:44:40.102333
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '1fa46383a9d3'
-down_revision = '30eca60587f1'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column('is_bot')

alembic/versions/2228d49c383f_add_cascade_rules_to_userportal.py

@@ -1,41 +0,0 @@
-"""Add cascade rules to UserPortal
-
-Revision ID: 2228d49c383f
-Revises: bcfefa1f1299
-Create Date: 2018-05-31 11:11:59.482112
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '2228d49c383f'
-down_revision = 'bcfefa1f1299'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    try:
-        with op.batch_alter_table("user_portal") as batch_op:
-            batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
-            batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
-            batch_op.create_foreign_key("user_portal_user_fkey", "user", ["user"], ["tgid"],
-                                        onupdate="CASCADE", ondelete="CASCADE")
-            batch_op.create_foreign_key("user_portal_portal_fkey", "portal",
-                                        ["portal", "portal_receiver"], ["tgid", "tg_receiver"],
-                                        onupdate="CASCADE", ondelete="CASCADE")
-    except ValueError:
-        return
-
-
-def downgrade():
-    try:
-        with op.batch_alter_table("user_portal") as batch_op:
-                batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
-                batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
-                batch_op.create_foreign_key("user_portal_user_fkey", "portal",
-                                            ["portal", "portal_receiver"], ["tgid", "tg_receiver"])
-                batch_op.create_foreign_key("user_portal_portal_fkey", "user", ["user"], ["tgid"])
-    except ValueError:
-        return

alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py

@@ -1,27 +0,0 @@
-"""Add encrypted field for portals
-
-Revision ID: 24f31fc8a72b
-Revises: a7c04a56041b
-Create Date: 2020-03-28 20:14:29.046699
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "24f31fc8a72b"
-down_revision = "a7c04a56041b"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("portal") as batch_op:
-        batch_op.add_column(sa.Column("encrypted", sa.Boolean(), nullable=False,
-                                      server_default=sa.sql.expression.false()))
-
-
-def downgrade():
-    with op.batch_alter_table("portal") as batch_op:
-        batch_op.drop_column("encrypted")

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -1,25 +0,0 @@
-"""Add megagroup field to portals
-
-Revision ID: 30eca60587f1
-Revises: cfc972368e50
-Create Date: 2018-04-29 15:51:04.656605
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '30eca60587f1'
-down_revision = 'cfc972368e50'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("portal") as batch_op:
-        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
-
-
-def downgrade():
-    with op.batch_alter_table("portal") as batch_op:
-        batch_op.drop_column('megagroup')

alembic/versions/3e3745baa458_store_matrix_avatar_url_in_database.py

@@ -1,32 +0,0 @@
-"""Store Matrix avatar URL in database
-
-Revision ID: 3e3745baa458
-Revises: dff56c93da8d
-Create Date: 2020-06-15 14:32:10.454033
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '3e3745baa458'
-down_revision = 'dff56c93da8d'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('portal', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('avatar_url', sa.String(), nullable=True))
-
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('portal', schema=None) as batch_op:
-        batch_op.drop_column('avatar_url')
-
-    # ### end Alembic commands ###

alembic/versions/4f7d7ed5792a_switch_mx_user_profile_to_native_enum.py

@@ -1,26 +0,0 @@
-"""Switch mx_user_profile to native enum
-
-Revision ID: 4f7d7ed5792a
-Revises: 9e9c89b0b877
-Create Date: 2019-08-04 17:47:36.568120
-
-"""
-from alembic import op
-
-
-# revision identifiers, used by Alembic.
-revision = '4f7d7ed5792a'
-down_revision = '9e9c89b0b877'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    conn = op.get_bind()
-    conn.execute("UPDATE mx_user_profile SET membership=UPPER(membership)")
-    conn.execute("UPDATE mx_user_profile SET membership='LEAVE' WHERE membership='LEFT'")
-
-
-def downgrade():
-    conn = op.get_bind()
-    conn.execute("UPDATE mx_user_profile SET membership=LOWER(membership)")

alembic/versions/501dad2868bc_move_sessions_to_main_database.py

@@ -1,111 +0,0 @@
-"""Move sessions to main database
-
-Revision ID: 501dad2868bc
-Revises: 7d47d84380b6
-Create Date: 2018-03-02 19:15:53.826985
-
-"""
-from alembic import op
-import sqlalchemy as sa
-import sqlite3
-import os
-
-# revision identifiers, used by Alembic.
-revision = '501dad2868bc'
-down_revision = '7d47d84380b6'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    Session = op.create_table('telethon_sessions',
-                              sa.Column('session_id', sa.String, nullable=False),
-                              sa.Column('dc_id', sa.Integer, nullable=False),
-                              sa.Column('server_address', sa.String, nullable=True),
-                              sa.Column('port', sa.Integer, nullable=True),
-                              sa.Column('auth_key', sa.LargeBinary, nullable=True),
-                              sa.PrimaryKeyConstraint('session_id', 'dc_id'))
-    SentFile = op.create_table('telethon_sent_files',
-                               sa.Column('session_id', sa.String, nullable=False),
-                               sa.Column('md5_digest', sa.LargeBinary, nullable=False),
-                               sa.Column('file_size', sa.Integer, nullable=False),
-                               sa.Column('type', sa.Integer, nullable=False),
-                               sa.Column('id', sa.BigInteger, nullable=True),
-                               sa.Column('hash', sa.BigInteger, nullable=True),
-                               sa.PrimaryKeyConstraint('session_id', 'md5_digest', 'file_size',
-                                                       'type'))
-    Entity = op.create_table('telethon_entities',
-                             sa.Column('session_id', sa.String, nullable=False),
-                             sa.Column('id', sa.Integer, nullable=False),
-                             sa.Column('hash', sa.Integer, nullable=False),
-                             sa.Column('username', sa.String, nullable=True),
-                             sa.Column('phone', sa.Integer, nullable=True),
-                             sa.Column('name', sa.String, nullable=True),
-                             sa.PrimaryKeyConstraint('session_id', 'id'))
-    Version = op.create_table('telethon_version',
-                              sa.Column('version', sa.Integer, nullable=False),
-                              sa.PrimaryKeyConstraint('version'))
-    conn = op.get_bind()
-    sessions = [os.path.basename(f) for f in os.listdir(".") if f.endswith(".session")]
-    for session in sessions:
-        session_to_sqlalchemy(conn, session, Session, SentFile, Entity)
-
-
-def session_to_sqlalchemy(conn, path, Session, SentFile, Entity):
-    session_conn = sqlite3.connect(path)
-    session_id = os.path.splitext(path)[0]
-    c = session_conn.cursor()
-
-    auth_data_tuples = c.execute("SELECT * FROM sessions").fetchall()
-    auth_data_dicts = []
-    for row in auth_data_tuples:
-        dc_id, server_address, port, auth_key = row
-        auth_data_dicts.append({
-            "session_id": session_id,
-            "dc_id": dc_id,
-            "server_address": server_address,
-            "port": port,
-            "auth_key": auth_key,
-        })
-    if auth_data_dicts:
-        conn.execute(Session.insert().values(auth_data_dicts))
-
-    sent_file_tuples = c.execute("SELECT * FROM sent_files").fetchall()
-    sent_file_dicts = []
-    for row in sent_file_tuples:
-        md5_digest, file_size, type, id, hash = row
-        sent_file_dicts.append({
-            "session_id": session_id,
-            "md5_digest": md5_digest,
-            "file_size": file_size,
-            "type": type,
-            "id": id,
-            "hash": hash,
-        })
-    if sent_file_dicts:
-        conn.execute(SentFile.insert().values(sent_file_dicts))
-
-    entity_tuples = c.execute("SELECT * FROM entities").fetchall()
-    entity_dicts = []
-    for row in entity_tuples:
-        id, hash, username, phone, name = row
-        entity_dicts.append({
-            "session_id": session_id,
-            "id": id,
-            "hash": hash,
-            "username": username,
-            "phone": phone,
-            "name": name,
-        })
-    if entity_dicts:
-        conn.execute(Entity.insert().values(entity_dicts))
-
-    c.close()
-    session_conn.close()
-
-
-def downgrade():
-    op.drop_table('telethon_version')
-    op.drop_table('telethon_entities')
-    op.drop_table('telethon_sent_files')
-    op.drop_table('telethon_sessions')

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -1,136 +0,0 @@
-"""Move state store to main database
-
-Revision ID: 6ca3d74d51e4
-Revises: 2228d49c383f
-Create Date: 2018-06-26 21:31:26.911307
-
-"""
-import json
-import re
-
-from alembic import context, op
-import sqlalchemy.orm as orm
-import sqlalchemy as sa
-
-from mautrix.util.db import Base
-
-from mautrix_telegram.config import Config
-
-# revision identifiers, used by Alembic.
-revision = "6ca3d74d51e4"
-down_revision = "2228d49c383f"
-branch_labels = None
-depends_on = None
-
-
-class RoomState(Base):
-    __tablename__ = "mx_room_state"
-    __table_args__ = {"extend_existing": True}
-
-    room_id = sa.Column(sa.String, primary_key=True)
-    power_levels = sa.Column("power_levels", sa.Text, nullable=True)
-
-
-class UserProfile(Base):
-    __tablename__ = "mx_user_profile"
-    __table_args__ = {"extend_existing": True}
-
-    room_id = sa.Column(sa.String, primary_key=True)
-    user_id = sa.Column(sa.String, primary_key=True)
-    membership = sa.Column(sa.String, nullable=False, default="leave")
-    displayname = sa.Column(sa.String, nullable=True)
-    avatar_url = sa.Column(sa.String, nullable=True)
-
-
-class Puppet(Base):
-    __tablename__ = "puppet"
-    __table_args__ = {"extend_existing": True}
-
-    id = sa.Column(sa.Integer, primary_key=True)
-    displayname = sa.Column(sa.String, nullable=True)
-    displayname_source = sa.Column(sa.Integer, nullable=True)
-    username = sa.Column(sa.String, nullable=True)
-    photo_id = sa.Column(sa.String, nullable=True)
-    is_bot = sa.Column(sa.Boolean, nullable=True)
-    matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False)
-
-
-def upgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
-                                      server_default=sa.sql.expression.false()))
-    op.create_table("mx_room_state",
-                    sa.Column("room_id", sa.String(), nullable=False),
-                    sa.Column("power_levels", sa.Text(), nullable=True),
-                    sa.PrimaryKeyConstraint("room_id"))
-    op.create_table("mx_user_profile",
-                    sa.Column("room_id", sa.String(), nullable=False),
-                    sa.Column("user_id", sa.String(), nullable=False),
-                    sa.Column("membership", sa.String(), nullable=False,
-                              default="leave"),
-                    sa.Column("displayname", sa.String(), nullable=True),
-                    sa.Column("avatar_url", sa.String(), nullable=True),
-                    sa.PrimaryKeyConstraint("room_id", "user_id"))
-
-    try:
-        migrate_state_store()
-    except Exception as e:
-        print("Failed to migrate state store:", e)
-        print("Migrating the state store isn't required, but you can retry by alembic downgrading "
-              "to revision 2228d49c383f and upgrading again.")
-
-
-def migrate_state_store():
-    conn = op.get_bind()
-    session: orm.Session = orm.sessionmaker(bind=conn)()
-
-    try:
-        with open("mx-state.json") as file:
-            data = json.load(file)
-    except FileNotFoundError:
-        return
-    if not data:
-        return
-    registrations = data.get("registrations", [])
-
-    mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
-    mxtg_config = Config(mxtg_config_path, None, None)
-    mxtg_config.load()
-
-    username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}")
-    hs_domain = mxtg_config["homeserver.domain"]
-    localpart = username_template.format(userid="(.+)")
-    mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain))
-    for user in registrations:
-        match = mxid_regex.match(user)
-        if not match:
-            continue
-
-        puppet = session.query(Puppet).get(match.group(1))
-        if not puppet:
-            continue
-
-        puppet.matrix_registered = True
-        session.merge(puppet)
-    session.commit()
-
-    user_profiles = [UserProfile(room_id=room, user_id=user,
-                                 membership=member.get("membership", "leave"),
-                                 displayname=member.get("displayname", None),
-                                 avatar_url=member.get("avatar_url", None))
-                     for room, members in data.get("members", {}).items()
-                     for user, member in members.items()]
-    session.add_all(user_profiles)
-    session.commit()
-
-    room_state = [RoomState(room_id=room, power_levels=json.dumps(levels))
-                  for room, levels in data.get("power_levels", {}).items()]
-    session.add_all(room_state)
-    session.commit()
-
-
-def downgrade():
-    op.drop_table("mx_user_profile")
-    op.drop_table("mx_room_state")
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column("matrix_registered")

alembic/versions/7d47d84380b6_add_timestamp_to_telegramfile.py

@@ -1,26 +0,0 @@
-"""Add timestamp to TelegramFile
-
-Revision ID: 7d47d84380b6
-Revises: 1b241f7e8530
-Create Date: 2018-02-19 23:53:18.050871
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '7d47d84380b6'
-down_revision = '1b241f7e8530'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column('telegram_file',
-                  sa.Column('timestamp', sa.BigInteger(), nullable=True, default=0,
-                            server_default="0"))
-
-
-def downgrade():
-    with op.batch_alter_table("telegram_file") as batch_op:
-        batch_op.drop_column('timestamp')

alembic/versions/7de69cf5809e_add_matrix_redaction_state_to_message_.py

@@ -1,25 +0,0 @@
-"""Add Matrix redaction state to message table
-
-Revision ID: 7de69cf5809e
-Revises: 888275d58e57
-Create Date: 2020-12-19 12:39:57.368568
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '7de69cf5809e'
-down_revision = '888275d58e57'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table('message', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('redacted', sa.Boolean(), server_default=sa.false(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table('message', schema=None) as batch_op:
-        batch_op.drop_column('redacted')

alembic/versions/888275d58e57_add_double_puppet_base_url_to_puppet_.py

@@ -1,30 +0,0 @@
-"""Add double puppet base URL to puppet table
-
-Revision ID: 888275d58e57
-Revises: a328bf4f0932
-Create Date: 2020-10-14 18:52:00.730666
-
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = '888275d58e57'
-down_revision = 'a328bf4f0932'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('puppet', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('base_url', sa.Text(), nullable=True))
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('puppet', schema=None) as batch_op:
-        batch_op.drop_column('base_url')
-    # ### end Alembic commands ###

alembic/versions/97d2a942bcf8_initial_revision.py

@@ -1,80 +0,0 @@
-"""initial revision
-
-Revision ID: 97d2a942bcf8
-Revises:
-Create Date: 2018-02-11 18:40:55.483842
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '97d2a942bcf8'
-down_revision = None
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table('portal',
-                    sa.Column('tgid', sa.Integer),
-                    sa.Column('tg_receiver', sa.Integer),
-                    sa.Column('peer_type', sa.String, nullable=False, default=""),
-                    sa.Column('mxid', sa.String, nullable=True),
-                    sa.Column('username', sa.String, nullable=True),
-                    sa.Column('title', sa.String, nullable=True),
-                    sa.Column('about', sa.String, nullable=True),
-                    sa.Column('photo_id', sa.String, nullable=True),
-                    sa.PrimaryKeyConstraint('tgid', 'tg_receiver'),
-                    sa.UniqueConstraint('mxid'))
-    op.create_table('user',
-                    sa.Column('mxid', sa.String),
-                    sa.Column('tgid', sa.Integer, nullable=True, unique=True),
-                    sa.Column('tg_username', sa.String, nullable=True),
-                    sa.Column('saved_contacts', sa.Integer, nullable=False, default=0),
-                    sa.PrimaryKeyConstraint('mxid'))
-    op.create_table('puppet',
-                    sa.Column('id', sa.Integer),
-                    sa.Column('displayname', sa.String, nullable=True),
-                    sa.Column('username', sa.String, nullable=True),
-                    sa.Column('photo_id', sa.String, nullable=True),
-                    sa.PrimaryKeyConstraint('id'))
-    op.create_table('contact',
-                    sa.Column('user', sa.Integer),
-                    sa.Column('contact', sa.Integer),
-                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
-                    sa.ForeignKeyConstraint(("contact",), ("puppet.id",)),
-                    sa.PrimaryKeyConstraint('user', 'contact'))
-    op.create_table('user_portal',
-                    sa.Column('user', sa.Integer),
-                    sa.Column('portal', sa.Integer),
-                    sa.Column('portal_receiver', sa.Integer),
-                    sa.PrimaryKeyConstraint('user', 'portal', 'portal_receiver'),
-                    sa.ForeignKeyConstraint(("user",), ("user.tgid",),
-                                            name="user_portal_user_fkey",
-                                            onupdate="CASCADE", ondelete="CASCADE"),
-                    sa.ForeignKeyConstraint(("portal", "portal_receiver"),
-                                            ("portal.tgid", "portal.tg_receiver"),
-                                            name="user_portal_portal_fkey",
-                                            onupdate="CASCADE", ondelete="CASCADE"))
-    op.create_table('message',
-                    sa.Column('mxid', sa.String),
-                    sa.Column('mx_room', sa.String),
-                    sa.Column('tgid', sa.Integer),
-                    sa.Column('tg_space', sa.Integer),
-                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
-                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
-    op.create_table('bot_chat',
-                    sa.Column('id', sa.Integer),
-                    sa.Column('type', sa.String, nullable=False, default=""),
-                    sa.PrimaryKeyConstraint('id'))
-
-
-def downgrade():
-    op.drop_table('bot_chat')
-    op.drop_table('message')
-    op.drop_table('user_portal')
-    op.drop_table('contact')
-    op.drop_table('puppet')
-    op.drop_table('user')
-    op.drop_table('portal')

alembic/versions/990f4395afc6_store_displayname_contact_status_in_.py

@@ -1,32 +0,0 @@
-"""Store displayname contact status in puppet table
-
-Revision ID: 990f4395afc6
-Revises: 7de69cf5809e
-Create Date: 2021-01-01 11:56:54.610681
-
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = '990f4395afc6'
-down_revision = '7de69cf5809e'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('puppet', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('displayname_contact', sa.Boolean(), server_default=sa.true(), nullable=False))
-
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('puppet', schema=None) as batch_op:
-        batch_op.drop_column('displayname_contact')
-
-    # ### end Alembic commands ###

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -1,48 +0,0 @@
-"""Add edit index to messages
-
-Revision ID: 9e9c89b0b877
-Revises: 17574c57f3f8
-Create Date: 2019-05-29 15:28:23.128377
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = '9e9c89b0b877'
-down_revision = '17574c57f3f8'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table('_message_temp',
-                    sa.Column('mxid', sa.String),
-                    sa.Column('mx_room', sa.String),
-                    sa.Column('tgid', sa.Integer),
-                    sa.Column('tg_space', sa.Integer),
-                    sa.Column('edit_index', sa.Integer),
-                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
-                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
-    c = op.get_bind()
-    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
-              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
-              "FROM message")
-    c.execute("DROP TABLE message")
-    c.execute("ALTER TABLE _message_temp RENAME TO message")
-
-
-def downgrade():
-    op.create_table('_message_temp',
-                    sa.Column('mxid', sa.String),
-                    sa.Column('mx_room', sa.String),
-                    sa.Column('tgid', sa.Integer),
-                    sa.Column('tg_space', sa.Integer),
-                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
-                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
-    c = op.get_bind()
-    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
-              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
-              "FROM message")
-    c.execute("DROP TABLE message")
-    c.execute("ALTER TABLE _message_temp RENAME TO message")

alembic/versions/a328bf4f0932_store_encryption_state_event_in_db.py

@@ -1,38 +0,0 @@
-"""Store encryption state event in db
-
-Revision ID: a328bf4f0932
-Revises: ccbaff858240
-Create Date: 2020-07-11 21:31:27.059813
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-from mautrix.client.state_store.sqlalchemy import SerializableType
-from mautrix.types import RoomEncryptionStateEventContent
-
-# revision identifiers, used by Alembic.
-revision = 'a328bf4f0932'
-down_revision = 'ccbaff858240'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('encryption',
-                                      SerializableType(RoomEncryptionStateEventContent),
-                                      nullable=True))
-        batch_op.add_column(sa.Column('has_full_member_list', sa.Boolean(), nullable=True))
-        batch_op.add_column(sa.Column('is_encrypted', sa.Boolean(), nullable=True))
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('mx_room_state', schema=None) as batch_op:
-        batch_op.drop_column('is_encrypted')
-        batch_op.drop_column('has_full_member_list')
-        batch_op.drop_column('encryption')
-    # ### end Alembic commands ###

alembic/versions/a7c04a56041b_store_custom_puppet_next_batch_in_.py

@@ -1,26 +0,0 @@
-"""Store custom puppet next_batch in database
-
-Revision ID: a7c04a56041b
-Revises: 4f7d7ed5792a
-Create Date: 2019-08-06 23:08:51.087651
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "a7c04a56041b"
-down_revision = "4f7d7ed5792a"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.add_column(sa.Column("next_batch", sa.String(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column("next_batch")

alembic/versions/a9119be92164_add_phone_number_field_to_users.py

@@ -1,25 +0,0 @@
-"""Add phone number field to users
-
-Revision ID: a9119be92164
-Revises: b54929c22c86
-Create Date: 2018-09-28 02:38:40.626282
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "a9119be92164"
-down_revision = "b54929c22c86"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column("user", sa.Column("tg_phone", sa.String(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("user") as batch_op:
-        batch_op.drop_column("tg_phone")

alembic/versions/b54929c22c86_add_portal_specific_config.py

@@ -1,25 +0,0 @@
-"""Add portal-specific config
-
-Revision ID: b54929c22c86
-Revises: d5f7b8b4b456
-Create Date: 2018-09-24 23:40:33.528710
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "b54929c22c86"
-down_revision = "d5f7b8b4b456"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column("portal", sa.Column("config", sa.Text(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("portal") as batch_op:
-        batch_op.drop_column("config")

alembic/versions/bcfefa1f1299_add_displayname_source_fields_for_.py

@@ -1,24 +0,0 @@
-"""Add displayname source fields for puppets
-
-Revision ID: bcfefa1f1299
-Revises: bdadd173ee02
-Create Date: 2018-05-19 17:00:21.078098
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = 'bcfefa1f1299'
-down_revision = 'bdadd173ee02'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column('puppet', sa.Column('displayname_source', sa.Integer(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column('displayname_source')

alembic/versions/bdadd173ee02_update_telethon_update_state_table.py

@@ -1,43 +0,0 @@
-"""Update telethon update state table
-
-Revision ID: bdadd173ee02
-Revises: eeaf0dae87ce
-Create Date: 2018-05-13 10:42:59.395597
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = 'bdadd173ee02'
-down_revision = 'eeaf0dae87ce'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("telethon_entities") as batch_op:
-        batch_op.alter_column("id", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("hash", existing_type=sa.Integer, type_=sa.BigInteger)
-
-    with op.batch_alter_table("telethon_update_state") as batch_op:
-        batch_op.alter_column("entity_id", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("pts", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("qts", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("date", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.alter_column("seq", existing_type=sa.Integer, type_=sa.BigInteger)
-        batch_op.add_column(sa.Column("unread_count", sa.Integer))
-
-
-def downgrade():
-    with op.batch_alter_table("telethon_entities") as batch_op:
-        batch_op.alter_column("id", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("hash", existing_type=sa.BigInteger, type_=sa.Integer)
-
-    with op.batch_alter_table("telethon_update_state") as batch_op:
-        batch_op.alter_column("entity_id", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("pts", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("qts", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("date", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.alter_column("seq", existing_type=sa.BigInteger, type_=sa.Integer)
-        batch_op.drop_column("unread_count")

alembic/versions/bfc0a39bfe02_store_displayname_quality_in_puppet_.py

@@ -1,32 +0,0 @@
-"""Store displayname quality in puppet table
-
-Revision ID: bfc0a39bfe02
-Revises: ec1d3dcc77e9
-Create Date: 2021-03-23 20:03:08.825333
-
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = 'bfc0a39bfe02'
-down_revision = 'ec1d3dcc77e9'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('puppet', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('displayname_quality', sa.Integer(), server_default='0', nullable=False))
-
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('puppet', schema=None) as batch_op:
-        batch_op.drop_column('displayname_quality')
-
-    # ### end Alembic commands ###

alembic/versions/ccbaff858240_switch_to_mautrix_python_crypto.py

@@ -1,71 +0,0 @@
-"""Switch to mautrix-python crypto
-
-Revision ID: ccbaff858240
-Revises: 3e3745baa458
-Create Date: 2020-07-08 19:06:12.588047
-
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = 'ccbaff858240'
-down_revision = '3e3745baa458'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('nio_account')
-    op.drop_table('nio_device_key')
-    op.drop_table('nio_outgoing_key_request')
-    op.drop_table('nio_olm_session')
-    op.drop_table('nio_megolm_inbound_session')
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('nio_megolm_inbound_session',
-    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('fp_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.Column('forwarded_chains', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('session_id', name='nio_megolm_inbound_session_pkey')
-    )
-    op.create_table('nio_olm_session',
-    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('sender_key', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('session', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
-    sa.Column('last_used', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('session_id', name='nio_olm_session_pkey')
-    )
-    op.create_table('nio_outgoing_key_request',
-    sa.Column('request_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('session_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('room_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('algorithm', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('request_id', name='nio_outgoing_key_request_pkey')
-    )
-    op.create_table('nio_device_key',
-    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('display_name', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('deleted', sa.BOOLEAN(), autoincrement=False, nullable=False),
-    sa.Column('keys', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_device_key_pkey')
-    )
-    op.create_table('nio_account',
-    sa.Column('user_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('device_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
-    sa.Column('shared', sa.BOOLEAN(), autoincrement=False, nullable=False),
-    sa.Column('sync_token', sa.TEXT(), autoincrement=False, nullable=False),
-    sa.Column('account', postgresql.BYTEA(), autoincrement=False, nullable=False),
-    sa.PrimaryKeyConstraint('user_id', 'device_id', name='nio_account_pkey')
-    )
-    # ### end Alembic commands ###

alembic/versions/cfc972368e50_add_metadata_to_telegramfile.py

@@ -1,35 +0,0 @@
-"""Add metadata to TelegramFile
-
-Revision ID: cfc972368e50
-Revises: 501dad2868bc
-Create Date: 2018-03-09 16:07:01.236712
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = 'cfc972368e50'
-down_revision = '501dad2868bc'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("telegram_file") as batch_op:
-        batch_op.add_column(sa.Column('size', sa.Integer(), nullable=True))
-        batch_op.add_column(sa.Column('width', sa.Integer(), nullable=True))
-        batch_op.add_column(sa.Column('height', sa.Integer(), nullable=True))
-        batch_op.add_column(sa.Column('thumbnail', sa.String(), nullable=True))
-        batch_op.create_foreign_key(constraint_name="fk_file_thumbnail",
-                                    referent_table="telegram_file",
-                                    local_cols=['thumbnail'],
-                                    remote_cols=['id'])
-
-
-def downgrade():
-    with op.batch_alter_table("telegram_file") as batch_op:
-        batch_op.drop_column('size')
-        batch_op.drop_column('width')
-        batch_op.drop_column('height')
-        batch_op.drop_column('thumbnail')

alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py

@@ -1,26 +0,0 @@
-"""Add decryption info field for reuploaded telegram files
-
-Revision ID: d3c922a6acd2
-Revises: 24f31fc8a72b
-Create Date: 2020-03-30 20:07:17.340346
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'd3c922a6acd2'
-down_revision = '24f31fc8a72b'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("telegram_file") as batch_op:
-        batch_op.add_column(sa.Column("decryption_info", sa.Text(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("telegram_file") as batch_op:
-        batch_op.drop_column("decryption_info")

alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py

@@ -1,26 +0,0 @@
-"""Add access_token and custom_mxid fields for puppets
-
-Revision ID: d5f7b8b4b456
-Revises: 6ca3d74d51e4
-Create Date: 2018-07-20 12:09:30.277960
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = "d5f7b8b4b456"
-down_revision = "6ca3d74d51e4"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
-    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
-
-
-def downgrade():
-    with op.batch_alter_table("puppet") as batch_op:
-        batch_op.drop_column("custom_mxid")
-        batch_op.drop_column("access_token")

alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py

@@ -1,71 +0,0 @@
-"""Add matrix-nio state store to main db
-
-Revision ID: dff56c93da8d
-Revises: d3c922a6acd2
-Create Date: 2020-03-31 22:04:04.014048
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'dff56c93da8d'
-down_revision = 'd3c922a6acd2'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('nio_account',
-    sa.Column('user_id', sa.String(length=255), nullable=False),
-    sa.Column('device_id', sa.String(length=255), nullable=False),
-    sa.Column('shared', sa.Boolean(), nullable=False),
-    sa.Column('sync_token', sa.Text(), nullable=False),
-    sa.Column('account', sa.LargeBinary(), nullable=False),
-    sa.PrimaryKeyConstraint('user_id', 'device_id')
-    )
-    op.create_table('nio_device_key',
-    sa.Column('user_id', sa.String(length=255), nullable=False),
-    sa.Column('device_id', sa.String(length=255), nullable=False),
-    sa.Column('display_name', sa.String(length=255), nullable=False),
-    sa.Column('deleted', sa.Boolean(), nullable=False),
-    sa.Column('keys', sa.PickleType(), nullable=False),
-    sa.PrimaryKeyConstraint('user_id', 'device_id')
-    )
-    op.create_table('nio_megolm_inbound_session',
-    sa.Column('session_id', sa.String(length=255), nullable=False),
-    sa.Column('sender_key', sa.String(length=255), nullable=False),
-    sa.Column('fp_key', sa.String(length=255), nullable=False),
-    sa.Column('room_id', sa.String(length=255), nullable=False),
-    sa.Column('session', sa.LargeBinary(), nullable=False),
-    sa.Column('forwarded_chains', sa.PickleType(), nullable=False),
-    sa.PrimaryKeyConstraint('session_id')
-    )
-    op.create_table('nio_olm_session',
-    sa.Column('session_id', sa.String(length=255), nullable=False),
-    sa.Column('sender_key', sa.String(length=255), nullable=False),
-    sa.Column('session', sa.LargeBinary(), nullable=False),
-    sa.Column('created_at', sa.DateTime(), nullable=False),
-    sa.Column('last_used', sa.DateTime(), nullable=False),
-    sa.PrimaryKeyConstraint('session_id')
-    )
-    op.create_table('nio_outgoing_key_request',
-    sa.Column('request_id', sa.String(length=255), nullable=False),
-    sa.Column('session_id', sa.String(length=255), nullable=False),
-    sa.Column('room_id', sa.String(length=255), nullable=False),
-    sa.Column('algorithm', sa.String(length=255), nullable=False),
-    sa.PrimaryKeyConstraint('request_id')
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('nio_outgoing_key_request')
-    op.drop_table('nio_olm_session')
-    op.drop_table('nio_megolm_inbound_session')
-    op.drop_table('nio_device_key')
-    op.drop_table('nio_account')
-    # ### end Alembic commands ###

alembic/versions/ec1d3dcc77e9_switch_telegram_ids_to_bigints.py

@@ -1,44 +0,0 @@
-"""Switch Telegram IDs to bigints
-
-Revision ID: ec1d3dcc77e9
-Revises: 990f4395afc6
-Create Date: 2021-03-09 21:36:58.443727
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = 'ec1d3dcc77e9'
-down_revision = '990f4395afc6'
-branch_labels = None
-depends_on = None
-
-
-columns_to_upgrade = (
-    ("bot_chat", "id"),
-    ("message", "tgid"),
-    ("message", "tg_space"),
-    ("portal", "tgid"),
-    ("portal", "tg_receiver"),
-    ("puppet", "id"),
-    ("puppet", "displayname_source"),
-    ("user", "tgid"),
-    ("user_portal", "user"),
-    ("user_portal", "portal"),
-    ("user_portal", "portal_receiver"),
-    ("contact", "user"),
-    ("contact", "contact"),
-)
-
-
-def upgrade():
-    if op.get_context().dialect.name == "postgresql":
-        for table, column in columns_to_upgrade:
-            op.alter_column(table, column, existing_type=sa.Integer, type_=sa.BigInteger)
-
-
-def downgrade():
-    if op.get_context().dialect.name == "postgresql":
-        for table, column in columns_to_upgrade:
-            op.alter_column(table, column, existing_type=sa.BigInteger, type_=sa.Integer)

alembic/versions/eeaf0dae87ce_add_telethon_update_state_table.py

@@ -1,34 +0,0 @@
-"""Add telethon update state table
-
-Revision ID: eeaf0dae87ce
-Revises: 1fa46383a9d3
-Create Date: 2018-04-30 17:30:59.610885
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = 'eeaf0dae87ce'
-down_revision = '1fa46383a9d3'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    with op.batch_alter_table("telethon_entities") as batch_op:
-        batch_op.alter_column('phone', existing_type=sa.Integer, type_=sa.BigInteger)
-    op.create_table('telethon_update_state',
-                    sa.Column('session_id', sa.String, nullable=False),
-                    sa.Column('entity_id', sa.Integer, nullable=False),
-                    sa.Column('pts', sa.Integer, nullable=True),
-                    sa.Column('qts', sa.Integer, nullable=True),
-                    sa.Column('date', sa.Integer, nullable=True),
-                    sa.Column('seq', sa.Integer, nullable=True),
-                    sa.PrimaryKeyConstraint('session_id', 'entity_id'))
-
-
-def downgrade():
-    with op.batch_alter_table("telethon_entities") as batch_op:
-        batch_op.alter_column('phone', existing_type=sa.BigInteger, type_=sa.Integer)
-    op.drop_table('telethon_update_state')

dev-requirements.txt

@@ -0,0 +1,3 @@
+pre-commit>=2.10.1,<3
+isort>=5.10.1,<6
+black>=22.3,<23

docker-run.sh

@@ -1,19 +1,32 @@
 #!/bin/sh
+if [ ! -z "$MAUTRIX_DIRECT_STARTUP" ]; then
+	if [ $(id -u) == 0 ]; then
+		echo "|------------------------------------------|"
+		echo "| Warning: running bridge unsafely as root |"
+		echo "|------------------------------------------|"
+	fi
+	exec python3 -m mautrix_telegram -c /data/config.yaml
+elif [ $(id -u) != 0 ]; then
+	echo "The startup script must run as root. It will use su-exec to drop permissions before running the bridge."
+	echo "To bypass the startup script, either set the `MAUTRIX_DIRECT_STARTUP` environment variable,"
+	echo "or just use `python3 -m mautrix_telegram -c /data/config.yaml` as the run command."
+	echo "Note that the config and registration will not be auto-generated when bypassing the startup script."
+	exit 1
+fi
 
 # Define functions.
 function fixperms {
-	chown -R $UID:$GID /data /opt/mautrix-telegram
+	chown -R $UID:$GID /data
+
+	# /opt/mautrix-telegram is read-only, so disable file logging if it's pointing there.
+	if [[ "$(yq e '.logging.handlers.file.filename' /data/config.yaml)" == "./mautrix-telegram.log" ]]; then
+		yq -I4 e -i 'del(.logging.root.handlers[] | select(. == "file"))' /data/config.yaml
+		yq -I4 e -i 'del(.logging.handlers.file)' /data/config.yaml
+	fi
 }
 
 cd /opt/mautrix-telegram
 
-# Replace database path in config.
-sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
-
-if [ -f /data/mx-state.json ]; then
-	ln -s /data/mx-state.json
-fi
-
 if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
 	echo "Didn't find a config file."
@@ -25,13 +38,13 @@ if [ ! -f /data/config.yaml ]; then
 fi
 
 if [ ! -f /data/registration.yaml ]; then
-	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
+	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml || exit $?
+	echo "Didn't find a registration file."
+	echo "Generated one for you."
+	echo "See https://docs.mau.fi/bridges/general/registering-appservices.html on how to use it."
 	fixperms
 	exit
 fi
 
-# Check that database is in the right state
-alembic -x config=/data/config.yaml upgrade head
-
 fixperms
 exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

helm/mautrix-telegram/.editorconfig

@@ -1,2 +0,0 @@
-[*.{yaml,yml}]
-indent_size = 2

helm/mautrix-telegram/.gitignore

@@ -1 +0,0 @@
-charts/*

helm/mautrix-telegram/.helmignore

@@ -1,22 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

helm/mautrix-telegram/Chart.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-name: mautrix-telegram
-version: 0.1.0
-appVersion: "0.7.0"
-description: A Matrix-Telegram hybrid puppeting/relaybot bridge.
-keywords:
-  - matrix
-  - bridge
-  - telegram
-maintainers:
-  - name: Tulir Asokan
-    email: tulir@maunium.net
-sources:
-  - https://github.com/tulir/mautrix-telegram

helm/mautrix-telegram/requirements.lock

@@ -1,6 +0,0 @@
-dependencies:
-- name: postgresql
-  repository: https://kubernetes-charts.storage.googleapis.com/
-  version: 6.5.0
-digest: sha256:85139e9d4207e49c11c5f84d7920d0135cffd3d427f3f3638d4e51258990de2a
-generated: "2019-10-23T22:11:37.005827507+03:00"

helm/mautrix-telegram/requirements.yaml

@@ -1,5 +0,0 @@
-dependencies:
-  - name: postgresql
-    version: 6.5.0
-    repository: https://kubernetes-charts.storage.googleapis.com/
-    condition: postgresql.enabled

helm/mautrix-telegram/templates/NOTES.txt

@@ -1,21 +0,0 @@
-Your registration file is below. Save it into a YAML file and give the path to that file to synapse:
-
-id: {{ .Values.appservice.id }}
-as_token: {{ .Values.appservice.asToken }}
-hs_token: {{ .Values.appservice.hsToken }}
-namespaces:
-    users:
-        - exclusive: true
-          regex: "@{{ .Values.bridge.username_template | replace "{userid}" ".+"}}:{{ .Values.homeserver.domain }}"
-          {{- if .Values.appservice.communityID }}
-          group_id: {{ .Values.appservice.communityID }}
-          {{- end }}
-    aliases:
-        - exclusive: true
-          regex: "@{{ .Values.bridge.alias_template | replace "{groupname}" ".+"}}:{{ .Values.homeserver.domain }}"
-          {{- if .Values.appservice.communityID }}
-          group_id: {{ .Values.appservice.communityID }}
-          {{- end }}
-url: {{ .Values.appservice.address }}
-sender_localpart: {{ .Values.appservice.botUsername }}
-rate_limited: false

helm/mautrix-telegram/templates/_helpers.tpl

@@ -1,55 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "mautrix-telegram.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "mautrix-telegram.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name -}}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "mautrix-telegram.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Common labels
-*/}}
-{{- define "mautrix-telegram.labels" -}}
-app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
-helm.sh/chart: {{ include "mautrix-telegram.chart" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end -}}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "mautrix-telegram.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create -}}
-    {{ default (include "mautrix-telegram.fullname" .) .Values.serviceAccount.name }}
-{{- else -}}
-    {{ default "default" .Values.serviceAccount.name }}
-{{- end -}}
-{{- end -}}

helm/mautrix-telegram/templates/configmap.yaml

@@ -1,57 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ template "mautrix-telegram.fullname" . }}
-  labels:
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/name: {{ template "mautrix-telegram.name" . }}
-data:
-  config.yaml: |
-    homeserver:
-      address: {{ .Values.homeserver.address }}
-      domain: {{ .Values.homeserver.domain }}
-      verify_ssl: {{ .Values.homeserver.verifySSL }}
-
-    appservice:
-      address: http://{{ include "mautrix-telegram.fullname" . }}:{{ .Values.service.port }}
-
-      hostname: 0.0.0.0
-      port: {{ .Values.service.port }}
-      max_body_size: {{ .Values.appservice.maxBodySize }}
-
-      {{- if .Values.postgresql.enabled }}
-      database: "postgres://postgres:{{ .Values.postgresql.postgresqlPassword }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.postgresqlDatabase }}"
-      {{- else }}
-      database: {{ .Values.appservice.database | quote }}
-      {{- end }}
-
-      public:
-        {{- toYaml .Values.appservice.public | nindent 8 }}
-
-      provisioning:
-        {{- toYaml .Values.appservice.provisioning | nindent 8 }}
-
-      id: {{ .Values.appservice.id }}
-      bot_username: {{ .Values.appservice.botUsername }}
-      bot_displayname: {{ .Values.appservice.botDisplayname }}
-      bot_avatar:  {{ .Values.appservice.botAvatar }}
-
-      community_id: {{ .Values.appservice.communityID }}
-
-      as_token: {{ .Values.appservice.asToken }}
-      hs_token: {{ .Values.appservice.hsToken }}
-
-    metrics:
-      {{- toYaml .Values.metrics | nindent 6 }}
-
-    bridge:
-      {{- toYaml .Values.bridge | nindent 6 }}
-
-    telegram:
-      {{- toYaml .Values.telegram | nindent 6 }}
-
-    logging:
-      {{- toYaml .Values.logging | nindent 6 }}
-  registration.yaml: ""

helm/mautrix-telegram/templates/deployment.yaml

@@ -1,69 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "mautrix-telegram.fullname" . }}
-  labels:
-    {{- include "mautrix-telegram.labels" . | nindent 4 }}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
-      app.kubernetes.io/instance: {{ .Release.Name }}
-  template:
-    {{- if .Values.podAnnotations }}
-    annotations:
-      {{- toYaml .Values.podAnnotations | nindent 6 }}
-    {{- end }}
-    metadata:
-      labels:
-        app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
-        app.kubernetes.io/instance: {{ .Release.Name }}
-    spec:
-      serviceAccountName: {{ template "mautrix-telegram.serviceAccountName" . }}
-      containers:
-        - name: {{ .Chart.Name }}
-          securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          volumeMounts:
-            - mountPath: /data
-              name: config-volume
-          ports:
-            - name: http
-              containerPort: {{ .Values.service.port }}
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              path: /_matrix/mau/live
-              port: http
-            initialDelaySeconds: 60
-            periodSeconds: 5
-          readinessProbe:
-            httpGet:
-              path: /_matrix/mau/ready
-              port: http
-            initialDelaySeconds: 60
-            periodSeconds: 5
-          resources:
-            {{- toYaml .Values.resources | nindent 12 }}
-      volumes:
-        - name: config-volume
-          configMap:
-            name: {{ template "mautrix-telegram.fullname" . }}
-
-      securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}

helm/mautrix-telegram/templates/service.yaml

@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "mautrix-telegram.fullname" . }}
-  labels:
-{{ include "mautrix-telegram.labels" . | indent 4 }}
-spec:
-  type: {{ .Values.service.type }}
-  ports:
-    - port: {{ .Values.service.port }}
-      targetPort: http
-      protocol: TCP
-      name: http
-  selector:
-    app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}

helm/mautrix-telegram/templates/serviceaccount.yaml

@@ -1,8 +0,0 @@
-{{- if .Values.serviceAccount.create -}}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ template "mautrix-telegram.serviceAccountName" . }}
-  labels:
-{{ include "mautrix-telegram.labels" . | indent 4 }}
-{{- end -}}

helm/mautrix-telegram/values.yaml

@@ -1,141 +0,0 @@
-image:
-  repository: dock.mau.dev/tulir/mautrix-telegram
-  tag: latest
-  pullPolicy: IfNotPresent
-
-nameOverride: ""
-fullnameOverride: ""
-
-serviceAccount:
-  # Specifies whether a service account should be created
-  create: true
-  # The name of the service account to use.
-  # If not set and create is true, a name is generated using the fullname template
-  name:
-
-service:
-  type: ClusterIP
-  port: 29317
-
-resources: {}
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
-
-nodeSelector: {}
-
-tolerations: []
-
-affinity: {}
-
-# Postgres pod configs
-postgresql:
-  enabled: true
-  postgresqlDatabase: mxtg
-  postgresqlPassword: SET TO RANDOM STRING
-  persistence:
-    size: 2Gi
-  resources:
-    requests:
-      memory: 256Mi
-      cpu: 100m
-
-# Homeserver details
-homeserver:
-  # The address that this appservice can use to connect to the homeserver.
-  address: https://example.com
-  # The domain of the homeserver (for MXIDs, etc).
-  domain: example.com
-  # Whether or not to verify the SSL certificate of the homeserver.
-  # Only applies if address starts with https://
-  verifySSL: true
-
-# Application service host/registration related details
-# Changing these values requires regeneration of the registration.
-appservice:
-  # The maximum body size of appservice API requests (from the homeserver) in mebibytes
-  # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
-  maxBodySize: 1
-
-  # Public part of web server for out-of-Matrix interaction with the bridge.
-  # Used for things like login if the user wants to make sure the 2FA password isn't stored in
-  # the HS database.
-  public:
-    # Whether or not the public-facing endpoints should be enabled.
-    enabled: true
-    # The prefix to use in the public-facing endpoints.
-    prefix: /public
-    # The base URL where the public-facing endpoints are available. The prefix is not added
-    # implicitly.
-    external: https://example.com/public
-
-  # Provisioning API part of the web server for automated portal creation and fetching information.
-  # Used by things like Dimension (https://dimension.t2bot.io/).
-  provisioning:
-    # Whether or not the provisioning API should be enabled.
-    enabled: true
-    # The prefix to use in the provisioning API endpoints.
-    prefix: /_matrix/provision/v1
-    # The shared secret to authorize users of the API.
-    shared_secret: SET TO RANDOM STRING
-
-  id: telegram
-  botUsername: telegrambot
-  # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
-  # to leave display name/avatar as-is.
-  botDisplayname: Telegram bridge bot
-  botAvatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
-
-  # Community ID for bridged users (changes registration file) and rooms.
-  # Must be created manually.
-  communityID: false
-
-  # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
-  asToken: SET TO RANDOM STRING
-  hsToken: SET TO RANDOM STRING
-
-# The keys below can be used to override the configs in the base config:
-# https://github.com/tulir/mautrix-telegram/blob/master/example-config.yaml
-# Note that the "appservice" and "homeserver" sections are above and slightly different than the base.
-
-# Bridge config
-bridge:
-  # Localpart template of MXIDs for Telegram users.
-  # {userid} is replaced with the user ID of the Telegram user.
-  username_template: "telegram_{userid}"
-  # Localpart template of room aliases for Telegram portal rooms.
-  # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} )
-  alias_template: "telegram_{groupname}"
-
-  # Permissions for using the bridge.
-  # Permitted values:
-  #   relaybot - Only use the bridge via the relaybot, no access to commands.
-  #       user - Relaybot level + access to commands to create bridges.
-  #  puppeting - User level + logging in with a Telegram account.
-  #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
-  #      admin - Full access to use the bridge and some extra administration commands.
-  # Permitted keys:
-  #        * - All Matrix users
-  #   domain - All users on that homeserver
-  #     mxid - Specific user
-  permissions:
-    "*": "relaybot"
-    "public.example.com": "user"
-    "example.com": "full"
-    "@admin:example.com": "admin"
-
-# Prometheus telemetry config.
-metrics:
-  enabled: false
-  listen_port: 8000
-
-# Telegram config
-telegram:
-  # Get your own API keys at https://my.telegram.org/apps
-  api_id: 12345
-  api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
-  # (Optional) Create your own bot at https://t.me/BotFather
-  # bot_token: 123456789:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.10.0rc1"
+__version__ = "0.12.1"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,33 +13,27 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from __future__ import annotations
 
-from alchemysession import AlchemySessionContainer
+from typing import Any
+
+from telethon import __version__ as __telethon_version__
 
-from mautrix.types import UserID, RoomID
 from mautrix.bridge import Bridge
-from mautrix.util.db import Base
+from mautrix.types import RoomID, UserID
 
+from .bot import Bot
+from .config import Config
+from .db import init as init_db, upgrade_table
+from .matrix import MatrixHandler
+from .portal import Portal
+from .puppet import Puppet
+from .user import User
+from .version import linkified_version, version
 from .web.provisioning import ProvisioningAPI
 from .web.public import PublicBridgeWebsite
-from .commands.manhole import ManholeState
-from .abstract_user import init as init_abstract_user
-from .bot import Bot, init as init_bot
-from .config import Config
-from .context import Context
-from .db import init as init_db
-from .formatter import init as init_formatter
-from .matrix import MatrixHandler
-from .portal import Portal, init as init_portal
-from .puppet import Puppet, init as init_puppet
-from .user import User, init as init_user
-from .version import version, linkified_version
 
-try:
-    import prometheus_client as prometheus
-except ImportError:
-    prometheus = None
+from .abstract_user import AbstractUser  # isort: skip
 
 
 class TelegramBridge(Bridge):
@@ -47,49 +41,52 @@ class TelegramBridge(Bridge):
     name = "mautrix-telegram"
     command = "python -m mautrix-telegram"
     description = "A Matrix-Telegram puppeting bridge."
-    repo_url = "https://github.com/tulir/mautrix-telegram"
-    real_user_content_key = "net.maunium.telegram.puppet"
+    repo_url = "https://github.com/mautrix/telegram"
     version = version
     markdown_version = linkified_version
     config_class = Config
     matrix_class = MatrixHandler
+    upgrade_table = upgrade_table
 
     config: Config
-    session_container: AlchemySessionContainer
-    bot: Bot
-    manhole: Optional[ManholeState]
+    bot: Bot | None
+    public_website: PublicBridgeWebsite | None
+    provisioning_api: ProvisioningAPI | None
 
     def prepare_db(self) -> None:
         super().prepare_db()
         init_db(self.db)
-        self.session_container = AlchemySessionContainer(
-            engine=self.db, table_base=Base, session=False,
-            table_prefix="telethon_", manage_tables=False)
-
-    def _prepare_website(self, context: Context) -> None:
-        if self.config["appservice.public.enabled"]:
-            public_website = PublicBridgeWebsite(self.loop)
-            self.az.app.add_subapp(self.config["appservice.public.prefix"], public_website.app)
-            context.public_website = public_website
 
+    def _prepare_website(self) -> None:
         if self.config["appservice.provisioning.enabled"]:
-            provisioning_api = ProvisioningAPI(context)
-            self.az.app.add_subapp(self.config["appservice.provisioning.prefix"],
-                                   provisioning_api.app)
-            context.provisioning_api = provisioning_api
+            self.provisioning_api = ProvisioningAPI(self)
+            self.az.app.add_subapp(
+                self.config["appservice.provisioning.prefix"], self.provisioning_api.app
+            )
+        else:
+            self.provisioning_api = None
+
+        if self.config["appservice.public.enabled"]:
+            self.public_website = PublicBridgeWebsite(self.loop)
+            self.az.app.add_subapp(
+                self.config["appservice.public.prefix"], self.public_website.app
+            )
+        else:
+            self.public_website = None
 
     def prepare_bridge(self) -> None:
-        self.bot = init_bot(self.config)
-        context = Context(self.az, self.config, self.loop, self.session_container, self, self.bot)
-        self._prepare_website(context)
-        self.matrix = context.mx = MatrixHandler(context)
-        self.manhole = None
-
-        init_abstract_user(context)
-        init_formatter(context)
-        init_portal(context)
-        self.add_startup_actions(init_puppet(context))
-        self.add_startup_actions(init_user(context))
+        self._prepare_website()
+        AbstractUser.init_cls(self)
+        bot_token: str = self.config["telegram.bot_token"]
+        if bot_token and not bot_token.lower().startswith("disable"):
+            self.bot = AbstractUser.relaybot = Bot(bot_token)
+        else:
+            self.bot = AbstractUser.relaybot = None
+        self.matrix = MatrixHandler(self)
+        Portal.init_cls(self)
+        self.add_startup_actions(Puppet.init_cls(self))
+        self.add_startup_actions(User.init_cls(self))
+        self.add_startup_actions(Portal.restart_scheduled_disappearing())
         if self.bot:
             self.add_startup_actions(self.bot.start())
         if self.config["bridge.resend_bridge_info"]:
@@ -99,35 +96,49 @@ class TelegramBridge(Bridge):
         self.config["bridge.resend_bridge_info"] = False
         self.config.save()
         self.log.info("Re-sending bridge info state event to all portals")
-        for portal in Portal.all():
+        async for portal in Portal.all():
             await portal.update_bridge_info()
         self.log.info("Finished re-sending bridge info state events")
 
     def prepare_stop(self) -> None:
         for puppet in Puppet.by_custom_mxid.values():
             puppet.stop()
-        self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
-        if self.manhole:
-            self.manhole.close()
-            self.manhole = None
+        self.add_shutdown_actions(user.stop() for user in User.by_tgid.values())
+        if self.bot:
+            self.add_shutdown_actions(self.bot.stop())
 
-    async def get_user(self, user_id: UserID, create: bool = True) -> User:
-        user = User.get_by_mxid(user_id, create=create)
+    async def get_user(self, user_id: UserID, create: bool = True) -> User | None:
+        user = await User.get_by_mxid(user_id, create=create)
         if user:
             await user.ensure_started()
         return user
 
-    async def get_portal(self, room_id: RoomID) -> Portal:
-        return Portal.get_by_mxid(room_id)
+    async def get_portal(self, room_id: RoomID) -> Portal | None:
+        return await Portal.get_by_mxid(room_id)
 
-    async def get_puppet(self, user_id: UserID, create: bool = False) -> Puppet:
+    async def get_puppet(self, user_id: UserID, create: bool = False) -> Puppet | None:
         return await Puppet.get_by_mxid(user_id, create=create)
 
-    async def get_double_puppet(self, user_id: UserID) -> Puppet:
+    async def get_double_puppet(self, user_id: UserID) -> Puppet | None:
         return await Puppet.get_by_custom_mxid(user_id)
 
     def is_bridge_ghost(self, user_id: UserID) -> bool:
         return bool(Puppet.get_id_from_mxid(user_id))
 
+    async def count_logged_in_users(self) -> int:
+        return len([user for user in User.by_tgid.values() if user.tgid])
+
+    async def manhole_global_namespace(self, user_id: UserID) -> dict[str, Any]:
+        return {
+            **await super().manhole_global_namespace(user_id),
+            "User": User,
+            "Portal": Portal,
+            "Puppet": Puppet,
+        }
+
+    @property
+    def manhole_banner_program_version(self) -> str:
+        return f"{super().manhole_banner_program_version} and Telethon {__telethon_version__}"
+
 
 TelegramBridge().run()

mautrix_telegram/abstract_user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,70 +13,119 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional, Union, Dict, Type, Any, TYPE_CHECKING
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Union
 from abc import ABC, abstractmethod
 import asyncio
 import logging
 import platform
 import time
 
+from telethon.errors import UnauthorizedError
+from telethon.network import (
+    Connection,
+    ConnectionTcpFull,
+    ConnectionTcpMTProxyRandomizedIntermediate,
+)
 from telethon.sessions import Session
-from telethon.network import (ConnectionTcpMTProxyRandomizedIntermediate, ConnectionTcpFull,
-                              Connection)
-from telethon.tl.patched import MessageService, Message
+from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
-    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdatePinnedMessages,
-    UpdatePinnedChannelMessages, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
-    UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
-    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
-    UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
-    UpdateUserTyping, User, UserStatusOffline, UserStatusOnline, UpdateReadHistoryInbox,
-    UpdateReadChannelInbox, MessageEmpty)
+    Channel,
+    Chat,
+    MessageActionChannelMigrateFrom,
+    MessageEmpty,
+    PeerChannel,
+    PeerChat,
+    PeerUser,
+    TypeUpdate,
+    UpdateChannel,
+    UpdateChannelUserTyping,
+    UpdateChatParticipantAdmin,
+    UpdateChatParticipants,
+    UpdateChatUserTyping,
+    UpdateDeleteChannelMessages,
+    UpdateDeleteMessages,
+    UpdateEditChannelMessage,
+    UpdateEditMessage,
+    UpdateFolderPeers,
+    UpdateMessageReactions,
+    UpdateNewChannelMessage,
+    UpdateNewMessage,
+    UpdateNotifySettings,
+    UpdatePinnedChannelMessages,
+    UpdatePinnedDialogs,
+    UpdatePinnedMessages,
+    UpdateReadChannelInbox,
+    UpdateReadHistoryInbox,
+    UpdateReadHistoryOutbox,
+    UpdateShort,
+    UpdateShortChatMessage,
+    UpdateShortMessage,
+    UpdateUserName,
+    UpdateUserPhoto,
+    UpdateUserStatus,
+    UpdateUserTyping,
+    User,
+    UserStatusOffline,
+    UserStatusOnline,
+)
 
-from mautrix.types import UserID, PresenceState
-from mautrix.errors import MatrixError
 from mautrix.appservice import AppService
+from mautrix.errors import MatrixError
+from mautrix.types import PresenceState, UserID
 from mautrix.util.logging import TraceLogger
-from mautrix.util.opt_prometheus import Histogram, Counter
-from alchemysession import AlchemySessionContainer
+from mautrix.util.opt_prometheus import Counter, Histogram
 
-from . import portal as po, puppet as pu, __version__
-from .db import Message as DBMessage
-from .types import TelegramID
+from . import __version__, portal as po, puppet as pu
+from .config import Config
+from .db import Message as DBMessage, PgSession
 from .tgclient import MautrixTelegramClient
+from .types import TelegramID
 
 if TYPE_CHECKING:
-    from .context import Context
-    from .config import Config
+    from .__main__ import TelegramBridge
     from .bot import Bot
 
-config: Optional['Config'] = None
-# Value updated from config in init()
-MAX_DELETIONS: int = 10
+UpdateMessage = Union[
+    UpdateShortChatMessage,
+    UpdateShortMessage,
+    UpdateNewChannelMessage,
+    UpdateNewMessage,
+    UpdateEditMessage,
+    UpdateEditChannelMessage,
+]
+UpdateMessageContent = Union[
+    UpdateShortMessage, UpdateShortChatMessage, Message, MessageService, MessageEmpty
+]
 
-UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
-                      UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
-UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
-
-UPDATE_TIME = Histogram("bridge_telegram_update", "Time spent processing Telegram updates",
-                        ("update_type",))
-UPDATE_ERRORS = Counter("bridge_telegram_update_error",
-                        "Number of fatal errors while handling Telegram updates", ("update_type",))
+UPDATE_TIME = Histogram(
+    name="bridge_telegram_update",
+    documentation="Time spent processing Telegram updates",
+    labelnames=("update_type",),
+)
+UPDATE_ERRORS = Counter(
+    name="bridge_telegram_update_error",
+    documentation="Number of fatal errors while handling Telegram updates",
+    labelnames=("update_type",),
+)
 
 
 class AbstractUser(ABC):
-    session_container: AlchemySessionContainer = None
     loop: asyncio.AbstractEventLoop = None
     log: TraceLogger
     az: AppService
-    relaybot: Optional['Bot']
+    bridge: "TelegramBridge"
+    config: Config
+    relaybot: "Bot"
     ignore_incoming_bot_events: bool = True
+    max_deletions: int = 10
 
-    client: Optional[MautrixTelegramClient]
-    mxid: Optional[UserID]
+    client: MautrixTelegramClient | None
+    mxid: UserID | None
 
-    tgid: Optional[TelegramID]
-    username: Optional['str']
+    tgid: TelegramID | None
+    username: str | None
     is_bot: bool
 
     is_relaybot: bool
@@ -102,14 +151,16 @@ class AbstractUser(ABC):
         return self.client and self.client.is_connected()
 
     @property
-    def _proxy_settings(self) -> Tuple[Type[Connection], Optional[Tuple[Any, ...]]]:
-        proxy_type = config["telegram.proxy.type"].lower()
+    def _proxy_settings(self) -> tuple[type[Connection], tuple[Any, ...] | None]:
+        proxy_type = self.config["telegram.proxy.type"].lower()
         connection = ConnectionTcpFull
-        connection_data = (config["telegram.proxy.address"],
-                           config["telegram.proxy.port"],
-                           config["telegram.proxy.rdns"],
-                           config["telegram.proxy.username"],
-                           config["telegram.proxy.password"])
+        connection_data = (
+            self.config["telegram.proxy.address"],
+            self.config["telegram.proxy.port"],
+            self.config["telegram.proxy.rdns"],
+            self.config["telegram.proxy.username"],
+            self.config["telegram.proxy.password"],
+        )
         if proxy_type == "disabled":
             connection_data = None
         elif proxy_type == "socks4":
@@ -124,53 +175,78 @@ class AbstractUser(ABC):
 
         return connection, connection_data
 
-    def _init_client(self) -> None:
+    @classmethod
+    def init_cls(cls, bridge: "TelegramBridge") -> None:
+        cls.bridge = bridge
+        cls.config = bridge.config
+        cls.loop = bridge.loop
+        cls.az = bridge.az
+        cls.ignore_incoming_bot_events = cls.config["bridge.relaybot.ignore_own_incoming_events"]
+        cls.max_deletions = cls.config["bridge.max_telegram_delete"]
+
+    async def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
 
-        self.session = self.session_container.new_session(self.name)
-        if config["telegram.server.enabled"]:
-            self.session.set_dc(config["telegram.server.dc"],
-                                config["telegram.server.ip"],
-                                config["telegram.server.port"])
+        session = await PgSession.get(self.name)
+        if self.config["telegram.server.enabled"]:
+            session.set_dc(
+                self.config["telegram.server.dc"],
+                self.config["telegram.server.ip"],
+                self.config["telegram.server.port"],
+            )
 
         if self.is_relaybot:
             base_logger = logging.getLogger("telethon.relaybot")
         else:
             base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
 
-        device = config["telegram.device_info.device_model"]
-        sysversion = config["telegram.device_info.system_version"]
-        appversion = config["telegram.device_info.app_version"]
+        device = self.config["telegram.device_info.device_model"]
+        sysversion = self.config["telegram.device_info.system_version"]
+        appversion = self.config["telegram.device_info.app_version"]
         connection, proxy = self._proxy_settings
 
-        assert isinstance(self.session, Session)
+        assert isinstance(session, Session)
 
         self.client = MautrixTelegramClient(
-            session=self.session,
-
-            api_id=config["telegram.api_id"],
-            api_hash=config["telegram.api_hash"],
-
+            session=session,
+            api_id=self.config["telegram.api_id"],
+            api_hash=self.config["telegram.api_hash"],
             app_version=__version__ if appversion == "auto" else appversion,
-            system_version=(MautrixTelegramClient.__version__
-                            if sysversion == "auto" else sysversion),
-            device_model=(f"{platform.system()} {platform.release()}"
-                          if device == "auto" else device),
-
-            timeout=config["telegram.connection.timeout"],
-            connection_retries=config["telegram.connection.retries"],
-            retry_delay=config["telegram.connection.retry_delay"],
-            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
-            request_retries=config["telegram.connection.request_retries"],
+            system_version=(
+                MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion
+            ),
+            device_model=(
+                f"{platform.system()} {platform.release()}" if device == "auto" else device
+            ),
+            timeout=self.config["telegram.connection.timeout"],
+            connection_retries=self.config["telegram.connection.retries"],
+            retry_delay=self.config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=self.config["telegram.connection.flood_sleep_threshold"],
+            request_retries=self.config["telegram.connection.request_retries"],
             connection=connection,
             proxy=proxy,
             raise_last_call_error=True,
-
+            catch_up=self.config["telegram.catch_up"],
+            sequential_updates=self.config["telegram.sequential_updates"],
             loop=self.loop,
-            base_logger=base_logger
+            base_logger=base_logger,
+            update_error_callback=self._telethon_update_error_callback,
         )
         self.client.add_event_handler(self._update_catch)
 
+    async def _telethon_update_error_callback(self, err: Exception) -> None:
+        if self.config["telegram.exit_on_update_error"]:
+            self.log.critical(f"Stopping due to update handling error {type(err).__name__}")
+            self.bridge.manual_stop(50)
+        else:
+            if isinstance(err, UnauthorizedError):
+                self.log.warning("Not recreating Telethon update loop")
+                return
+            self.log.info("Recreating Telethon update loop in 60 seconds")
+            await asyncio.sleep(60)
+            self.log.debug("Now recreating Telethon update loop")
+            self.client._updates_handle = self.loop.create_task(self.client._update_loop())
+
     @abstractmethod
     async def update(self, update: TypeUpdate) -> bool:
         return False
@@ -194,7 +270,7 @@ class AbstractUser(ABC):
             if not await self.update(update):
                 await self._update(update)
         except Exception:
-            self.log.exception(f"Failed to handle Telegram update {update}")
+            self.log.exception("Failed to handle Telegram update")
             UPDATE_ERRORS.labels(update_type=update_type).inc()
         UPDATE_TIME.labels(update_type=update_type).observe(time.time() - start_time)
 
@@ -204,47 +280,65 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     async def is_logged_in(self) -> bool:
-        return (self.client and self.client.is_connected()
-                and await self.client.is_user_authorized())
+        return (
+            self.client and self.client.is_connected() and await self.client.is_user_authorized()
+        )
 
     async def has_full_access(self, allow_bot: bool = False) -> bool:
-        return (self.puppet_whitelisted
-                and (not self.is_bot or allow_bot)
-                and await self.is_logged_in())
+        return (
+            self.puppet_whitelisted
+            and (not self.is_bot or allow_bot)
+            and await self.is_logged_in()
+        )
 
-    async def start(self, delete_unless_authenticated: bool = False) -> 'AbstractUser':
+    async def start(self, delete_unless_authenticated: bool = False) -> AbstractUser:
         if not self.client:
-            self._init_client()
+            await self._init_client()
         await self.client.connect()
         self.log.debug(f"{'Bot' if self.is_relaybot else self.mxid} connected: {self.connected}")
         return self
 
-    async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
+    async def ensure_started(self, even_if_no_session=False) -> AbstractUser:
         if self.connected:
             return self
-        if even_if_no_session or self.session_container.has_session(self.mxid):
-            self.log.debug("Starting client due to ensure_started"
-                           f"(even_if_no_session={even_if_no_session})")
+        session_exists = await PgSession.has(self.mxid)
+        if even_if_no_session or session_exists:
+            self.log.debug(
+                f"Starting client due to ensure_started({even_if_no_session=}, {session_exists=})"
+            )
             await self.start(delete_unless_authenticated=not even_if_no_session)
         return self
 
     async def stop(self) -> None:
-        await self.client.disconnect()
-        self.client = None
+        if self.client:
+            await self.client.disconnect()
+            self.client = None
 
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
-        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
-                              loop=self.loop)
-        if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
-                               UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
+        if isinstance(update, UpdateShort):
+            update = update.update
+        asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
+        if isinstance(
+            update,
+            (
+                UpdateShortChatMessage,
+                UpdateShortMessage,
+                UpdateNewChannelMessage,
+                UpdateNewMessage,
+                UpdateEditMessage,
+                UpdateEditChannelMessage,
+            ),
+        ):
             await self.update_message(update)
         elif isinstance(update, UpdateDeleteMessages):
             await self.delete_message(update)
         elif isinstance(update, UpdateDeleteChannelMessages):
             await self.delete_channel_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
+        elif isinstance(update, UpdateMessageReactions):
+            await self.update_reactions(update)
+        elif isinstance(update, (UpdateChatUserTyping, UpdateChannelUserTyping, UpdateUserTyping)):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
@@ -260,22 +354,41 @@ class AbstractUser(ABC):
             await self.update_read_receipt(update)
         elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
             await self.update_own_read_receipt(update)
+        elif isinstance(update, UpdateFolderPeers):
+            await self.update_folder_peers(update)
+        elif isinstance(update, UpdatePinnedDialogs):
+            await self.update_pinned_dialogs(update)
+        elif isinstance(update, UpdateNotifySettings):
+            await self.update_notify_settings(update)
+        elif isinstance(update, UpdateChannel):
+            await self.update_channel(update)
         else:
             self.log.trace("Unhandled update: %s", update)
 
-    async def update_pinned_messages(self, update: Union[UpdatePinnedMessages,
-                                                         UpdatePinnedChannelMessages]) -> None:
+    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
+        pass
+
+    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
+        pass
+
+    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
+        pass
+
+    async def update_pinned_messages(
+        self, update: UpdatePinnedMessages | UpdatePinnedChannelMessages
+    ) -> None:
         if isinstance(update, UpdatePinnedMessages):
-            portal = po.Portal.get_by_entity(update.peer, receiver_id=self.tgid)
+            portal = await po.Portal.get_by_entity(update.peer, tg_receiver=self.tgid)
         else:
-            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_ids(update.messages, self.tgid,
-                                                  remove=not update.pinned)
+            await portal.receive_telegram_pin_ids(
+                update.messages, self.tgid, remove=not update.pinned
+            )
 
     @staticmethod
     async def update_participants(update: UpdateChatParticipants) -> None:
-        portal = po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
+        portal = await po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
         if portal and portal.mxid:
             await portal.update_power_levels(update.participants.participants)
 
@@ -284,30 +397,37 @@ class AbstractUser(ABC):
             self.log.debug("Unexpected read receipt peer: %s", update.peer)
             return
 
-        portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
+        portal = await po.Portal.get_by_tgid(
+            TelegramID(update.peer.user_id), tg_receiver=self.tgid
+        )
         if not portal or not portal.mxid:
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
+        message = await DBMessage.get_one_by_tgid(
+            TelegramID(update.max_id), self.tgid, edit_index=-1
+        )
         if not message:
             return
 
-        puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
+        puppet = await pu.Puppet.get_by_peer(update.peer)
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
-    async def update_own_read_receipt(self, update: Union[UpdateReadHistoryInbox,
-                                                          UpdateReadChannelInbox]) -> None:
-        puppet = pu.Puppet.get(self.tgid)
+    async def update_own_read_receipt(
+        self, update: UpdateReadHistoryInbox | UpdateReadChannelInbox
+    ) -> None:
+        puppet = await pu.Puppet.get_by_tgid(self.tgid)
         if not puppet.is_real_user:
             return
 
         if isinstance(update, UpdateReadChannelInbox):
-            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
         elif isinstance(update.peer, PeerChat):
-            portal = po.Portal.get_by_tgid(TelegramID(update.peer.chat_id))
+            portal = await po.Portal.get_by_tgid(TelegramID(update.peer.chat_id))
         elif isinstance(update.peer, PeerUser):
-            portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
+            portal = await po.Portal.get_by_tgid(
+                TelegramID(update.peer.user_id), tg_receiver=self.tgid
+            )
         else:
             self.log.debug("Unexpected own read receipt peer: %s", update.peer)
             return
@@ -316,7 +436,9 @@ class AbstractUser(ABC):
             return
 
         tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
-        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), tg_space, edit_index=-1)
+        message = await DBMessage.get_one_by_tgid(
+            TelegramID(update.max_id), tg_space, edit_index=-1
+        )
         if not message:
             return
 
@@ -324,49 +446,63 @@ class AbstractUser(ABC):
 
     async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
         # TODO duplication not checked
-        portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
         if not portal or not portal.mxid:
             return
 
         await portal.set_telegram_admin(TelegramID(update.user_id))
 
-    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+    async def update_typing(
+        self, update: UpdateUserTyping | UpdateChatUserTyping | UpdateChannelUserTyping
+    ) -> None:
+        sender = None
         if isinstance(update, UpdateUserTyping):
-            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
+            portal = await po.Portal.get_by_tgid(
+                TelegramID(update.user_id), tg_receiver=self.tgid, peer_type="user"
+            )
+            sender = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
+        elif isinstance(update, UpdateChannelUserTyping):
+            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        elif isinstance(update, UpdateChatUserTyping):
+            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
         else:
-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
-
-        if not portal or not portal.mxid:
             return
 
-        sender = pu.Puppet.get(TelegramID(update.user_id))
+        if isinstance(update, (UpdateChannelUserTyping, UpdateChatUserTyping)):
+            sender = await pu.Puppet.get_by_peer(update.from_id)
+
+        if not sender or not portal or not portal.mxid:
+            return
+
         await portal.handle_telegram_typing(sender, update)
 
-    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
-                                     ) -> None:
+    async def _handle_entity_updates(self, entities: dict[int, User | Chat | Channel]) -> None:
         try:
-            users = (entity for entity in entities.values() if isinstance(entity, User))
-            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
-            await asyncio.gather(*[puppet.try_update_info(self, info)
-                                   for puppet, info in puppets if puppet])
+            users = (entity for entity in entities.values() if isinstance(entity, (User, Channel)))
+            puppets = ((await pu.Puppet.get_by_peer(user), user) for user in users)
+            await asyncio.gather(
+                *[puppet.try_update_info(self, info) async for puppet, info in puppets if puppet]
+            )
         except Exception:
             self.log.exception("Failed to handle entity updates")
 
-    async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
+    async def update_others_info(self, update: UpdateUserName | UpdateUserPhoto) -> None:
         # TODO duplication not checked
-        puppet = pu.Puppet.get(TelegramID(update.user_id))
+        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
         if isinstance(update, UpdateUserName):
             puppet.username = update.username
             if await puppet.update_displayname(self, update):
                 await puppet.save()
+                await puppet.update_portals_meta()
         elif isinstance(update, UpdateUserPhoto):
             if await puppet.update_avatar(self, update.photo):
                 await puppet.save()
+                await puppet.update_portals_meta()
         else:
             self.log.warning(f"Unexpected other user info update: {type(update)}")
 
     async def update_status(self, update: UpdateUserStatus) -> None:
-        puppet = pu.Puppet.get(TelegramID(update.user_id))
+        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
         if isinstance(update.status, UserStatusOnline):
             await puppet.default_mxid_intent.set_presence(PresenceState.ONLINE)
         elif isinstance(update.status, UserStatusOffline):
@@ -375,38 +511,48 @@ class AbstractUser(ABC):
             self.log.warning(f"Unexpected user status update: type({update})")
         return
 
-    def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
-                                                                  Optional[pu.Puppet],
-                                                                  Optional[po.Portal]]:
+    async def get_message_details(
+        self, update: UpdateMessage
+    ) -> tuple[UpdateMessageContent, pu.Puppet | None, po.Portal | None]:
         if isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
             if not portal:
                 self.log.warning(f"Received message in chat with unknown type {update.chat_id}")
-            sender = pu.Puppet.get(TelegramID(update.from_id))
+            sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id))
         elif isinstance(update, UpdateShortMessage):
-            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
-            sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
-        elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
-                                 UpdateEditMessage, UpdateEditChannelMessage)):
+            portal = await po.Portal.get_by_tgid(
+                TelegramID(update.user_id), tg_receiver=self.tgid, peer_type="user"
+            )
+            sender = await pu.Puppet.get_by_tgid(self.tgid if update.out else update.user_id)
+        elif isinstance(
+            update,
+            (
+                UpdateNewMessage,
+                UpdateNewChannelMessage,
+                UpdateEditMessage,
+                UpdateEditChannelMessage,
+            ),
+        ):
             update = update.message
             if isinstance(update, MessageEmpty):
                 return update, None, None
-            portal = po.Portal.get_by_entity(update.peer_id, receiver_id=self.tgid)
+            portal = await po.Portal.get_by_entity(update.peer_id, tg_receiver=self.tgid)
             if update.out:
-                sender = pu.Puppet.get(self.tgid)
-            elif isinstance(update.from_id, PeerUser):
-                sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
+                sender = await pu.Puppet.get_by_tgid(self.tgid)
+            elif isinstance(update.from_id, (PeerUser, PeerChannel)):
+                sender = await pu.Puppet.get_by_peer(update.from_id)
             else:
                 sender = None
         else:
-            self.log.warning("Unexpected message type in User#get_message_details: "
-                             f"{type(update)}")
+            self.log.warning(
+                f"Unexpected message type in User#get_message_details: {type(update)}"
+            )
             return update, None, None
         return update, sender, portal
 
     @staticmethod
     async def _try_redact(message: DBMessage) -> None:
-        portal = po.Portal.get_by_mxid(message.mx_room)
+        portal = await po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
@@ -415,33 +561,69 @@ class AbstractUser(ABC):
             pass
 
     async def delete_message(self, update: UpdateDeleteMessages) -> None:
-        if len(update.messages) > MAX_DELETIONS:
+        if len(update.messages) > self.max_deletions:
             return
 
         for message_id in update.messages:
-            for message in DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
+            for message in await DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
                 if message.redacted:
                     continue
-                message.delete()
-                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                await message.delete()
+                number_left = await DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
                 if number_left == 0:
                     await self._try_redact(message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
-        if len(update.messages) > MAX_DELETIONS:
+        if len(update.messages) > self.max_deletions:
             return
 
         channel_id = TelegramID(update.channel_id)
 
         for message_id in update.messages:
-            for message in DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
+            for message in await DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
                 if message.redacted:
                     continue
-                message.delete()
+                await message.delete()
                 await self._try_redact(message)
 
+    async def update_reactions(self, update: UpdateMessageReactions) -> None:
+        portal = await po.Portal.get_by_entity(update.peer, tg_receiver=self.tgid)
+        if not portal or not portal.mxid or not portal.allow_bridging:
+            return
+        await portal.handle_telegram_reactions(self, TelegramID(update.msg_id), update.reactions)
+
+    async def update_channel(self, update: UpdateChannel) -> None:
+        portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        if not portal:
+            return
+        if getattr(update, "mau_telethon_is_leave", False):
+            self.log.debug("UpdateChannel has mau_telethon_is_leave, leaving portal")
+            await portal.delete_telegram_user(self.tgid, sender=None)
+        elif chan := getattr(update, "mau_channel", None):
+            if not portal.mxid:
+                asyncio.create_task(self._delayed_create_channel(chan))
+            else:
+                self.log.debug("Updating channel info with data fetched by Telethon")
+                await portal.update_info(self, chan)
+                await portal.invite_to_matrix(self.mxid)
+
+    async def _delayed_create_channel(self, chan: Channel) -> None:
+        self.log.debug("Waiting 5 seconds before handling UpdateChannel for non-existent portal")
+        await asyncio.sleep(5)
+        portal = await po.Portal.get_by_tgid(TelegramID(chan.id))
+        if portal.mxid:
+            self.log.debug(
+                "Portal started existing after waiting 5 seconds, dropping UpdateChannel"
+            )
+            return
+        else:
+            self.log.info(
+                "Creating Matrix room with data fetched by Telethon due to UpdateChannel"
+            )
+            await portal.create_matrix_room(self, chan)
+
     async def update_message(self, original_update: UpdateMessage) -> None:
-        update, sender, portal = self.get_message_details(original_update)
+        update, sender, portal = await self.get_message_details(original_update)
         if not portal:
             return
         elif portal and not portal.allow_bridging:
@@ -450,30 +632,44 @@ class AbstractUser(ABC):
 
         if self.is_relaybot:
             if update.is_private:
-                if not config["bridge.relaybot.private_chat.invite"]:
-                    self.log.debug(f"Ignoring private message to bot from {sender.id}")
+                if not self.config["bridge.relaybot.private_chat.invite"]:
+                    if sender:
+                        self.log.debug(f"Ignoring private message to bot from {sender.id}")
                     return
-            elif not portal.mxid and config["bridge.relaybot.ignore_unbridged_group_chat"]:
-                self.log.debug("Ignoring message received by bot"
-                               f" in unbridged chat {portal.tgid_log}")
+            elif not portal.mxid and self.config["bridge.relaybot.ignore_unbridged_group_chat"]:
+                self.log.debug(
+                    f"Ignoring message received by bot in unbridged chat {portal.tgid_log}"
+                )
                 return
 
-        if ((self.ignore_incoming_bot_events and self.relaybot
-             and sender and sender.id == self.relaybot.tgid)):
-            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
+        if (
+            self.ignore_incoming_bot_events
+            and self.relaybot
+            and sender
+            and sender.id == self.relaybot.tgid
+        ):
+            self.log.debug("Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
             return
 
-        await portal.backfill_lock.wait(update.id)
+        await portal.backfill_lock.wait(f"update {update.id}")
 
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.trace(f"Received %s in %s by %d, unregistering portal...",
-                               update.action, portal.tgid_log, sender.id)
+                self.log.trace(
+                    "Received %s in %s by %d, unregistering portal...",
+                    update.action,
+                    portal.tgid_log,
+                    sender.id,
+                )
                 await self.unregister_portal(update.action.chat_id, update.action.chat_id)
                 await self.register_portal(portal)
                 return
-            self.log.trace("Handling action %s to %s by %d", update.action, portal.tgid_log,
-                           (sender.id if sender else 0))
+            self.log.trace(
+                "Handling action %s to %s by %d",
+                update.action,
+                portal.tgid_log,
+                (sender.id if sender else 0),
+            )
             return await portal.handle_telegram_action(self, sender, update)
 
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
@@ -481,11 +677,3 @@ class AbstractUser(ABC):
         return await portal.handle_telegram_message(self, sender, update)
 
     # endregion
-
-
-def init(context: 'Context') -> None:
-    global config, MAX_DELETIONS
-    AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
-    AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
-    AbstractUser.session_container = context.session_container
-    MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/bot.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,43 +13,86 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING
-import logging
+from __future__ import annotations
 
+from typing import Awaitable, Callable, Literal
+import logging
+import time
+
+from telethon.errors import ChannelInvalidError, ChannelPrivateError
+from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
+from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
-    ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
-    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser, PeerUser,
-    MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
-    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
-from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
-from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
-from telethon.errors import ChannelInvalidError, ChannelPrivateError
+    ChannelParticipantAdmin,
+    ChannelParticipantCreator,
+    ChatForbidden,
+    ChatParticipantAdmin,
+    ChatParticipantCreator,
+    ChatParticipantsForbidden,
+    InputChannel,
+    InputUser,
+    MessageActionChatAddUser,
+    MessageActionChatDeleteUser,
+    MessageActionChatMigrateTo,
+    MessageEntityBotCommand,
+    PeerChannel,
+    PeerChat,
+    PeerUser,
+    TypeChannelParticipant,
+    TypeChatParticipant,
+    TypeInputPeer,
+    TypePeer,
+    UpdateNewChannelMessage,
+    UpdateNewMessage,
+    User,
+)
+from telethon.utils import add_surrogate, del_surrogate
 
-from mautrix.types import UserID
+from mautrix.errors import MBadState, MForbidden
+from mautrix.types import RoomID, UserID
 
+from . import portal as po, puppet as pu, user as u
 from .abstract_user import AbstractUser
-from .db import BotChat
+from .db import BotChat, Message as DBMessage
 from .types import TelegramID
-from . import puppet as pu, portal as po, user as u
-
-if TYPE_CHECKING:
-    from .config import Config
-
-config: Optional['Config'] = None
 
 ReplyFunc = Callable[[str], Awaitable[Message]]
+BanFunc = Callable[[RoomID, UserID, str], Awaitable[None]]
+TelegramAdminPermission = Literal[
+    "change_info",
+    "post_messages",
+    "edit_messages",
+    "delete_messages",
+    "ban_users",
+    "invite_users",
+    "pin_messages",
+    "add_admins",
+    "anonymous",
+    "manage_call",
+    "other",
+]
 
 
 class Bot(AbstractUser):
     log: logging.Logger = logging.getLogger("mau.user.bot")
 
     token: str
-    chats: Dict[int, str]
-    tg_whitelist: List[int]
+    chats: dict[int, str]
+    tg_whitelist: list[int]
     whitelist_group_admins: bool
-    _me_info: Optional[User]
-    _me_mxid: Optional[UserID]
+    _me_info: User | None
+    _me_mxid: UserID | None
+    _admin_cache: dict[
+        tuple[int, int],
+        tuple[ChatParticipantAdmin | ChatParticipantCreator | None, float],
+    ]
+    required_permissions: dict[str, TelegramAdminPermission] = {
+        "portal": None,
+        "invite": "invite_users",
+        "mxban": "ban_users",
+        "mxkick": "ban_users",
+    }
 
     def __init__(self, token: str) -> None:
         super().__init__()
@@ -59,24 +102,27 @@ class Bot(AbstractUser):
         self.puppet_whitelisted = True
         self.whitelisted = True
         self.relaybot_whitelisted = True
-        self.username = None
+        self.tg_username = None
         self.is_relaybot = True
         self.is_bot = True
         self.chats = {}
+        self._admin_cache = {}
         self.tg_whitelist = []
-        self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
-                                       or False)
+        self.whitelist_group_admins = (
+            self.config["bridge.relaybot.whitelist_group_admins"] or False
+        )
         self._me_info = None
         self._me_mxid = None
+        self._login_wait_fut = self.loop.create_future()
 
-    async def get_me(self, use_cache: bool = True) -> Tuple[User, UserID]:
+    async def get_me(self, use_cache: bool = True) -> tuple[User, UserID]:
         if not use_cache or not self._me_mxid:
             self._me_info = await self.client.get_me()
             self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
         return self._me_info, self._me_mxid
 
     async def init_permissions(self) -> None:
-        whitelist = config["bridge.relaybot.whitelist"] or []
+        whitelist = self.config["bridge.relaybot.whitelist"] or []
         for user_id in whitelist:
             if isinstance(user_id, str):
                 entity = await self.client.get_input_entity(user_id)
@@ -87,8 +133,8 @@ class Bot(AbstractUser):
             if isinstance(user_id, int):
                 self.tg_whitelist.append(user_id)
 
-    async def start(self, delete_unless_authenticated: bool = False) -> 'Bot':
-        self.chats = {chat.id: chat.type for chat in BotChat.all()}
+    async def start(self, delete_unless_authenticated: bool = False) -> Bot:
+        self.chats = {chat.id: chat.type for chat in await BotChat.all()}
         await super().start(delete_unless_authenticated)
         if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
@@ -99,71 +145,115 @@ class Bot(AbstractUser):
         await self.init_permissions()
         info = await self.client.get_me()
         self.tgid = TelegramID(info.id)
-        self.username = info.username
+        self.tg_username = info.username
         self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
+        self._login_wait_fut.set_result(None)
+        self._login_wait_fut = None
 
         chat_ids = [chat_id for chat_id, chat_type in self.chats.items() if chat_type == "chat"]
         response = await self.client(GetChatsRequest(chat_ids))
         for chat in response.chats:
             if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
-                self.remove_chat(TelegramID(chat.id))
+                await self.remove_chat(TelegramID(chat.id))
 
-        channel_ids = [InputChannel(chat_id, 0)
-                       for chat_id, chat_type in self.chats.items()
-                       if chat_type == "channel"]
+        channel_ids = [
+            InputChannel(chat_id, 0)
+            for chat_id, chat_type in self.chats.items()
+            if chat_type == "channel"
+        ]
         for channel_id in channel_ids:
             try:
                 await self.client(GetChannelsRequest([channel_id]))
             except (ChannelPrivateError, ChannelInvalidError):
-                self.remove_chat(TelegramID(channel_id.channel_id))
+                await self.remove_chat(TelegramID(channel_id.channel_id))
 
     async def register_portal(self, portal: po.Portal) -> None:
-        self.add_chat(portal.tgid, portal.peer_type)
+        await self.add_chat(portal.tgid, portal.peer_type)
 
-    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
-        self.remove_chat(tgid)
+    async def unregister_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
+        await self.remove_chat(tgid)
 
-    def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
+    async def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
         if chat_id not in self.chats:
             self.chats[chat_id] = chat_type
-            BotChat(id=TelegramID(chat_id), type=chat_type).insert()
+            await BotChat(id=chat_id, type=chat_type).insert()
 
-    def remove_chat(self, chat_id: TelegramID) -> None:
+    async def remove_chat(self, chat_id: TelegramID) -> None:
         try:
             del self.chats[chat_id]
         except KeyError:
             pass
-        BotChat.delete_by_id(chat_id)
+        await BotChat.delete_by_id(chat_id)
 
-    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
+    async def _get_admin_participant(
+        self, chat: TypePeer | TypeInputPeer, tgid: TelegramID
+    ) -> TypeChatParticipant | TypeChannelParticipant | None:
+        chan_id = chat.channel_id if isinstance(chat, PeerChannel) else chat.chat_id
+        try:
+            cached, created = self._admin_cache[chan_id, tgid]
+            if created + 60 < time.time():
+                return cached
+        except KeyError:
+            pass
+        if isinstance(chat, PeerChannel):
+            p = await self.client(GetParticipantRequest(chat, tgid))
+            pcp = p.participant
+            self._admin_cache[chat.channel_id, tgid] = (pcp, time.time())
+            return pcp
+        elif isinstance(chat, PeerChat):
+            chat = await self.client(GetFullChatRequest(chat.chat_id))
+            if isinstance(chat.full_chat.participants, ChatParticipantsForbidden):
+                return None
+            participants = chat.full_chat.participants.participants
+            for p in participants:
+                self._admin_cache[chat.channel_id, tgid] = (p, time.time())
+                if p.user_id == tgid:
+                    return p
+        return None
+
+    @staticmethod
+    def _has_participant_permission(
+        pcp: TypeChatParticipant | TypeChannelParticipant | None,
+        permission: TelegramAdminPermission | None,
+    ) -> bool:
+        if isinstance(pcp, (ChannelParticipantCreator, ChannelParticipantAdmin)):
+            return permission is None or getattr(pcp.admin_rights, permission, False)
+        elif isinstance(pcp, (ChatParticipantCreator, ChatParticipantAdmin)):
+            return True
+        return False
+
+    async def _can_use_commands(
+        self, chat: TypePeer, tgid: TelegramID, permission: TelegramAdminPermission | None = None
+    ) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
-        user = u.User.get_by_tgid(tgid)
+        user = await u.User.get_by_tgid(tgid)
         if user and user.is_admin:
             self.tg_whitelist.append(user.tgid)
             return True
 
         if self.whitelist_group_admins:
-            if isinstance(chat, PeerChannel):
-                p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin))
-            elif isinstance(chat, PeerChat):
-                chat = await self.client(GetFullChatRequest(chat.chat_id))
-                participants = chat.full_chat.participants.participants
-                for p in participants:
-                    if p.user_id == tgid:
-                        return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+            pcp = await self._get_admin_participant(chat, tgid)
+            return self._has_participant_permission(pcp, permission)
         return False
 
-    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
-        if not await self._can_use_commands(event.to_id, TelegramID(event.from_id)):
+    async def check_can_use_command(self, event: Message, reply: ReplyFunc, command: str) -> bool:
+        if command not in self.required_permissions:
+            # Unknown command
+            return False
+        elif not isinstance(event.from_id, PeerUser):
+            await reply("Channels can't use commands")
+            return False
+        elif not await self._can_use_commands(
+            event.to_id, TelegramID(event.from_id.user_id), self.required_permissions[command]
+        ):
             await reply("You do not have the permission to use that command.")
             return False
         return True
 
     async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> Message:
-        if not config["bridge.relaybot.authless_portals"]:
+        if not self.config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
         if not portal.allow_bridging:
@@ -173,31 +263,85 @@ class Bot(AbstractUser):
         if portal.mxid:
             if portal.username:
                 return await reply(
-                    f"Portal is public: [{portal.alias}](https://matrix.to/#/{portal.alias})")
+                    f"Portal is public: [{portal.alias}](https://matrix.to/#/{portal.alias})"
+                )
             else:
-                return await reply(
-                    "Portal is not public. Use `/invite <mxid>` to get an invite.")
+                return await reply("Portal is not public. Use `/invite <mxid>` to get an invite.")
+        else:
+            return await reply("Couldn't create portal room")
 
-    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc,
-                                    mxid_input: UserID) -> Message:
+    async def handle_command_invite(
+        self, portal: po.Portal, reply: ReplyFunc, mxid_input: UserID
+    ) -> Message:
         if len(mxid_input) == 0:
             return await reply("Usage: `/invite <mxid>`")
         elif not portal.mxid:
-            return await reply("Portal does not have Matrix room. "
-                               "Create one with /portal first.")
-        if mxid_input[0] != '@' or mxid_input.find(':') < 2:
+            return await reply("Portal does not have Matrix room. Create one with /portal first.")
+        if mxid_input[0] != "@" or mxid_input.find(":") < 2:
             return await reply("That doesn't look like a Matrix ID.")
-        user = await u.User.get_by_mxid(mxid_input).ensure_started()
+        user = await u.User.get_and_start_by_mxid(mxid_input)
         if not user.relaybot_whitelisted:
             return await reply("That user is not whitelisted to use the bridge.")
         elif await user.is_logged_in():
-            displayname = f"@{user.username}" if user.username else user.displayname
-            return await reply("That user seems to be logged in. "
-                               f"Just invite [{displayname}](tg://user?id={user.tgid})")
+            displayname = f"@{user.tg_username}" if user.tg_username else user.displayname
+            return await reply(
+                "That user seems to be logged in. "
+                f"Just invite [{displayname}](tg://user?id={user.tgid})"
+            )
         else:
-            await portal.main_intent.invite_user(portal.mxid, user.mxid)
+            try:
+                await portal.invite_to_matrix(user.mxid)
+            except MBadState:
+                try:
+                    await portal.main_intent.unban_user(
+                        portal.mxid, user.mxid, reason="Invited from Telegram"
+                    )
+                except Exception:
+                    return await reply(f"Failed to unban `{user.mxid}` from the portal.")
+                await portal.invite_to_matrix(user.mxid)
+                return await reply(f"Unbanned and invited `{user.mxid}` to the portal.")
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
+    async def handle_command_ban(
+        self,
+        message: Message,
+        portal: po.Portal,
+        reply: ReplyFunc,
+        reason: str,
+        action: Literal["kick", "ban"] = "ban",
+    ) -> Message:
+        if not message.reply_to:
+            return await reply("You must reply to a relaybot message when using that command")
+        reply_to_id = TelegramID(message.reply_to.reply_to_msg_id)
+        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
+        msg = await DBMessage.get_one_by_tgid(reply_to_id, tg_space)
+        if not msg or msg.sender != self.tgid or not msg.sender_mxid:
+            return await reply("Target message is not a relayed message")
+        puppet = await pu.Puppet.get_by_peer(message.from_id)
+        actioned = "Banned" if action == "ban" else "Kicked"
+        try:
+            intent = puppet.intent_for(portal)
+            func: BanFunc = intent.ban_user if action == "ban" else intent.kick_user
+            await func(portal.mxid, msg.sender_mxid, reason)
+        except MForbidden as e:
+            self.log.warning(
+                f"Failed to {action} {msg.sender_mxid} from {portal.mxid} as {puppet.mxid}: {e}, "
+                f"falling back to bridge bot"
+            )
+            reason_prefix = f"{actioned} by {puppet.displayname or puppet.tgid}"
+            reason = f"{reason_prefix}: {reason}" if reason else reason_prefix
+            try:
+                func: BanFunc = (
+                    self.az.intent.ban_user if action == "ban" else self.az.intent.kick_user
+                )
+                await func(portal.mxid, msg.sender_mxid, reason)
+            except MForbidden as e:
+                self.log.warning(
+                    f"Failed to {action} {msg.sender_mxid} from {portal.mxid} as bridge bot: {e}"
+                )
+                return await reply(f"Failed to {action} `{msg.sender_mxid}`")
+        return await reply(f"Successfully {actioned.lower()} `{msg.sender_mxid}`")
+
     @staticmethod
     def handle_command_id(message: Message, reply: ReplyFunc) -> Awaitable[Message]:
         # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
@@ -207,59 +351,56 @@ class Bot(AbstractUser):
         elif isinstance(message.to_id, PeerChat):
             return reply(str(-message.to_id.chat_id))
         elif isinstance(message.to_id, PeerUser):
-            return reply(f"Your user ID is {message.from_id}.")
+            return reply(
+                f"Your user ID is {message.to_id.user_id}.\n\n"
+                f"If you're trying to bridge a group chat to Matrix, you must run the command in "
+                f"the group, not here. **The ID above will not work** with `!tg bridge`."
+            )
         else:
             return reply("Failed to find chat ID.")
 
-    def match_command(self, text: str, command: str) -> bool:
-        text = text.lower()
-        command = f"/{command.lower()}"
-        command_targeted = f"{command}@{self.username.lower()}"
+    def parse_command(self, message: Message) -> tuple[str | None, str | None]:
+        if not message.entities or len(message.entities) < 1 or not message.message:
+            return None, None
+        cmd_entity = message.entities[0]
+        if not isinstance(cmd_entity, MessageEntityBotCommand) or cmd_entity.offset != 0:
+            return None, None
+        surrogated_text = add_surrogate(message.message)
+        command: str = del_surrogate(surrogated_text[: cmd_entity.length]).lower()
+        rest_of_message: str = ""
+        if len(surrogated_text) > cmd_entity.length + 1:
+            rest_of_message: str = del_surrogate(surrogated_text[cmd_entity.length + 1 :])
+        command, *target = command.split("@", 1)
+        if not command.startswith("/"):
+            return None, None
+        elif target and target[0] != self.tg_username.lower():
+            return None, None
+        return command[1:], rest_of_message
 
-        is_plain_command = text == command or text == command_targeted
-        if is_plain_command:
-            return True
-
-        is_arg_command = text.startswith(command + " ") or text.startswith(command_targeted + " ")
-        if is_arg_command:
-            return True
-
-        return False
-
-    async def handle_command(self, message: Message) -> None:
+    async def handle_command(self, message: Message, command: str, args: str) -> None:
         def reply(reply_text: str) -> Awaitable[Message]:
             return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)
 
-        text = message.message
-
-        if self.match_command(text, "start"):
-            pcm = config["bridge.relaybot.private_chat.message"]
+        if command == "start":
+            pcm = self.config["bridge.relaybot.private_chat.message"]
             if pcm:
                 await reply(pcm)
-            return
-        elif self.match_command(text, "id"):
+        elif command == "id":
             await self.handle_command_id(message, reply)
-            return
-        elif message.is_private:
-            return
-
-        portal = po.Portal.get_by_entity(message.to_id)
-
-        is_portal_cmd = self.match_command(text, "portal")
-        is_invite_cmd = self.match_command(text, "invite")
-        if is_portal_cmd or is_invite_cmd:
-            if not await self.check_can_use_commands(message, reply):
+        elif not message.is_private:
+            if not await self.check_can_use_command(message, reply, command):
                 return
-            if is_portal_cmd:
+            portal = await po.Portal.get_by_entity(message.to_id)
+            if command == "portal":
                 await self.handle_command_portal(portal, reply)
-            elif is_invite_cmd:
-                try:
-                    mxid = text[text.index(" ") + 1:]
-                except ValueError:
-                    mxid = ""
-                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
+            elif command == "invite":
+                await self.handle_command_invite(portal, reply, mxid_input=UserID(args))
+            elif command == "mxban":
+                await self.handle_command_ban(message, portal, reply, reason=args)
+            elif command == "mxkick":
+                await self.handle_command_ban(message, portal, reply, reason=args, action="kick")
 
-    def handle_service_message(self, message: MessageService) -> None:
+    async def handle_service_message(self, message: MessageService) -> None:
         to_peer = message.to_id
         if isinstance(to_peer, PeerChannel):
             to_id = TelegramID(to_peer.channel_id)
@@ -272,26 +413,26 @@ class Bot(AbstractUser):
 
         action = message.action
         if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
-            self.add_chat(to_id, chat_type)
+            await self.add_chat(to_id, chat_type)
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
-            self.remove_chat(to_id)
+            await self.remove_chat(to_id)
         elif isinstance(action, MessageActionChatMigrateTo):
-            self.remove_chat(to_id)
-            self.add_chat(TelegramID(action.channel_id), "channel")
+            await self.remove_chat(to_id)
+            await self.add_chat(TelegramID(action.channel_id), "channel")
 
     async def update(self, update) -> bool:
+        if self._login_wait_fut:
+            await self._login_wait_fut
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
             return False
         if isinstance(update.message, MessageService):
-            self.handle_service_message(update.message)
+            await self.handle_service_message(update.message)
             return False
 
-        is_command = (isinstance(update.message, Message)
-                      and update.message.entities and len(update.message.entities) > 0
-                      and isinstance(update.message.entities[0], MessageEntityBotCommand)
-                      and update.message.entities[0].offset == 0)
-        if is_command:
-            await self.handle_command(update.message)
+        if isinstance(update.message, Message):
+            command, args = self.parse_command(update.message)
+            if command:
+                await self.handle_command(update.message, command, args)
         return False
 
     def is_in_chat(self, peer_id) -> bool:
@@ -300,12 +441,3 @@ class Bot(AbstractUser):
     @property
     def name(self) -> str:
         return "bot"
-
-
-def init(cfg: 'Config') -> Optional[Bot]:
-    global config
-    config = cfg
-    token = config["telegram.bot_token"]
-    if token and not token.lower().startswith("disable"):
-        return Bot(token)
-    return None

mautrix_telegram/commands/__init__.py

@@ -1,8 +1,26 @@
-from .handler import (command_handler, CommandHandler, CommandProcessor, CommandEvent,
-                      SECTION_AUTH, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT,
-                      SECTION_MISC, SECTION_ADMIN)
-from . import portal, telegram, matrix_auth, manhole
+from .handler import (
+    SECTION_ADMIN,
+    SECTION_AUTH,
+    SECTION_CREATING_PORTALS,
+    SECTION_MISC,
+    SECTION_PORTAL_MANAGEMENT,
+    CommandEvent,
+    CommandHandler,
+    CommandProcessor,
+    command_handler,
+)
 
-__all__ = ["command_handler", "CommandHandler", "CommandProcessor", "CommandEvent",
-           "SECTION_AUTH", "SECTION_MISC", "SECTION_ADMIN", "SECTION_CREATING_PORTALS",
-           "SECTION_PORTAL_MANAGEMENT"]
+# This has to happen after the handler imports
+from . import matrix_auth, portal, telegram  # isort: skip
+
+__all__ = [
+    "command_handler",
+    "CommandHandler",
+    "CommandProcessor",
+    "CommandEvent",
+    "SECTION_AUTH",
+    "SECTION_MISC",
+    "SECTION_ADMIN",
+    "SECTION_CREATING_PORTALS",
+    "SECTION_PORTAL_MANAGEMENT",
+]

mautrix_telegram/commands/handler.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,19 +13,27 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-"""This module contains classes handling commands issued by Matrix users."""
-from typing import Awaitable, Callable, List, Optional, NamedTuple, Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, NamedTuple
 
 from telethon.errors import FloodWaitError
 
-from mautrix.types import RoomID, EventID, MessageEventContent
-from mautrix.bridge.commands import (HelpSection, CommandEvent as BaseCommandEvent,
-                                     CommandHandler as BaseCommandHandler,
-                                     CommandProcessor as BaseCommandProcessor,
-                                     CommandHandlerFunc, command_handler as base_command_handler)
+from mautrix.bridge.commands import (
+    CommandEvent as BaseCommandEvent,
+    CommandHandler as BaseCommandHandler,
+    CommandHandlerFunc,
+    CommandProcessor as BaseCommandProcessor,
+    HelpSection,
+    command_handler as base_command_handler,
+)
+from mautrix.types import EventID, MessageEventContent, RoomID
+from mautrix.util.format_duration import format_duration
 
-from ..util import format_duration
-from .. import user as u, context as c, portal as po
+from .. import portal as po, user as u
+
+if TYPE_CHECKING:
+    from ..__main__ import TelegramBridge
 
 
 class HelpCacheKey(NamedTuple):
@@ -48,11 +56,31 @@ class CommandEvent(BaseCommandEvent):
     sender: u.User
     portal: po.Portal
 
-    def __init__(self, processor: 'CommandProcessor', room_id: RoomID, event_id: EventID,
-                 sender: u.User, command: str, args: List[str], content: MessageEventContent,
-                 portal: Optional['po.Portal'], is_management: bool, has_bridge_bot: bool) -> None:
-        super().__init__(processor, room_id, event_id, sender, command, args, content,
-                         portal, is_management, has_bridge_bot)
+    def __init__(
+        self,
+        processor: CommandProcessor,
+        room_id: RoomID,
+        event_id: EventID,
+        sender: u.User,
+        command: str,
+        args: list[str],
+        content: MessageEventContent,
+        portal: po.Portal | None,
+        is_management: bool,
+        has_bridge_bot: bool,
+    ) -> None:
+        super().__init__(
+            processor,
+            room_id,
+            event_id,
+            sender,
+            command,
+            args,
+            content,
+            portal,
+            is_management,
+            has_bridge_bot,
+        )
         self.bridge = processor.bridge
         self.tgbot = processor.tgbot
         self.config = processor.config
@@ -63,9 +91,14 @@ class CommandEvent(BaseCommandEvent):
         return self.sender.is_admin
 
     async def get_help_key(self) -> HelpCacheKey:
-        return HelpCacheKey(self.is_management, self.portal is not None,
-                            self.sender.puppet_whitelisted, self.sender.matrix_puppet_whitelisted,
-                            self.sender.is_admin, await self.sender.is_logged_in())
+        return HelpCacheKey(
+            self.is_management,
+            self.portal is not None,
+            self.sender.puppet_whitelisted,
+            self.sender.matrix_puppet_whitelisted,
+            self.sender.is_admin,
+            await self.sender.is_logged_in(),
+        )
 
 
 class CommandHandler(BaseCommandHandler):
@@ -74,49 +107,87 @@ class CommandHandler(BaseCommandHandler):
     needs_puppeting: bool
     needs_matrix_puppeting: bool
 
-    def __init__(self, handler: Callable[[CommandEvent], Awaitable[EventID]],
-                 management_only: bool, name: str, help_text: str, help_args: str,
-                 help_section: HelpSection, needs_auth: bool, needs_puppeting: bool,
-                 needs_matrix_puppeting: bool, needs_admin: bool) -> None:
-        super().__init__(handler, management_only, name, help_text, help_args, help_section,
-                         needs_auth=needs_auth, needs_puppeting=needs_puppeting,
-                         needs_matrix_puppeting=needs_matrix_puppeting, needs_admin=needs_admin)
+    def __init__(
+        self,
+        handler: Callable[[CommandEvent], Awaitable[EventID]],
+        management_only: bool,
+        name: str,
+        help_text: str,
+        help_args: str,
+        help_section: HelpSection,
+        needs_auth: bool,
+        needs_puppeting: bool,
+        needs_matrix_puppeting: bool,
+        needs_admin: bool,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            handler,
+            management_only,
+            name,
+            help_text,
+            help_args,
+            help_section,
+            needs_auth=needs_auth,
+            needs_puppeting=needs_puppeting,
+            needs_matrix_puppeting=needs_matrix_puppeting,
+            needs_admin=needs_admin,
+            **kwargs,
+        )
 
-    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+    async def get_permission_error(self, evt: CommandEvent) -> str | None:
         if self.needs_puppeting and not evt.sender.puppet_whitelisted:
-            return "This command requires puppeting privileges."
+            return "That command is limited to users with puppeting privileges."
         elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
-            return "This command requires Matrix puppeting privileges."
+            return "That command is limited to users with full puppeting privileges."
         return await super().get_permission_error(evt)
 
     def has_permission(self, key: HelpCacheKey) -> bool:
-        return (super().has_permission(key) and
-                (not self.needs_puppeting or key.puppet_whitelisted) and
-                (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted))
+        return (
+            super().has_permission(key)
+            and (not self.needs_puppeting or key.puppet_whitelisted)
+            and (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted)
+        )
 
 
-def command_handler(_func: Optional[CommandHandlerFunc] = None, *, needs_auth: bool = True,
-                    needs_puppeting: bool = True, needs_matrix_puppeting: bool = False,
-                    needs_admin: bool = False, management_only: bool = False,
-                    name: Optional[str] = None, help_text: str = "", help_args: str = "",
-                    help_section: HelpSection = None) -> Callable[[CommandHandlerFunc],
-                                                                  CommandHandler]:
+def command_handler(
+    _func: CommandHandlerFunc | None = None,
+    *,
+    needs_auth: bool = True,
+    needs_puppeting: bool = True,
+    needs_matrix_puppeting: bool = False,
+    needs_admin: bool = False,
+    management_only: bool = False,
+    name: str | None = None,
+    help_text: str = "",
+    help_args: str = "",
+    help_section: HelpSection = None,
+) -> Callable[[CommandHandlerFunc], CommandHandler]:
     return base_command_handler(
-        _func, _handler_class=CommandHandler, name=name, help_text=help_text, help_args=help_args,
-        help_section=help_section, management_only=management_only, needs_auth=needs_auth,
-        needs_admin=needs_admin, needs_puppeting=needs_puppeting,
-        needs_matrix_puppeting=needs_matrix_puppeting)
+        _func,
+        _handler_class=CommandHandler,
+        name=name,
+        help_text=help_text,
+        help_args=help_args,
+        help_section=help_section,
+        management_only=management_only,
+        needs_auth=needs_auth,
+        needs_admin=needs_admin,
+        needs_puppeting=needs_puppeting,
+        needs_matrix_puppeting=needs_matrix_puppeting,
+    )
 
 
 class CommandProcessor(BaseCommandProcessor):
-    def __init__(self, context: c.Context) -> None:
-        super().__init__(event_class=CommandEvent, bridge=context.bridge)
-        self.tgbot = context.bot
-        self.public_website = context.public_website
+    def __init__(self, bridge: "TelegramBridge") -> None:
+        super().__init__(event_class=CommandEvent, bridge=bridge)
+        self.tgbot = bridge.bot
+        self.public_website = bridge.public_website
 
     @staticmethod
-    async def _run_handler(handler: Callable[[CommandEvent], Awaitable[Any]], evt: CommandEvent
-                           ) -> Any:
+    async def _run_handler(
+        handler: Callable[[CommandEvent], Awaitable[Any]], evt: CommandEvent
+    ) -> Any:
         try:
             return await handler(evt)
         except FloodWaitError as e:

mautrix_telegram/commands/manhole.py

@@ -1,128 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Set, Callable
-import asyncio
-import sys
-import os
-
-from attr import dataclass
-
-from telethon import __version__ as __telethon_version__
-
-from mautrix import __version__ as __mautrix_version__
-from mautrix.types import UserID
-from mautrix.errors import MatrixConnectionError
-from mautrix.util.manhole import start_manhole
-
-from .. import __version__
-from . import command_handler, CommandEvent, SECTION_ADMIN
-
-
-@dataclass
-class ManholeState:
-    server: asyncio.AbstractServer
-    opened_by: UserID
-    close: Callable[[], None]
-    whitelist: Set[int]
-
-
-@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
-                 help_text="Open a manhole into the bridge.", help_args="<_uid..._>")
-async def open_manhole(evt: CommandEvent) -> None:
-    if not evt.config["manhole.enabled"]:
-        await evt.reply("The manhole has been disabled in the config.")
-        return
-    elif len(evt.args) == 0:
-        await evt.reply("**Usage:** `$cmdprefix+sp open-manhole <uid...>`")
-        return
-
-    whitelist = set()
-    whitelist_whitelist = evt.config["manhole.whitelist"]
-    for arg in evt.args:
-        try:
-            uid = int(arg)
-        except ValueError:
-            await evt.reply(f"{arg} is not an integer.")
-            return
-        if whitelist_whitelist and uid not in whitelist_whitelist:
-            await evt.reply(f"{uid} is not in the list of allowed UIDs.")
-            return
-        whitelist.add(uid)
-
-    if evt.bridge.manhole:
-        added = [uid for uid in whitelist
-                 if uid not in evt.bridge.manhole.whitelist]
-        evt.bridge.manhole.whitelist |= set(added)
-        if len(added) == 0:
-            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
-                            " and all the given UIDs are already whitelisted.")
-        else:
-            added_str = (f"{', '.join(str(uid) for uid in added[:-1])} and {added[-1]}"
-                         if len(added) > 1 else added[0])
-            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
-                            f". Added {added_str} to the whitelist.")
-            evt.log.info(f"{evt.sender.mxid} added {added_str} to the manhole whitelist.")
-        return
-
-    from ..portal import Portal
-    from ..puppet import Puppet
-    from ..user import User
-    namespace = {
-        "bridge": evt.bridge,
-        "User": User,
-        "Portal": Portal,
-        "Puppet": Puppet,
-    }
-    banner = (f"Python {sys.version} on {sys.platform}\n"
-              f"mautrix-telegram {__version__} with mautrix-python {__mautrix_version__} "
-              f"and Telethon {__telethon_version__}\n\nManhole opened by {evt.sender.mxid}\n")
-    path = evt.config["manhole.path"]
-
-    wl_list = list(whitelist)
-    whitelist_str = (f"{', '.join(str(uid) for uid in wl_list[:-1])} and {wl_list[-1]}"
-                     if len(wl_list) > 1 else wl_list[0])
-    evt.log.info(f"{evt.sender.mxid} opened a manhole with {whitelist_str} whitelisted.")
-    server, close = await start_manhole(path=path, banner=banner, namespace=namespace,
-                                        loop=evt.loop, whitelist=whitelist)
-    evt.bridge.manhole = ManholeState(server=server, opened_by=evt.sender.mxid, close=close,
-                                      whitelist=whitelist)
-    plrl = "s" if len(whitelist) != 1 else ""
-    await evt.reply(f"Opened manhole at unix://{path} with UID{plrl} {whitelist_str} whitelisted")
-    await server.wait_closed()
-    evt.bridge.manhole = None
-    try:
-        os.unlink(path)
-    except FileNotFoundError:
-        pass
-    evt.log.info(f"{evt.sender.mxid}'s manhole was closed.")
-    try:
-        await evt.reply("Your manhole was closed.")
-    except (AttributeError, MatrixConnectionError) as e:
-        evt.log.warning(f"Failed to send manhole close notification: {e}")
-
-
-@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
-                 help_text="Close an open manhole.")
-async def close_manhole(evt: CommandEvent) -> None:
-    if not evt.bridge.manhole:
-        await evt.reply("There is no open manhole.")
-        return
-
-    opened_by = evt.bridge.manhole.opened_by
-    evt.bridge.manhole.close()
-    evt.bridge.manhole = None
-    if opened_by != evt.sender.mxid:
-        await evt.reply(f"Closed manhole opened by {opened_by}")

mautrix_telegram/commands/matrix_auth.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,33 +13,27 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.types import EventID
 from mautrix.bridge import InvalidAccessToken, OnlyLoginSelf
+from mautrix.types import EventID
 
-from . import command_handler, CommandEvent, SECTION_AUTH
 from .. import puppet as pu
+from . import SECTION_AUTH, CommandEvent, command_handler
 
 
-@command_handler(needs_auth=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH, help_text="Revert your Telegram account's Matrix "
-                                                      "puppet to use the default Matrix account.")
-async def logout_matrix(evt: CommandEvent) -> EventID:
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if not puppet.is_real_user:
-        return await evt.reply("You are not logged in with your Matrix account.")
-    await puppet.switch_mxid(None, None)
-    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
-
-
-@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
-                           "account.")
+@command_handler(
+    needs_auth=True,
+    management_only=True,
+    needs_matrix_puppeting=True,
+    help_section=SECTION_AUTH,
+    help_text="Replace your Telegram account's Matrix puppet with your own Matrix account.",
+)
 async def login_matrix(evt: CommandEvent) -> EventID:
-    puppet = pu.Puppet.get(evt.sender.tgid)
+    puppet = await pu.Puppet.get_by_tgid(evt.sender.tgid)
     if puppet.is_real_user:
-        return await evt.reply("You have already logged in with your Matrix account. "
-                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+        return await evt.reply(
+            "You have already logged in with your Matrix account. "
+            "Log out with `$cmdprefix+sp logout-matrix` first."
+        )
     allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
     if allow_matrix_login:
         evt.sender.command_status = {
@@ -57,57 +51,35 @@ async def login_matrix(evt: CommandEvent) -> EventID:
                 "here.\n"
                 f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
                 "Logging in outside of Matrix is recommended, because in-Matrix login would save "
-                "your access token in the message history.")
-        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
-                               f"Please visit [the login page]({url}) to log in.")
+                "your access token in the message history."
+            )
+        return await evt.reply(
+            "This bridge instance does not allow logging in inside Matrix.\n\n"
+            f"Please visit [the login page]({url}) to log in."
+        )
     elif allow_matrix_login:
         return await evt.reply(
             "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your Matrix access token here to log in.")
+            "Please send your Matrix access token here to log in."
+        )
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 
-@command_handler(needs_auth=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Pings the server with the stored matrix authentication.")
-async def ping_matrix(evt: CommandEvent) -> EventID:
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if not puppet.is_real_user:
-        return await evt.reply("You are not logged in with your Matrix account.")
-    try:
-        await puppet.start()
-    except InvalidAccessToken:
-        return await evt.reply("Your access token is invalid.")
-    return await evt.reply("Your Matrix login is working.")
-
-
-@command_handler(needs_auth=True, needs_matrix_puppeting=True, help_section=SECTION_AUTH,
-                 help_text="Clear the Matrix sync token stored for your custom puppet.")
-async def clear_cache_matrix(evt: CommandEvent) -> EventID:
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if not puppet.is_real_user:
-        return await evt.reply("You are not logged in with your Matrix account.")
-    try:
-        puppet.stop()
-        puppet.next_batch = None
-        await puppet.start()
-    except InvalidAccessToken:
-        return await evt.reply("Your access token is invalid.")
-    return await evt.reply("Cleared cache successfully.")
-
-
 async def enter_matrix_token(evt: CommandEvent) -> EventID:
     evt.sender.command_status = None
 
-    puppet = pu.Puppet.get(evt.sender.tgid)
+    puppet = await pu.Puppet.get_by_tgid(evt.sender.tgid)
     if puppet.is_real_user:
-        return await evt.reply("You have already logged in with your Matrix account. "
-                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+        return await evt.reply(
+            "You have already logged in with your Matrix account. "
+            "Log out with `$cmdprefix+sp logout-matrix` first."
+        )
     try:
         await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
     except OnlyLoginSelf:
         return await evt.reply("You can only log in as your own Matrix user.")
     except InvalidAccessToken:
         return await evt.reply("Failed to verify access token.")
-    return await evt.reply("Replaced your Telegram account's Matrix puppet "
-                           f"with {puppet.custom_mxid}.")
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}."
+    )

mautrix_telegram/commands/portal/admin.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,13 +18,16 @@ import asyncio
 from mautrix.types import EventID
 
 from ... import portal as po, puppet as pu, user as u
-from .. import command_handler, CommandEvent, SECTION_ADMIN
+from .. import SECTION_ADMIN, CommandEvent, command_handler
 
 
-@command_handler(needs_admin=True, needs_auth=False,
-                 help_section=SECTION_ADMIN,
-                 help_args="<`portal`|`puppet`|`user`>",
-                 help_text="Clear internal bridge caches")
+@command_handler(
+    needs_admin=True,
+    needs_auth=False,
+    help_section=SECTION_ADMIN,
+    help_args="<`portal`|`puppet`|`user`>",
+    help_text="Clear internal bridge caches",
+)
 async def clear_db_cache(evt: CommandEvent) -> EventID:
     try:
         section = evt.args[0].lower()
@@ -35,41 +38,43 @@ async def clear_db_cache(evt: CommandEvent) -> EventID:
         po.Portal.by_mxid = {}
         await evt.reply("Cleared portal cache")
     elif section == "puppet":
-        pu.Puppet.cache = {}
+        pu.Puppet.by_tgid = {}
         for puppet in pu.Puppet.by_custom_mxid.values():
-            puppet.sync_task.cancel()
+            puppet.stop()
         pu.Puppet.by_custom_mxid = {}
-        await asyncio.gather(*[puppet.try_start() for puppet in pu.Puppet.all_with_custom_mxid()],
-                             loop=evt.loop)
+        await asyncio.gather(
+            *[puppet.try_start() async for puppet in pu.Puppet.all_with_custom_mxid()]
+        )
         await evt.reply("Cleared puppet cache and restarted custom puppet syncers")
     elif section == "user":
-        u.User.by_mxid = {
-            user.mxid: user
-            for user in u.User.by_tgid.values()
-        }
+        u.User.by_mxid = {user.mxid: user for user in u.User.by_tgid.values()}
         await evt.reply("Cleared non-logged-in user cache")
     else:
         return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
 
 
-@command_handler(needs_admin=True, needs_auth=False,
-                 help_section=SECTION_ADMIN,
-                 help_args="[_mxid_]",
-                 help_text="Reload and reconnect a user")
+@command_handler(
+    needs_admin=True,
+    needs_auth=False,
+    help_section=SECTION_ADMIN,
+    help_args="[_mxid_]",
+    help_text="Reload and reconnect a user",
+)
 async def reload_user(evt: CommandEvent) -> EventID:
     if len(evt.args) > 0:
         mxid = evt.args[0]
     else:
         mxid = evt.sender.mxid
-    user = u.User.get_by_mxid(mxid, create=False)
+    user = await u.User.get_by_mxid(mxid, create=False)
     if not user:
         return await evt.reply("User not found")
     puppet = await pu.Puppet.get_by_custom_mxid(mxid)
     if puppet:
-        puppet.sync_task.cancel()
+        puppet.stop()
     await user.stop()
-    user.delete(delete_db=False)
-    user = u.User.get_by_mxid(mxid)
+    del u.User.by_tgid[user.tgid]
+    del u.User.by_mxid[user.mxid]
+    user = await u.User.get_by_mxid(mxid)
     await user.ensure_started()
     if puppet:
         await puppet.start()

mautrix_telegram/commands/portal/bridge.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,29 +13,36 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Tuple, Awaitable
+from __future__ import annotations
+
+from typing import Awaitable
 import asyncio
 
-from telethon.tl.types import ChatForbidden, ChannelForbidden
+from telethon.tl.types import ChannelForbidden, ChatForbidden
 
 from mautrix.types import EventID, RoomID
 
-from ...types import TelegramID
 from ... import portal as po
-from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
-from .util import user_has_power_level, get_initial_state
+from ...types import TelegramID
+from .. import SECTION_CREATING_PORTALS, CommandEvent, command_handler
+from .util import get_initial_state, user_has_power_level, warn_missing_power
 
 
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_CREATING_PORTALS,
-                 help_args="[_id_]",
-                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
-                           "ID. The ID must be the prefixed version that you get with the `/id` "
-                           "command of the Telegram-side bot.")
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="[_id_]",
+    help_text=(
+        "Bridge the current Matrix room to the Telegram chat with the given ID. The ID must be "
+        "the prefixed version that you get with the `/id` command of the Telegram-side bot."
+    ),
+)
 async def bridge(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
-        return await evt.reply("**Usage:** "
-                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`"
+        )
     force_use_bot = False
     if evt.args[0] == "--usebot" and evt.sender.is_admin:
         force_use_bot = True
@@ -43,7 +50,7 @@ async def bridge(evt: CommandEvent) -> EventID:
     room_id = RoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
     that_this = "This" if room_id == evt.room_id else "That"
 
-    portal = po.Portal.get_by_mxid(room_id)
+    portal = await po.Portal.get_by_mxid(room_id)
     if portal:
         return await evt.reply(f"{that_this} room is already a portal room.")
 
@@ -52,128 +59,183 @@ async def bridge(evt: CommandEvent) -> EventID:
 
     # The /id bot command provides the prefixed ID, so we assume
     tgid_str = evt.args[0]
-    if tgid_str.startswith("-100"):
-        tgid = TelegramID(int(tgid_str[4:]))
-        peer_type = "channel"
-    elif tgid_str.startswith("-"):
-        tgid = TelegramID(-int(tgid_str))
-        peer_type = "chat"
-    else:
-        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
-                               "If you did not get the ID using the `/id` bot command, please "
-                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
-                               "Bridging private chats to existing rooms is not allowed.")
+    tgid = None
+    try:
+        if tgid_str.startswith("-100"):
+            tgid = TelegramID(int(tgid_str[4:]))
+            peer_type = "channel"
+        elif tgid_str.startswith("-"):
+            tgid = TelegramID(-int(tgid_str))
+            peer_type = "chat"
+    except ValueError:
+        # Invalid integer
+        pass
+    if not tgid:
+        return await evt.reply(
+            "That doesn't seem like a prefixed Telegram chat ID.\n\n"
+            "If you did not get the ID using the `/id` bot command, please prefix"
+            "channel/supergroup IDs with `-100` and non-super group IDs with `-`.\n\n"
+            "Bridging private chats to existing rooms is not allowed."
+        )
 
-    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    portal = await po.Portal.get_by_tgid(tgid, peer_type=peer_type)
     if not portal.allow_bridging:
-        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
-                               "If you're the bridge admin, try "
-                               "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
-    if portal.mxid:
+        return await evt.reply(
+            "This bridge doesn't allow bridging that Telegram chat.\n"
+            "If you're the bridge admin, try "
+            "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first."
+        )
+    elif portal.mxid:
         has_portal_message = (
             "That Telegram chat already has a portal at "
-            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). "
+        )
         if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-            return await evt.reply(f"{has_portal_message}"
-                                   "Additionally, you do not have the permissions to unbridge "
-                                   "that room.")
+            return await evt.reply(
+                f"{has_portal_message}"
+                "Additionally, you do not have the permissions to unbridge that room."
+            )
         evt.sender.command_status = {
             "next": confirm_bridge,
             "action": "Room bridging",
             "mxid": portal.mxid,
             "bridge_to_mxid": room_id,
             "tgid": portal.tgid,
-            "peer_type": portal.peer_type,
+            "peer_type": peer_type,
             "force_use_bot": force_use_bot,
         }
-        return await evt.reply(f"{has_portal_message}"
-                               "However, you have the permissions to unbridge that room.\n\n"
-                               "To delete that portal completely and continue bridging, use "
-                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
-                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
-                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
+        return await evt.reply(
+            f"{has_portal_message}"
+            "However, you have the permissions to unbridge that room.\n\n"
+            "To delete that portal completely and continue bridging, use "
+            "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+            "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+            "continue`. To cancel, use `$cmdprefix+sp cancel`"
+        )
     evt.sender.command_status = {
         "next": confirm_bridge,
         "action": "Room bridging",
         "bridge_to_mxid": room_id,
         "tgid": portal.tgid,
-        "peer_type": portal.peer_type,
+        "peer_type": peer_type,
         "force_use_bot": force_use_bot,
     }
-    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
-                           "chat to this room, use `$cmdprefix+sp continue`")
+    return await evt.reply(
+        "That Telegram chat has no existing portal. To confirm bridging the "
+        "chat to this room, use `$cmdprefix+sp continue`"
+    )
 
 
-async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
-                                            ) -> Tuple[bool, Optional[Awaitable[None]]]:
+async def cleanup_old_portal_while_bridging(
+    evt: CommandEvent, portal: po.Portal
+) -> tuple[bool, Awaitable[None] | None]:
     if not portal.mxid:
-        await evt.reply("The portal seems to have lost its Matrix room between you"
-                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                        "Continuing without touching previous Matrix room...")
+        await evt.reply(
+            "The portal seems to have lost its Matrix room between you"
+            "calling `$cmdprefix+sp bridge` and this command.\n\n"
+            "Continuing without touching previous Matrix room..."
+        )
         return True, None
     elif evt.args[0] == "delete-and-continue":
         return True, portal.cleanup_portal("Portal deleted (moving to another room)", delete=False)
     elif evt.args[0] == "unbridge-and-continue":
-        return True, portal.cleanup_portal("Room unbridged (portal moving to another room)",
-                                           puppets_only=True, delete=False)
+        return True, portal.cleanup_portal(
+            "Room unbridged (portal moving to another room)", puppets_only=True, delete=False
+        )
     else:
         await evt.reply(
             "The chat you were trying to bridge already has a Matrix portal room.\n\n"
             "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
             "continue` to either delete or unbridge the existing room (respectively) and "
             "continue with the bridging.\n\n"
-            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel."
+        )
         return False, None
 
 
-async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
+async def confirm_bridge(evt: CommandEvent) -> EventID | None:
     status = evt.sender.command_status
     try:
-        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        portal = await po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
         bridge_to_mxid = status["bridge_to_mxid"]
     except KeyError:
         evt.sender.command_status = None
-        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
-                               "This shouldn't happen unless you're messing with the command "
-                               "handler code.")
+        return await evt.reply(
+            "Fatal error: tgid or peer_type missing from command_status. "
+            "This shouldn't happen unless you're messing with the command handler code."
+        )
 
     is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
 
     if "mxid" in status:
+        if portal.peer_type != status["peer_type"]:
+            evt.log.warning(
+                "Portal %d in database has mismatching peer type %s (expected %s),"
+                " trusting database as a room already existed",
+                portal.tgid,
+                portal.peer_type,
+                status["peer_type"],
+            )
+            await evt.reply(
+                "Mismatching peer type in command and portal table, "
+                "trusting portal as room already existed"
+            )
         ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
         if not ok:
             return None
         elif coro:
-            asyncio.ensure_future(coro, loop=evt.loop)
+            asyncio.create_task(coro)
             await evt.reply("Cleaning up previous portal room...")
     elif portal.mxid:
         evt.sender.command_status = None
-        return await evt.reply("The portal seems to have created a Matrix room between you "
-                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                               "Please start over by calling the bridge command again.")
+        return await evt.reply(
+            "The portal seems to have created a Matrix room between you "
+            "calling `$cmdprefix+sp bridge` and this command.\n\n"
+            "Please start over by calling the bridge command again."
+        )
     elif evt.args[0] != "continue":
-        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
-                               "`$cmdprefix+sp cancel` to cancel.")
+        return await evt.reply(
+            "Please use `$cmdprefix+sp continue` to confirm the bridging or "
+            "`$cmdprefix+sp cancel` to cancel."
+        )
+    elif portal.peer_type != status["peer_type"]:
+        evt.log.warning(
+            "Portal %d in database has mismatching peer type %s (expected %s),"
+            " trusting new peer type as there's no existing room",
+            portal.tgid,
+            portal.peer_type,
+            status["peer_type"],
+        )
+        await evt.reply(
+            "Mismatching peer type in command and portal table, "
+            "trusting you as portal room doesn't exist"
+        )
+        portal.peer_type = status["peer_type"]
 
     evt.sender.command_status = None
     async with portal._room_create_lock:
-        await _locked_confirm_bridge(evt, portal=portal, room_id=bridge_to_mxid,
-                                     is_logged_in=is_logged_in)
+        await _locked_confirm_bridge(
+            evt, portal=portal, room_id=bridge_to_mxid, is_logged_in=is_logged_in
+        )
 
 
-async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id: RoomID,
-                                 is_logged_in: bool) -> Optional[EventID]:
+async def _locked_confirm_bridge(
+    evt: CommandEvent, portal: po.Portal, room_id: RoomID, is_logged_in: bool
+) -> EventID | None:
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)
     except Exception:
         evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
         if is_logged_in:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You are logged in, are you in that chat?")
+            return await evt.reply(
+                "Failed to get info of telegram chat. You are logged in, are you in that chat?"
+            )
         else:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You're not logged in, is the relay bot in the chat?")
+            return await evt.reply(
+                "Failed to get info of telegram chat. "
+                "You're not logged in, is the relay bot in the chat?"
+            )
     if isinstance(entity, (ChatForbidden, ChannelForbidden)):
         if is_logged_in:
             return await evt.reply("You don't seem to be in that chat.")
@@ -182,12 +244,15 @@ async def _locked_confirm_bridge(evt: CommandEvent, portal: 'po.Portal', room_id
 
     portal.mxid = room_id
     portal.by_mxid[portal.mxid] = portal
-    (portal.title, portal.about, levels,
-     portal.encrypted) = await get_initial_state(evt.az.intent, evt.room_id)
+    (portal.title, portal.about, levels, portal.encrypted) = await get_initial_state(
+        evt.az.intent, evt.room_id
+    )
     portal.photo_id = ""
     await portal.save()
+    await portal.update_bridge_info()
 
-    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct=False, levels=levels),
-                          loop=evt.loop)
+    asyncio.create_task(portal.update_matrix_room(user, entity, levels=levels))
+
+    await warn_missing_power(levels, evt)
 
     return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,21 +13,27 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Any
+from __future__ import annotations
+
+from typing import Any, Awaitable
 from io import StringIO
 
 from ruamel.yaml import YAMLError
 
-from mautrix.util.config import yaml
 from mautrix.types import EventID
+from mautrix.util.config import yaml
 
 from ... import portal as po, util
-from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+from .. import SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
 
 
-@command_handler(needs_auth=False, help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="View or change per-portal settings.",
-                 help_args="<`help`|_subcommand_> [...]")
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="View or change per-portal settings.",
+    help_args="<`help`|_subcommand_> [...]",
+)
 async def config(evt: CommandEvent) -> None:
     cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
     if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
@@ -37,7 +43,7 @@ async def config(evt: CommandEvent) -> None:
         await config_defaults(evt)
         return
 
-    portal = po.Portal.get_by_mxid(evt.room_id)
+    portal = await po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         await evt.reply("This is not a portal room.")
         return
@@ -67,7 +73,8 @@ async def config(evt: CommandEvent) -> None:
 
 
 def config_help(evt: CommandEvent) -> Awaitable[EventID]:
-    return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
+    return evt.reply(
+        """**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
 
 * **help** - View this help text.
 * **view** - View the current config data.
@@ -76,7 +83,8 @@ def config_help(evt: CommandEvent) -> Awaitable[EventID]:
 * **unset** <_key_> - Remove a config value.
 * **add** <_key_> <_value_> - Add a value to an array.
 * **del** <_key_> <_value_> - Remove a value from an array.
-""")
+"""
+    )
 
 
 def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[EventID]:
@@ -84,18 +92,20 @@ def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[EventID]:
 
 
 def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
-    value = _str_value({
-        "bridge_notices": {
-            "default": evt.config["bridge.bridge_notices.default"],
-            "exceptions": evt.config["bridge.bridge_notices.exceptions"],
-        },
-        "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
-        "inline_images": evt.config["bridge.inline_images"],
-        "message_formats": evt.config["bridge.message_formats"],
-        "emote_format": evt.config["bridge.emote_format"],
-        "state_event_formats": evt.config["bridge.state_event_formats"],
-        "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
-    })
+    value = _str_value(
+        {
+            "bridge_notices": {
+                "default": evt.config["bridge.bridge_notices.default"],
+                "exceptions": evt.config["bridge.bridge_notices.exceptions"],
+            },
+            "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
+            "caption_in_message": evt.config["bridge.caption_in_message"],
+            "message_formats": evt.config["bridge.message_formats"],
+            "emote_format": evt.config["bridge.emote_format"],
+            "state_event_formats": evt.config["bridge.state_event_formats"],
+            "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
+        }
+    )
     return evt.reply(f"Bridge instance wide config:\n{value.rstrip()}")
 
 
@@ -115,8 +125,7 @@ def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: Any) -> Aw
     elif util.recursive_set(portal.local_config, key, value):
         return evt.reply(f"Successfully set the value of `{key}` to {_str_value(value)}".rstrip())
     else:
-        return evt.reply(f"Failed to set value of `{key}`. "
-                         "Does the path contain non-map types?")
+        return evt.reply(f"Failed to set value of `{key}`. Does the path contain non-map types?")
 
 
 def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[EventID]:
@@ -128,15 +137,17 @@ def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Ev
         return evt.reply(f"`{key}` not found in config.")
 
 
-def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
-                   ) -> Awaitable[EventID]:
+def config_add_del(
+    evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
+) -> Awaitable[EventID]:
     if not key or value is None:
         return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
 
     arr = util.recursive_get(portal.local_config, key)
     if not arr:
-        return evt.reply(f"`{key}` not found in config. "
-                         f"Maybe do `$cmdprefix+sp config set {key} []` first?")
+        return evt.reply(
+            f"`{key}` not found in config. Maybe do `$cmdprefix+sp config set {key} []` first?"
+        )
     elif not isinstance(arr, list):
         return evt.reply("`{key}` does not seem to be an array.")
     elif cmd == "add":

mautrix_telegram/commands/portal/create_chat.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,26 +13,32 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
 from mautrix.types import EventID
 
 from ... import portal as po
 from ...types import TelegramID
-from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
-from .util import user_has_power_level, get_initial_state
+from .. import SECTION_CREATING_PORTALS, CommandEvent, command_handler
+from .util import get_initial_state, user_has_power_level, warn_missing_power
 
 
-@command_handler(help_section=SECTION_CREATING_PORTALS,
-                 help_args="[_type_]",
-                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
-                           "The type is either `group`, `supergroup` or `channel` (defaults to "
-                           "`supergroup`).")
+@command_handler(
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="[_type_]",
+    help_text=(
+        "Create a Telegram chat of the given type for the current Matrix room. "
+        "The type is either `group`, `supergroup` or `channel` (defaults to `supergroup`)."
+    ),
+)
 async def create(evt: CommandEvent) -> EventID:
     type = evt.args[0] if len(evt.args) > 0 else "supergroup"
     if type not in ("chat", "group", "supergroup", "channel"):
         return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`"
+        )
 
-    if po.Portal.get_by_mxid(evt.room_id):
+    if await po.Portal.get_by_mxid(evt.room_id):
         return await evt.reply("This is already a portal room.")
 
     if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
@@ -50,17 +56,28 @@ async def create(evt: CommandEvent) -> EventID:
         "group": "chat",
     }[type]
 
-    portal = po.Portal(tgid=TelegramID(0), peer_type=type, mxid=evt.room_id,
-                       title=title, about=about, encrypted=encrypted)
-    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender)
+    portal = po.Portal(
+        tgid=TelegramID(0),
+        tg_receiver=TelegramID(0),
+        peer_type=type,
+        mxid=evt.room_id,
+        title=title,
+        about=about,
+        encrypted=encrypted,
+    )
+    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender, pre_create=True)
     if len(errors) > 0:
         error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
-        await evt.reply(f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
-                        "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
-                        "those users.")
+        await evt.reply(
+            f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
+            "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
+            "those users."
+        )
+
+    await warn_missing_power(levels, evt)
+
     try:
         await portal.create_telegram_chat(evt.sender, invites=invites, supergroup=supergroup)
     except ValueError as e:
         await portal.delete()
         return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/filter.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,17 +13,20 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
 from mautrix.types import EventID
 
 from ... import portal as po
-from .. import command_handler, CommandEvent, SECTION_ADMIN
+from .. import SECTION_ADMIN, CommandEvent, command_handler
 
 
-@command_handler(needs_admin=True,
-                 help_section=SECTION_ADMIN,
-                 help_args="<`whitelist`|`blacklist`>",
-                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
-                           "default.")
+@command_handler(
+    needs_admin=True,
+    help_section=SECTION_ADMIN,
+    help_args="<`whitelist`|`blacklist`>",
+    help_text="Change whether the bridge will allow or disallow bridging rooms by default.",
+)
 async def filter_mode(evt: CommandEvent) -> EventID:
     try:
         mode = evt.args[0]
@@ -36,19 +39,26 @@ async def filter_mode(evt: CommandEvent) -> EventID:
     evt.config.save()
     po.Portal.filter_mode = mode
     if mode == "whitelist":
-        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
-                               "To allow bridging a specific chat, use"
-                               "`!filter whitelist <chat ID>`.")
+        return await evt.reply(
+            "The bridge will now disallow bridging chats by default.\n"
+            "To allow bridging a specific chat, use"
+            "`!filter whitelist <chat ID>`."
+        )
     else:
-        return await evt.reply("The bridge will now allow bridging chats by default.\n"
-                               "To disallow bridging a specific chat, use"
-                               "`!filter blacklist <chat ID>`.")
+        return await evt.reply(
+            "The bridge will now allow bridging chats by default.\n"
+            "To disallow bridging a specific chat, use"
+            "`!filter blacklist <chat ID>`."
+        )
 
 
-@command_handler(name="filter", needs_admin=True,
-                 help_section=SECTION_ADMIN,
-                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
-                 help_text="Allow or disallow bridging a specific chat.")
+@command_handler(
+    name="filter",
+    needs_admin=True,
+    help_section=SECTION_ADMIN,
+    help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+    help_text="Allow or disallow bridging a specific chat.",
+)
 async def edit_filter(evt: CommandEvent) -> EventID:
     try:
         action = evt.args[0]
@@ -67,7 +77,7 @@ async def edit_filter(evt: CommandEvent) -> EventID:
 
     mode = evt.config["bridge.filter.mode"]
     if mode not in ("blacklist", "whitelist"):
-        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
+        return await evt.reply(f'Unknown filter mode "{mode}". Please fix the bridge config.')
 
     filter_id_list = evt.config["bridge.filter.list"]
 

mautrix_telegram/commands/portal/misc.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,27 +13,47 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, List, Tuple
-from datetime import timedelta, datetime
+from __future__ import annotations
+
+from datetime import datetime, timedelta
 import re
 
+from telethon.errors import (
+    ChatAdminRequiredError,
+    RPCError,
+    UsernameInvalidError,
+    UsernameNotModifiedError,
+    UsernameOccupiedError,
+)
+from telethon.helpers import add_surrogate
 from telethon.tl.functions.channels import GetFullChannelRequest
-from telethon.tl.functions.messages import GetFullChatRequest
-from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
-                             UsernameNotModifiedError, UsernameOccupiedError, RPCError)
+from telethon.tl.functions.messages import GetExportedChatInvitesRequest, GetFullChatRequest
+from telethon.tl.types import (
+    ChatInviteExported,
+    InputMessageEntityMentionName,
+    InputUserSelf,
+    MessageEntityMention,
+    TypeInputPeer,
+    TypeInputUser,
+)
+from telethon.tl.types.messages import ExportedChatInvites
 
 from mautrix.types import EventID
 
-from ... import portal as po
-from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT, SECTION_MISC
+from ... import formatter as fmt, portal as po, puppet as pu
+from .. import SECTION_MISC, SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
 from .util import user_has_power_level
 
 
-@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
-                 help_section=SECTION_MISC,
-                 help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.")
+@command_handler(
+    needs_admin=False,
+    needs_puppeting=False,
+    needs_auth=False,
+    help_section=SECTION_MISC,
+    help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.",
+)
 async def sync_state(evt: CommandEvent) -> EventID:
-    portal = po.Portal.get_by_mxid(evt.room_id)
+    portal = await po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
     elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
@@ -43,10 +63,11 @@ async def sync_state(evt: CommandEvent) -> EventID:
     await evt.reply("Synchronization complete")
 
 
-@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
-                 help_section=SECTION_MISC)
+@command_handler(
+    needs_admin=False, needs_puppeting=False, needs_auth=False, help_section=SECTION_MISC
+)
 async def sync_full(evt: CommandEvent) -> EventID:
-    portal = po.Portal.get_by_mxid(evt.room_id)
+    portal = await po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
 
@@ -69,11 +90,16 @@ async def sync_full(evt: CommandEvent) -> EventID:
     return await evt.reply("Portal synced successfully.")
 
 
-@command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
-                 help_section=SECTION_MISC,
-                 help_text="Get the ID of the Telegram chat where this room is bridged.")
+@command_handler(
+    name="id",
+    needs_admin=False,
+    needs_puppeting=False,
+    needs_auth=False,
+    help_section=SECTION_MISC,
+    help_text="Get the ID of the Telegram chat where this room is bridged.",
+)
 async def get_id(evt: CommandEvent) -> EventID:
-    portal = po.Portal.get_by_mxid(evt.room_id)
+    portal = await po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
     tgid = portal.tgid
@@ -84,39 +110,50 @@ async def get_id(evt: CommandEvent) -> EventID:
     await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
 
 
-invite_link_usage = ("**Usage:** `$cmdprefix+sp invite-link [--uses=<amount>] [--expire=<delta>]`"
-                     "\n\n"
-                     "* `--uses`: the number of times the invite link can be used."
-                     "            Defaults to unlimited.\n"
-                     "* `--expire`: the duration after which the link will expire."
-                     "              A number suffixed with d(ay), h(our), m(inute) or s(econd)")
+invite_link_usage = (
+    "**Usage:** `$cmdprefix+sp invite-link "
+    "[--uses=<amount>] [--expire=<delta>] [--request-needed] -- [title]`"
+    "\n\n"
+    "* `--uses`: the number of times the invite link can be used."
+    "            Defaults to unlimited.\n"
+    "* `--expire`: the duration after which the link will expire."
+    "              A number suffixed with d(ay), h(our), m(inute) or s(econd)\n"
+    "* `--request-needed`: should the link require admins to approve joins?\n"
+    "* `title`: a description of the link (only shown to admins)."
+)
 
 
-def _parse_flag(args: List[str]) -> Tuple[str, str]:
+def _parse_flag(args: list[str]) -> tuple[str, str]:
     arg = args.pop(0).lower()
+    if arg == "--":
+        return "", ""
+    value = ""
     if arg.startswith("--"):
-        value_start = arg.index("=")
-        if value_start:
+        value_start = arg.find("=")
+        if value_start > 0:
             flag = arg[2:value_start]
-            value = arg[value_start+1:]
+            value = arg[value_start + 1 :]
         else:
             flag = arg[2:]
-            value = args.pop(0).lower()
+            if arg not in ("request", "request-needed"):
+                value = args.pop(0).lower()
     elif arg.startswith("-"):
         flag = arg[1]
         if len(arg) > 3 and arg[2] == "=":
             value = arg[3:]
-        else:
+        elif arg != "r":
             value = args.pop(0).lower()
     else:
         raise ValueError("invalid flag")
     return flag, value
 
 
-delta_regex = re.compile("([0-9]+)(w(?:eek)?|d(?:ay)?|h(?:our)?|m(?:in(?:ute)?)?|s(?:ec(?:ond)?)?)")
+delta_regex = re.compile(
+    "([0-9]+)(w(?:eek)?|d(?:ay)?|h(?:our)?|m(?:in(?:ute)?)?|s(?:ec(?:ond)?)?)"
+)
 
 
-def _parse_delta(value: str) -> Optional[timedelta]:
+def _parse_delta(value: str) -> timedelta | None:
     match = delta_regex.fullmatch(value)
     if not match:
         return None
@@ -136,19 +173,27 @@ def _parse_delta(value: str) -> Optional[timedelta]:
         return None
 
 
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Get a Telegram invite link to the current chat.",
-                 help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>]")
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="Get a Telegram invite link to the current chat.",
+    help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>] [--request-needed] -- [title]",
+)
 async def invite_link(evt: CommandEvent) -> EventID:
+    if not evt.is_portal:
+        return await evt.reply("This is not a portal room.")
+
     # TODO once we switch to Python 3.9 minimum, use argparse with exit_on_error=False
     uses = None
     expire = None
+    request_needed = False
     while evt.args:
         try:
             flag, value = _parse_flag(evt.args)
         except (ValueError, IndexError):
             return await evt.reply(invite_link_usage)
-        if flag in ("uses", "u"):
+        if not flag:
+            break
+        elif flag in ("uses", "u"):
             try:
                 uses = int(value)
             except ValueError:
@@ -158,27 +203,96 @@ async def invite_link(evt: CommandEvent) -> EventID:
             if not expire_delta:
                 await evt.reply("Invalid format for expiry time delta")
             expire = datetime.now() + expire_delta
+        elif flag in ("request", "request-needed", "r"):
+            request_needed = True
+    title = " ".join(evt.args)
 
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
+    if evt.portal.peer_type == "user":
         return await evt.reply("You can't invite users to private chats.")
 
     try:
-        link = await portal.get_invite_link(evt.sender, uses=uses, expire=expire)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
+        link = await evt.portal.get_invite_link(
+            evt.sender, uses=uses, expire=expire, request_needed=request_needed, title=title
+        )
+        return await evt.reply(f"Invite link to {evt.portal.title}: {link}")
     except ValueError as e:
         return await evt.reply(e.args[0])
     except ChatAdminRequiredError:
         return await evt.reply("You don't have the permission to create an invite link.")
 
 
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Upgrade a normal Telegram group to a supergroup.")
+async def _format_invite_link(link: ChatInviteExported) -> str:
+    desc = f"* {link.link}"
+    if link.title:
+        desc += f" - {link.title}"
+    if link.expire_date:
+        desc += f"  \n  Expires at {link.expire_date.isoformat()}"
+    if link.usage_limit:
+        desc += f"  \n  Used {link.usage or 0} out of {link.usage_limit} times"
+    elif link.usage:
+        desc += f"  \n  Used {link.usage} times"
+    else:
+        desc += "  \n  Never used"
+    if link.request_needed:
+        desc += "  \n  Join requests enabled - using link requires admin approval"
+    return desc
+
+
+async def _hacky_find_mention(evt: CommandEvent) -> TypeInputUser | TypeInputPeer | None:
+    if len(evt.args) == 0:
+        return None
+    text, entities = await fmt.matrix_to_telegram(
+        evt.sender.client, text=evt.content.body, html=evt.content.formatted_body
+    )
+    for entity in entities:
+        if isinstance(entity, MessageEntityMention):
+            admin_username = add_surrogate(text)[entity.offset + 1 : entity.offset + entity.length]
+            return await evt.sender.client.get_input_entity(admin_username)
+        elif isinstance(entity, InputMessageEntityMentionName):
+            return entity.user_id
+    return None
+
+
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="List existing Telegram invite links to the current chat.",
+    help_args="[creator]",
+)
+async def list_invite_links(evt: CommandEvent) -> EventID:
+    admin_id = InputUserSelf()
+    try:
+        admin_id = await _hacky_find_mention(evt) or InputUserSelf()
+    except Exception:
+        pass
+    resp: ExportedChatInvites = await evt.sender.client(
+        GetExportedChatInvitesRequest(
+            peer=await evt.portal.get_input_entity(evt.sender),
+            admin_id=admin_id,
+            limit=100,
+        )
+    )
+    if resp.count == 0:
+        if isinstance(admin_id, InputUserSelf):
+            return await evt.reply("You haven't created any invite links to the current chat")
+        else:
+            return await evt.reply("That user hasn't created any invite links to the current chat")
+    formatted_links = "\n".join([await _format_invite_link(link) for link in resp.invites])
+    if isinstance(admin_id, InputUserSelf):
+        await evt.reply(f"Your links to this chat:\n\n{formatted_links}")
+    else:
+        puppet = await pu.Puppet.get_by_peer(admin_id)
+        await evt.reply(
+            f"[{puppet.displayname}](https://matrix.to/#/{puppet.mxid})'s links to this chat:\n\n"
+            f"{formatted_links}"
+        )
+
+
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="Upgrade a normal Telegram group to a supergroup.",
+)
 async def upgrade(evt: CommandEvent) -> EventID:
-    portal = po.Portal.get_by_mxid(evt.room_id)
+    portal = await po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
     elif portal.peer_type == "channel":
@@ -195,30 +309,33 @@ async def upgrade(evt: CommandEvent) -> EventID:
         return await evt.reply(e.args[0])
 
 
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_args="<_name_|`-`>",
-                 help_text="Change the username of a supergroup/channel. "
-                           "To disable, use a dash (`-`) as the name.")
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_args="<_name_|`-`>",
+    help_text=(
+        "Change the username of a supergroup/channel. To disable, use a dash (`-`) as the name."
+    ),
+)
 async def group_name(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
 
-    portal = po.Portal.get_by_mxid(evt.room_id)
+    portal = await po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
     elif portal.peer_type != "channel":
         return await evt.reply("Only channels and supergroups have usernames.")
 
     try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
+        await portal.set_telegram_username(evt.sender, evt.args[0] if evt.args[0] != "-" else "")
         if portal.username:
             return await evt.reply(f"Username of channel changed to {portal.username}.")
         else:
             return await evt.reply(f"Channel is now private.")
     except ChatAdminRequiredError:
         return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
+            "You don't have the permission to set the username of this channel."
+        )
     except UsernameNotModifiedError:
         if portal.username:
             return await evt.reply("That is already the username of this channel.")

mautrix_telegram/commands/portal/unbridge.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,19 +13,21 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, Callable, Optional
+from __future__ import annotations
 
-from mautrix.types import RoomID, EventID
+from typing import Callable
+
+from mautrix.types import EventID, RoomID
 
 from ... import portal as po
-from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+from .. import SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
 from .util import user_has_power_level
 
 
-async def _get_portal_and_check_permission(evt: CommandEvent) -> Optional[po.Portal]:
+async def _get_portal_and_check_permission(evt: CommandEvent) -> po.Portal | None:
     room_id = RoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
 
-    portal = po.Portal.get_by_mxid(room_id)
+    portal = await po.Portal.get_by_mxid(room_id)
     if not portal:
         that_this = "This" if room_id == evt.room_id else "That"
         await evt.reply(f"{that_this} is not a portal room.")
@@ -43,9 +45,10 @@ async def _get_portal_and_check_permission(evt: CommandEvent) -> Optional[po.Por
     return portal
 
 
-def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
-                                completed_message: str) -> Dict:
-    async def post_confirm(confirm) -> Optional[EventID]:
+def _get_portal_murder_function(
+    action: str, room_id: str, function: Callable, command: str, completed_message: str
+) -> dict:
+    async def post_confirm(confirm) -> EventID | None:
         confirm.sender.command_status = None
         if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
             await function()
@@ -61,40 +64,55 @@ def _get_portal_murder_function(action: str, room_id: str, function: Callable, c
     }
 
 
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Remove all users from the current portal room and forget the portal. "
-                           "Only works for group chats; to delete a private chat portal, simply "
-                           "leave the room.")
-async def delete_portal(evt: CommandEvent) -> Optional[EventID]:
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text=(
+        "Remove all users from the current portal room and forget the portal. "
+        "Only works for group chats; to delete a private chat portal, simply leave the room."
+    ),
+)
+async def delete_portal(evt: CommandEvent) -> EventID | None:
     portal = await _get_portal_and_check_permission(evt)
     if not portal:
         return None
 
-    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
-                                                            portal.cleanup_and_delete, "delete",
-                                                            "Portal successfully deleted.")
-    return await evt.reply("Please confirm deletion of portal "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           f"to Telegram chat \"{portal.title}\" "
-                           "by typing `$cmdprefix+sp confirm-delete`"
-                           "\n\n"
-                           "**WARNING:** If the bridge bot has the power level to do so, **this "
-                           "will kick ALL users** in the room. If you just want to remove the "
-                           "bridge, use `$cmdprefix+sp unbridge` instead.")
+    evt.sender.command_status = _get_portal_murder_function(
+        "Portal deletion",
+        portal.mxid,
+        portal.cleanup_and_delete,
+        "delete",
+        "Portal successfully deleted.",
+    )
+    return await evt.reply(
+        "Please confirm deletion of portal "
+        f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+        f'to Telegram chat "{portal.title}" '
+        "by typing `$cmdprefix+sp confirm-delete`"
+        "\n\n"
+        "**WARNING:** If the bridge bot has the power level to do so, **this "
+        "will kick ALL users** in the room. If you just want to remove the "
+        "bridge, use `$cmdprefix+sp unbridge` instead."
+    )
 
 
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Remove puppets from the current portal room and forget the portal.")
-async def unbridge(evt: CommandEvent) -> Optional[EventID]:
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_text="Remove puppets from the current portal room and forget the portal.",
+)
+async def unbridge(evt: CommandEvent) -> EventID | None:
     portal = await _get_portal_and_check_permission(evt)
     if not portal:
         return None
 
-    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
-                                                            portal.unbridge, "unbridge",
-                                                            "Room successfully unbridged.")
-    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           "by typing `$cmdprefix+sp confirm-unbridge`")
+    evt.sender.command_status = _get_portal_murder_function(
+        "Room unbridging", portal.mxid, portal.unbridge, "unbridge", "Room successfully unbridged."
+    )
+    return await evt.reply(
+        f'Please confirm unbridging chat "{portal.title}" from room '
+        f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+        "by typing `$cmdprefix+sp confirm-unbridge`"
+    )

mautrix_telegram/commands/portal/util.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,23 +13,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional
+from __future__ import annotations
 
-from mautrix.errors import MatrixRequestError
 from mautrix.appservice import IntentAPI
-from mautrix.types import RoomID, EventType, PowerLevelStateEventContent
+from mautrix.errors import MatrixRequestError
+from mautrix.types import EventType, PowerLevelStateEventContent, RoomID
 
 from ... import user as u
-
-OptStr = Optional[str]
+from .. import CommandEvent
 
 
-async def get_initial_state(intent: IntentAPI, room_id: RoomID
-                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent], bool]:
+async def get_initial_state(
+    intent: IntentAPI, room_id: RoomID
+) -> tuple[str | None, str | None, PowerLevelStateEventContent | None, bool]:
     state = await intent.get_state(room_id)
-    title: OptStr = None
-    about: OptStr = None
-    levels: Optional[PowerLevelStateEventContent] = None
+    title: str | None = None
+    about: str | None = None
+    levels: PowerLevelStateEventContent | None = None
     encrypted: bool = False
     for event in state:
         try:
@@ -49,8 +49,18 @@ async def get_initial_state(intent: IntentAPI, room_id: RoomID
     return title, about, levels, encrypted
 
 
-async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.User,
-                               event: str) -> bool:
+async def warn_missing_power(levels: PowerLevelStateEventContent, evt: CommandEvent) -> None:
+    if levels.get_user_level(evt.az.bot_mxid) < levels.redact:
+        await evt.reply(
+            "Warning: The bot does not have privileges to redact messages on Matrix. "
+            "Message deletions from Telegram will not be bridged unless you give "
+            f"redaction permissions to [{evt.az.bot_mxid}](https://matrix.to/#/{evt.az.bot_mxid})"
+        )
+
+
+async def user_has_power_level(
+    room_id: RoomID, intent: IntentAPI, sender: u.User, event: str
+) -> bool:
     if sender.is_admin:
         return True
     # Make sure the state store contains the power levels.
@@ -58,5 +68,5 @@ async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.Use
         await intent.get_power_levels(room_id)
     except MatrixRequestError:
         return False
-    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
+    event_type = EventType.find(f"fi.mau.telegram.{event}", t_class=EventType.Class.STATE)
     return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)

mautrix_telegram/commands/telegram/account.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,23 +13,36 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional
+from __future__ import annotations
 
-from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError, AuthKeyError, FirstNameInvalidError, AboutTooLongError)
+from telethon.errors import (
+    AboutTooLongError,
+    AuthKeyError,
+    FirstNameInvalidError,
+    HashInvalidError,
+    UsernameInvalidError,
+    UsernameNotModifiedError,
+    UsernameOccupiedError,
+)
+from telethon.tl.functions.account import (
+    GetAuthorizationsRequest,
+    ResetAuthorizationRequest,
+    UpdateProfileRequest,
+    UpdateUsernameRequest,
+)
 from telethon.tl.types import Authorization
-from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
-                                           ResetAuthorizationRequest, UpdateProfileRequest)
 
 from mautrix.types import EventID
 
-from .. import command_handler, CommandEvent, SECTION_AUTH
+from .. import SECTION_AUTH, CommandEvent, command_handler
 
 
-@command_handler(needs_auth=True,
-                 help_section=SECTION_AUTH,
-                 help_args="<_new username_>",
-                 help_text="Change your Telegram username.")
+@command_handler(
+    needs_auth=True,
+    help_section=SECTION_AUTH,
+    help_args="<_new username_>",
+    help_text="Change your Telegram username.",
+)
 async def username(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp username <new username>`")
@@ -41,22 +54,26 @@ async def username(evt: CommandEvent) -> EventID:
     try:
         await evt.sender.client(UpdateUsernameRequest(username=new_name))
     except UsernameInvalidError:
-        return await evt.reply("Invalid username. Usernames must be between 5 and 30 alphanumeric "
-                               "characters.")
+        return await evt.reply(
+            "Invalid username. Usernames must be between 5 and 30 alphanumeric characters."
+        )
     except UsernameNotModifiedError:
         return await evt.reply("That is your current username.")
     except UsernameOccupiedError:
         return await evt.reply("That username is already in use.")
     await evt.sender.update_info()
-    if not evt.sender.username:
+    if not evt.sender.tg_username:
         await evt.reply("Username removed")
     else:
-        await evt.reply(f"Username changed to {evt.sender.username}")
+        await evt.reply(f"Username changed to {evt.sender.tg_username}")
 
-@command_handler(needs_auth=True,
-                 help_section=SECTION_AUTH,
-                 help_args="<_new about_>",
-                 help_text="Change your Telegram about section.")
+
+@command_handler(
+    needs_auth=True,
+    help_section=SECTION_AUTH,
+    help_args="<_new about_>",
+    help_text="Change your Telegram about section.",
+)
 async def about(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp about <new about>`")
@@ -71,17 +88,22 @@ async def about(evt: CommandEvent) -> EventID:
         return await evt.reply("The provided about section is too long")
     return await evt.reply("About section updated")
 
-@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
-                 help_text="Change your Telegram displayname.")
+
+@command_handler(
+    needs_auth=True,
+    help_section=SECTION_AUTH,
+    help_args="<_new displayname_>",
+    help_text="Change your Telegram displayname.",
+)
 async def displayname(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
     if evt.sender.is_bot:
         return await evt.reply("Bots can't set their own displayname.")
 
-    first_name, last_name = ((evt.args[0], "")
-                             if len(evt.args) == 1
-                             else (" ".join(evt.args[:-1]), evt.args[-1]))
+    first_name, last_name = (
+        (evt.args[0], "") if len(evt.args) == 1 else (" ".join(evt.args[:-1]), evt.args[-1])
+    )
     try:
         await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
     except FirstNameInvalidError:
@@ -91,16 +113,20 @@ async def displayname(evt: CommandEvent) -> EventID:
 
 
 def _format_session(sess: Authorization) -> str:
-    return (f"**{sess.app_name} {sess.app_version}**  \n"
-            f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
-            f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
-            f"  **From:** {sess.ip} - {sess.region}, {sess.country}")
+    return (
+        f"**{sess.app_name} {sess.app_version}**  \n"
+        f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
+        f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
+        f"  **From:** {sess.ip} - {sess.region}, {sess.country}"
+    )
 
 
-@command_handler(needs_auth=True,
-                 help_section=SECTION_AUTH,
-                 help_args="<`list`|`terminate`> [_hash_]",
-                 help_text="View or delete other Telegram sessions.")
+@command_handler(
+    needs_auth=True,
+    help_section=SECTION_AUTH,
+    help_args="<`list`|`terminate`> [_hash_]",
+    help_text="View or delete other Telegram sessions.",
+)
 async def session(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")
@@ -112,14 +138,18 @@ async def session(evt: CommandEvent) -> EventID:
         session_list = res.authorizations
         current = [s for s in session_list if s.current][0]
         current_text = _format_session(current)
-        other_text = "\n".join(f"* {_format_session(sess)}  \n"
-                               f"  **Hash:** {sess.hash}"
-                               for sess in session_list if not sess.current)
-        return await evt.reply(f"### Current session\n"
-                               f"{current_text}\n"
-                               f"\n"
-                               f"### Other active sessions\n"
-                               f"{other_text}")
+        other_text = "\n".join(
+            f"* {_format_session(sess)}  \n  **Hash:** {sess.hash}"
+            for sess in session_list
+            if not sess.current
+        )
+        return await evt.reply(
+            f"### Current session\n"
+            f"{current_text}\n"
+            f"\n"
+            f"### Other active sessions\n"
+            f"{other_text}"
+        )
     elif cmd == "terminate" and len(evt.args) > 1:
         try:
             session_hash = int(evt.args[1])
@@ -131,8 +161,9 @@ async def session(evt: CommandEvent) -> EventID:
             return await evt.reply("Invalid session hash.")
         except AuthKeyError as e:
             if e.message == "FRESH_RESET_AUTHORISATION_FORBIDDEN":
-                return await evt.reply("New sessions can't terminate other sessions. "
-                                       "Please wait a while.")
+                return await evt.reply(
+                    "New sessions can't terminate other sessions. Please wait a while."
+                )
             raise
         if ok:
             return await evt.reply("Session terminated successfully.")

mautrix_telegram/commands/telegram/auth.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,63 +13,94 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Any, Dict, Optional
+from __future__ import annotations
+
+from typing import Any
 import asyncio
 import io
 
-from telethon.errors import (  # isort: skip
-    AccessTokenExpiredError, AccessTokenInvalidError, FirstNameInvalidError, FloodWaitError,
-    PasswordHashInvalidError, PhoneCodeExpiredError, PhoneCodeInvalidError,
-    PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
-    PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError,
-    PhoneNumberInvalidError)
+from telethon.errors import (
+    AccessTokenExpiredError,
+    AccessTokenInvalidError,
+    FirstNameInvalidError,
+    FloodWaitError,
+    PasswordHashInvalidError,
+    PhoneCodeExpiredError,
+    PhoneCodeInvalidError,
+    PhoneNumberAppSignupForbiddenError,
+    PhoneNumberBannedError,
+    PhoneNumberFloodError,
+    PhoneNumberInvalidError,
+    PhoneNumberOccupiedError,
+    PhoneNumberUnoccupiedError,
+    SessionPasswordNeededError,
+)
 from telethon.tl.types import User
 
-from mautrix.types import (EventID, UserID, MediaMessageEventContent, ImageInfo, MessageType,
-                           TextMessageEventContent)
+from mautrix.client import Client
+from mautrix.errors import MForbidden
+from mautrix.types import (
+    EventID,
+    ImageInfo,
+    MediaMessageEventContent,
+    MessageType,
+    TextMessageEventContent,
+    UserID,
+)
+from mautrix.util.format_duration import format_duration as fmt_duration
 
 from ... import user as u
+from ...commands import SECTION_AUTH, CommandEvent, command_handler
 from ...types import TelegramID
-from ...commands import command_handler, CommandEvent, SECTION_AUTH
-from ...util import format_duration as fmt_duration
 
 try:
-    import qrcode
-    import PIL as _
     from telethon.tl.custom import QRLogin
+    import PIL as _
+    import qrcode
 except ImportError:
     qrcode = None
     QRLogin = None
 
 
-@command_handler(needs_auth=False,
-                 help_section=SECTION_AUTH,
-                 help_text="Check if you're logged into Telegram.")
+@command_handler(
+    needs_auth=False, help_section=SECTION_AUTH, help_text="Check if you're logged into Telegram."
+)
 async def ping(evt: CommandEvent) -> EventID:
-    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
-    if me:
-        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
-        return await evt.reply(f"You're logged in as {human_tg_id}")
+    if await evt.sender.is_logged_in():
+        me = await evt.sender.get_me()
+        if me:
+            human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
+            return await evt.reply(f"You're logged in as {human_tg_id}")
+        else:
+            return await evt.reply("You were logged in, but there appears to have been an error.")
     else:
         return await evt.reply("You're not logged in.")
 
 
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_AUTH,
-                 help_text="Get the info of the message relay Telegram bot.")
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_AUTH,
+    help_text="Get the info of the message relay Telegram bot.",
+)
 async def ping_bot(evt: CommandEvent) -> EventID:
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
     info, mxid = await evt.tgbot.get_me(use_cache=False)
-    return await evt.reply("Telegram message relay bot is active: "
-                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
-                           "To use the bot, simply invite it to a portal room.")
+    return await evt.reply(
+        "Telegram message relay bot is active: "
+        f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
+        "To use the bot, simply invite it to a portal room."
+    )
 
 
-@command_handler(needs_auth=False, management_only=True,
-                 help_section=SECTION_AUTH,
-                 help_args="<_phone_> <_full name_>",
-                 help_text="Register to Telegram")
+@command_handler(
+    needs_auth=False,
+    management_only=True,
+    help_section=SECTION_AUTH,
+    help_args="<_phone_> <_full name_>",
+    help_text="Register to Telegram",
+)
 async def register(evt: CommandEvent) -> EventID:
     if await evt.sender.is_logged_in():
         return await evt.reply("You are already logged in.")
@@ -82,13 +113,19 @@ async def register(evt: CommandEvent) -> EventID:
     else:
         full_name = " ".join(evt.args[1:-1]), evt.args[-1]
 
-    await _request_code(evt, phone_number, {
-        "next": enter_code_register,
-        "action": "Register",
-        "full_name": full_name,
-    })
-    return await evt.reply("By signing up for Telegram, you agree to "
-                           "the terms of service: https://telegram.org/tos")
+    await _request_code(
+        evt,
+        phone_number,
+        {
+            "next": enter_code_register,
+            "action": "Register",
+            "full_name": full_name,
+        },
+    )
+    return await evt.reply(
+        "By signing up for Telegram, you agree to "
+        "the terms of service: https://telegram.org/tos"
+    )
 
 
 async def enter_code_register(evt: CommandEvent) -> EventID:
@@ -98,31 +135,39 @@ async def enter_code_register(evt: CommandEvent) -> EventID:
         await evt.sender.ensure_started(even_if_no_session=True)
         first_name, last_name = evt.sender.command_status["full_name"]
         user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
-        asyncio.ensure_future(evt.sender.post_login(user, first_login=True), loop=evt.loop)
+        asyncio.create_task(evt.sender.post_login(user, first_login=True))
         evt.sender.command_status = None
         return await evt.reply(f"Successfully registered to Telegram.")
     except PhoneNumberOccupiedError:
-        return await evt.reply("That phone number has already been registered. "
-                               "You can log in with `$cmdprefix+sp login`.")
+        return await evt.reply(
+            "That phone number has already been registered. "
+            "You can log in with `$cmdprefix+sp login`."
+        )
     except FirstNameInvalidError:
         return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
     except PhoneCodeExpiredError:
         return await evt.reply(
-            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`.")
+            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`."
+        )
     except PhoneCodeInvalidError:
         return await evt.reply("Invalid phone code.")
     except Exception:
         evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code. "
-                               "Check console for more details.")
+        return await evt.reply(
+            "Unhandled exception while sending code. Check console for more details."
+        )
 
 
-@command_handler(needs_auth=False, management_only=True, help_section=SECTION_AUTH,
-                 help_text="Log in by scanning a QR code.")
+@command_handler(
+    needs_auth=False,
+    management_only=True,
+    help_section=SECTION_AUTH,
+    help_text="Log in by scanning a QR code.",
+)
 async def login_qr(evt: CommandEvent) -> EventID:
     login_as = evt.sender
     if len(evt.args) > 0 and evt.sender.is_admin:
-        login_as = u.User.get_by_mxid(UserID(evt.args[0]))
+        login_as = await u.User.get_by_mxid(UserID(evt.args[0]))
     if not qrcode or not QRLogin:
         return await evt.reply("This bridge instance does not support logging in with a QR code.")
     if await login_as.is_logged_in():
@@ -130,7 +175,7 @@ async def login_qr(evt: CommandEvent) -> EventID:
 
     await login_as.ensure_started(even_if_no_session=True)
     qr_login = QRLogin(login_as.client, ignored_ids=[])
-    qr_event_id: Optional[EventID] = None
+    qr_event_id: EventID | None = None
 
     async def upload_qr() -> None:
         nonlocal qr_event_id
@@ -140,9 +185,12 @@ async def login_qr(evt: CommandEvent) -> EventID:
         image.save(buffer, "PNG")
         qr = buffer.getvalue()
         mxc = await evt.az.intent.upload_media(qr, "image/png", "login-qr.png", len(qr))
-        content = MediaMessageEventContent(body=qr_login.url, url=mxc, msgtype=MessageType.IMAGE,
-                                           info=ImageInfo(mimetype="image/png", size=len(qr),
-                                                          width=size, height=size))
+        content = MediaMessageEventContent(
+            body=qr_login.url,
+            url=mxc,
+            msgtype=MessageType.IMAGE,
+            info=ImageInfo(mimetype="image/png", size=len(qr), width=size, height=size),
+        )
         if qr_event_id:
             content.set_edit(qr_event_id)
             await evt.az.intent.send_message(evt.room_id, content)
@@ -165,8 +213,13 @@ async def login_qr(evt: CommandEvent) -> EventID:
                 "login_as": login_as if login_as != evt.sender else None,
                 "action": "Login (password entry)",
             }
-            return await evt.reply("Your account has two-factor authentication. "
-                                   "Please send your password here.")
+            return await evt.reply(
+                "Your account has two-factor authentication. Please send your password here."
+            )
+        try:
+            await evt.main_intent.redact(evt.room_id, qr_event_id, reason="QR code scanned")
+        except Exception:
+            pass
     else:
         timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
         timeout.set_edit(qr_event_id)
@@ -175,14 +228,31 @@ async def login_qr(evt: CommandEvent) -> EventID:
     return await _finish_sign_in(evt, user, login_as=login_as)
 
 
-@command_handler(needs_auth=False, management_only=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Get instructions on how to log in.")
+@command_handler(
+    needs_auth=False,
+    management_only=True,
+    help_section=SECTION_AUTH,
+    help_text="Get instructions on how to log in.",
+)
 async def login(evt: CommandEvent) -> EventID:
     override_sender = False
-    if len(evt.args) > 0 and evt.sender.is_admin:
-        evt.sender = await u.User.get_by_mxid(UserID(evt.args[0])).ensure_started()
+    if len(evt.args) > 0 and evt.sender.is_admin and evt.args[0]:
+        override_user_id = UserID(evt.args[0])
+        try:
+            Client.parse_user_id(override_user_id)
+        except ValueError:
+            return await evt.reply(
+                f"**Usage:** `$cmdprefix+sp login [override user ID]`\n\n"
+                f"{override_user_id!r} is not a valid Matrix user ID"
+            )
+        orig_user_id = evt.sender.mxid
+        evt.sender = await u.User.get_and_start_by_mxid(override_user_id)
         override_sender = True
+        if orig_user_id != evt.sender:
+            await evt.reply(
+                f"Admin override: logging in as {evt.sender.mxid} instead of {orig_user_id}"
+            )
+
     if await evt.sender.is_logged_in():
         return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 
@@ -198,24 +268,32 @@ async def login(evt: CommandEvent) -> EventID:
         prefix = evt.config["appservice.public.external"]
         url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
         if override_sender:
-            return await evt.reply(f"[Click here to log in]({url}) as "
-                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+            return await evt.reply(
+                f"[Click here to log in]({url}) as "
+                f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid})."
+            )
         elif allow_matrix_login:
-            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
-                                   f" number (or bot auth token) here to log in.\n\n{nb}")
+            return await evt.reply(
+                f"[Click here to log in]({url}). Alternatively, send your phone"
+                f" number (or bot auth token) here to log in.\n\n{nb}"
+            )
         return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
     elif allow_matrix_login:
         if override_sender:
             return await evt.reply(
                 "This bridge instance does not allow you to log in outside of Matrix. "
-                "Logging in as another user inside Matrix is not currently possible.")
-        return await evt.reply("Please send your phone number (or bot auth token) here to start "
-                               f"the login process.\n\n{nb}")
+                "Logging in as another user inside Matrix is not currently possible."
+            )
+        return await evt.reply(
+            "Please send your phone number (or bot auth token) here to start "
+            f"the login process.\n\n{nb}"
+        )
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 
-async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
-                        ) -> EventID:
+async def _request_code(
+    evt: CommandEvent, phone_number: str, next_status: dict[str, Any]
+) -> EventID:
     ok = False
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
@@ -225,33 +303,42 @@ async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[
     except PhoneNumberAppSignupForbiddenError:
         return await evt.reply("Your phone number does not allow 3rd party apps to sign in.")
     except PhoneNumberFloodError:
-        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
-                               "The ban is usually applied for around a day.")
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            "The ban is usually applied for around a day."
+        )
     except FloodWaitError as e:
-        return await evt.reply("Your phone number has been temporarily blocked for flooding. "
-                               f"Please wait for {fmt_duration(e.seconds)} before trying again.")
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            f"Please wait for {fmt_duration(e.seconds)} before trying again."
+        )
     except PhoneNumberBannedError:
         return await evt.reply("Your phone number has been banned from Telegram.")
     except PhoneNumberUnoccupiedError:
-        return await evt.reply("That phone number has not been registered. "
-                               "Please register with `$cmdprefix+sp register <phone>`.")
+        return await evt.reply(
+            "That phone number has not been registered. "
+            "Please register with `$cmdprefix+sp register <phone>`."
+        )
     except PhoneNumberInvalidError:
         return await evt.reply("That phone number is not valid.")
     except Exception:
         evt.log.exception("Error requesting phone code")
-        return await evt.reply("Unhandled exception while requesting code. "
-                               "Check console for more details.")
+        return await evt.reply(
+            "Unhandled exception while requesting code. Check console for more details."
+        )
     finally:
         evt.sender.command_status = next_status if ok else None
 
 
 @command_handler(needs_auth=False)
-async def enter_phone_or_token(evt: CommandEvent) -> Optional[EventID]:
+async def enter_phone_or_token(evt: CommandEvent) -> EventID | None:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
+        return await evt.reply(
+            "This bridge instance does not allow in-Matrix login. "
+            "Please use `$cmdprefix+sp login` to get login instructions"
+        )
 
     # phone numbers don't contain colons but telegram bot auth tokens do
     if evt.args[0].find(":") > 0:
@@ -259,54 +346,62 @@ async def enter_phone_or_token(evt: CommandEvent) -> Optional[EventID]:
             await _sign_in(evt, bot_token=evt.args[0])
         except Exception:
             evt.log.exception("Error sending auth token")
-            return await evt.reply("Unhandled exception while sending auth token. "
-                                   "Check console for more details.")
+            return await evt.reply(
+                "Unhandled exception while sending auth token. Check console for more details."
+            )
     else:
-        await _request_code(evt, evt.args[0], {
-            "next": enter_code,
-            "action": "Login",
-        })
+        await _request_code(evt, evt.args[0], {"next": enter_code, "action": "Login"})
     return None
 
 
 @command_handler(needs_auth=False)
-async def enter_code(evt: CommandEvent) -> Optional[EventID]:
+async def enter_code(evt: CommandEvent) -> EventID | None:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
+        return await evt.reply(
+            "This bridge instance does not allow in-Matrix login. "
+            "Please use `$cmdprefix+sp login` to get login instructions"
+        )
     try:
         await _sign_in(evt, code=evt.args[0])
     except Exception:
         evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code. "
-                               "Check console for more details.")
+        return await evt.reply(
+            "Unhandled exception while sending code. Check console for more details."
+        )
     return None
 
 
 @command_handler(needs_auth=False)
-async def enter_password(evt: CommandEvent) -> Optional[EventID]:
+async def enter_password(evt: CommandEvent) -> EventID | None:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
+        return await evt.reply(
+            "This bridge instance does not allow in-Matrix login. "
+            "Please use `$cmdprefix+sp login` to get login instructions"
+        )
+    await evt.redact()
     try:
-        await _sign_in(evt, login_as=evt.sender.command_status.get("login_as", None),
-                       password=" ".join(evt.args))
+        await _sign_in(
+            evt,
+            login_as=evt.sender.command_status.get("login_as", None),
+            password=" ".join(evt.args),
+        )
     except AccessTokenInvalidError:
         return await evt.reply("That bot token is not valid.")
     except AccessTokenExpiredError:
         return await evt.reply("That bot token has expired.")
     except Exception:
         evt.log.exception("Error sending password")
-        return await evt.reply("Unhandled exception while sending password. "
-                               "Check console for more details.")
+        return await evt.reply(
+            "Unhandled exception while sending password. Check console for more details."
+        )
     return None
 
 
-async def _sign_in(evt: CommandEvent, login_as: 'u.User' = None, **sign_in_info) -> EventID:
+async def _sign_in(evt: CommandEvent, login_as: u.User = None, **sign_in_info) -> EventID:
     login_as = login_as or evt.sender
     try:
         await login_as.ensure_started(even_if_no_session=True)
@@ -323,33 +418,37 @@ async def _sign_in(evt: CommandEvent, login_as: 'u.User' = None, **sign_in_info)
             "next": enter_password,
             "action": "Login (password entry)",
         }
-        return await evt.reply("Your account has two-factor authentication. "
-                               "Please send your password here.")
+        return await evt.reply(
+            "Your account has two-factor authentication. Please send your password here."
+        )
 
 
-async def _finish_sign_in(evt: CommandEvent, user: User, login_as: 'u.User' = None) -> EventID:
+async def _finish_sign_in(evt: CommandEvent, user: User, login_as: u.User = None) -> EventID:
     login_as = login_as or evt.sender
-    existing_user = u.User.get_by_tgid(TelegramID(user.id))
+    existing_user = await u.User.get_by_tgid(TelegramID(user.id))
     if existing_user and existing_user != login_as:
         await existing_user.log_out()
-        await evt.reply(f"[{existing_user.displayname}]"
-                        f"(https://matrix.to/#/{existing_user.mxid})"
-                        " was logged out from the account.")
-    asyncio.ensure_future(login_as.post_login(user, first_login=True), loop=evt.loop)
+        await evt.reply(
+            f"[{existing_user.displayname}] (https://matrix.to/#/{existing_user.mxid})"
+            " was logged out from the account."
+        )
+    asyncio.create_task(login_as.post_login(user, first_login=True))
     evt.sender.command_status = None
     name = f"@{user.username}" if user.username else f"+{user.phone}"
     if login_as != evt.sender:
-        msg = (f"Successfully logged in [{login_as.mxid}](https://matrix.to/#/{login_as.mxid})"
-               f" as {name}")
+        msg = (
+            f"Successfully logged in [{login_as.mxid}](https://matrix.to/#/{login_as.mxid})"
+            f" as {name}"
+        )
     else:
         msg = f"Successfully logged in as {name}"
     return await evt.reply(msg)
 
 
-@command_handler(needs_auth=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Log out from Telegram.")
+@command_handler(needs_auth=False, help_section=SECTION_AUTH, help_text="Log out from Telegram.")
 async def logout(evt: CommandEvent) -> EventID:
+    if not evt.sender.tgid:
+        return await evt.reply("You're not logged in")
     if await evt.sender.log_out():
         return await evt.reply("Logged out successfully.")
     return await evt.reply("Failed to log out.")

mautrix_telegram/commands/telegram/misc.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,37 +13,64 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Optional, Tuple, cast
-import codecs
+from __future__ import annotations
+
+from typing import cast
 import base64
+import codecs
 import re
 
 from aiohttp import ClientSession, InvalidURL
-
-from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
-                             UserAlreadyParticipantError, ChatIdInvalidError,
-                             TakeoutInitDelayError, EmoticonInvalidError)
-from telethon.tl.patched import Message
-from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
-                               TypeInputPeer, InputMediaDice)
-from telethon.tl.types.messages import BotCallbackAnswer
-from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
-                                            GetBotCallbackAnswerRequest, SendVoteRequest)
+from telethon.errors import (
+    ChatIdInvalidError,
+    EmoticonInvalidError,
+    InviteHashExpiredError,
+    InviteHashInvalidError,
+    InviteRequestSentError,
+    OptionsTooMuchError,
+    TakeoutInitDelayError,
+    UserAlreadyParticipantError,
+)
 from telethon.tl.functions.channels import JoinChannelRequest
+from telethon.tl.functions.messages import (
+    CheckChatInviteRequest,
+    GetBotCallbackAnswerRequest,
+    ImportChatInviteRequest,
+    SendVoteRequest,
+)
+from telethon.tl.patched import Message
+from telethon.tl.types import (
+    InputMediaDice,
+    MessageMediaGame,
+    MessageMediaPoll,
+    TypeInputPeer,
+    TypeUpdates,
+    User as TLUser,
+)
+from telethon.tl.types.messages import BotCallbackAnswer
 
 from mautrix.types import EventID, Format
 
-from ... import puppet as pu, portal as po
+from ... import portal as po, puppet as pu
 from ...abstract_user import AbstractUser
+from ...commands import (
+    SECTION_CREATING_PORTALS,
+    SECTION_MISC,
+    SECTION_PORTAL_MANAGEMENT,
+    CommandEvent,
+    command_handler,
+)
 from ...db import Message as DBMessage
 from ...types import TelegramID
-from ...commands import (command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS,
-                         SECTION_PORTAL_MANAGEMENT)
 
 
-@command_handler(needs_auth=False,
-                 help_section=SECTION_MISC, help_args="<_caption_>",
-                 help_text="Set a caption for the next image you send")
+@command_handler(
+    needs_auth=False,
+    needs_puppeting=False,
+    help_section=SECTION_MISC,
+    help_args="<_caption_>",
+    help_text="Set a caption for the next image you send",
+)
 async def caption(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp caption <caption>`")
@@ -53,13 +80,17 @@ async def caption(evt: CommandEvent) -> EventID:
         evt.content.formatted_body = evt.content.formatted_body.replace(prefix, "", 1)
     evt.content.body = evt.content.body.replace(prefix, "", 1)
     evt.sender.command_status = {"caption": evt.content, "action": "Caption"}
-    return await evt.reply("Your next image or file will be sent with that caption. "
-                           "Use `$cmdprefix+sp cancel` to cancel the caption.")
+    return await evt.reply(
+        "Your next image or file will be sent with that caption. "
+        "Use `$cmdprefix+sp cancel` to cancel the caption."
+    )
 
 
-@command_handler(help_section=SECTION_MISC,
-                 help_args="[_-r|--remote_] <_query_>",
-                 help_text="Search your contacts or the Telegram servers for users.")
+@command_handler(
+    help_section=SECTION_MISC,
+    help_args="[_-r|--remote_] <_query_>",
+    help_text="Search your contacts or the Telegram servers for users.",
+)
 async def search(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
@@ -77,29 +108,37 @@ async def search(evt: CommandEvent) -> EventID:
 
     if not results:
         if len(query) < 5 and remote:
-            return await evt.reply("No local results. "
-                                   "Minimum length of remote query is 5 characters.")
+            return await evt.reply(
+                "No local results. Minimum length of remote query is 5 characters."
+            )
         return await evt.reply("No results 3:")
 
-    reply: List[str] = []
+    reply: list[str] = []
     if remote:
         reply += ["**Results from Telegram server:**", ""]
     else:
         reply += ["**Results in contacts:**", ""]
-    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
-               f"{puppet.id} ({similarity}% match)")
-              for puppet, similarity in results]
+    reply += [
+        (
+            f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
+            f"{puppet.id} ({similarity}% match)"
+        )
+        for puppet, similarity in results
+    ]
 
     # TODO somehow show remote channel results when joining by alias is possible?
 
     return await evt.reply("\n".join(reply))
 
 
-@command_handler(help_section=SECTION_CREATING_PORTALS, help_args="<_identifier_>",
-                 help_text="Open a private chat with the given Telegram user. The identifier is "
-                           "either the internal user ID, the username or the phone number. "
-                           "**N.B.** The phone numbers you start chats with must already be in "
-                           "your contacts.")
+@command_handler(
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="<_identifier_>",
+    help_text="Open a private chat with the given Telegram user. The identifier is "
+    "either the internal user ID, the username or the phone number. "
+    "**N.B.** The phone numbers you start chats with must already be in "
+    "your contacts.",
+)
 async def pm(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
@@ -114,14 +153,15 @@ async def pm(evt: CommandEvent) -> EventID:
         return await evt.reply("User not found.")
     elif not isinstance(user, TLUser):
         return await evt.reply("That doesn't seem to be a user.")
-    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
+    portal = await po.Portal.get_by_entity(user, tg_receiver=evt.sender.tgid)
     await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
     displayname, _ = pu.Puppet.get_displayname(user, False)
     return await evt.reply(f"Created private chat room with {displayname}")
 
 
-async def _join(evt: CommandEvent, identifier: str, link_type: str
-                ) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
+async def _join(
+    evt: CommandEvent, identifier: str, link_type: str
+) -> tuple[TypeUpdates | None, EventID | None]:
     if link_type == "joinchat":
         try:
             await evt.sender.client(CheckChatInviteRequest(identifier))
@@ -133,6 +173,8 @@ async def _join(evt: CommandEvent, identifier: str, link_type: str
             return (await evt.sender.client(ImportChatInviteRequest(identifier))), None
         except UserAlreadyParticipantError:
             return None, await evt.reply("You are already in that chat.")
+        except InviteRequestSentError:
+            return None, await evt.reply("Invite request sent successfully.")
     else:
         channel = await evt.sender.client.get_entity(identifier)
         if not channel:
@@ -140,10 +182,12 @@ async def _join(evt: CommandEvent, identifier: str, link_type: str
         return await evt.sender.client(JoinChannelRequest(channel)), None
 
 
-@command_handler(help_section=SECTION_CREATING_PORTALS,
-                 help_args="<_link_>",
-                 help_text="Join a chat with an invite link.")
-async def join(evt: CommandEvent) -> Optional[EventID]:
+@command_handler(
+    help_section=SECTION_CREATING_PORTALS,
+    help_args="<_link_>",
+    help_text="Join a chat with an invite link.",
+)
+async def join(evt: CommandEvent) -> EventID | None:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
 
@@ -155,8 +199,10 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
         except InvalidURL:
             return await evt.reply("That doesn't look like a Telegram invite link.")
 
-    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)"
-                       r"(?:/(?P<type>joinchat|s))?/(?P<id>[^/]+)/?", flags=re.IGNORECASE)
+    regex = re.compile(
+        r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:/(?P<type>joinchat|s))?/(?P<id>[^/]+)/?",
+        flags=re.IGNORECASE,
+    )
     arg = regex.match(url)
     if not arg:
         return await evt.reply("That doesn't look like a Telegram invite link.")
@@ -166,12 +212,15 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
     link_type = data["type"]
     if link_type:
         link_type = link_type.lower()
+    elif identifier.startswith("+"):
+        link_type = "joinchat"
+        identifier = identifier[1:]
     updates, _ = await _join(evt, identifier, link_type)
     if not updates:
         return None
 
     for chat in updates.chats:
-        portal = po.Portal.get_by_entity(chat)
+        portal = await po.Portal.get_by_entity(chat)
         if portal.mxid:
             await portal.invite_to_matrix([evt.sender.mxid])
             return await evt.reply(f"Invited you to portal of {portal.title}")
@@ -180,16 +229,23 @@ async def join(evt: CommandEvent) -> Optional[EventID]:
             try:
                 await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
             except ChatIdInvalidError as e:
-                evt.log.trace("ChatIdInvalidError while creating portal from !tg join command: %s",
-                              updates.stringify())
+                evt.log.trace(
+                    "ChatIdInvalidError while creating portal from !tg join command: %s",
+                    updates.stringify(),
+                )
                 raise e
-            return await evt.reply(f"Created room for {portal.title}")
+            if portal.mxid:
+                return await evt.reply(f"Created room for {portal.title}")
+            else:
+                return await evt.reply(f"Couldn't create room for {portal.title}")
     return None
 
 
-@command_handler(help_section=SECTION_MISC,
-                 help_args="[`chats`|`contacts`|`me`]",
-                 help_text="Synchronize your chat portals, contacts and/or own info.")
+@command_handler(
+    help_section=SECTION_MISC,
+    help_args="[`chats`|`contacts`|`me`]",
+    help_text="Synchronize your chat portals, contacts and/or own info.",
+)
 async def sync(evt: CommandEvent) -> EventID:
     if len(evt.args) > 0:
         sync_only = evt.args[0]
@@ -218,8 +274,9 @@ class MessageIDError(ValueError):
         self.message = message
 
 
-async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
-                               ) -> Tuple[TypeInputPeer, Message]:
+async def _parse_encoded_msgid(
+    user: AbstractUser, enc_id: str, type_name: str
+) -> tuple[TypeInputPeer, Message]:
     try:
         enc_id += (4 - len(enc_id) % 4) * "="
         enc_id = base64.b64decode(enc_id)
@@ -233,10 +290,10 @@ async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
         raise MessageIDError(f"Invalid {type_name} ID (format)") from e
 
     if peer_type == PEER_TYPE_CHAT:
-        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
+        orig_msg = await DBMessage.get_one_by_tgid(msg_id, space)
         if not orig_msg:
             raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
-        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
+        new_msg = await DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
         if not new_msg:
             raise MessageIDError(f"Invalid {type_name} ID (your copy of message not found in db)")
         msg_id = new_msg.tgid
@@ -251,9 +308,9 @@ async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
     return peer, cast(Message, msg)
 
 
-@command_handler(help_section=SECTION_MISC,
-                 help_args="<_play ID_>",
-                 help_text="Play a Telegram game.")
+@command_handler(
+    help_section=SECTION_MISC, help_args="<_play ID_>", help_text="Play a Telegram game."
+)
 async def play(evt: CommandEvent) -> EventID:
     if len(evt.args) < 1:
         return await evt.reply("**Usage:** `$cmdprefix+sp play <play ID>`")
@@ -271,18 +328,23 @@ async def play(evt: CommandEvent) -> EventID:
         return await evt.reply("Invalid play ID (message doesn't look like a game)")
 
     game = await evt.sender.client(
-        GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
+        GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True)
+    )
     if not isinstance(game, BotCallbackAnswer):
         return await evt.reply("Game request response invalid")
 
-    return await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
-                           f"{msg.media.game.description}")
+    return await evt.reply(
+        f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
+        f"{msg.media.game.description}"
+    )
 
 
-@command_handler(help_section=SECTION_MISC,
-                 help_args="<_poll ID_> <_choice number_>",
-                 help_text="Vote in a Telegram poll.")
-async def vote(evt: CommandEvent) -> EventID:
+@command_handler(
+    help_section=SECTION_MISC,
+    help_args="<_poll ID_> <_choice number_>",
+    help_text="Vote in a Telegram poll.",
+)
+async def vote(evt: CommandEvent) -> EventID | None:
     if len(evt.args) < 1:
         return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
     elif not await evt.sender.is_logged_in():
@@ -307,32 +369,38 @@ async def vote(evt: CommandEvent) -> EventID:
         except ValueError:
             option_index = None
         if option_index is None:
-            return await evt.reply(f"Invalid option number \"{option}\"",
-                                   render_markdown=False, allow_html=False)
+            return await evt.reply(
+                f'Invalid option number "{option}"', render_markdown=False, allow_html=False
+            )
         elif option_index < 0:
-            return await evt.reply(f"Invalid option number {option}. "
-                                   f"Option numbers must be positive.")
+            return await evt.reply(
+                f"Invalid option number {option}. Option numbers must be positive."
+            )
         elif option_index >= len(msg.media.poll.answers):
-            return await evt.reply(f"Invalid option number {option}. "
-                                   f"The poll only has {len(msg.media.poll.answers)} options.")
+            return await evt.reply(
+                f"Invalid option number {option}. "
+                f"The poll only has {len(msg.media.poll.answers)} options."
+            )
         options.append(msg.media.poll.answers[option_index].option)
-    options = [msg.media.poll.answers[int(option) - 1].option
-               for option in evt.args[1:]]
+    options = [msg.media.poll.answers[int(option) - 1].option for option in evt.args[1:]]
     try:
-        resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
+        await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
     except OptionsTooMuchError:
         return await evt.reply("You passed too many options.")
     # TODO use response
     return await evt.mark_read()
 
 
-@command_handler(help_section=SECTION_MISC, help_args="<_emoji_>",
-                 help_text="Roll a dice (\U0001F3B2), kick a football (\u26BD\uFE0F) or throw a "
-                           "dart (\U0001F3AF) or basketball (\U0001F3C0) on the Telegram servers.")
+@command_handler(
+    help_section=SECTION_MISC,
+    help_args="<_emoji_>",
+    help_text="Roll a dice (\U0001F3B2), kick a football (\u26BD\uFE0F) or throw a "
+    "dart (\U0001F3AF) or basketball (\U0001F3C0) on the Telegram servers.",
+)
 async def random(evt: CommandEvent) -> EventID:
     if not evt.is_portal:
         return await evt.reply("You can only randomize values in portal rooms")
-    portal = po.Portal.get_by_mxid(evt.room_id)
+    portal = await po.Portal.get_by_mxid(evt.room_id)
     arg = evt.args[0] if len(evt.args) > 0 else "dice"
     emoticon = {
         "dart": "\U0001F3AF",
@@ -343,14 +411,18 @@ async def random(evt: CommandEvent) -> EventID:
         "soccer": "\u26BD",
     }.get(arg, arg)
     try:
-        await evt.sender.client.send_media(await portal.get_input_entity(evt.sender),
-                                           InputMediaDice(emoticon))
+        await evt.sender.client.send_media(
+            await portal.get_input_entity(evt.sender), InputMediaDice(emoticon)
+        )
     except EmoticonInvalidError:
         return await evt.reply("Invalid emoji for randomization")
 
 
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT, help_args="[_limit_]",
-                 help_text="Backfill messages from Telegram history.")
+@command_handler(
+    help_section=SECTION_PORTAL_MANAGEMENT,
+    help_args="[_limit_]",
+    help_text="Backfill messages from Telegram history.",
+)
 async def backfill(evt: CommandEvent) -> None:
     if not evt.is_portal:
         await evt.reply("You can only use backfill in portal rooms")
@@ -359,17 +431,20 @@ async def backfill(evt: CommandEvent) -> None:
         limit = int(evt.args[0])
     except (ValueError, IndexError):
         limit = -1
-    portal = po.Portal.get_by_mxid(evt.room_id)
+    portal = await po.Portal.get_by_mxid(evt.room_id)
     if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
         await evt.reply("Backfilling normal groups is disabled in the bridge config")
         return
     try:
         await portal.backfill(evt.sender, limit=limit)
     except TakeoutInitDelayError:
-        msg = ("Please accept the data export request from a mobile device, "
-               "then re-run the backfill command.")
+        msg = (
+            "Please accept the data export request from a mobile device, "
+            "then re-run the backfill command."
+        )
         if portal.peer_type == "user":
             from mautrix.appservice import IntentAPI
+
             await portal.main_intent.send_notice(evt.room_id, msg)
         else:
             await evt.reply(msg)

mautrix_telegram/config.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2020 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,31 +14,40 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Any, List, NamedTuple
-from ruamel.yaml.comments import CommentedMap
 import os
 
-from mautrix.types import UserID
-from mautrix.client import Client
-from mautrix.bridge.config import BaseBridgeConfig
-from mautrix.util.config import ForbiddenKey, ForbiddenDefault, ConfigUpdateHelper
+from ruamel.yaml.comments import CommentedMap
 
-Permissions = NamedTuple("Permissions", relaybot=bool, user=bool, puppeting=bool,
-                         matrix_puppeting=bool, admin=bool, level=str)
+from mautrix.bridge.config import BaseBridgeConfig
+from mautrix.client import Client
+from mautrix.types import UserID
+from mautrix.util.config import ConfigUpdateHelper, ForbiddenDefault, ForbiddenKey
+
+Permissions = NamedTuple(
+    "Permissions",
+    relaybot=bool,
+    user=bool,
+    puppeting=bool,
+    matrix_puppeting=bool,
+    admin=bool,
+    level=str,
+)
 
 
 class Config(BaseBridgeConfig):
-    def __getitem__(self, key: str) -> Any:
-        try:
-            return os.environ[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
-        except KeyError:
-            return super().__getitem__(key)
-
     @property
     def forbidden_defaults(self) -> List[ForbiddenDefault]:
         return [
             *super().forbidden_defaults,
-            ForbiddenDefault("appservice.public.external", "https://example.com/public",
-                             condition="appservice.public.enabled"),
+            ForbiddenDefault(
+                "appservice.database",
+                "postgres://username:password@hostname/dbname",
+            ),
+            ForbiddenDefault(
+                "appservice.public.external",
+                "https://example.com/public",
+                condition="appservice.public.enabled",
+            ),
             ForbiddenDefault("bridge.permissions", ForbiddenKey("example.com")),
             ForbiddenDefault("telegram.api_id", 12345),
             ForbiddenDefault("telegram.api_hash", "tjyd5yge35lbodk1xwzw2jstp90k55qz"),
@@ -48,11 +57,12 @@ class Config(BaseBridgeConfig):
         super().do_update(helper)
         copy, copy_dict, base = helper
 
-        copy("homeserver.asmux")
-
         if "appservice.protocol" in self and "appservice.address" not in self:
-            protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
-                                        self["appservice.port"])
+            protocol, hostname, port = (
+                self["appservice.protocol"],
+                self["appservice.hostname"],
+                self["appservice.port"],
+            )
             base["appservice.address"] = f"{protocol}://{hostname}:{port}"
         if "appservice.debug" in self and "logging" not in self:
             level = "DEBUG" if self["appservice.debug"] else "INFO"
@@ -66,19 +76,24 @@ class Config(BaseBridgeConfig):
 
         copy("appservice.provisioning.enabled")
         copy("appservice.provisioning.prefix")
+        if base["appservice.provisioning.prefix"].endswith("/v1"):
+            base["appservice.provisioning.prefix"] = base["appservice.provisioning.prefix"][
+                : -len("/v1")
+            ]
         copy("appservice.provisioning.shared_secret")
         if base["appservice.provisioning.shared_secret"] == "generate":
             base["appservice.provisioning.shared_secret"] = self._new_token()
 
-        copy("appservice.community_id")
+        if "pool_size" in base["appservice.database_opts"]:
+            pool_size = base["appservice.database_opts"].pop("pool_size")
+            base["appservice.database_opts.min_size"] = pool_size
+            base["appservice.database_opts.max_size"] = pool_size
+        if "pool_pre_ping" in base["appservice.database_opts"]:
+            del base["appservice.database_opts.pool_pre_ping"]
 
         copy("metrics.enabled")
         copy("metrics.listen_port")
 
-        copy("manhole.enabled")
-        copy("manhole.path")
-        copy("manhole.whitelist")
-
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
@@ -88,6 +103,7 @@ class Config(BaseBridgeConfig):
         copy("bridge.allow_avatar_remove")
 
         copy("bridge.max_initial_member_sync")
+        copy("bridge.max_member_count")
         copy("bridge.sync_channel_members")
         copy("bridge.skip_deleted_members")
         copy("bridge.startup_sync")
@@ -101,12 +117,12 @@ class Config(BaseBridgeConfig):
         copy("bridge.max_telegram_delete")
         copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
-        copy("bridge.plaintext_highlights")
         copy("bridge.public_portals")
         copy("bridge.sync_with_custom_puppets")
         copy("bridge.sync_direct_chat_list")
         copy("bridge.double_puppet_server_map")
         copy("bridge.double_puppet_allow_discovery")
+        copy("bridge.create_group_on_invite")
         if "bridge.login_shared_secret" in self:
             base["bridge.login_shared_secret_map"] = {
                 base["homeserver.domain"]: self["bridge.login_shared_secret"]
@@ -115,23 +131,32 @@ class Config(BaseBridgeConfig):
             copy("bridge.login_shared_secret_map")
         copy("bridge.telegram_link_preview")
         copy("bridge.invite_link_resolve")
-        copy("bridge.inline_images")
+        copy("bridge.caption_in_message")
         copy("bridge.image_as_file_size")
-        copy("bridge.max_document_size")
+        copy("bridge.image_as_file_pixels")
         copy("bridge.parallel_file_transfer")
         copy("bridge.federate_rooms")
         copy("bridge.animated_sticker.target")
-        copy("bridge.animated_sticker.args")
-        copy("bridge.encryption.allow")
-        copy("bridge.encryption.default")
-        copy("bridge.encryption.database")
-        copy("bridge.encryption.key_sharing.allow")
-        copy("bridge.encryption.key_sharing.require_cross_signing")
-        copy("bridge.encryption.key_sharing.require_verification")
+        copy("bridge.animated_sticker.convert_from_webm")
+        copy("bridge.animated_sticker.args.width")
+        copy("bridge.animated_sticker.args.height")
+        copy("bridge.animated_sticker.args.fps")
+        copy("bridge.animated_emoji.target")
+        copy("bridge.animated_emoji.args.width")
+        copy("bridge.animated_emoji.args.height")
+        copy("bridge.animated_emoji.args.fps")
         copy("bridge.private_chat_portal_meta")
         copy("bridge.delivery_receipts")
         copy("bridge.delivery_error_reports")
+        copy("bridge.message_status_events")
         copy("bridge.resend_bridge_info")
+        copy("bridge.mute_bridging")
+        copy("bridge.pinned_tag")
+        copy("bridge.archive_tag")
+        copy("bridge.tag_only_on_create")
+        copy("bridge.bridge_matrix_leave")
+        copy("bridge.kick_on_logout")
+        copy("bridge.always_read_joined_telegram_notice")
         copy("bridge.backfill.invite_own_puppet")
         copy("bridge.backfill.takeout_limit")
         copy("bridge.backfill.initial_limit")
@@ -144,20 +169,16 @@ class Config(BaseBridgeConfig):
 
         copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):
-            base["bridge.bridge_notices"] = {
-                "default": self["bridge.bridge_notices"],
-                "exceptions": ["@importantbot:example.com"],
-            }
+            base["bridge.bridge_notices"]["default"] = self["bridge.bridge_notices"]
         else:
-            copy("bridge.bridge_notices")
-
-        copy("bridge.deduplication.pre_db_check")
-        copy("bridge.deduplication.cache_queue_length")
+            copy("bridge.bridge_notices.default")
+            copy("bridge.bridge_notices.exceptions")
 
         if "bridge.message_formats.m_text" in self:
             del self["bridge.message_formats"]
         copy_dict("bridge.message_formats", override_existing_map=False)
         copy("bridge.emote_format")
+        copy("bridge.relay_user_distinguishers")
 
         copy("bridge.state_event_formats.join")
         copy("bridge.state_event_formats.leave")
@@ -168,9 +189,11 @@ class Config(BaseBridgeConfig):
 
         copy("bridge.command_prefix")
 
-        migrate_permissions = ("bridge.permissions" not in self
-                               or "bridge.whitelist" in self
-                               or "bridge.admins" in self)
+        migrate_permissions = (
+            "bridge.permissions" not in self
+            or "bridge.whitelist" in self
+            or "bridge.admins" in self
+        )
         if migrate_permissions:
             permissions = self["bridge.permissions"] or CommentedMap()
             for entry in self["bridge.whitelist"] or []:
@@ -179,7 +202,7 @@ class Config(BaseBridgeConfig):
                 permissions[entry] = "admin"
             base["bridge.permissions"] = permissions
         else:
-            copy_dict("bridge.permissions")
+            copy_dict("bridge.permissions", override_existing_map=True)
 
         if "bridge.relaybot" not in self:
             copy("bridge.authless_relaybot_portals", "bridge.relaybot.authless_portals")
@@ -198,6 +221,10 @@ class Config(BaseBridgeConfig):
         copy("telegram.api_hash")
         copy("telegram.bot_token")
 
+        copy("telegram.catch_up")
+        copy("telegram.sequential_updates")
+        copy("telegram.exit_on_update_error")
+
         copy("telegram.connection.timeout")
         copy("telegram.connection.retries")
         copy("telegram.connection.retry_delay")

mautrix_telegram/context.py

@@ -1,57 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Tuple, TYPE_CHECKING
-import asyncio
-
-from alchemysession import AlchemySessionContainer
-
-from mautrix.appservice import AppService
-
-if TYPE_CHECKING:
-    from .web import PublicBridgeWebsite, ProvisioningAPI
-    from .config import Config
-    from .bot import Bot
-    from .matrix import MatrixHandler
-    from .__main__ import TelegramBridge
-
-
-class Context:
-    az: AppService
-    config: 'Config'
-    loop: asyncio.AbstractEventLoop
-    bridge: 'TelegramBridge'
-    bot: Optional['Bot']
-    mx: Optional['MatrixHandler']
-    session_container: AlchemySessionContainer
-    public_website: Optional['PublicBridgeWebsite']
-    provisioning_api: Optional['ProvisioningAPI']
-
-    def __init__(self, az: AppService, config: 'Config', loop: asyncio.AbstractEventLoop,
-                 session_container: AlchemySessionContainer, bridge: 'TelegramBridge',
-                 bot: Optional['Bot']) -> None:
-        self.az = az
-        self.config = config
-        self.loop = loop
-        self.bridge = bridge
-        self.bot = bot
-        self.mx = None
-        self.session_container = session_container
-        self.public_website = None
-        self.provisioning_api = None
-
-    @property
-    def core(self) -> Tuple[AppService, 'Config', asyncio.AbstractEventLoop, Optional['Bot']]:
-        return self.az, self.config, self.loop, self.bot

mautrix_telegram/db/__init__.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,19 +13,45 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy.engine.base import Engine
-
-from mautrix.client.state_store.sqlalchemy import UserProfile, RoomState
+from mautrix.util.async_db import Database
 
 from .bot_chat import BotChat
+from .disappearing_message import DisappearingMessage
 from .message import Message
 from .portal import Portal
 from .puppet import Puppet
+from .reaction import Reaction
 from .telegram_file import TelegramFile
-from .user import User, UserPortal, Contact
+from .telethon_session import PgSession
+from .upgrade import upgrade_table
+from .user import User
 
 
-def init(db_engine: Engine) -> None:
-    for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
-                  RoomState, BotChat):
-        table.bind(db_engine)
+def init(db: Database) -> None:
+    for table in (
+        Portal,
+        Message,
+        Reaction,
+        User,
+        Puppet,
+        TelegramFile,
+        BotChat,
+        PgSession,
+        DisappearingMessage,
+    ):
+        table.db = db
+
+
+__all__ = [
+    "upgrade_table",
+    "init",
+    "Portal",
+    "Message",
+    "Reaction",
+    "User",
+    "Puppet",
+    "TelegramFile",
+    "BotChat",
+    "PgSession",
+    "DisappearingMessage",
+]

mautrix_telegram/db/bot_chat.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,26 +13,43 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Iterable
+from __future__ import annotations
 
-from sqlalchemy import Column, BigInteger, String
+from typing import TYPE_CHECKING, ClassVar
 
-from mautrix.util.db import Base
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.util.async_db import Database
 
 from ..types import TelegramID
 
+fake_db = Database.create("") if TYPE_CHECKING else None
+
 
 # Fucking Telegram not telling bots what chats they are in 3:<
-class BotChat(Base):
-    __tablename__ = "bot_chat"
-    id: TelegramID = Column(BigInteger, primary_key=True)
-    type: str = Column(String, nullable=False)
+@dataclass
+class BotChat:
+    db: ClassVar[Database] = fake_db
+
+    id: TelegramID
+    type: str
 
     @classmethod
-    def delete_by_id(cls, chat_id: TelegramID) -> None:
-        with cls.db.begin() as conn:
-            conn.execute(cls.t.delete().where(cls.c.id == chat_id))
+    def _from_row(cls, row: Record | None) -> BotChat | None:
+        if row is None:
+            return None
+        return cls(**row)
 
     @classmethod
-    def all(cls) -> Iterable['BotChat']:
-        return cls._select_all()
+    async def delete_by_id(cls, chat_id: TelegramID) -> None:
+        await cls.db.execute("DELETE FROM bot_chat WHERE id=$1", chat_id)
+
+    @classmethod
+    async def all(cls) -> list[BotChat]:
+        rows = await cls.db.fetch("SELECT id, type FROM bot_chat")
+        return [cls._from_row(row) for row in rows]
+
+    async def insert(self) -> None:
+        q = "INSERT INTO bot_chat (id, type) VALUES ($1, $2)"
+        await self.db.execute(q, self.id, self.type)

mautrix_telegram/db/disappearing_message.py

@@ -0,0 +1,78 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Sumner Evans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+import asyncpg
+
+from mautrix.bridge import AbstractDisappearingMessage
+from mautrix.types import EventID, RoomID
+from mautrix.util.async_db import Database
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+class DisappearingMessage(AbstractDisappearingMessage):
+    db: ClassVar[Database] = fake_db
+
+    async def insert(self) -> None:
+        q = """
+            INSERT INTO disappearing_message (room_id, event_id, expiration_seconds, expiration_ts)
+            VALUES ($1, $2, $3, $4)
+        """
+        await self.db.execute(
+            q, self.room_id, self.event_id, self.expiration_seconds, self.expiration_ts
+        )
+
+    async def update(self) -> None:
+        q = "UPDATE disappearing_message SET expiration_ts=$3 WHERE room_id=$1 AND event_id=$2"
+        await self.db.execute(q, self.room_id, self.event_id, self.expiration_ts)
+
+    async def delete(self) -> None:
+        q = "DELETE from disappearing_message WHERE room_id=$1 AND event_id=$2"
+        await self.db.execute(q, self.room_id, self.event_id)
+
+    @classmethod
+    def _from_row(cls, row: asyncpg.Record) -> DisappearingMessage:
+        return cls(**row)
+
+    @classmethod
+    async def get(cls, room_id: RoomID, event_id: EventID) -> DisappearingMessage | None:
+        q = """
+            SELECT room_id, event_id, expiration_seconds, expiration_ts FROM disappearing_message
+            WHERE room_id=$1 AND mxid=$2
+        """
+        try:
+            return cls._from_row(await cls.db.fetchrow(q, room_id, event_id))
+        except Exception:
+            return None
+
+    @classmethod
+    async def get_all_scheduled(cls) -> list[DisappearingMessage]:
+        q = """
+            SELECT room_id, event_id, expiration_seconds, expiration_ts FROM disappearing_message
+            WHERE expiration_ts IS NOT NULL
+        """
+        return [cls._from_row(r) for r in await cls.db.fetch(q)]
+
+    @classmethod
+    async def get_unscheduled_for_room(cls, room_id: RoomID) -> list[DisappearingMessage]:
+        q = """
+            SELECT room_id, event_id, expiration_seconds, expiration_ts FROM disappearing_message
+            WHERE room_id = $1 AND expiration_ts IS NULL
+        """
+        return [cls._from_row(r) for r in await cls.db.fetch(q, room_id)]

mautrix_telegram/db/message.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,96 +13,194 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Iterator, List
+from __future__ import annotations
 
-from sqlalchemy import (Column, UniqueConstraint, BigInteger, Integer, String, Boolean, and_, func,
-                        desc, select, false)
+from typing import TYPE_CHECKING, ClassVar
 
-from mautrix.types import RoomID, EventID
-from mautrix.util.db import Base
+from asyncpg import Record
+from attr import dataclass
+
+from mautrix.types import EventID, RoomID, UserID
+from mautrix.util.async_db import Database, Scheme
 
 from ..types import TelegramID
 
+fake_db = Database.create("") if TYPE_CHECKING else None
 
-class Message(Base):
-    __tablename__ = "message"
 
-    mxid: EventID = Column(String)
-    mx_room: RoomID = Column(String)
-    tgid: TelegramID = Column(BigInteger, primary_key=True)
-    tg_space: TelegramID = Column(BigInteger, primary_key=True)
-    edit_index: int = Column(Integer, primary_key=True)
-    redacted: bool = Column(Boolean, server_default=false())
+@dataclass
+class Message:
+    db: ClassVar[Database] = fake_db
 
-    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"),)
+    mxid: EventID
+    mx_room: RoomID
+    tgid: TelegramID
+    tg_space: TelegramID
+    edit_index: int
+    redacted: bool = False
+    content_hash: bytes | None = None
+    sender_mxid: UserID | None = None
+    sender: TelegramID | None = None
 
     @classmethod
-    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Iterator['Message']:
-        return cls._select_all(cls.c.tgid == tgid, cls.c.tg_space == tg_space)
+    def _from_row(cls, row: Record | None) -> Message | None:
+        if row is None:
+            return None
+        return cls(**row)
+
+    columns: ClassVar[str] = ", ".join(
+        (
+            "mxid",
+            "mx_room",
+            "tgid",
+            "tg_space",
+            "edit_index",
+            "redacted",
+            "content_hash",
+            "sender_mxid",
+            "sender",
+        )
+    )
 
     @classmethod
-    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
-                        ) -> Optional['Message']:
+    async def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> list[Message]:
+        q = f"SELECT {cls.columns} FROM message WHERE tgid=$1 AND tg_space=$2"
+        rows = await cls.db.fetch(q, tgid, tg_space)
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def get_one_by_tgid(
+        cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+    ) -> Message | None:
         if edit_index < 0:
-            return cls._one_or_none(cls.db.execute(
-                cls.t.select()
-                    .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
-                    .order_by(desc(cls.c.edit_index))
-                    .limit(1).offset(-edit_index - 1)))
+            q = (
+                f"SELECT {cls.columns} FROM message WHERE tgid=$1 AND tg_space=$2 "
+                f"ORDER BY edit_index DESC LIMIT 1 OFFSET {-edit_index - 1}"
+            )
+            row = await cls.db.fetchrow(q, tgid, tg_space)
         else:
-            return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
-                                           cls.c.edit_index == edit_index)
+            q = (
+                f"SELECT {cls.columns} FROM message"
+                " WHERE tgid=$1 AND tg_space=$2 AND edit_index=$3"
+            )
+            row = await cls.db.fetchrow(q, tgid, tg_space, edit_index)
+        return cls._from_row(row)
 
     @classmethod
-    def get_first_by_tgids(cls, tgids: List[TelegramID], tg_space: TelegramID
-                           ) -> Iterator['Message']:
-        return cls._select_all(cls.c.tgid.in_(tgids), cls.c.tg_space == tg_space,
-                               cls.c.edit_index == 0)
+    async def get_first_by_tgids(
+        cls, tgids: list[TelegramID], tg_space: TelegramID
+    ) -> list[Message]:
+        if cls.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+            q = (
+                f"SELECT {cls.columns} FROM message"
+                " WHERE tgid=ANY($1) AND tg_space=$2 AND edit_index=0"
+            )
+            rows = await cls.db.fetch(q, tgids, tg_space)
+        else:
+            tgid_placeholders = ("?," * len(tgids)).rstrip(",")
+            q = (
+                f"SELECT {cls.columns} FROM message "
+                f"WHERE tg_space=? AND edit_index=0 AND tgid IN ({tgid_placeholders})"
+            )
+            rows = await cls.db.fetch(q, tg_space, *tgids)
+        return [cls._from_row(row) for row in rows]
 
     @classmethod
-    def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
-        rows = cls.db.execute(select([func.count(cls.c.tg_space)])
-                              .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
-        try:
-            count, = next(rows)
-            return count
-        except StopIteration:
-            return 0
+    async def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
+        return (
+            await cls.db.fetchval(
+                "SELECT COUNT(tg_space) FROM message WHERE mxid=$1 AND mx_room=$2", mxid, mx_room
+            )
+            or 0
+        )
 
     @classmethod
-    def find_last(cls, mx_room: RoomID, tg_space: TelegramID) -> Optional['Message']:
-        return cls._one_or_none(cls.db.execute(
-            cls._make_simple_select(cls.c.mx_room == mx_room, cls.c.tg_space == tg_space)
-                .order_by(desc(cls.c.tgid)).limit(1)))
+    async def find_last(cls, mx_room: RoomID, tg_space: TelegramID) -> Message | None:
+        q = (
+            f"SELECT {cls.columns} FROM message WHERE mx_room=$1 AND tg_space=$2 "
+            f"ORDER BY tgid DESC LIMIT 1"
+        )
+        return cls._from_row(await cls.db.fetchrow(q, mx_room, tg_space))
 
     @classmethod
-    def delete_all(cls, mx_room: RoomID) -> None:
-        cls.db.execute(cls.t.delete().where(cls.c.mx_room == mx_room))
+    async def delete_all(cls, mx_room: RoomID) -> None:
+        await cls.db.execute("DELETE FROM message WHERE mx_room=$1", mx_room)
 
     @classmethod
-    def get_by_mxid(cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
-                    ) -> Optional['Message']:
-        return cls._select_one_or_none(cls.c.mxid == mxid, cls.c.mx_room == mx_room,
-                                       cls.c.tg_space == tg_space)
+    async def get_by_mxid(
+        cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
+    ) -> Message | None:
+        q = f"SELECT {cls.columns} FROM message WHERE mxid=$1 AND mx_room=$2 AND tg_space=$3"
+        return cls._from_row(await cls.db.fetchrow(q, mxid, mx_room, tg_space))
 
     @classmethod
-    def get_by_mxids(cls, mxids: List[EventID], mx_room: RoomID, tg_space: TelegramID
-                     ) -> Iterator['Message']:
-        return cls._select_all(cls.c.mxid.in_(mxids), cls.c.mx_room == mx_room,
-                               cls.c.tg_space == tg_space)
+    async def get_by_mxids(
+        cls, mxids: list[EventID], mx_room: RoomID, tg_space: TelegramID
+    ) -> list[Message]:
+        if cls.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+            q = (
+                f"SELECT {cls.columns} FROM message"
+                " WHERE mxid=ANY($1) AND mx_room=$2 AND tg_space=$3"
+            )
+            rows = await cls.db.fetch(q, mxids, mx_room, tg_space)
+        else:
+            mxid_placeholders = ("?," * len(mxids)).rstrip(",")
+            q = (
+                f"SELECT {cls.columns} FROM message "
+                f"WHERE mx_room=? AND tg_space=? AND mxid IN ({mxid_placeholders})"
+            )
+            rows = await cls.db.fetch(q, mx_room, tg_space, *mxids)
+        return [cls._from_row(row) for row in rows]
 
     @classmethod
-    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
-                       **values) -> None:
-        with cls.db.begin() as conn:
-            conn.execute(cls.t.update()
-                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
-                                     cls.c.edit_index == s_edit_index))
-                         .values(**values))
+    async def find_recent(
+        cls, mx_room: RoomID, not_sender: TelegramID, limit: int = 20
+    ) -> list[Message]:
+        q = f"""
+        SELECT {cls.columns} FROM message
+        WHERE mx_room=$1 AND sender<>$2
+        ORDER BY tgid DESC LIMIT $3
+        """
+        return [cls._from_row(row) for row in await cls.db.fetch(q, mx_room, not_sender, limit)]
 
     @classmethod
-    def update_by_mxid(cls, s_mxid: EventID, s_mx_room: RoomID, **values) -> None:
-        with cls.db.begin() as conn:
-            conn.execute(cls.t.update()
-                         .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
-                         .values(**values))
+    async def replace_temp_mxid(cls, temp_mxid: str, mx_room: RoomID, real_mxid: EventID) -> None:
+        q = "UPDATE message SET mxid=$1 WHERE mxid=$2 AND mx_room=$3"
+        await cls.db.execute(q, real_mxid, temp_mxid, mx_room)
+
+    @classmethod
+    async def delete_temp_mxid(cls, temp_mxid: str, mx_room: RoomID) -> None:
+        q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2"
+        await cls.db.execute(q, temp_mxid, mx_room)
+
+    @property
+    def _values(self):
+        return (
+            self.mxid,
+            self.mx_room,
+            self.tgid,
+            self.tg_space,
+            self.edit_index,
+            self.redacted,
+            self.content_hash,
+            self.sender_mxid,
+            self.sender,
+        )
+
+    async def insert(self) -> None:
+        q = """
+            INSERT INTO message (
+                mxid, mx_room, tgid, tg_space, edit_index, redacted, content_hash,
+                sender_mxid, sender
+            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+        """
+        await self.db.execute(q, *self._values)
+
+    async def delete(self) -> None:
+        q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2 AND tg_space=$3"
+        await self.db.execute(q, self.mxid, self.mx_room, self.tg_space)
+
+    async def mark_redacted(self) -> None:
+        self.redacted = True
+        q = "UPDATE message SET redacted=true WHERE mxid=$1 AND mx_room=$2"
+        await self.db.execute(q, self.mxid, self.mx_room)

mautrix_telegram/db/portal.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,54 +13,177 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Iterable
+from __future__ import annotations
 
-from sqlalchemy import Column, BigInteger, String, Boolean, Text, func, sql
+from typing import TYPE_CHECKING, Any, ClassVar
+import json
 
-from mautrix.types import RoomID, ContentURI
-from mautrix.util.db import Base
+from asyncpg import Record
+from attr import dataclass
+import attr
+
+from mautrix.types import BatchID, ContentURI, EventID, RoomID
+from mautrix.util.async_db import Database
 
 from ..types import TelegramID
 
+fake_db = Database.create("") if TYPE_CHECKING else None
 
-class Portal(Base):
-    __tablename__ = "portal"
+
+@dataclass
+class Portal:
+    db: ClassVar[Database] = fake_db
 
     # Telegram chat information
-    tgid: TelegramID = Column(BigInteger, primary_key=True)
-    tg_receiver: TelegramID = Column(BigInteger, primary_key=True)
-    peer_type: str = Column(String, nullable=False)
-    megagroup: bool = Column(Boolean)
+    tgid: TelegramID
+    tg_receiver: TelegramID
+    peer_type: str
+    megagroup: bool
 
     # Matrix portal information
-    mxid: Optional[RoomID] = Column(String, unique=True, nullable=True)
-    avatar_url: Optional[ContentURI] = Column(String, nullable=True)
-    encrypted: bool = Column(Boolean, nullable=False, server_default=sql.expression.false())
+    mxid: RoomID | None
+    avatar_url: ContentURI | None
+    encrypted: bool
+    first_event_id: EventID | None
+    next_batch_id: BatchID | None
+    base_insertion_id: EventID | None
 
-    config: str = Column(Text, nullable=True)
+    sponsored_event_id: EventID | None
+    sponsored_event_ts: int | None
+    sponsored_msg_random_id: bytes | None
 
     # Telegram chat metadata
-    username: str = Column(String, nullable=True)
-    title: str = Column(String, nullable=True)
-    about: str = Column(String, nullable=True)
-    photo_id: str = Column(String, nullable=True)
+    username: str | None
+    title: str | None
+    about: str | None
+    photo_id: str | None
+    name_set: bool
+    avatar_set: bool
+
+    local_config: dict[str, Any] = attr.ib(factory=lambda: {})
 
     @classmethod
-    def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
-        return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver)
+    def _from_row(cls, row: Record | None) -> Portal | None:
+        if row is None:
+            return None
+        data = {**row}
+        data["local_config"] = json.loads(data.pop("config", None) or "{}")
+        return cls(**data)
+
+    columns: ClassVar[str] = ", ".join(
+        (
+            "tgid",
+            "tg_receiver",
+            "peer_type",
+            "megagroup",
+            "mxid",
+            "avatar_url",
+            "encrypted",
+            "first_event_id",
+            "next_batch_id",
+            "base_insertion_id",
+            "sponsored_event_id",
+            "sponsored_event_ts",
+            "sponsored_msg_random_id",
+            "username",
+            "title",
+            "about",
+            "photo_id",
+            "name_set",
+            "avatar_set",
+            "config",
+        )
+    )
 
     @classmethod
-    def find_private_chats(cls, tg_receiver: TelegramID) -> Iterable['Portal']:
-        yield from cls._select_all(cls.c.tg_receiver == tg_receiver, cls.c.peer_type == "user")
+    async def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Portal | None:
+        q = f"SELECT {cls.columns} FROM portal WHERE tgid=$1 AND tg_receiver=$2"
+        return cls._from_row(await cls.db.fetchrow(q, tgid, tg_receiver))
 
     @classmethod
-    def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
-        return cls._select_one_or_none(cls.c.mxid == mxid)
+    async def get_by_mxid(cls, mxid: RoomID) -> Portal | None:
+        q = f"SELECT {cls.columns} FROM portal WHERE mxid=$1"
+        return cls._from_row(await cls.db.fetchrow(q, mxid))
 
     @classmethod
-    def get_by_username(cls, username: str) -> Optional['Portal']:
-        return cls._select_one_or_none(func.lower(cls.c.username) == username)
+    async def find_by_username(cls, username: str) -> Portal | None:
+        q = f"SELECT {cls.columns} FROM portal WHERE lower(username)=$1"
+        return cls._from_row(await cls.db.fetchrow(q, username.lower()))
 
     @classmethod
-    def all(cls) -> Iterable['Portal']:
-        yield from cls._select_all()
+    async def find_private_chats_of(cls, tg_receiver: TelegramID) -> list[Portal]:
+        q = f"SELECT {cls.columns} FROM portal WHERE tg_receiver=$1 AND peer_type='user'"
+        return [cls._from_row(row) for row in await cls.db.fetch(q, tg_receiver)]
+
+    @classmethod
+    async def find_private_chats_with(cls, tgid: TelegramID) -> list[Portal]:
+        q = f"SELECT {cls.columns} FROM portal WHERE tgid=$1 AND peer_type='user'"
+        return [cls._from_row(row) for row in await cls.db.fetch(q, tgid)]
+
+    @classmethod
+    async def all(cls) -> list[Portal]:
+        rows = await cls.db.fetch(f"SELECT {cls.columns} FROM portal")
+        return [cls._from_row(row) for row in rows]
+
+    @property
+    def _values(self):
+        return (
+            self.tgid,
+            self.tg_receiver,
+            self.peer_type,
+            self.mxid,
+            self.avatar_url,
+            self.encrypted,
+            self.first_event_id,
+            self.next_batch_id,
+            self.base_insertion_id,
+            self.sponsored_event_id,
+            self.sponsored_event_ts,
+            self.sponsored_msg_random_id,
+            self.username,
+            self.title,
+            self.about,
+            self.photo_id,
+            self.name_set,
+            self.avatar_set,
+            self.megagroup,
+            json.dumps(self.local_config) if self.local_config else None,
+        )
+
+    async def save(self) -> None:
+        q = """
+        UPDATE portal
+        SET mxid=$4, avatar_url=$5, encrypted=$6,
+            first_event_id=$7, next_batch_id=$8, base_insertion_id=$9,
+            sponsored_event_id=$10, sponsored_event_ts=$11, sponsored_msg_random_id=$12,
+            username=$13, title=$14, about=$15, photo_id=$16, name_set=$17, avatar_set=$18,
+            megagroup=$19, config=$20
+        WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)
+        """
+        await self.db.execute(q, *self._values)
+
+    async def update_id(self, id: TelegramID, peer_type: str) -> None:
+        q = (
+            "UPDATE portal SET tgid=$1, tg_receiver=$1, peer_type=$2 "
+            "WHERE tgid=$3 AND tg_receiver=$3"
+        )
+        await self.db.execute(q, id, peer_type, self.tgid)
+        self.tgid = id
+        self.tg_receiver = id
+        self.peer_type = peer_type
+
+    async def insert(self) -> None:
+        q = """
+        INSERT INTO portal (
+            tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,
+            first_event_id, base_insertion_id, next_batch_id,
+            sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,
+            username, title, about, photo_id, name_set, avatar_set, megagroup, config
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
+                  $19, $20)
+        """
+        await self.db.execute(q, *self._values)
+
+    async def delete(self) -> None:
+        q = "DELETE FROM portal WHERE tgid=$1 AND tg_receiver=$2"
+        await self.db.execute(q, self.tgid, self.tg_receiver)

mautrix_telegram/db/puppet.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,51 +13,129 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, Iterable
+from __future__ import annotations
 
-from sqlalchemy import Column, Integer, BigInteger, String, Text, Boolean
-from sqlalchemy.sql import expression, func
+from typing import TYPE_CHECKING, ClassVar
 
-from mautrix.types import UserID, SyncToken
-from mautrix.util.db import Base
+from asyncpg import Record
+from attr import dataclass
+from yarl import URL
+
+from mautrix.types import ContentURI, SyncToken, UserID
+from mautrix.util.async_db import Database
 
 from ..types import TelegramID
 
+fake_db = Database.create("") if TYPE_CHECKING else None
 
-class Puppet(Base):
-    __tablename__ = "puppet"
 
-    id: TelegramID = Column(BigInteger, primary_key=True)
-    custom_mxid: UserID = Column(String, nullable=True)
-    access_token: str = Column(String, nullable=True)
-    next_batch: SyncToken = Column(String, nullable=True)
-    base_url: str = Column(Text, nullable=True)
-    displayname: str = Column(String, nullable=True)
-    displayname_source: TelegramID = Column(BigInteger, nullable=True)
-    displayname_contact: bool = Column(Boolean, nullable=False, server_default=expression.true())
-    displayname_quality: int = Column(Integer, nullable=False, server_default="0")
-    username: str = Column(String, nullable=True)
-    photo_id: str = Column(String, nullable=True)
-    is_bot: bool = Column(Boolean, nullable=True)
-    matrix_registered: bool = Column(Boolean, nullable=False, server_default=expression.false())
-    disable_updates: bool = Column(Boolean, nullable=False, server_default=expression.false())
+@dataclass
+class Puppet:
+    db: ClassVar[Database] = fake_db
+
+    id: TelegramID
+
+    is_registered: bool
+
+    displayname: str | None
+    displayname_source: TelegramID | None
+    displayname_contact: bool
+    displayname_quality: int
+    disable_updates: bool
+    username: str | None
+    phone: str | None
+    photo_id: str | None
+    avatar_url: ContentURI | None
+    name_set: bool
+    avatar_set: bool
+    is_bot: bool | None
+    is_channel: bool
+    is_premium: bool
+
+    custom_mxid: UserID | None
+    access_token: str | None
+    next_batch: SyncToken | None
+    base_url: URL | None
 
     @classmethod
-    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
-        yield from cls._select_all(cls.c.custom_mxid != None)
+    def _from_row(cls, row: Record | None) -> Puppet | None:
+        if row is None:
+            return None
+        data = {**row}
+        base_url = data.pop("base_url", None)
+        return cls(**data, base_url=URL(base_url) if base_url else None)
+
+    columns: ClassVar[str] = (
+        "id, is_registered, displayname, displayname_source, displayname_contact, "
+        "displayname_quality, disable_updates, username, phone, photo_id, avatar_url, "
+        "name_set, avatar_set, is_bot, is_channel, is_premium, "
+        "custom_mxid, access_token, next_batch, base_url"
+    )
 
     @classmethod
-    def get_by_tgid(cls, tgid: TelegramID) -> Optional['Puppet']:
-        return cls._select_one_or_none(cls.c.id == tgid)
+    async def all_with_custom_mxid(cls) -> list[Puppet]:
+        q = f"SELECT {cls.columns} FROM puppet WHERE custom_mxid<>''"
+        return [cls._from_row(row) for row in await cls.db.fetch(q)]
 
     @classmethod
-    def get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
-        return cls._select_one_or_none(cls.c.custom_mxid == mxid)
+    async def get_by_tgid(cls, tgid: TelegramID) -> Puppet | None:
+        q = f"SELECT {cls.columns} FROM puppet WHERE id=$1"
+        return cls._from_row(await cls.db.fetchrow(q, tgid))
 
     @classmethod
-    def get_by_username(cls, username: str) -> Optional['Puppet']:
-        return cls._select_one_or_none(func.lower(cls.c.username) == username)
+    async def get_by_custom_mxid(cls, mxid: UserID) -> Puppet | None:
+        q = f"SELECT {cls.columns} FROM puppet WHERE custom_mxid=$1"
+        return cls._from_row(await cls.db.fetchrow(q, mxid))
 
     @classmethod
-    def get_by_displayname(cls, displayname: str) -> Optional['Puppet']:
-        return cls._select_one_or_none(cls.c.displayname == displayname)
+    async def find_by_username(cls, username: str) -> Puppet | None:
+        q = f"SELECT {cls.columns} FROM puppet WHERE lower(username)=$1"
+        return cls._from_row(await cls.db.fetchrow(q, username.lower()))
+
+    @property
+    def _values(self):
+        return (
+            self.id,
+            self.is_registered,
+            self.displayname,
+            self.displayname_source,
+            self.displayname_contact,
+            self.displayname_quality,
+            self.disable_updates,
+            self.username,
+            self.phone,
+            self.photo_id,
+            self.avatar_url,
+            self.name_set,
+            self.avatar_set,
+            self.is_bot,
+            self.is_channel,
+            self.is_premium,
+            self.custom_mxid,
+            self.access_token,
+            self.next_batch,
+            str(self.base_url) if self.base_url else None,
+        )
+
+    async def save(self) -> None:
+        q = """
+        UPDATE puppet
+        SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,
+            displayname_quality=$6, disable_updates=$7, username=$8, phone=$9, photo_id=$10,
+            avatar_url=$11, name_set=$12, avatar_set=$13, is_bot=$14, is_channel=$15,
+            is_premium=$16, custom_mxid=$17, access_token=$18, next_batch=$19, base_url=$20
+        WHERE id=$1
+        """
+        await self.db.execute(q, *self._values)
+
+    async def insert(self) -> None:
+        q = """
+        INSERT INTO puppet (
+            id, is_registered, displayname, displayname_source, displayname_contact,
+            displayname_quality, disable_updates, username, phone, photo_id, avatar_url, name_set,
+            avatar_set, is_bot, is_channel, is_premium, custom_mxid, access_token, next_batch,
+            base_url
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
+                  $19, $20)
+        """
+        await self.db.execute(q, *self._values)

mautrix_telegram/db/reaction.py

@@ -0,0 +1,100 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from asyncpg import Record
+from attr import dataclass
+from telethon.tl.types import ReactionCustomEmoji, ReactionEmoji, TypeReaction
+
+from mautrix.types import EventID, RoomID
+from mautrix.util.async_db import Database
+
+from ..types import TelegramID
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+@dataclass
+class Reaction:
+    db: ClassVar[Database] = fake_db
+
+    mxid: EventID
+    mx_room: RoomID
+    msg_mxid: EventID
+    tg_sender: TelegramID
+    reaction: str
+
+    @classmethod
+    def _from_row(cls, row: Record | None) -> Reaction | None:
+        if row is None:
+            return None
+        return cls(**row)
+
+    columns: ClassVar[str] = "mxid, mx_room, msg_mxid, tg_sender, reaction"
+
+    @classmethod
+    async def delete_all(cls, mx_room: RoomID) -> None:
+        await cls.db.execute("DELETE FROM reaction WHERE mx_room=$1", mx_room)
+
+    @classmethod
+    async def get_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> Reaction | None:
+        q = f"SELECT {cls.columns} FROM reaction WHERE mxid=$1 AND mx_room=$2"
+        return cls._from_row(await cls.db.fetchrow(q, mxid, mx_room))
+
+    @classmethod
+    async def get_by_sender(
+        cls, mxid: EventID, mx_room: RoomID, tg_sender: TelegramID
+    ) -> list[Reaction]:
+        q = f"SELECT {cls.columns} FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3"
+        rows = await cls.db.fetch(q, mxid, mx_room, tg_sender)
+        return [cls._from_row(row) for row in rows]
+
+    @classmethod
+    async def get_all_by_message(cls, mxid: EventID, mx_room: RoomID) -> list[Reaction]:
+        q = f"SELECT {cls.columns} FROM reaction WHERE msg_mxid=$1 AND mx_room=$2"
+        rows = await cls.db.fetch(q, mxid, mx_room)
+        return [cls._from_row(row) for row in rows]
+
+    @property
+    def telegram(self) -> TypeReaction:
+        if self.reaction.isdecimal():
+            return ReactionCustomEmoji(document_id=int(self.reaction))
+        else:
+            return ReactionEmoji(emoticon=self.reaction)
+
+    @property
+    def _values(self):
+        return (
+            self.mxid,
+            self.mx_room,
+            self.msg_mxid,
+            self.tg_sender,
+            self.reaction,
+        )
+
+    async def save(self) -> None:
+        q = """
+            INSERT INTO reaction (mxid, mx_room, msg_mxid, tg_sender, reaction)
+            VALUES ($1, $2, $3, $4, $5) ON CONFLICT (msg_mxid, mx_room, tg_sender, reaction)
+                DO UPDATE SET mxid=excluded.mxid
+        """
+        await self.db.execute(q, *self._values)
+
+    async def delete(self) -> None:
+        q = "DELETE FROM reaction WHERE msg_mxid=$1 AND mx_room=$2 AND tg_sender=$3 AND reaction=$4"
+        await self.db.execute(q, self.msg_mxid, self.mx_room, self.tg_sender, self.reaction)

mautrix_telegram/db/telegram_file.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2021 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,69 +13,98 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Optional, cast, Dict, Any, TYPE_CHECKING
+from __future__ import annotations
 
-from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
-                        TypeDecorator)
+from typing import TYPE_CHECKING, ClassVar
+
+from asyncpg import Record
+from attr import dataclass
 
 from mautrix.types import ContentURI, EncryptedFile
-from mautrix.util.db import Base
+from mautrix.util.async_db import Database, Scheme
 
-if TYPE_CHECKING:
-    from sqlalchemy.engine.result import RowProxy
+fake_db = Database.create("") if TYPE_CHECKING else None
 
 
-class DBEncryptedFile(TypeDecorator):
-    impl = Text
+@dataclass
+class TelegramFile:
+    db: ClassVar[Database] = fake_db
 
-    @property
-    def python_type(self):
-        return EncryptedFile
+    id: str
+    mxc: ContentURI
+    mime_type: str
+    was_converted: bool
+    timestamp: int
+    size: int | None
+    width: int | None
+    height: int | None
+    decryption_info: EncryptedFile | None
+    thumbnail: TelegramFile | None = None
 
-    def process_bind_param(self, value: EncryptedFile, dialect) -> Optional[str]:
-        if value is not None:
-            return value.json()
-        return None
-
-    def process_result_value(self, value: str, dialect) -> Optional[EncryptedFile]:
-        if value is not None:
-            return EncryptedFile.parse_json(value)
-        return None
-
-    def process_literal_param(self, value, dialect):
-        return value
-
-
-class TelegramFile(Base):
-    __tablename__ = "telegram_file"
-
-    id: str = Column(String, primary_key=True)
-    mxc: ContentURI = Column(String)
-    mime_type: str = Column(String)
-    was_converted: bool = Column(Boolean)
-    timestamp: int = Column(BigInteger)
-    size: Optional[int] = Column(Integer, nullable=True)
-    width: Optional[int] = Column(Integer, nullable=True)
-    height: Optional[int] = Column(Integer, nullable=True)
-    decryption_info: Optional[Dict[str, Any]] = Column(DBEncryptedFile, nullable=True)
-    thumbnail_id: str = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
-    thumbnail: Optional['TelegramFile'] = None
+    columns: ClassVar[str] = (
+        "id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail, "
+        "decryption_info"
+    )
 
     @classmethod
-    def scan(cls, row: 'RowProxy') -> 'TelegramFile':
-        telegram_file = cast(TelegramFile, super().scan(row))
-        if isinstance(telegram_file.thumbnail, str):
-            telegram_file.thumbnail = cls.get(telegram_file.thumbnail)
-        return telegram_file
+    def _from_row(cls, row: Record | None) -> TelegramFile | None:
+        if row is None:
+            return None
+        data = {**row}
+        data.pop("thumbnail", None)
+        decryption_info = data.pop("decryption_info", None)
+        return cls(
+            **data,
+            thumbnail=None,
+            decryption_info=EncryptedFile.parse_json(decryption_info) if decryption_info else None,
+        )
 
     @classmethod
-    def get(cls, loc_id: str) -> Optional['TelegramFile']:
-        return cls._select_one_or_none(cls.c.id == loc_id)
+    async def get_many(cls, loc_ids: list[str]) -> list[TelegramFile]:
+        if cls.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+            q = f"SELECT {cls.columns} FROM telegram_file WHERE id=ANY($1)"
+            rows = await cls.db.fetch(q, loc_ids)
+        else:
+            tgid_placeholders = ("?," * len(loc_ids)).rstrip(",")
+            q = f"SELECT {cls.columns} FROM telegram_file WHERE id IN ({tgid_placeholders})"
+            rows = await cls.db.fetch(q, *loc_ids)
+        return [cls._from_row(row) for row in rows]
 
-    def insert(self) -> None:
-        with self.db.begin() as conn:
-            conn.execute(self.t.insert().values(
-                id=self.id, mxc=self.mxc, mime_type=self.mime_type,
-                was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
-                width=self.width, height=self.height, decryption_info=self.decryption_info,
-                thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))
+    @classmethod
+    async def get(cls, loc_id: str, *, _thumbnail: bool = False) -> TelegramFile | None:
+        q = f"SELECT {cls.columns} FROM telegram_file WHERE id=$1"
+        row = await cls.db.fetchrow(q, loc_id)
+        file = cls._from_row(row)
+        if file is None:
+            return None
+        try:
+            thumbnail_id = row["thumbnail"]
+        except KeyError:
+            thumbnail_id = None
+        if thumbnail_id and not _thumbnail:
+            file.thumbnail = await cls.get(thumbnail_id, _thumbnail=True)
+        return file
+
+    @classmethod
+    async def find_by_mxc(cls, mxc: ContentURI) -> TelegramFile | None:
+        q = f"SELECT {cls.columns} FROM telegram_file WHERE mxc=$1"
+        return cls._from_row(await cls.db.fetchrow(q, mxc))
+
+    async def insert(self) -> None:
+        q = (
+            "INSERT INTO telegram_file (id, mxc, mime_type, was_converted, size, width, height, "
+            "                           thumbnail, decryption_info) "
+            "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"
+        )
+        await self.db.execute(
+            q,
+            self.id,
+            self.mxc,
+            self.mime_type,
+            self.was_converted,
+            self.size,
+            self.width,
+            self.height,
+            self.thumbnail.id if self.thumbnail else None,
+            self.decryption_info.json() if self.decryption_info else None,
+        )

mautrix_telegram/db/telethon_session.py

@@ -0,0 +1,230 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar, Iterable
+import asyncio
+import datetime
+
+from telethon import utils
+from telethon.crypto import AuthKey
+from telethon.sessions import MemorySession
+from telethon.tl.types import PeerChannel, PeerChat, PeerUser, updates
+
+from mautrix.util.async_db import Database, Scheme
+
+fake_db = Database.create("") if TYPE_CHECKING else None
+
+
+class PgSession(MemorySession):
+    db: ClassVar[Database] = fake_db
+
+    session_id: str
+    _dc_id: int
+    _server_address: str | None
+    _port: int | None
+    _auth_key: AuthKey | None
+    _takeout_id: int | None
+    _process_entities_lock: asyncio.Lock
+
+    def __init__(
+        self,
+        session_id: str,
+        dc_id: int = 0,
+        server_address: str | None = None,
+        port: int | None = None,
+        auth_key: AuthKey | None = None,
+        takeout_id: int | None = None,
+    ) -> None:
+        super().__init__()
+        self.session_id = session_id
+        self._dc_id = dc_id
+        self._server_address = server_address
+        self._port = port
+        self._auth_key = auth_key
+        self._takeout_id = takeout_id
+        self._process_entities_lock = asyncio.Lock()
+
+    def clone(self, to_instance=None) -> MemorySession:
+        # We don't want to store data of clones
+        # (which are used for temporarily connecting to different DCs)
+        return super().clone(MemorySession())
+
+    @property
+    def auth_key_bytes(self) -> bytes | None:
+        return self._auth_key.key if self._auth_key else None
+
+    @classmethod
+    async def get(cls, session_id: str) -> PgSession:
+        q = (
+            "SELECT session_id, dc_id, server_address, port, auth_key FROM telethon_sessions "
+            "WHERE session_id=$1"
+        )
+        row = await cls.db.fetchrow(q, session_id)
+        if row is None:
+            return cls(session_id)
+        data = {**row}
+        auth_key = AuthKey(data.pop("auth_key", None))
+        return cls(**data, auth_key=auth_key)
+
+    @classmethod
+    async def has(cls, session_id: str) -> bool:
+        q = "SELECT COUNT(*) FROM telethon_sessions WHERE session_id=$1"
+        count = await cls.db.fetchval(q, session_id)
+        return count > 0
+
+    async def save(self) -> None:
+        q = (
+            "INSERT INTO telethon_sessions (session_id, dc_id, server_address, port, auth_key) "
+            "VALUES ($1, $2, $3, $4, $5) ON CONFLICT (session_id) "
+            "DO UPDATE SET dc_id=$2, server_address=$3, port=$4, auth_key=$5"
+        )
+        await self.db.execute(
+            q, self.session_id, self.dc_id, self.server_address, self.port, self.auth_key_bytes
+        )
+
+    _tables: ClassVar[tuple[str, ...]] = (
+        "telethon_sessions",
+        "telethon_entities",
+        "telethon_sent_files",
+        "telethon_update_state",
+    )
+
+    async def delete(self) -> None:
+        async with self.db.acquire() as conn, conn.transaction():
+            for table in self._tables:
+                await conn.execute(f"DELETE FROM {table} WHERE session_id=$1", self.session_id)
+
+    async def close(self) -> None:
+        # Nothing to do here, DB connection is global
+        pass
+
+    async def get_update_state(self, entity_id: int) -> updates.State | None:
+        q = (
+            "SELECT pts, qts, date, seq, unread_count FROM telethon_update_state "
+            "WHERE session_id=$1 AND entity_id=$2"
+        )
+        row = await self.db.fetchrow(q, self.session_id, entity_id)
+        if row is None:
+            return None
+        date = datetime.datetime.utcfromtimestamp(row["date"])
+        return updates.State(row["pts"], row["qts"], date, row["seq"], row["unread_count"])
+
+    async def set_update_state(self, entity_id: int, row: updates.State) -> None:
+        q = """
+        INSERT INTO telethon_update_state(session_id, entity_id, pts, qts, date, seq, unread_count)
+        VALUES ($1, $2, $3, $4, $5, $6, $7)
+        ON CONFLICT (session_id, entity_id) DO UPDATE SET
+            pts=excluded.pts, qts=excluded.qts, date=excluded.date, seq=excluded.seq,
+            unread_count=excluded.unread_count
+        """
+        ts = row.date.timestamp()
+        await self.db.execute(
+            q, self.session_id, entity_id, row.pts, row.qts, ts, row.seq, row.unread_count
+        )
+
+    async def delete_update_state(self, entity_id: int) -> None:
+        q = "DELETE FROM telethon_update_state WHERE session_id=$1 AND entity_id=$2"
+        await self.db.execute(q, self.session_id, entity_id)
+
+    async def get_update_states(self) -> Iterable[tuple[int, updates.State], ...]:
+        q = (
+            "SELECT entity_id, pts, qts, date, seq, unread_count FROM telethon_update_state "
+            "WHERE session_id=$1"
+        )
+        rows = await self.db.fetch(q, self.session_id)
+        return (
+            (
+                row["entity_id"],
+                updates.State(
+                    row["pts"],
+                    row["qts"],
+                    datetime.datetime.utcfromtimestamp(row["date"]),
+                    row["seq"],
+                    row["unread_count"],
+                ),
+            )
+            for row in rows
+        )
+
+    def _entity_values_to_row(
+        self, id: int, hash: int, username: str | None, phone: str | int | None, name: str | None
+    ) -> tuple[str, int, int, str | None, str | None, str | None]:
+        return self.session_id, id, hash, username, str(phone) if phone else None, name
+
+    async def process_entities(self, tlo) -> None:
+        # Postgres likes to deadlock on simultaneous upserts, so just lock the whole thing here
+        # TODO: make sure postgres doesn't deadlock on upserts when session_id is different
+        async with self._process_entities_lock:
+            await self._locked_process_entities(tlo)
+
+    async def _locked_process_entities(self, tlo) -> None:
+        rows: list[
+            tuple[str, int, int, str | None, str | None, str | None]
+        ] = self._entities_to_rows(tlo)
+        if not rows:
+            return
+        if self.db.scheme == Scheme.POSTGRES:
+            q = (
+                "INSERT INTO telethon_entities (session_id, id, hash, username, phone, name) "
+                "VALUES ($1, unnest($2::bigint[]), unnest($3::bigint[]), "
+                "        unnest($4::text[]), unnest($5::text[]), unnest($6::text[])) "
+                "ON CONFLICT (session_id, id) DO UPDATE"
+                "  SET hash=excluded.hash, username=excluded.username,"
+                "      phone=excluded.phone, name=excluded.name"
+            )
+            _, ids, hashes, usernames, phones, names = zip(*rows)
+            await self.db.execute(q, self.session_id, ids, hashes, usernames, phones, names)
+        else:
+            q = (
+                "INSERT INTO telethon_entities (session_id, id, hash, username, phone, name) "
+                "VALUES ($1, $2, $3, $4, $5, $6) "
+                "ON CONFLICT (session_id, id) DO UPDATE "
+                "    SET hash=$3, username=$4, phone=$5, name=$6"
+            )
+            await self.db.executemany(q, rows)
+
+    async def _select_entity(
+        self, constraint: str, *args: str | int | tuple[int, ...]
+    ) -> tuple[int, int] | None:
+        q = f"SELECT id, hash FROM telethon_entities WHERE session_id=$1 AND {constraint}"
+        row = await self.db.fetchrow(q, self.session_id, *args)
+        if row is None:
+            return None
+        return row["id"], row["hash"]
+
+    async def get_entity_rows_by_phone(self, key: str | int) -> tuple[int, int] | None:
+        return await self._select_entity("phone=$2", str(key))
+
+    async def get_entity_rows_by_username(self, key: str) -> tuple[int, int] | None:
+        return await self._select_entity("username=$2", key)
+
+    async def get_entity_rows_by_name(self, key: str) -> tuple[int, int] | None:
+        return await self._select_entity("name=$2", key)
+
+    async def get_entity_rows_by_id(self, key: int, exact: bool = True) -> tuple[int, int] | None:
+        if exact:
+            return await self._select_entity("id=$2", key)
+
+        ids = (
+            utils.get_peer_id(PeerUser(key)),
+            utils.get_peer_id(PeerChat(key)),
+            utils.get_peer_id(PeerChannel(key)),
+        )
+        if self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+            return await self._select_entity("id=ANY($2)", ids)
+        else:
+            return await self._select_entity(f"id IN ($2, $3, $4)", *ids)

mautrix_telegram/db/upgrade/__init__.py

@@ -0,0 +1,19 @@
+from mautrix.util.async_db import UpgradeTable
+
+upgrade_table = UpgradeTable()
+
+from . import (
+    v01_initial_revision,
+    v02_sponsored_events,
+    v03_reactions,
+    v04_disappearing_messages,
+    v05_channel_ghosts,
+    v06_puppet_avatar_url,
+    v07_puppet_phone_number,
+    v08_portal_first_event,
+    v09_puppet_username_index,
+    v10_more_backfill_fields,
+    v11_backfill_queue,
+    v12_message_sender,
+    v13_multiple_reactions,
+)

mautrix_telegram/db/upgrade/v00_latest_revision.py

@@ -0,0 +1,236 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection, Scheme
+
+latest_version = 13
+
+
+async def create_latest_tables(conn: Connection, scheme: Scheme) -> int:
+    await conn.execute(
+        """CREATE TABLE "user" (
+            mxid TEXT   PRIMARY KEY,
+            tgid BIGINT UNIQUE,
+            tg_username    TEXT,
+            tg_phone       TEXT,
+            is_bot         BOOLEAN NOT NULL DEFAULT false,
+            is_premium     BOOLEAN NOT NULL DEFAULT false,
+            saved_contacts INTEGER NOT NULL DEFAULT 0
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE portal (
+            tgid        BIGINT,
+            tg_receiver BIGINT,
+            peer_type   TEXT NOT NULL,
+            mxid        TEXT UNIQUE,
+            avatar_url  TEXT,
+            encrypted   BOOLEAN NOT NULL DEFAULT false,
+            username    TEXT,
+            title       TEXT,
+            about       TEXT,
+            photo_id    TEXT,
+            name_set    BOOLEAN NOT NULL DEFAULT false,
+            avatar_set  BOOLEAN NOT NULL DEFAULT false,
+            megagroup   BOOLEAN,
+            config      jsonb,
+
+            first_event_id    TEXT,
+            next_batch_id     TEXT,
+            base_insertion_id TEXT,
+
+            sponsored_event_id     TEXT,
+            sponsored_event_ts     BIGINT,
+            sponsored_msg_random_id bytea,
+
+            PRIMARY KEY (tgid, tg_receiver)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE message (
+            mxid         TEXT   NOT NULL,
+            mx_room      TEXT   NOT NULL,
+            tgid         BIGINT,
+            tg_space     BIGINT,
+            edit_index   INTEGER,
+            redacted     BOOLEAN NOT NULL DEFAULT false,
+            content_hash bytea,
+            sender_mxid  TEXT,
+            sender       BIGINT,
+            PRIMARY KEY (tgid, tg_space, edit_index),
+            UNIQUE (mxid, mx_room, tg_space)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE reaction (
+            mxid      TEXT NOT NULL,
+            mx_room   TEXT NOT NULL,
+            msg_mxid  TEXT NOT NULL,
+            tg_sender BIGINT,
+            reaction  TEXT NOT NULL,
+
+            PRIMARY KEY (msg_mxid, mx_room, tg_sender, reaction),
+            UNIQUE (mxid, mx_room)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE disappearing_message (
+            room_id             TEXT,
+            event_id            TEXT,
+            expiration_seconds  BIGINT,
+            expiration_ts       BIGINT,
+
+            PRIMARY KEY (room_id, event_id)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE puppet (
+            id BIGINT PRIMARY KEY,
+
+            is_registered BOOLEAN NOT NULL DEFAULT false,
+
+            displayname         TEXT,
+            displayname_source  BIGINT,
+            displayname_contact BOOLEAN NOT NULL DEFAULT true,
+            displayname_quality INTEGER NOT NULL DEFAULT 0,
+            disable_updates     BOOLEAN NOT NULL DEFAULT false,
+            username            TEXT,
+            phone               TEXT,
+            photo_id            TEXT,
+            avatar_url          TEXT,
+            name_set            BOOLEAN NOT NULL DEFAULT false,
+            avatar_set          BOOLEAN NOT NULL DEFAULT false,
+            is_bot              BOOLEAN,
+            is_channel          BOOLEAN NOT NULL DEFAULT false,
+            is_premium          BOOLEAN NOT NULL DEFAULT false,
+
+            access_token TEXT,
+            custom_mxid  TEXT,
+            next_batch   TEXT,
+            base_url     TEXT
+        )"""
+    )
+    await conn.execute("CREATE INDEX puppet_username_idx ON puppet(LOWER(username))")
+    await conn.execute(
+        """CREATE TABLE telegram_file (
+            id              TEXT PRIMARY KEY,
+            mxc             TEXT NOT NULL,
+            mime_type       TEXT,
+            was_converted   BOOLEAN NOT NULL DEFAULT false,
+            timestamp       BIGINT  NOT NULL DEFAULT 0,
+            size            BIGINT,
+            width           INTEGER,
+            height          INTEGER,
+            thumbnail       TEXT,
+            decryption_info jsonb,
+            FOREIGN KEY (thumbnail) REFERENCES telegram_file(id)
+                ON UPDATE CASCADE ON DELETE SET NULL
+        )"""
+    )
+    await conn.execute("CREATE INDEX telegram_file_mxc_idx ON telegram_file(mxc)")
+    await conn.execute(
+        """CREATE TABLE bot_chat (
+            id   BIGINT PRIMARY KEY,
+            type TEXT NOT NULL
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE user_portal (
+            "user"          BIGINT,
+            portal          BIGINT,
+            portal_receiver BIGINT,
+            PRIMARY KEY ("user", portal, portal_receiver),
+            FOREIGN KEY ("user") REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (portal, portal_receiver) REFERENCES portal(tgid, tg_receiver)
+                 ON DELETE CASCADE ON UPDATE CASCADE
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE contact (
+            "user"  BIGINT,
+            contact BIGINT,
+            PRIMARY KEY ("user", contact),
+            FOREIGN KEY ("user")  REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (contact) REFERENCES puppet(id)   ON DELETE CASCADE ON UPDATE CASCADE
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_sessions (
+            session_id     TEXT PRIMARY KEY,
+            dc_id          INTEGER,
+            server_address TEXT,
+            port           INTEGER,
+            auth_key       bytea
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_entities (
+            session_id TEXT,
+            id         BIGINT,
+            hash       BIGINT NOT NULL,
+            username   TEXT,
+            phone      TEXT,
+            name       TEXT,
+            PRIMARY KEY (session_id, id)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_sent_files (
+            session_id TEXT,
+            md5_digest bytea,
+            file_size  INTEGER,
+            type       INTEGER,
+            id         BIGINT,
+            hash       BIGINT,
+            PRIMARY KEY (session_id, md5_digest, file_size, type)
+        )"""
+    )
+    await conn.execute(
+        """CREATE TABLE telethon_update_state (
+            session_id   TEXT,
+            entity_id    BIGINT,
+            pts          BIGINT,
+            qts          BIGINT,
+            date         BIGINT,
+            seq          BIGINT,
+            unread_count INTEGER,
+            PRIMARY KEY (session_id, entity_id)
+        )"""
+    )
+    gen = ""
+    if scheme in (Scheme.POSTGRES, Scheme.COCKROACH):
+        gen = "GENERATED ALWAYS AS IDENTITY"
+    await conn.execute(
+        f"""
+        CREATE TABLE backfill_queue (
+            queue_id            INTEGER PRIMARY KEY {gen},
+            user_mxid           TEXT,
+            priority            INTEGER NOT NULL,
+            portal_tgid         BIGINT,
+            portal_tg_receiver  BIGINT,
+            messages_per_batch  INTEGER NOT NULL,
+            post_batch_delay    INTEGER NOT NULL,
+            max_batches         INTEGER NOT NULL,
+            dispatch_time       TIMESTAMP,
+            completed_at        TIMESTAMP,
+            cooldown_timeout    TIMESTAMP,
+            FOREIGN KEY (user_mxid) REFERENCES "user"(mxid) ON DELETE CASCADE ON UPDATE CASCADE,
+            FOREIGN KEY (portal_tgid, portal_tg_receiver)
+                REFERENCES portal(tgid, tg_receiver) ON DELETE CASCADE
+        )
+        """
+    )
+
+    return latest_version

mautrix_telegram/db/upgrade/v01_initial_revision.py

@@ -0,0 +1,181 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+from .v00_latest_revision import create_latest_tables, latest_version
+
+legacy_version_query = "SELECT version_num FROM alembic_version"
+last_legacy_version = "bfc0a39bfe02"
+
+
+async def first_upgrade_target(conn: Connection, scheme: Scheme) -> int:
+    is_legacy = await conn.table_exists("alembic_version")
+    # If it's a legacy db, the upgrade process will go to v1 and run each migration up to latest.
+    # If it's a new db, we'll create the latest tables directly (see create_latest_tables call).
+    return 1 if is_legacy else latest_version
+
+
+@upgrade_table.register(description="Initial asyncpg revision", upgrades_to=first_upgrade_target)
+async def upgrade_v1(conn: Connection, scheme: Scheme) -> int:
+    is_legacy = await conn.table_exists("alembic_version")
+    if is_legacy:
+        await migrate_legacy_to_v1(conn, scheme)
+        return 1
+    else:
+        return await create_latest_tables(conn, scheme)
+
+
+async def drop_constraints(conn: Connection, table: str, contype: str) -> None:
+    q = (
+        "SELECT conname FROM pg_constraint con INNER JOIN pg_class rel ON rel.oid=con.conrelid "
+        f"WHERE rel.relname='{table}' AND contype='{contype}'"
+    )
+    names = [row["conname"] for row in await conn.fetch(q)]
+    drops = ", ".join(f"DROP CONSTRAINT {name}" for name in names)
+    await conn.execute(f"ALTER TABLE {table} {drops}")
+
+
+async def migrate_legacy_to_v1(conn: Connection, scheme: Scheme) -> None:
+    legacy_version = await conn.fetchval(legacy_version_query)
+    if legacy_version != last_legacy_version:
+        raise RuntimeError(
+            "Legacy database is not on last version. "
+            "Please upgrade the old database with alembic or drop it completely first."
+        )
+    if scheme != Scheme.SQLITE:
+        await drop_constraints(conn, "contact", contype="f")
+        await conn.execute(
+            """
+            ALTER TABLE contact
+              ADD CONSTRAINT contact_user_fkey FOREIGN KEY (contact) REFERENCES puppet(id)
+                ON DELETE CASCADE ON UPDATE CASCADE,
+              ADD CONSTRAINT contact_contact_fkey FOREIGN KEY ("user") REFERENCES "user"(tgid)
+                ON DELETE CASCADE ON UPDATE CASCADE
+            """
+        )
+        await drop_constraints(conn, "telethon_sessions", contype="p")
+        await conn.execute(
+            """
+            ALTER TABLE telethon_sessions
+              ADD CONSTRAINT telethon_sessions_pkey PRIMARY KEY (session_id)
+            """
+        )
+        await drop_constraints(conn, "telegram_file", contype="f")
+        await conn.execute(
+            """
+            ALTER TABLE telegram_file
+              ADD CONSTRAINT fk_file_thumbnail
+                FOREIGN KEY (thumbnail) REFERENCES telegram_file(id)
+                ON UPDATE CASCADE ON DELETE SET NULL
+            """
+        )
+        await conn.execute("ALTER TABLE puppet ALTER COLUMN id DROP IDENTITY IF EXISTS")
+        await conn.execute("ALTER TABLE puppet ALTER COLUMN id DROP DEFAULT")
+        await conn.execute("DROP SEQUENCE IF EXISTS puppet_id_seq")
+        await conn.execute("ALTER TABLE bot_chat ALTER COLUMN id DROP IDENTITY IF EXISTS")
+        await conn.execute("ALTER TABLE bot_chat ALTER COLUMN id DROP DEFAULT")
+        await conn.execute("DROP SEQUENCE IF EXISTS bot_chat_id_seq")
+        await conn.execute("ALTER TABLE portal ALTER COLUMN config TYPE jsonb USING config::jsonb")
+        await conn.execute(
+            "ALTER TABLE telegram_file ALTER COLUMN decryption_info TYPE jsonb "
+            "USING decryption_info::jsonb"
+        )
+        await varchar_to_text(conn)
+    else:
+        await conn.execute(
+            """CREATE TABLE telethon_sessions_new (
+                session_id     TEXT PRIMARY KEY,
+                dc_id          INTEGER,
+                server_address TEXT,
+                port           INTEGER,
+                auth_key       bytea
+            )"""
+        )
+        await conn.execute(
+            """
+            INSERT INTO telethon_sessions_new (session_id, dc_id, server_address, port, auth_key)
+            SELECT session_id, dc_id, server_address, port, auth_key FROM telethon_sessions
+            """
+        )
+        await conn.execute("DROP TABLE telethon_sessions")
+        await conn.execute("ALTER TABLE telethon_sessions_new RENAME TO telethon_sessions")
+
+    await update_state_store(conn, scheme)
+    await conn.execute('ALTER TABLE "user" ADD COLUMN is_bot BOOLEAN NOT NULL DEFAULT false')
+    await conn.execute("ALTER TABLE puppet RENAME COLUMN matrix_registered TO is_registered")
+    await conn.execute("DROP TABLE telethon_version")
+    await conn.execute("DROP TABLE alembic_version")
+
+
+async def update_state_store(conn: Connection, scheme: Scheme) -> None:
+    # The Matrix state store already has more or less the correct schema, so set the version
+    await conn.execute("CREATE TABLE mx_version (version INTEGER PRIMARY KEY)")
+    await conn.execute("INSERT INTO mx_version (version) VALUES (2)")
+    await conn.execute("UPDATE mx_user_profile SET membership='LEAVE' WHERE membership='LEFT'")
+    if scheme != Scheme.SQLITE:
+        # Also add the membership type on postgres
+        await conn.execute(
+            "CREATE TYPE membership AS ENUM ('join', 'leave', 'invite', 'ban', 'knock')"
+        )
+        await conn.execute(
+            "ALTER TABLE mx_user_profile ALTER COLUMN membership TYPE membership "
+            "USING LOWER(membership)::membership"
+        )
+    else:
+        # On SQLite there's no custom type, but we still want to lowercase everything
+        await conn.execute("UPDATE mx_user_profile SET membership=LOWER(membership)")
+
+
+async def varchar_to_text(conn: Connection) -> None:
+    columns_to_adjust = {
+        "user": ("mxid", "tg_username", "tg_phone"),
+        "portal": (
+            "peer_type",
+            "mxid",
+            "username",
+            "title",
+            "about",
+            "photo_id",
+            "avatar_url",
+            "config",
+        ),
+        "message": ("mxid", "mx_room"),
+        "puppet": (
+            "displayname",
+            "username",
+            "photo_id",
+            "access_token",
+            "custom_mxid",
+            "next_batch",
+            "base_url",
+        ),
+        "bot_chat": ("type",),
+        "telegram_file": ("id", "mxc", "mime_type", "thumbnail"),
+        # Phone is a bigint in the old schema, which is safe, but we don't do math on it,
+        # so let's change it to a string
+        "telethon_entities": ("session_id", "username", "name", "phone"),
+        "telethon_sent_files": ("session_id",),
+        "telethon_sessions": ("session_id", "server_address"),
+        "telethon_update_state": ("session_id",),
+        "mx_room_state": ("room_id",),
+        "mx_user_profile": ("room_id", "user_id", "displayname", "avatar_url"),
+    }
+    for table, columns in columns_to_adjust.items():
+        for column in columns:
+            await conn.execute(f'ALTER TABLE "{table}" ALTER COLUMN {column} TYPE TEXT')

mautrix_telegram/db/upgrade/v02_sponsored_events.py

@@ -0,0 +1,25 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add column to store sponsored message event ID in channels")
+async def upgrade_v2(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE portal ADD COLUMN sponsored_event_id TEXT")
+    await conn.execute("ALTER TABLE portal ADD COLUMN sponsored_event_ts BIGINT")
+    await conn.execute("ALTER TABLE portal ADD COLUMN sponsored_msg_random_id bytea")

mautrix_telegram/db/upgrade/v03_reactions.py

@@ -0,0 +1,39 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2021 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add support for reactions")
+async def upgrade_v3(conn: Connection, scheme: str) -> None:
+    await conn.execute(
+        """CREATE TABLE reaction (
+            mxid      TEXT NOT NULL,
+            mx_room   TEXT NOT NULL,
+            msg_mxid  TEXT NOT NULL,
+            tg_sender BIGINT,
+            reaction  TEXT NOT NULL,
+
+            PRIMARY KEY (msg_mxid, mx_room, tg_sender),
+            UNIQUE (mxid, mx_room)
+        )"""
+    )
+    if scheme != "sqlite":
+        await conn.execute("DELETE FROM message WHERE mxid IS NULL OR mx_room IS NULL")
+        await conn.execute("ALTER TABLE message ALTER COLUMN mxid SET NOT NULL")
+        await conn.execute("ALTER TABLE message ALTER COLUMN mx_room SET NOT NULL")
+    await conn.execute("ALTER TABLE message ADD COLUMN content_hash bytea")

mautrix_telegram/db/upgrade/v04_disappearing_messages.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2022 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,23 +13,20 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
 
 
-def format_duration(seconds: int) -> str:
-    def pluralize(count: int, singular: str) -> str:
-        return singular if count == 1 else singular + "s"
+@upgrade_table.register(description="Add support for disappearing messages")
+async def upgrade_v4(conn: Connection) -> None:
+    await conn.execute(
+        """CREATE TABLE disappearing_message (
+            room_id             TEXT,
+            event_id            TEXT,
+            expiration_seconds  BIGINT,
+            expiration_ts       BIGINT,
 
-    def include(count: int, word: str) -> str:
-        return f"{count} {pluralize(count, word)}" if count > 0 else ""
-
-    minutes, seconds = divmod(seconds, 60)
-    hours, minutes = divmod(minutes, 60)
-    days, hours = divmod(hours, 24)
-    parts = [a for a in [
-        include(days, "day"),
-        include(hours, "hour"),
-        include(minutes, "minute"),
-        include(seconds, "second")] if a]
-    if len(parts) > 2:
-        return "{} and {}".format(", ".join(parts[:-1]), parts[-1])
-    return " and ".join(parts)
+            PRIMARY KEY (room_id, event_id)
+        )"""
+    )

mautrix_telegram/db/upgrade/v05_channel_ghosts.py

@@ -0,0 +1,25 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from mautrix.util.async_db import Connection, Scheme
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Add separate ghost users for channel senders")
+async def upgrade_v5(conn: Connection, scheme: str) -> None:
+    await conn.execute("ALTER TABLE puppet ADD COLUMN is_channel BOOLEAN NOT NULL DEFAULT false")
+    if scheme == Scheme.POSTGRES:
+        await conn.execute("ALTER TABLE puppet ALTER COLUMN is_channel DROP DEFAULT")