Bump version to 0.7.2

Fixes #417

.codeclimate.yml

@@ -0,0 +1,8 @@
+engines:
+  sonar-python:
+    enabled: true
+    checks:
+      python:S107:
+        enabled: false
+exclude_patterns:
+- "alembic/"

.editorconfig

@@ -13,6 +13,3 @@ max_line_length = 99
 
 [*.{yaml,yml,py}]
 indent_style = space
-
-[.gitlab-ci.yml]
-indent_size = 2

.github/workflows/python-lint.yml

@@ -1,18 +0,0 @@
-name: Python lint
-
-on: [push, pull_request]
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: "3.10"
-    - uses: isort/isort-action@master
-      with:
-        sortPaths: "./mautrix_telegram"
-    - uses: psf/black@21.12b0
-      with:
-        src: "./mautrix_telegram"

.gitignore

@@ -14,5 +14,4 @@ __pycache__
 /registration.yaml
 *.log*
 *.db
-*.pickle
 *.bak

.gitlab-ci.yml

@@ -2,65 +2,37 @@ image: docker:stable
 
 stages:
 - build
-- manifest
+- push
 
 default:
   before_script:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
-build amd64:
+build:
   stage: build
-  tags:
-  - amd64
   script:
   - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=amd64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  after_script:
-  - |
-    if [[ "$CI_COMMIT_BRANCH" == "master" && "$CI_JOB_STATUS" == "success" ]]; then
-      apk add --update curl jq
-      rm -rf /var/cache/apk/*
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
 
-      jq -n '
-         {
-           password:  env.BEEPER_DEV_ADMIN_NIGHTLY_PASS,
-           bridge:    env.BEEPER_BRIDGE_TYPE,
-           image:     "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
-           channel:   "STABLE"
-         }
-        ' | curl "$BEEPER_DEV_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
-
-      jq -n '
-         {
-           password:    env.BEEPER_PROD_ADMIN_NIGHTLY_PASS,
-           bridge:      env.BEEPER_BRIDGE_TYPE,
-           image:       "\(env.CI_REGISTRY_IMAGE):\(env.CI_COMMIT_SHA)-amd64",
-           channel:     "INTERNAL",
-           deployNext:  true
-         }
-         ' | curl "$BEEPER_PROD_ADMIN_API_URL" -H "Content-Type: application/json" -d @-
-    fi
-
-build arm64:
-  stage: build
-  tags:
-  - arm64
+push latest:
+  stage: push
+  only:
+  - master
+  variables:
+    GIT_STRATEGY: none
   script:
-  - docker pull $CI_REGISTRY_IMAGE:latest || true
-  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --build-arg TARGETARCH=arm64 --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 .
-  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:latest
 
-manifest:
-  stage: manifest
-  before_script:
-  - "mkdir -p $HOME/.docker && echo '{\"experimental\": \"enabled\"}' > $HOME/.docker/config.json"
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+push tag:
+  stage: push
+  variables:
+    GIT_STRATEGY: none
+  except:
+  - master
   script:
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64
-  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
-  - if [ "$CI_COMMIT_BRANCH" = "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:latest; fi
-  - if [ "$CI_COMMIT_BRANCH" != "master" ]; then docker manifest create $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64 && docker manifest push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME; fi
-  - docker rmi $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-amd64 $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA-arm64
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME

.isort.cfg

@@ -0,0 +1,16 @@
+[settings]
+line_length=99
+indent=4
+
+multi_line_output=5
+
+sections=FUTURE,STDLIB,THIRDPARTY,TELETHON,MAUTRIX,FIRSTPARTY,LOCALFOLDER
+no_lines_before=LOCALFOLDER
+default_section=FIRSTPARTY
+
+known_thirdparty=aiohttp,sqlalchemy,alembic,commonmark,ruamel.yaml,PIL,moviepy,prometheus_client,yarl,mako,pkg_resources
+known_telethon=telethon,alchemysession,cryptg
+known_mautrix=mautrix
+
+balanced_wrapping=True
+length_sort=True

CHANGELOG.md

@@ -1,704 +0,0 @@
-# (unreleased)
-
-* Added support for message reactions
-* Added support for spoiler text.
-
-# v0.11.0 (2021-12-28)
-
-* Switched from SQLAlchemy to asyncpg/aiosqlite.
-  * The default database is now Postgres. If using SQLite, make sure you install
-    the `sqlite` [optional dependency](https://docs.mau.fi/bridges/python/optional-dependencies.html).
-  * **Alembic is no longer used**, schema migrations happen automatically on startup.
-  * **The automatic database migration requires you to be on the latest legacy
-    database version.** If you were running any v0.10.x version, you should be on
-    the latest version already. Otherwise, update to v0.10.2 first, upgrade the
-    database with `alembic`, then upgrade to v0.11.0 (or higher).
-* Added support for contact messages.
-* Added support for Telegram sponsored messages in channels.
-  * Only applies to broadcast channels with 1000+ members
-    (as per <https://t.me/durov/172>).
-  * Only applies if you're using puppeting with a normal user account,
-    because bots can't get sponsored messages.
-* Fixed non-supergroup member sync incorrectly kicking one user from the Matrix
-  side if there was no limit on the number of members to sync (broke in v0.10.2).
-* Updated animated sticker conversion to support [lottieconverter r0.2]
-  (thanks to [@sot-tech] in [#694]).
-* Updated Docker image to Alpine 3.15.
-* Formatted all code using [black](https://github.com/psf/black)
-  and [isort](https://github.com/PyCQA/isort).
-
-[lottieconverter r0.2]: https://github.com/sot-tech/LottieConverter/releases/tag/r0.2
-[#694]: https://github.com/mautrix/telegram/pull/694
-
-# v0.10.2 (2021-11-13)
-
-### Added
-* Added extensions when bridging unnamed files from Telegram.
-* Added support for custom bridge bot welcome messages
-  (thanks to [@justinbot] in [#676]).
-
-### Improved
-* Improved handling authorization errors if the bridge was logged out remotely.
-* Updated room syncer to use existing power levels to find appropriate levels
-  for admins and normal users instead of hardcoding 50 and 0.
-* Updated to Telegram API layer 133 to handle 64-bit user/chat/channel IDs.
-* Stopped logging message contents when message handling failed
-  (thanks to [@justinbot] in [#681]).
-* Removed Element iOS compatibility hack from non-sticker files.
-* Made `max_initial_member_sync` work for non-supergroups too
-  (thanks to [@tadzik] in [#680]).
-* SQLite is now supported for the crypto database. Pickle is no longer supported.
-  If you were using pickle, the bridge will create a new e2ee session and store
-  the data in SQLite this time.
-
-### Fixed
-* Fixed generating reply fallbacks to encrypted messages.
-* Fixed chat sync failing if the member list contained banned users.
-
-[@justinbot]: https://github.com/justinbot
-[@tadzik]: https://github.com/tadzik
-[#676]: https://github.com/mautrix/telegram/pull/676
-[#680]: https://github.com/mautrix/telegram/pull/680
-[#681]: https://github.com/mautrix/telegram/pull/681
-
-# v0.10.1 (2021-08-19)
-
-**N.B.** Docker images have moved from `dock.mau.dev/tulir/mautrix-telegram`
-to `dock.mau.dev/mautrix/telegram`. New versions are only available at the new
-path.
-
-### Added
-
-* Warning when bridging existing room if bridge bot doesn't have redaction
-  permissions.
-* Custom flag to invite events that will be auto-accepted using double puppeting.
-* Custom flags for animated stickers (same as what gifs already had).
-
-### Improved
-* Updated to Telethon 1.22.
-* Updated Docker image to Alpine 3.14.
-
-### Fixed
-* Fixed Bridging Matrix location messages with additional flags in `geo_uri`.
-* Editing encrypted messages will no longer add an asterisk on Telegram.
-* Matrix typing notifications won't be echoed back for double puppeted users anymore.
-* `AuthKeyDuplicatedError` is now handled properly instead of making the user
-  get stuck.
-* Fixed `public_portals` setting not being respected on room creation.
-
-# v0.10.0 (2021-06-14)
-
-* Added options to bridge archive, pin and mute status from Telegram to Matrix.
-* Added custom fields in Matrix events indicating Telegram gifs.
-* Allowed zero-width joiners in displaynames so things like multi-part emoji
-  would work correctly.
-* Fixed Telegram->Matrix typing notifications.
-
-## rc1 (2021-04-05)
-
-### Added
-* Support for multiple pins from/to Telegram.
-* Option to resolve redirects when joining invite links, for people who use
-  custom URLs as invite links.
-* Command to update about section in Telegram profile info
-  (thanks to [@MadhuranS] in [#599]).
-* Own read marker/unread status from Telegram is now synced to Matrix after backfilling.
-* Support for showing the individual slots in 🎰 dice rolls from Telegram.
-
-### Improved
-* Improved invite link regex to allow joining with less precise invite links.
-* Invite links can be customized with the `--uses=<amount>` and
-  `--expire=<delta>` flags for `!tg invite-link`.
-* Read receipts where the target message is unknown will now cause the chat to
-  be marked as fully read instead of the read receipt event being ignored.
-* WebP stickers are now sent as-is without converting to png.
-* Default power levels in rooms now allow enabling encryption with PL 50 if
-  e2be is enabled in config (thanks to [@Rafaeltheraven] in [#550]).
-* Updated Docker image to Alpine 3.13 and removed all edge repo stuff.
-
-### Fixed
-* Matrix->Telegram location message bridging no longer flips the coordinates.
-* Fixed some user displaynames constantly changing between contact/non-contact
-  names and other similar cases.
-
-[@Rafaeltheraven]: https://github.com/Rafaeltheraven
-[@MadhuranS]: https://github.com/MadhuranS
-[#550]: https://github.com/mautrix/telegram/pull/550
-[#599]: https://github.com/mautrix/telegram/pull/599
-
-# v0.9.0 (2020-11-17)
-
-* Fixed cleaning unidentified rooms.
-
-## rc3 (2020-11-12)
-
-### Added
-* Added retrying message sending if server returns 502.
-
-### Fixed
-* Fixed Matrix → Telegram name mentions.
-* Fixed some bugs with replies.
-
-## rc2 (2020-11-06)
-
-### Improved
-* Ephemeral event handling should be faster by not checking the database for
-  user existence.
-* Using the register command now sends a link to the Telegram terms of service.
-* The `bridge_connected` metric is now only set for users who are logged in.
-
-### Fixed
-* Fixed bug where syncing members sometimes kicked ghosts of users who were
-  actually still in the chat.
-* Fixed sending captions to Telegram with `!tg caption` (broken in rc1).
-* Logging out will now delete private chat portals, instead of only kicking the
-  user and leaving the portal in a broken state.
-* Unbridging direct chat portals is now possible.
-
-## rc1 (2020-10-24)
-
-### Breaking changes
-* Prometheus metric names are now prefixed with `bridge_`.
-* An entrypoint script is no longer automatically generated. This won't affect
-  most users, as `python -m mautrix_telegram` has been the official way to start
-  the bridge for a long time.
-
-### Added
-* Support for logging in by scanning a QR code from another Telegram client.
-* Automatic backfilling of old messages when creating portals.
-* Automatic backfilling of missed messages when starting bridge.
-* Option to update `m.direct` list when using double puppeting.
-* PNG thumbnails for animated stickers when converted to webm.
-* Support for receiving ephemeral events pushed directly with [MSC2409]
-  (requires Synapse 1.22 or higher).
-
-### Improved
-* Switched end-to-bridge encryption to mautrix-python instead of a hacky
-  matrix-nio solution.
-* End-to-bridge encryption no longer requires `login_shared_secret`, it uses
-  [MSC2778] instead (requires Synapse 1.21 or higher).
-* The bridge info state event is now updated whenever the chat name or avatar changes.
-* Double puppeting is no longer limited to users on the same homeserver as the bridge.
-* Delivery receipts are no longer sent in unencrypted private chat portals, as
-  the bridge bot is usually not present in them.
-
-### Fixed
-* File captions are now sent as a separate message like photo captions.
-* The relaybot no longer drops Telegram messages with commands.
-* Bridging events of a user whose power level is malformed (i.e. a string
-  instead of an integer) now works.
-
-[MSC2409]: https://github.com/matrix-org/matrix-doc/pull/2409
-[MSC2778]: https://github.com/matrix-org/matrix-doc/pull/2778
-
-# v0.8.2 (2020-07-27)
-
-* Fixed deleting messages from Matrix.
-* Fixed Alpine edge dependencies in Docker image.
-
-Note: this release is not on PyPI, as the only changes were a mautrix-python
-update (v0.5.8) and a fix to the Docker image.
-
-# v0.8.1 (2020-06-08)
-
-* Fixed starting bridge for the first time failing due to not registering the bridge bot.
-* Updated Docker image to Alpine 3.12.
-
-# v0.8.0 (2020-06-03)
-
-* Updated to mautrix-python 0.5.0 and matrix-nio 0.12.0.
-
-## rc5 (2020-05-30)
-
-* Added option to disable removing avatars from Telegram ghosts.
-* Added option to send delivery error notices.
-* Added option to send delivery receipts.
-* Bumped maximum Telethon version to 1.14.
-* Possibly fixed infinite loop of avatar changes when using double puppeting.
-
-## rc3 (2020-05-22)
-
-* Moved private information to trace log level.
-* Added `private_chat_portal_meta` option. This is implicitly enabled when
-  encryption is enabled, it was only added as an option for instances with
-  encryption disabled.
-* Removed avatars are now synced properly from Telegram, instead of leaving the
-  last known avatar forever.
-* Fixed admin detection on Telegram-side relaybot commands
-  (thanks to [@davidmehren] in [#468]).
-* Fixed bug handling `ChatForbidden` when syncing chats.
-
-[@davidmehren]: https://github.com/davidmehren
-[#468]: https://github.com/mautrix/telegram/pull/468
-
-## rc2 (2020-05-20)
-
-* Implemented [MSC2346]: Bridge information state event for newly created rooms.
-* Fixed `sync_direct_chats` option creating non-working portals.
-* Fixed video thumbnailing sometimes leaving behind downloaded videos in `/tmp`.
-
-[MSC2346]: https://github.com/matrix-org/matrix-doc/pull/2346
-
-## rc1 (2020-04-25)
-
-### Added
-* Command for backfilling room history from Telegram.
-* arm64 support in docker images.
-* Optional end-to-bridge encryption support.
-  See [docs](https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html) for more info.
-* Bridging for Telegram dice roll messages.
-
-### Fixed
-* Riot iOS not showing stickers properly.
-* Updated to Telethon 1.13 to fix bugs like [#443].
-
-[#443]: https://github.com/mautrix/telegram/issues/443
-
-# v0.7.2 (2020-04-04)
-
-* No changes since rc1.
-
-## rc1 (2020-02-08)
-
-* Fixed enabling double puppeting causing saved messages to become unusable.
-* Fixed receiving channel messages when `ignore_own_incoming_events` was enabled.
-
-# v0.7.1 (2020-02-04)
-
-* Fixed missing responses in logout provisioning API.
-
-## rc2 (2020-01-25)
-
-* Fixed import in database migration script (thanks to [@cubesky] in [#409]).
-* Fixed relaybot messages being allowed through to Matrix even when
-  `ignore_own_incoming_events` was set to `true`.
-
-[@cubesky]: https://github.com/cubesky
-[#409]: https://github.com/mautrix/telegram/pull/409
-
-## rc1 (2020-01-11)
-
-* Fixed incorrect parameter name causing `!tg config set` to throw an error.
-* Fixed potential dictionary size changed during iteration crash.
-
-# v0.7.0 (2019-12-28)
-
-* No changes since rc4.
-
-## rc4 (2019-12-25)
-
-* Fixed handling of Matrix `m.emote` events.
-
-## rc3 (2019-12-25)
-
-* Added option to log in to custom puppet with shared secret
-  (<https://github.com/devture/matrix-synapse-shared-secret-auth>).
-* Updated Docker image to Alpine 3.11.
-* Improved displayname syncing by trusting any displayname if user is not a contact.
-* Fixed error when cleaning up rooms.
-* Fixed stack traces being printed to non-admin users.
-* Fixed invite rejections being handles as leaves.
-* Fixed `version` command output in CI docker builds not showing the correct
-  git commit hash.
-
-## rc2 (2019-12-01)
-
-* Added command to get bridge version.
-* Made bridge refuse to start if config contains example values.
-* Removed some debug stack traces.
-* Ignored `ChatForbidden` when syncing chats that was causing the sync to fail.
-* Fixed DB migration causing some incorrect values to be left behind.
-
-## rc1 (2019-11-30)
-
-### Important changes
-* Dropped Python 3.5 compatibility.
-* Moved docker registry to [dock.mau.dev](https://mau.dev/tulir/mautrix-telegram/container_registry).
-
-### Added
-* Support for bridging animated stickers (thanks to [@sot-tech] in [#366]).
-  * Requires [LottieConverter](https://github.com/sot-tech/LottieConverter),
-    which is included in the docker image.
-  * Can be [configured](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L174-L187).
-* Support for MTProxy (thanks to [@sot-tech] in [#344]).
-* [Config option](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L117-L118)
-  for max length of displayname, with the default being 100.
-* Separate [config option](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L232-L238)
-  for `m.emote` formatting of logged in users.
-* Streamed file transfers and parallel telegram file download/upload.
-  * Files are streamed from telegram servers to the media repo rather than
-    downloading the whole file into memory.
-  * File transfers use multiple connections to telegram servers to transfer faster.
-  * Parallel and streamed file transfers can be enabled in the
-    [config](https://github.com/mautrix/telegram/blob/v0.7.0-rc1/example-config.yaml#L166-L169).
-* Command to set caption for files and images when sending to telegram.
-* Bridging bans to telegram.
-* Helm chart.
-
-### Improved
-* Switched from mautrix-appservice-python to [mautrix-python](https://github.com/mautrix/python).
-* Users with Matrix puppeting can now bridge their "Saved Messages" chat.
-* The bridge will refuse to start without access to the example config file.
-* Changed default port to 29317.
-* Mentions are now marked as read on Telegram when bridging read receipts using
-  double puppeting.
-* Kicking or banning the bridge bot will now unbridge the room.
-* Shrinked Docker image from 151mb to 77mb.
-
-### Fixed
-* The bridge will no longer crash if one user's startup fails.
-* (hopefully) Incorrect peer type being saved into database in some cases.
-* File names when bridging to Telegram.
-* Alembic config interpolating passwords with `%`.
-* A single chat failing to sync preventing any chat from syncing.
-* Users logged in as a bot not receiving any messages.
-* Username matching being case-sensitive in the database and preventing
-  telegram->matrix pillifying.
-* IndexError if running `!tg set-pl` with no parameters.
-
-[@sot-tech]: https://github.com/sot-tech
-[#344]: https://github.com/mautrix/telegram/pull/344
-[#366]: https://github.com/mautrix/telegram/pull/366
-
-# v0.6.0 (2019-07-09)
-
-* Fixed vulnerability in event handling.
-
-## rc2 (2019-07-06)
-
-* Nested formatting is now supported by Telegram, so the bridge also supports it.
-* Strikethrough and underline are now bridged into native Telegram formatting
-  rather than unicode hacks.
-* Fixed displayname not updating for users who the bridge only saw via a logged
-  in user who had the problematic user in their contacts.
-* Fixed handling unsupported media.
-* Added handling for `FileIdInvalidError` in file transfers that could disrupt
-  `sync`s.
-
-## rc1 (2019-06-22)
-
-### Added
-* Native Matrix edit support and new fallback format.
-* Config options for `retry_delay` and other TelegramClient constructor fields.
-* Config option for maximum document size to let through the bridge.
-* External URL field for chat and private channel messages.
-* Telegram user info (puppet displayname & avatar) is now updated every time
-  the user sends a message.
-* Command to change Telegram displayname.
-* Possibility to override config fields with environment variables
-  (thanks to [@pacien] in [#332]).
-
-### Improved
-* Simplified bridged poll message.
-* Telegram user info updates are now accepted from any logged in user as long
-  as the logged in user doesn't see a phone number for the Telegram user.
-* Some image errors are now handled by resending the image as a document.
-* Made getting started more user-friendly.
-* Updated to Telethon 1.8.
-
-### Fixed
-* Portal peer type not being saved in database after Telegram chat upgrade.
-* Newlines in unformatted messages not being bridged when using relaybot.
-* Mime type info field for stickers converted to PNG.
-* Content after newlines being stripped in messages sent by some clients.
-* Potential `NoneType is not iterable` exception when logging out
-  (thanks to [@turt2live] in [#315]).
-* Handling of Matrix messages where `m.relates_to` is null.
-* Internal server error when logging in with an account on another DC.
-* Spaces between command and arguments are now trimmed.
-* Changed migrations to use `batch_alter_table` for adding columns to have less
-  warnings with SQLite.
-* Error when `ping`ing without being logged in.
-* Terminating sessions with negative hashes.
-* State cache not being updated when sending events, causing invalid cache if
-  the server doesn't echo the sent events.
-
-[@pacien]: https://github.com/pacien
-[#315]: https://github.com/mautrix/telegram/pull/315
-[#332]: https://github.com/mautrix/telegram/pull/332
-
-# v0.5.2 (2019-05-25)
-
-* Fixed null `m.relates_to`'s that break Synapse 0.99.5.
-
-# v0.5.1 (2019-03-21)
-
-* Fixed Python 3.5 compatibility.
-* Fixed DBMS migration script.
-
-# v0.5.0 (2019-03-19)
-
-* Replaced rawgit with cdnjs in public website as rawgit is deprecated.
-* Fixed login command throwing error when web login is enabled.
-* Updated telethon-session-sqlalchemy to fix logging into an account on another DC.
-* Stopped adding reply fallback to caption when sending caption and image as
-  separate messages.
-
-## rc4 (2019-03-16)
-
-* Added verbose flag to migration script.
-* Added pytest setup and some tests (thanks to [@V02460] in [#290]).
-* Fixed scripts (DBMS migration and Telematrix import) not being included in builds.
-* Fixed some database problems.
-* Removed remaining traces of ORM that might have been the causes of some other
-  database problems.
-* Removed option to use lxml in HTML parsing as it was messing up emoji offset
-  handling. The new HTML parser supports using the default python HTMLParser
-  class since 0.5.0rc1, so lxml wasn't really useful anway.
-
-[#290]: https://github.com/mautrix/telegram/pull/290
-
-## rc3 (2019-02-16)
-
-* Fixed bridging documents without thumbnails to Matrix.
-* Added option to set maximum size of image to send to Telegram. Images above
-  the size limit will be sent as documents without the compression Telegram
-  applies to images.
-* Fixed saving user portals and contacts.
-* Added Telegram -> Matrix poll bridging and a command to vote in polls.
-
-## rc2 (2019-02-15)
-
-* Added missing future-fstrings comments that caused the bridge to not start on
-  Python 3.5.
-* Fixed handling of document thumbnails.
-* Fixed private chat portals failing to be created.
-* Made relaybot handle Telegram chat upgrade events.
-
-## rc1 (2019-02-14)
-
-### Added
-* More config options
-  * Option to to use Telegram test servers.
-  * Option to disable link previews on Telegram.
-  * Option to disable startup sync.
-  * Option to skip deleted members when syncing member lists.
-  * Option to change number of dialogs to handle in startup sync.
-* More commands
-  * `username` for setting Telegram username.
-  * `sync-state` for updating Matrix room state cache.
-  * `matrix-ping` for checking Matrix login status (thanks to [@krombel] in [#271]).
-  * `clear-db-cache` for clearing internal database caches.
-  * `reload-user` for reloading and reconnecting a Telegram user.
-  * `session` for listing and terminating other Telegram sessions.
-  * Added argument to `login` to allow admins to log in for other users.
-* Added warning when logging in that it grants the bridge full access to the
-  telegram account.
-* Telegram->Matrix bridging:
-  * Telegram games
-  * Message pins in normal groups
-  * Custom message for unsupported media like polls
-* Added client ID in logs when making requests to telegram.
-* Added handling for Matrix room upgrades.
-
-### Improved
-* Removed lxml dependency from the new HTML parser and removed the old parser
-  completely.
-* Switched mautrix-appservice-python state store and most mautrix-telegram
-  tables to SQLAlchemy core. This should speed things up and reduce problems
-  with the ORM getting stuck.
-* `ensure_started` is now only called for logged in users, which should improve
-  performance for large instances.
-* Displayname template extras (e.g. the `(Telegram)` suffix) are now stripped
-  when mentioning Telegram users with no username.
-* Updated Telethon.
-* Switched Dockerfile to use setup.py for dependencies to avoid dependency
-  updates breaking stuff.
-* The telematrix import script will now warn about and skip over duplicate portals.
-* Relaybot will now be used for users who have logged in, but are not in the chat.
-
-### Fixed
-* Bug where stickers with an unidentified emoji failed to bridge.
-* Invalid letter prefixes in clean-rooms output.
-* Messages forwarded from channels showing up as "Unknown source".
-* Matrix->Telegram room avatar bridging.
-
-[@krombel]: https://github.com/krombel
-[#271]: https://github.com/mautrix/telegram/pull/271
-
-# v0.4.0 (2018-11-28)
-
-* No changes since rc2.
-
-## rc2 (2018-11-15)
-
-* Fixed kicking Telegram puppets from Matrix.
-
-## rc1 (2018-11-15)
-
-### Added
-* Flag to indicate if user can unbridge portal in provisioning API
-  (thanks to [@turt2live] in [#225]).
-* Option to send captions as second message (replaces option to send caption
-  in `body`.
-* Room-specific settings.
-
-### Improved
-* (internal) Added type hints everywhere (mostly thanks to [@V02460] in [#206]).
-* Telegram->Matrix formatter now uses `<pre>` tags for multiline code even if
-  said code was in the telegram equivalent of inline code tags.
-* Better bullets and linebreak handling in Matrix->Telegram formatter.
-* Logging in will now show your phone number instead of `@None` if you don't
-  have a username.
-* Significantly improved performance on high-load instances (t2bot.io) by
-  moving most used database tables to SQLAlchemy Core.
-
-### Fixed
-* Bugs that caused database migrations to fail in some cases.
-* Editing the config (e.g. whitelisting chats) corrupting the config.
-* Negative numbers (chat IDs) in `/connect` of the provisioning API
-  (thanks to [@turt2live] in [#223]).
-* Relaybot creating portals automatically when receiving message.
-* Not being able to use a bridge bot localpart that would also match the puppet
-  localpart format.
-* Matrix login sync failing completely if the homeserver stopped during a sync
-  response.
-* Errors when cleaning rooms.
-* Bridging code blocks without a language.
-* Error and lost messages when trying to bridge PM from new users in some cases.
-* Logging in with an account that someone has already logged in failing
-  silently and then breaking the bridge.
-* Relaybot message when adding/removing Matrix displaynames.
-
-[@V02460]: https://github.com/V02460
-[#206]: https://github.com/mautrix/telegram/pull/206
-[#223]: https://github.com/mautrix/telegram/pull/223
-[#225]: https://github.com/mautrix/telegram/pull/225
-
-# v0.3.0 (2018-08-15)
-
-* Added database URI format examples.
-* Bumped maximum Telethon version to 1.2, possibly fixing the catch_up option.
-
-## rc3 (2018-08-08)
-
-* Improved Telegram message deduplication options.
-  * Added pre-send message database check for deduplication.
-  * Made dedup cache queue length configurable.
-
-## rc2 (2018-08-06)
-
-* Added option to change max body size for AS API.
-* Fixed a minor error regarding power level changes (thanks to [@turt2live] in [#203]).
-* Updated minimum mautrix-appservice version to include some recent bugfixes.
-
-[@turt2live]: https://github.com/turt2live
-[#203]: https://github.com/mautrix/telegram/pull/203
-
-## rc1 (2018-08-05)
-
-### Added
-* Logging in with a bot
-  (see [docs](https://docs.mau.fi/bridges/python/telegram/authentication.html#bot-token) for usage).
-  * You can log in with a personal Telegram bot to appear almost like a real
-    user without logging in with a real Telegram account.
-* Replacing your Telegram account's Matrix puppet with your Matrix account
-  (see [docs](https://docs.mau.fi/bridges/general/double-puppeting.html) for usage).
-* Formatting options for relaybot messages.
-  * Real displaynames are now supported and enabled by default.
-  * State events (join/leave/name change) can be independently disabled by
-    setting the format to a blank string.
-* New config sections
-  * Proper log config, including logging to file (by default)
-  * Proxy support (requires installing PySocks)
-  * Separate field for appservice address for homeserver
-    (useful if using a reverse proxy).
-* New permission levels to allow initiating bridges without allowing puppeting
-  and to allow Telegram puppeting without allowing Matrix puppeting.
-* Telematrix import script (see [docs](https://docs.mau.fi/bridges/python/telegram/migrating-from-telematrix.html) for usage).
-* Provisioning API (see [docs](https://docs.mau.fi/bridges/python/telegram/provisioning-api.html) for more info).
-* DBMS migration script (see [docs](https://docs.mau.fi/bridges/python/telegram/dbms-migration.html) for usage).
-
-### Improved
-* Tabs are now replaced with 4 spaces so that Telegram servers wouldn't change
-  the message.
-* Help page now detects your permissions and only shows commands you can use.
-* Moved Matrix state cache to the main database. This means that the
-  `mx-state.json` file is no longer needed and all non-config data is
-  stored in the main database.
-* Better lxml-based HTML parser for Matrix->Telegram formatting bridging.
-  lxml is still optional, so the old parser is used as fallback if lxml is not
-  installed.
-* Disabled Telegram->Matrix bridging of messages sent by the relaybot.
-  Can be re-enabled in config if necessary.
-
-### Fixed
-* A `ValueError` in some cases when syncing power levels.
-* Telegram connections being created for unauthenticated users possibly
-  triggering spam protection connection delays in the Telegram servers.
-* Logging out if a portal had been deleted/unbridged.
-
-# v0.2.0 (2018-06-08)
-
-* No changes since rc6.
-
-## rc6 (2018-06-06)
-
-* Added warning about `delete-portal` kicking all room members.
-* Fixed error when upgrading/creating SQLite database.
-
-## rc5 (2018-06-01)
-
-* Fixed relaybot automatically creating portal rooms when invited to Telegram chat ([#145]).
-* Fixed kicking Telegram puppets and fix error message when bridging chats you've left.
-* Fixed integrity error deleting portals from database.
-
-[#145]: https://github.com/mautrix/telegram/issues/145
-
-## rc4 (2018-05-29)
-
-* ~~Fixed~~ Added Postgres compatibility.
-* Fixed manual bridging (`!tg bridge`) for unauthenticated users.
-* Fixed inviting unauthenticated Matrix users from Telegram (via `/invite <mxid>`).
-* Changed Alembic to read database path from the config, so editing `alembic.ini`
-  is no longer necessary. Use `alembic -x config=/path/to/config.yaml ...` to
-  specify the config path.
-
-## rc3 (2018-05-25)
-
-* Reworked Dockerfile to remove virtualenv and use Alpine packages (thanks to
-  [@jcgruenhage] in [#142]). This fixes webp->png conversion for stickers.
-
-[#142]: https://github.com/mautrix/telegram/pull/142
-
-## rc2 (2018-05-21)
-
-* Added Dockerfile (thanks to [@jcgruenhage] in [#136]).
-
-[#136]: https://github.com/mautrix/telegram/pull/136
-[@jcgruenhage]: https://github.com/jcgruenhage
-
-## rc1 (2018-05-19)
-
-* Added
-  * Option to exclude telegram chats from being bridged.
-  * Support for using a relay bot to relay messages for unauthenticated users
-  * Bridging for message pinning and room mentions/pills.
-  * Matrix->Telegram sticker bridging.
-  * `!command` to `/command` conversion at the start of Matrix message text.
-  * Conversion of t.me message links to matrix.to message links
-  * Timestamp massaging (bridge Telegram timestamps to Matrix)
-  * Support for out-of-Matrix login (useful if you don't want your 2FA password to be stored in the homeserver)
-  * Optional HQ gif/video thumbnails using moviepy.
-  * Option to send bot messages as `m.notice`
-* Improved deduplication
-  * Matrix file uploads are now reused if the same Telegram file (e.g. a sticker) is sent multiple times
-  * Room metadata changes and other non-message actions are now deduplicated
-* Improved formatting bridging
-* Improved Telegram user display name handling in cases where one or more users have set custom display names for other users.
-* Fixed Alembic setup and removed automatic database generation.
-* Fixed outgoing message deduplication in cases where message is sent to other clients before responding to the sender.
-* Moved mautrix-appservice-python to separate repository.
-* Switched to telethon-session-sqlalchemy to have the session databases in the main database.
-* Switched license from GPLv3 to AGPLv3
-* Probably a bunch of other stuff I forgot
-
-# v0.1.1 (2018-02-18)
-
-Fixed bridging formatted messages from Matrix to Telegram
-
-# v0.1.0 (2018-02-17)
-
-First release.
-
-Things work.

Dockerfile

@@ -1,61 +1,77 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.15
+FROM docker.io/alpine:3.10 AS lottieconverter
 
-ARG TARGETARCH=amd64
+WORKDIR /build
 
-RUN apk add --no-cache \
-      python3 py3-pip py3-setuptools py3-wheel \
+RUN apk add --no-cache git build-base cmake \
+  && git clone https://github.com/Samsung/rlottie.git \
+  && cd rlottie \
+  && mkdir build \
+  && cd build \
+  && cmake .. \
+  && make -j2 \
+  && make install \
+  && cd ../..
+
+RUN apk add --no-cache libpng libpng-dev zlib zlib-dev \
+  && git clone https://github.com/Eramde/LottieConverter.git \
+  && cd LottieConverter \
+  && git checkout 543c1d23ac9322f4f03c7fb6612ea7d026d44ac0 \
+  && make
+
+FROM docker.io/alpine:3.11
+
+ENV UID=1337 \
+    GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
+
+COPY --from=lottieconverter /usr/lib/librlottie* /usr/lib/
+COPY --from=lottieconverter /build/LottieConverter/dist/Debug/GNU-Linux/lottieconverter /usr/local/bin/lottieconverter
+
+COPY . /opt/mautrix-telegram
+WORKDIR /opt/mautrix-telegram
+RUN apk add --no-cache --virtual .build-deps \
+      python3-dev \
+      libffi-dev \
+      build-base \
+      git \
+  && apk add --no-cache \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-magic \
+      py3-sqlalchemy \
+      py3-psycopg2 \
       py3-ruamel.yaml \
-      py3-commonmark \
-      py3-prometheus-client \
       # Indirect dependencies
       py3-idna \
+      #commonmark
+        py3-future \
+      #alembic
+        py3-mako \
+        py3-dateutil \
+        py3-markupsafe \
       #moviepy
         py3-decorator \
         py3-tqdm \
         py3-requests \
         #imageio
           py3-numpy \
-      #py3-telethon \ (outdated)
-        # Optional for socks proxies
-        py3-pysocks \
-        py3-pyaes \
+      #telethon
+        py3-rsa \
         # cryptg
           py3-cffi \
-	  py3-qrcode \
-      py3-brotli \
       # Other dependencies
       ffmpeg \
       ca-certificates \
       su-exec \
       netcat-openbsd \
-      # encryption
-      py3-olm \
-      py3-pycryptodome \
-      py3-unpaddedbase64 \
-      py3-future \
-      bash \
-      curl \
-      jq \
-      yq
-
-COPY requirements.txt /opt/mautrix-telegram/requirements.txt
-COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
-WORKDIR /opt/mautrix-telegram
-RUN apk add --virtual .build-deps python3-dev libffi-dev build-base \
- && pip3 install -r requirements.txt -r optional-requirements.txt \
+      # lottieconverter
+      zlib libpng \
+ && pip3 install .[speedups,hq_thumbnails,metrics] \
+ # pip installs the sources to /usr/lib/python3.8/site-packages, so we don't need them here
+ && rm -rf /opt/mautrix-telegram/mautrix_telegram \
  && apk del .build-deps
 
-COPY . /opt/mautrix-telegram
-RUN apk add git && pip3 install .[all] && apk del git \
-  # This doesn't make the image smaller, but it's needed so that the `version` command works properly
-  && cp mautrix_telegram/example-config.yaml . && rm -rf mautrix_telegram
-
 VOLUME /data
-ENV UID=1337 GID=1337 \
-    FFMPEG_BINARY=/usr/bin/ffmpeg
 
 CMD ["/opt/mautrix-telegram/docker-run.sh"]

MANIFEST.in

@@ -1,5 +0,0 @@
-include README.md
-include CHANGELOG.md
-include LICENSE
-include requirements.txt
-include optional-requirements.txt

README.md

@@ -1,30 +1,15 @@
 # mautrix-telegram
-![Languages](https://img.shields.io/github/languages/top/mautrix/telegram.svg)
-[![License](https://img.shields.io/github/license/mautrix/telegram.svg)](LICENSE)
-[![Release](https://img.shields.io/github/release/mautrix/telegram/all.svg)](https://github.com/mautrix/telegram/releases)
-[![GitLab CI](https://mau.dev/mautrix/telegram/badges/master/pipeline.svg)](https://mau.dev/mautrix/telegram/container_registry)
-[![Code style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![Imports](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+![Languages](https://img.shields.io/github/languages/top/tulir/mautrix-telegram.svg)
+[![License](https://img.shields.io/github/license/tulir/mautrix-telegram.svg)](LICENSE)
+[![Release](https://img.shields.io/github/release/tulir/mautrix-telegram/all.svg)](https://github.com/tulir/mautrix-telegram/releases)
+[![GitLab CI](https://mau.dev/tulir/mautrix-telegram/badges/master/pipeline.svg)](https://mau.dev/tulir/mautrix-telegram/container_registry)
+[![Maintainability](https://img.shields.io/codeclimate/maintainability/tulir/mautrix-telegram.svg)](https://codeclimate.com/github/tulir/mautrix-telegram)
 
 A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
-## Sponsors
-* [Joel Lehtonen / Zouppen](https://github.com/zouppen)
+### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)
 
-### Documentation
-All setup and usage instructions are located on
-[docs.mau.fi](https://docs.mau.fi/bridges/python/telegram/index.html).
-Some quick links:
-
-* [Bridge setup](https://docs.mau.fi/bridges/python/setup/index.html?bridge=telegram)
-  (or [with Docker](https://docs.mau.fi/bridges/python/setup/docker.html?bridge=telegram))
-* Basic usage: [Authentication](https://docs.mau.fi/bridges/python/telegram/authentication.html),
-  [Creating chats](https://docs.mau.fi/bridges/python/telegram/creating-and-managing-chats.html),
-  [Relaybot setup](https://docs.mau.fi/bridges/python/telegram/relay-bot.html)
-
-### Features & Roadmap
-[ROADMAP.md](https://github.com/mautrix/telegram/blob/master/ROADMAP.md)
-contains a general overview of what is supported by the bridge.
+### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
 
 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)
@@ -32,4 +17,4 @@ Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net
 Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
 
 ## Preview
-![Preview](preview.png)
+![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)

ROADMAP.md

@@ -6,9 +6,9 @@
   * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
-  * [x] Typing notifications
-  * [x] Read receipts
-  * [x] Pinning messages
+  * [x] Typing notifications*
+  * [x] Read receipts*
+  * [x] Pinning messages*
   * [x] Power level
     * [x] Normal chats
       * [ ] Non-hardcoded PL requirements
@@ -28,10 +28,7 @@
 	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
-  * [x] Message history
-    * [x] Manually (`!tg backfill`)
-    * [x] Automatically when creating portal
-    * [x] Automatically for missed messages
+  * [ ] Message history
   * [x] Avatars
   * [x] Presence
   * [x] Typing notifications
@@ -53,11 +50,12 @@
     * [x] At startup
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [x] Option to use bot to relay messages for unauthenticated Matrix users (relaybot)
-  * [x] Option to use own Matrix account for messages sent from other Telegram clients (double puppeting)
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
-  * [ ] ‡ Secret chats (i.e. End-to-bridge encryption on Telegram)
-  * [x] End-to-bridge encryption in Matrix rooms (see [wiki](https://github.com/tulir/mautrix-telegram/wiki/End%E2%80%90to%E2%80%90bridge-encryption))
+  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
+\* Requires [double puppeting](https://github.com/tulir/mautrix-telegram/wiki/Authentication#replacing-telegram-accounts-matrix-puppet-with-matrix-account) to be enabled  
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic.ini

@@ -0,0 +1,71 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

alembic/env.py

@@ -0,0 +1,89 @@
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+import sys
+from os.path import abspath, dirname
+
+sys.path.insert(0, dirname(dirname(abspath(__file__))))
+
+from mautrix.util.db import Base
+import mautrix_telegram.db
+from mautrix_telegram.config import Config
+from alchemysession import AlchemySessionContainer
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+mxtg_config = Config(mxtg_config_path, None, None)
+mxtg_config.load()
+config.set_main_option("sqlalchemy.url", mxtg_config["appservice.database"].replace("%", "%%"))
+
+
+AlchemySessionContainer.create_table_classes(None, "telethon_", Base)
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = Base.metadata
+
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -0,0 +1,27 @@
+"""Add disable_updates field for puppets
+
+Revision ID: 17574c57f3f8
+Revises: a9119be92164
+Create Date: 2019-05-15 00:24:46.967529
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '17574c57f3f8'
+down_revision = 'a9119be92164'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("disable_updates")

alembic/versions/1b241f7e8530_add_telegramfile_table.py

@@ -0,0 +1,28 @@
+"""Add TelegramFile table
+
+Revision ID: 1b241f7e8530
+Revises: 97d2a942bcf8
+Create Date: 2018-02-19 23:52:06.605741
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '1b241f7e8530'
+down_revision = '97d2a942bcf8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('telegram_file',
+                    sa.Column('id', sa.String(), nullable=False),
+                    sa.Column('mxc', sa.String(), nullable=True),
+                    sa.Column('mime_type', sa.String(), nullable=True),
+                    sa.Column('was_converted', sa.Boolean(), nullable=True),
+                    sa.PrimaryKeyConstraint('id'))
+
+
+def downgrade():
+    op.drop_table('telegram_file')

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -0,0 +1,26 @@
+"""Add is_bot field to puppets
+
+Revision ID: 1fa46383a9d3
+Revises: 30eca60587f1
+Create Date: 2018-04-29 23:44:40.102333
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1fa46383a9d3'
+down_revision = '30eca60587f1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('is_bot')

alembic/versions/2228d49c383f_add_cascade_rules_to_userportal.py

@@ -0,0 +1,41 @@
+"""Add cascade rules to UserPortal
+
+Revision ID: 2228d49c383f
+Revises: bcfefa1f1299
+Create Date: 2018-05-31 11:11:59.482112
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '2228d49c383f'
+down_revision = 'bcfefa1f1299'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+            batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+            batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+            batch_op.create_foreign_key("user_portal_user_fkey", "user", ["user"], ["tgid"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+            batch_op.create_foreign_key("user_portal_portal_fkey", "portal",
+                                        ["portal", "portal_receiver"], ["tgid", "tg_receiver"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+    except ValueError:
+        return
+
+
+def downgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+                batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+                batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+                batch_op.create_foreign_key("user_portal_user_fkey", "portal",
+                                            ["portal", "portal_receiver"], ["tgid", "tg_receiver"])
+                batch_op.create_foreign_key("user_portal_portal_fkey", "user", ["user"], ["tgid"])
+    except ValueError:
+        return

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -0,0 +1,25 @@
+"""Add megagroup field to portals
+
+Revision ID: 30eca60587f1
+Revises: cfc972368e50
+Create Date: 2018-04-29 15:51:04.656605
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '30eca60587f1'
+down_revision = 'cfc972368e50'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column('megagroup')

alembic/versions/4f7d7ed5792a_switch_mx_user_profile_to_native_enum.py

@@ -0,0 +1,26 @@
+"""Switch mx_user_profile to native enum
+
+Revision ID: 4f7d7ed5792a
+Revises: 9e9c89b0b877
+Create Date: 2019-08-04 17:47:36.568120
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = '4f7d7ed5792a'
+down_revision = '9e9c89b0b877'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    conn = op.get_bind()
+    conn.execute("UPDATE mx_user_profile SET membership=UPPER(membership)")
+    conn.execute("UPDATE mx_user_profile SET membership='LEAVE' WHERE membership='LEFT'")
+
+
+def downgrade():
+    conn = op.get_bind()
+    conn.execute("UPDATE mx_user_profile SET membership=LOWER(membership)")

alembic/versions/501dad2868bc_move_sessions_to_main_database.py

@@ -0,0 +1,111 @@
+"""Move sessions to main database
+
+Revision ID: 501dad2868bc
+Revises: 7d47d84380b6
+Create Date: 2018-03-02 19:15:53.826985
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlite3
+import os
+
+# revision identifiers, used by Alembic.
+revision = '501dad2868bc'
+down_revision = '7d47d84380b6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    Session = op.create_table('telethon_sessions',
+                              sa.Column('session_id', sa.String, nullable=False),
+                              sa.Column('dc_id', sa.Integer, nullable=False),
+                              sa.Column('server_address', sa.String, nullable=True),
+                              sa.Column('port', sa.Integer, nullable=True),
+                              sa.Column('auth_key', sa.LargeBinary, nullable=True),
+                              sa.PrimaryKeyConstraint('session_id', 'dc_id'))
+    SentFile = op.create_table('telethon_sent_files',
+                               sa.Column('session_id', sa.String, nullable=False),
+                               sa.Column('md5_digest', sa.LargeBinary, nullable=False),
+                               sa.Column('file_size', sa.Integer, nullable=False),
+                               sa.Column('type', sa.Integer, nullable=False),
+                               sa.Column('id', sa.BigInteger, nullable=True),
+                               sa.Column('hash', sa.BigInteger, nullable=True),
+                               sa.PrimaryKeyConstraint('session_id', 'md5_digest', 'file_size',
+                                                       'type'))
+    Entity = op.create_table('telethon_entities',
+                             sa.Column('session_id', sa.String, nullable=False),
+                             sa.Column('id', sa.Integer, nullable=False),
+                             sa.Column('hash', sa.Integer, nullable=False),
+                             sa.Column('username', sa.String, nullable=True),
+                             sa.Column('phone', sa.Integer, nullable=True),
+                             sa.Column('name', sa.String, nullable=True),
+                             sa.PrimaryKeyConstraint('session_id', 'id'))
+    Version = op.create_table('telethon_version',
+                              sa.Column('version', sa.Integer, nullable=False),
+                              sa.PrimaryKeyConstraint('version'))
+    conn = op.get_bind()
+    sessions = [os.path.basename(f) for f in os.listdir(".") if f.endswith(".session")]
+    for session in sessions:
+        session_to_sqlalchemy(conn, session, Session, SentFile, Entity)
+
+
+def session_to_sqlalchemy(conn, path, Session, SentFile, Entity):
+    session_conn = sqlite3.connect(path)
+    session_id = os.path.splitext(path)[0]
+    c = session_conn.cursor()
+
+    auth_data_tuples = c.execute("SELECT * FROM sessions").fetchall()
+    auth_data_dicts = []
+    for row in auth_data_tuples:
+        dc_id, server_address, port, auth_key = row
+        auth_data_dicts.append({
+            "session_id": session_id,
+            "dc_id": dc_id,
+            "server_address": server_address,
+            "port": port,
+            "auth_key": auth_key,
+        })
+    if auth_data_dicts:
+        conn.execute(Session.insert().values(auth_data_dicts))
+
+    sent_file_tuples = c.execute("SELECT * FROM sent_files").fetchall()
+    sent_file_dicts = []
+    for row in sent_file_tuples:
+        md5_digest, file_size, type, id, hash = row
+        sent_file_dicts.append({
+            "session_id": session_id,
+            "md5_digest": md5_digest,
+            "file_size": file_size,
+            "type": type,
+            "id": id,
+            "hash": hash,
+        })
+    if sent_file_dicts:
+        conn.execute(SentFile.insert().values(sent_file_dicts))
+
+    entity_tuples = c.execute("SELECT * FROM entities").fetchall()
+    entity_dicts = []
+    for row in entity_tuples:
+        id, hash, username, phone, name = row
+        entity_dicts.append({
+            "session_id": session_id,
+            "id": id,
+            "hash": hash,
+            "username": username,
+            "phone": phone,
+            "name": name,
+        })
+    if entity_dicts:
+        conn.execute(Entity.insert().values(entity_dicts))
+
+    c.close()
+    session_conn.close()
+
+
+def downgrade():
+    op.drop_table('telethon_version')
+    op.drop_table('telethon_entities')
+    op.drop_table('telethon_sent_files')
+    op.drop_table('telethon_sessions')

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -0,0 +1,136 @@
+"""Move state store to main database
+
+Revision ID: 6ca3d74d51e4
+Revises: 2228d49c383f
+Create Date: 2018-06-26 21:31:26.911307
+
+"""
+import json
+import re
+
+from alembic import context, op
+import sqlalchemy.orm as orm
+import sqlalchemy as sa
+
+from mautrix.util.db import Base
+
+from mautrix_telegram.config import Config
+
+# revision identifiers, used by Alembic.
+revision = "6ca3d74d51e4"
+down_revision = "2228d49c383f"
+branch_labels = None
+depends_on = None
+
+
+class RoomState(Base):
+    __tablename__ = "mx_room_state"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    power_levels = sa.Column("power_levels", sa.Text, nullable=True)
+
+
+class UserProfile(Base):
+    __tablename__ = "mx_user_profile"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    user_id = sa.Column(sa.String, primary_key=True)
+    membership = sa.Column(sa.String, nullable=False, default="leave")
+    displayname = sa.Column(sa.String, nullable=True)
+    avatar_url = sa.Column(sa.String, nullable=True)
+
+
+class Puppet(Base):
+    __tablename__ = "puppet"
+    __table_args__ = {"extend_existing": True}
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    displayname = sa.Column(sa.String, nullable=True)
+    displayname_source = sa.Column(sa.Integer, nullable=True)
+    username = sa.Column(sa.String, nullable=True)
+    photo_id = sa.Column(sa.String, nullable=True)
+    is_bot = sa.Column(sa.Boolean, nullable=True)
+    matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False)
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+    op.create_table("mx_room_state",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("power_levels", sa.Text(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id"))
+    op.create_table("mx_user_profile",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("user_id", sa.String(), nullable=False),
+                    sa.Column("membership", sa.String(), nullable=False,
+                              default="leave"),
+                    sa.Column("displayname", sa.String(), nullable=True),
+                    sa.Column("avatar_url", sa.String(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id", "user_id"))
+
+    try:
+        migrate_state_store()
+    except Exception as e:
+        print("Failed to migrate state store:", e)
+        print("Migrating the state store isn't required, but you can retry by alembic downgrading "
+              "to revision 2228d49c383f and upgrading again.")
+
+
+def migrate_state_store():
+    conn = op.get_bind()
+    session: orm.Session = orm.sessionmaker(bind=conn)()
+
+    try:
+        with open("mx-state.json") as file:
+            data = json.load(file)
+    except FileNotFoundError:
+        return
+    if not data:
+        return
+    registrations = data.get("registrations", [])
+
+    mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+    mxtg_config = Config(mxtg_config_path, None, None)
+    mxtg_config.load()
+
+    username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}")
+    hs_domain = mxtg_config["homeserver.domain"]
+    localpart = username_template.format(userid="(.+)")
+    mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain))
+    for user in registrations:
+        match = mxid_regex.match(user)
+        if not match:
+            continue
+
+        puppet = session.query(Puppet).get(match.group(1))
+        if not puppet:
+            continue
+
+        puppet.matrix_registered = True
+        session.merge(puppet)
+    session.commit()
+
+    user_profiles = [UserProfile(room_id=room, user_id=user,
+                                 membership=member.get("membership", "leave"),
+                                 displayname=member.get("displayname", None),
+                                 avatar_url=member.get("avatar_url", None))
+                     for room, members in data.get("members", {}).items()
+                     for user, member in members.items()]
+    session.add_all(user_profiles)
+    session.commit()
+
+    room_state = [RoomState(room_id=room, power_levels=json.dumps(levels))
+                  for room, levels in data.get("power_levels", {}).items()]
+    session.add_all(room_state)
+    session.commit()
+
+
+def downgrade():
+    op.drop_table("mx_user_profile")
+    op.drop_table("mx_room_state")
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("matrix_registered")

alembic/versions/7d47d84380b6_add_timestamp_to_telegramfile.py

@@ -0,0 +1,26 @@
+"""Add timestamp to TelegramFile
+
+Revision ID: 7d47d84380b6
+Revises: 1b241f7e8530
+Create Date: 2018-02-19 23:53:18.050871
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '7d47d84380b6'
+down_revision = '1b241f7e8530'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('telegram_file',
+                  sa.Column('timestamp', sa.BigInteger(), nullable=True, default=0,
+                            server_default="0"))
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column('timestamp')

alembic/versions/97d2a942bcf8_initial_revision.py

@@ -0,0 +1,80 @@
+"""initial revision
+
+Revision ID: 97d2a942bcf8
+Revises:
+Create Date: 2018-02-11 18:40:55.483842
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '97d2a942bcf8'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('portal',
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_receiver', sa.Integer),
+                    sa.Column('peer_type', sa.String, nullable=False, default=""),
+                    sa.Column('mxid', sa.String, nullable=True),
+                    sa.Column('username', sa.String, nullable=True),
+                    sa.Column('title', sa.String, nullable=True),
+                    sa.Column('about', sa.String, nullable=True),
+                    sa.Column('photo_id', sa.String, nullable=True),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_receiver'),
+                    sa.UniqueConstraint('mxid'))
+    op.create_table('user',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('tgid', sa.Integer, nullable=True, unique=True),
+                    sa.Column('tg_username', sa.String, nullable=True),
+                    sa.Column('saved_contacts', sa.Integer, nullable=False, default=0),
+                    sa.PrimaryKeyConstraint('mxid'))
+    op.create_table('puppet',
+                    sa.Column('id', sa.Integer),
+                    sa.Column('displayname', sa.String, nullable=True),
+                    sa.Column('username', sa.String, nullable=True),
+                    sa.Column('photo_id', sa.String, nullable=True),
+                    sa.PrimaryKeyConstraint('id'))
+    op.create_table('contact',
+                    sa.Column('user', sa.Integer),
+                    sa.Column('contact', sa.Integer),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
+                    sa.ForeignKeyConstraint(("contact",), ("puppet.id",)),
+                    sa.PrimaryKeyConstraint('user', 'contact'))
+    op.create_table('user_portal',
+                    sa.Column('user', sa.Integer),
+                    sa.Column('portal', sa.Integer),
+                    sa.Column('portal_receiver', sa.Integer),
+                    sa.PrimaryKeyConstraint('user', 'portal', 'portal_receiver'),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",),
+                                            name="user_portal_user_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"),
+                    sa.ForeignKeyConstraint(("portal", "portal_receiver"),
+                                            ("portal.tgid", "portal.tg_receiver"),
+                                            name="user_portal_portal_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"))
+    op.create_table('message',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    op.create_table('bot_chat',
+                    sa.Column('id', sa.Integer),
+                    sa.Column('type', sa.String, nullable=False, default=""),
+                    sa.PrimaryKeyConstraint('id'))
+
+
+def downgrade():
+    op.drop_table('bot_chat')
+    op.drop_table('message')
+    op.drop_table('user_portal')
+    op.drop_table('contact')
+    op.drop_table('puppet')
+    op.drop_table('user')
+    op.drop_table('portal')

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,48 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

alembic/versions/a7c04a56041b_store_custom_puppet_next_batch_in_.py

@@ -0,0 +1,26 @@
+"""Store custom puppet next_batch in database
+
+Revision ID: a7c04a56041b
+Revises: 4f7d7ed5792a
+Create Date: 2019-08-06 23:08:51.087651
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "a7c04a56041b"
+down_revision = "4f7d7ed5792a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("next_batch", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("next_batch")

alembic/versions/a9119be92164_add_phone_number_field_to_users.py

@@ -0,0 +1,25 @@
+"""Add phone number field to users
+
+Revision ID: a9119be92164
+Revises: b54929c22c86
+Create Date: 2018-09-28 02:38:40.626282
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "a9119be92164"
+down_revision = "b54929c22c86"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("user", sa.Column("tg_phone", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("user") as batch_op:
+        batch_op.drop_column("tg_phone")

alembic/versions/b54929c22c86_add_portal_specific_config.py

@@ -0,0 +1,25 @@
+"""Add portal-specific config
+
+Revision ID: b54929c22c86
+Revises: d5f7b8b4b456
+Create Date: 2018-09-24 23:40:33.528710
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "b54929c22c86"
+down_revision = "d5f7b8b4b456"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("portal", sa.Column("config", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column("config")

alembic/versions/bcfefa1f1299_add_displayname_source_fields_for_.py

@@ -0,0 +1,24 @@
+"""Add displayname source fields for puppets
+
+Revision ID: bcfefa1f1299
+Revises: bdadd173ee02
+Create Date: 2018-05-19 17:00:21.078098
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'bcfefa1f1299'
+down_revision = 'bdadd173ee02'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('puppet', sa.Column('displayname_source', sa.Integer(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('displayname_source')

alembic/versions/bdadd173ee02_update_telethon_update_state_table.py

@@ -0,0 +1,43 @@
+"""Update telethon update state table
+
+Revision ID: bdadd173ee02
+Revises: eeaf0dae87ce
+Create Date: 2018-05-13 10:42:59.395597
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'bdadd173ee02'
+down_revision = 'eeaf0dae87ce'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column("id", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("hash", existing_type=sa.Integer, type_=sa.BigInteger)
+
+    with op.batch_alter_table("telethon_update_state") as batch_op:
+        batch_op.alter_column("entity_id", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("pts", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("qts", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("date", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("seq", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.add_column(sa.Column("unread_count", sa.Integer))
+
+
+def downgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column("id", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("hash", existing_type=sa.BigInteger, type_=sa.Integer)
+
+    with op.batch_alter_table("telethon_update_state") as batch_op:
+        batch_op.alter_column("entity_id", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("pts", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("qts", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("date", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("seq", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.drop_column("unread_count")

alembic/versions/cfc972368e50_add_metadata_to_telegramfile.py

@@ -0,0 +1,35 @@
+"""Add metadata to TelegramFile
+
+Revision ID: cfc972368e50
+Revises: 501dad2868bc
+Create Date: 2018-03-09 16:07:01.236712
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'cfc972368e50'
+down_revision = '501dad2868bc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.add_column(sa.Column('size', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('width', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('height', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('thumbnail', sa.String(), nullable=True))
+        batch_op.create_foreign_key(constraint_name="fk_file_thumbnail",
+                                    referent_table="telegram_file",
+                                    local_cols=['thumbnail'],
+                                    remote_cols=['id'])
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column('size')
+        batch_op.drop_column('width')
+        batch_op.drop_column('height')
+        batch_op.drop_column('thumbnail')

alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py

@@ -0,0 +1,26 @@
+"""Add access_token and custom_mxid fields for puppets
+
+Revision ID: d5f7b8b4b456
+Revises: 6ca3d74d51e4
+Create Date: 2018-07-20 12:09:30.277960
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d5f7b8b4b456"
+down_revision = "6ca3d74d51e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
+    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("custom_mxid")
+        batch_op.drop_column("access_token")

alembic/versions/eeaf0dae87ce_add_telethon_update_state_table.py

@@ -0,0 +1,34 @@
+"""Add telethon update state table
+
+Revision ID: eeaf0dae87ce
+Revises: 1fa46383a9d3
+Create Date: 2018-04-30 17:30:59.610885
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'eeaf0dae87ce'
+down_revision = '1fa46383a9d3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column('phone', existing_type=sa.Integer, type_=sa.BigInteger)
+    op.create_table('telethon_update_state',
+                    sa.Column('session_id', sa.String, nullable=False),
+                    sa.Column('entity_id', sa.Integer, nullable=False),
+                    sa.Column('pts', sa.Integer, nullable=True),
+                    sa.Column('qts', sa.Integer, nullable=True),
+                    sa.Column('date', sa.Integer, nullable=True),
+                    sa.Column('seq', sa.Integer, nullable=True),
+                    sa.PrimaryKeyConstraint('session_id', 'entity_id'))
+
+
+def downgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column('phone', existing_type=sa.BigInteger, type_=sa.Integer)
+    op.drop_table('telethon_update_state')

docker-run.sh

@@ -7,6 +7,15 @@ function fixperms {
 
 cd /opt/mautrix-telegram
 
+# Replace database path in config.
+sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
+
+if [ -f /data/mx-state.json ]; then
+	ln -s /data/mx-state.json
+fi
+# Check that database is in the right state
+alembic -x config=/data/config.yaml upgrade head
+
 if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
 	echo "Didn't find a config file."
@@ -19,6 +28,9 @@ fi
 
 if [ ! -f /data/registration.yaml ]; then
 	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
+	echo "Didn't find a registration file."
+	echo "Generated one for you."
+	echo "Copy that over to synapses app service directory."
 	fixperms
 	exit
 fi

example-config.yaml

@@ -7,24 +7,12 @@ homeserver:
     # Whether or not to verify the SSL certificate of the homeserver.
     # Only applies if address starts with https://
     verify_ssl: true
-    asmux: false
-    # Number of retries for all HTTP requests if the homeserver isn't reachable.
-    http_retry_count: 4
-    # The URL to push real-time bridge status to.
-    # If set, the bridge will make POST requests to this URL whenever a user's Telegram connection state changes.
-    # The bridge will use the appservice as_token to authorize requests.
-    status_endpoint: null
-    # Endpoint for reporting per-message status.
-    message_send_checkpoint_endpoint: null
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
 appservice:
     # The address that the homeserver can use to connect to this appservice.
     address: http://localhost:29317
-    # When using https:// the TLS certificate and key files for the address.
-    tls_cert: false
-    tls_key: false
 
     # The hostname and port where this appservice should listen.
     hostname: 0.0.0.0
@@ -33,18 +21,12 @@ appservice:
     # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
     max_body_size: 1
 
-    # The full URI to the database. SQLite and Postgres are supported.
+    # The full URI to the database. SQLite and Postgres are fully supported.
+    # Other DBMSes supported by SQLAlchemy may or may not work.
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: postgres://username:password@hostname/dbname
-    # Additional arguments for asyncpg.create_pool() or sqlite3.connect()
-    # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool
-    # https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
-    # For sqlite, min_size is used as the connection thread pool size and max_size is ignored.
-    database_opts:
-        min_size: 1
-        max_size: 10
+    database: sqlite:///mautrix-telegram.db
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in
@@ -59,7 +41,7 @@ appservice:
         external: https://example.com/public
 
     # Provisioning API part of the web server for automated portal creation and fetching information.
-    # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager).
+    # Used by things like Dimension (https://dimension.t2bot.io/).
     provisioning:
         # Whether or not the provisioning API should be enabled.
         enabled: true
@@ -80,15 +62,8 @@ appservice:
 
     # Community ID for bridged users (changes registration file) and rooms.
     # Must be created manually.
-    #
-    # Example: "+telegram:example.com". Set to false to disable.
     community_id: false
 
-    # Whether or not to receive ephemeral events via appservice transactions.
-    # Requires MSC2409 support (i.e. Synapse 1.22+).
-    # You should disable bridge -> sync_with_custom_puppets when this is enabled.
-    ephemeral_events: false
-
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
@@ -141,16 +116,12 @@ bridge:
     - phone number
     # Maximum length of displayname
     displayname_max_length: 100
-    # Remove avatars from Telegram ghost users when removed on Telegram. This is disabled by default
-    # as there's no way to determine whether an avatar is removed or just hidden from some users. If
-    # you're on a single-user instance, this should be safe to enable.
-    allow_avatar_remove: false
 
     # Maximum number of members to sync per portal when starting up. Other members will be
     # synced when they send messages. The maximum is 10000, after which the Telegram server
     # will not send any more members.
-    # -1 means no limit (which means it's limited to 10000 by the server)
-    max_initial_member_sync: 100
+    # Defaults to no local limit (-> limited to 10000 by server)
+    max_initial_member_sync: -1
     # Whether or not to sync the member list in channels.
     # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
     # list regardless of this setting.
@@ -162,10 +133,7 @@ bridge:
     startup_sync: true
     # Number of most recently active dialogs to check when syncing chats.
     # Set to 0 to remove limit.
-    sync_update_limit: 0
-    # Number of most recently active dialogs to create portals for when syncing chats.
-    # Set to 0 to remove limit.
-    sync_create_limit: 30
+    sync_dialog_limit: 30
     # Whether or not to sync and create portals for direct chats at startup.
     sync_direct_chats: false
     # The maximum number of simultaneous Telegram deletions to handle.
@@ -174,8 +142,8 @@ bridge:
     # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
     # at startup and when creating a bridge.
     sync_matrix_state: true
-    # Allow logging in within Matrix. If false, users can only log in using login-qr or the
-    # out-of-Matrix login website (see appservice.public config section)
+    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
+    # login website (see appservice.public config section)
     allow_matrix_login: true
     # Whether or not to bridge plaintext highlights.
     # Only enable this if your displayname_template has some static part that the bridge can use to
@@ -183,34 +151,19 @@ bridge:
     plaintext_highlights: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
-    # Whether or not to use /sync to get presence, read receipts and typing notifications
-    # when double puppeting is enabled
+    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # your own Matrix account as the Matrix puppet for your Telegram account.
     sync_with_custom_puppets: true
-    # Whether or not to update the m.direct account data event when double puppeting is enabled.
-    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
-    # and is therefore prone to race conditions.
-    sync_direct_chat_list: false
-    # Servers to always allow double puppeting from
-    double_puppet_server_map:
-        example.com: https://example.com
-    # Allow using double puppeting from any server with a valid client .well-known file.
-    double_puppet_allow_discovery: false
-    # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth
+    # Shared secret for https://github.com/devture/matrix-synapse-shared-secret-auth
     #
     # If set, custom puppets will be enabled automatically for local users
     # instead of users having to find an access token and run `login-matrix`
     # manually.
-    # If using this for other servers than the bridge's server,
-    # you must also set the URL in the double_puppet_server_map.
-    login_shared_secret_map:
-        example.com: foobar
+    login_shared_secret: null
     # Set to false to disable link previews in messages sent to Telegram.
     telegram_link_preview: true
-    # Whether or not the !tg join command should do a HTTP request
-    # to resolve redirects in invite links.
-    invite_link_resolve: false
     # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Element iOS/Android).
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
@@ -219,7 +172,6 @@ bridge:
     # Enable experimental parallel file transfer, which makes uploads/downloads much faster by
     # streaming from/to Matrix and using many connections for Telegram.
     # Note that generating HQ thumbnails for videos is not possible with streamed transfers.
-    # This option uses internal Telethon implementation details and may break with minor updates.
     parallel_file_transfer: false
     # Whether or not created rooms should have federation enabled.
     # If false, created portal rooms will never be federated.
@@ -229,91 +181,16 @@ bridge:
         # Format to which animated stickers should be converted.
         # disable - No conversion, send as-is (gzipped lottie)
         # png - converts to non-animated png (fastest),
-        # gif - converts to animated gif
+        # gif - converts to animated gif, but loses transparency
         # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support
         target: gif
         # Arguments for converter. All converters take width and height.
+        # GIF converter takes background as a hex color.
         args:
             width: 256
             height: 256
-            fps: 25 # only for webm and gif (2, 5, 10, 20 or 25 recommended)
-    # End-to-bridge encryption support options.
-    #
-    # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info.
-    encryption:
-        # Allow encryption, work in group chat rooms with e2ee enabled
-        allow: false
-        # Default to encryption, force-enable encryption in all portals the bridge creates
-        # This will cause the bridge bot to be in private chats for the encryption to work properly.
-        default: false
-        # Database for the encryption data. If set to `default`, will use the appservice database.
-        database: default
-        # Options for automatic key sharing.
-        key_sharing:
-            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
-            # You must use a client that supports requesting keys from other users to use this feature.
-            allow: false
-            # Require the requesting device to have a valid cross-signing signature?
-            # This doesn't require that the bridge has verified the device, only that the user has verified it.
-            # Not yet implemented.
-            require_cross_signing: false
-            # Require devices to be verified by the bridge?
-            # Verification by the bridge is not yet implemented.
-            require_verification: true
-    # Whether or not to explicitly set the avatar and room name for private
-    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
-    private_chat_portal_meta: false
-    # Whether or not the bridge should send a read receipt from the bridge bot when a message has
-    # been sent to Telegram.
-    delivery_receipts: false
-    # Whether or not delivery errors should be reported as messages in the Matrix room.
-    delivery_error_reports: false
-    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
-    # This field will automatically be changed back to false after it,
-    # except if the config file is not writable.
-    resend_bridge_info: false
-    # When using double puppeting, should muted chats be muted in Matrix?
-    mute_bridging: false
-    # When using double puppeting, should pinned chats be moved to a specific tag in Matrix?
-    # The favorites tag is `m.favourite`.
-    pinned_tag: null
-    # Same as above for archived chats, the low priority tag is `m.lowpriority`.
-    archive_tag: null
-    # Whether or not mute status and tags should only be bridged when the portal room is created.
-    tag_only_on_create: true
-    # Should leaving the room on Matrix make the user leave on Telegram?
-    bridge_matrix_leave: true
-    # Should the user be kicked out of all portals when logging out of the bridge?
-    kick_on_logout: true
-    # Settings for backfilling messages from Telegram.
-    backfill:
-        # Whether or not the Telegram ghosts of logged in Matrix users should be
-        # invited to private chats when backfilling history from Telegram. This is
-        # usually needed to prevent rate limits and to allow timestamp massaging.
-        invite_own_puppet: true
-        # Maximum number of messages to backfill without using a takeout.
-        # The first time a takeout is used, the user has to manually approve it from a different
-        # device. If initial_limit or missed_limit are higher than this value, the bridge will ask
-        # the user to accept the takeout after logging in before syncing any chats.
-        takeout_limit: 100
-        # Maximum number of messages to backfill initially.
-        # Set to 0 to disable backfilling when creating portal, or -1 to disable the limit.
-        #
-        # N.B. Initial backfill will only start after member sync. Make sure your
-        #      max_initial_member_sync is set to a low enough value so it doesn't take forever.
-        initial_limit: 0
-        # Maximum number of messages to backfill if messages were missed while the bridge was
-        # disconnected. Note that this only works for logged in users and only if the chat isn't
-        # older than sync_update_limit
-        # Set to 0 to disable backfilling missed messages.
-        missed_limit: 50
-        # If using double puppeting, should notifications be disabled
-        # while the initial backfill is in progress?
-        disable_notifications: false
-        # Whether or not to enable backfilling in normal groups.
-        # Normal groups have numerous technical problems in Telegram, and backfilling normal groups
-        # will likely cause problems if there are multiple Matrix users in the group.
-        normal_groups: false
+            background: "020202"  # only for gif
+            fps: 30               # only for webm
 
     # Overrides for base power levels.
     initial_power_level_overrides:
@@ -393,21 +270,6 @@ bridge:
     # The prefix for commands. Only required in non-management rooms.
     command_prefix: "!tg"
 
-    # Messages sent upon joining a management room.
-    # Markdown is supported. The defaults are listed below.
-    management_room_text:
-        # Sent when joining a room.
-        welcome: "Hello, I'm a Telegram bridge bot."
-        # Sent when joining a management room and the user is already logged in.
-        welcome_connected: "Use `help` for help."
-        # Sent when joining a management room and the user is not logged in.
-        welcome_unconnected: "Use `help` for help or `login` to log in."
-        # Optional extra text sent when joining a management room.
-        additional_help: ""
-
-    # Send each message separately (for readability in some clients)
-    management_room_multiple_messages: false
-
     # Permissions for using the bridge.
     # Permitted values:
     #   relaybot - Only use the bridge via the relaybot, no access to commands.
@@ -547,7 +409,7 @@ logging:
         mau:
             level: DEBUG
         telethon:
-            level: INFO
+            level: DEBUG
         aiohttp:
             level: INFO
     root:

helm/mautrix-telegram/.editorconfig

@@ -0,0 +1,2 @@
+[*.{yaml,yml}]
+indent_size = 2

helm/mautrix-telegram/.gitignore

@@ -0,0 +1 @@
+charts/*

helm/mautrix-telegram/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm/mautrix-telegram/Chart.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+name: mautrix-telegram
+version: 0.1.0
+appVersion: "0.7.0"
+description: A Matrix-Telegram hybrid puppeting/relaybot bridge.
+keywords:
+  - matrix
+  - bridge
+  - telegram
+maintainers:
+  - name: Tulir Asokan
+    email: tulir@maunium.net
+sources:
+  - https://github.com/tulir/mautrix-telegram

helm/mautrix-telegram/requirements.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: postgresql
+  repository: https://kubernetes-charts.storage.googleapis.com/
+  version: 6.5.0
+digest: sha256:85139e9d4207e49c11c5f84d7920d0135cffd3d427f3f3638d4e51258990de2a
+generated: "2019-10-23T22:11:37.005827507+03:00"

helm/mautrix-telegram/requirements.yaml

@@ -0,0 +1,5 @@
+dependencies:
+  - name: postgresql
+    version: 6.5.0
+    repository: https://kubernetes-charts.storage.googleapis.com/
+    condition: postgresql.enabled

helm/mautrix-telegram/templates/NOTES.txt

@@ -0,0 +1,21 @@
+Your registration file is below. Save it into a YAML file and give the path to that file to synapse:
+
+id: {{ .Values.appservice.id }}
+as_token: {{ .Values.appservice.asToken }}
+hs_token: {{ .Values.appservice.hsToken }}
+namespaces:
+    users:
+        - exclusive: true
+          regex: "@{{ .Values.bridge.username_template | replace "{userid}" ".+"}}:{{ .Values.homeserver.domain }}"
+          {{- if .Values.appservice.communityID }}
+          group_id: {{ .Values.appservice.communityID }}
+          {{- end }}
+    aliases:
+        - exclusive: true
+          regex: "@{{ .Values.bridge.alias_template | replace "{groupname}" ".+"}}:{{ .Values.homeserver.domain }}"
+          {{- if .Values.appservice.communityID }}
+          group_id: {{ .Values.appservice.communityID }}
+          {{- end }}
+url: {{ .Values.appservice.address }}
+sender_localpart: {{ .Values.appservice.botUsername }}
+rate_limited: false

helm/mautrix-telegram/templates/_helpers.tpl

@@ -0,0 +1,55 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mautrix-telegram.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mautrix-telegram.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mautrix-telegram.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "mautrix-telegram.labels" -}}
+app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
+helm.sh/chart: {{ include "mautrix-telegram.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mautrix-telegram.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "mautrix-telegram.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}

helm/mautrix-telegram/templates/configmap.yaml

@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "mautrix-telegram.fullname" . }}
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    app.kubernetes.io/name: {{ template "mautrix-telegram.name" . }}
+data:
+  config.yaml: |
+    homeserver:
+      address: {{ .Values.homeserver.address }}
+      domain: {{ .Values.homeserver.domain }}
+      verify_ssl: {{ .Values.homeserver.verifySSL }}
+
+    appservice:
+      address: http://{{ include "mautrix-telegram.fullname" . }}:{{ .Values.service.port }}
+
+      hostname: 0.0.0.0
+      port: {{ .Values.service.port }}
+      max_body_size: {{ .Values.appservice.maxBodySize }}
+
+      {{- if .Values.postgresql.enabled }}
+      database: "postgres://postgres:{{ .Values.postgresql.postgresqlPassword }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.postgresqlDatabase }}"
+      {{- else }}
+      database: {{ .Values.appservice.database | quote }}
+      {{- end }}
+
+      public:
+        {{- toYaml .Values.appservice.public | nindent 8 }}
+
+      provisioning:
+        {{- toYaml .Values.appservice.provisioning | nindent 8 }}
+
+      id: {{ .Values.appservice.id }}
+      bot_username: {{ .Values.appservice.botUsername }}
+      bot_displayname: {{ .Values.appservice.botDisplayname }}
+      bot_avatar:  {{ .Values.appservice.botAvatar }}
+
+      community_id: {{ .Values.appservice.communityID }}
+
+      as_token: {{ .Values.appservice.asToken }}
+      hs_token: {{ .Values.appservice.hsToken }}
+
+    metrics:
+      {{- toYaml .Values.metrics | nindent 6 }}
+
+    bridge:
+      {{- toYaml .Values.bridge | nindent 6 }}
+
+    telegram:
+      {{- toYaml .Values.telegram | nindent 6 }}
+
+    logging:
+      {{- toYaml .Values.logging | nindent 6 }}
+  registration.yaml: ""

helm/mautrix-telegram/templates/deployment.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "mautrix-telegram.fullname" . }}
+  labels:
+    {{- include "mautrix-telegram.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    {{- if .Values.podAnnotations }}
+    annotations:
+      {{- toYaml .Values.podAnnotations | nindent 6 }}
+    {{- end }}
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      serviceAccountName: {{ template "mautrix-telegram.serviceAccountName" . }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          volumeMounts:
+            - mountPath: /data
+              name: config-volume
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /_matrix/mau/live
+              port: http
+            initialDelaySeconds: 60
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /_matrix/mau/ready
+              port: http
+            initialDelaySeconds: 60
+            periodSeconds: 5
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ template "mautrix-telegram.fullname" . }}
+
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

helm/mautrix-telegram/templates/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "mautrix-telegram.fullname" . }}
+  labels:
+{{ include "mautrix-telegram.labels" . | indent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/name: {{ include "mautrix-telegram.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}

helm/mautrix-telegram/templates/serviceaccount.yaml

@@ -0,0 +1,8 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "mautrix-telegram.serviceAccountName" . }}
+  labels:
+{{ include "mautrix-telegram.labels" . | indent 4 }}
+{{- end -}}

helm/mautrix-telegram/values.yaml

@@ -0,0 +1,141 @@
+image:
+  repository: dock.mau.dev/tulir/mautrix-telegram
+  tag: latest
+  pullPolicy: IfNotPresent
+
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:
+
+service:
+  type: ClusterIP
+  port: 29317
+
+resources: {}
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+# Postgres pod configs
+postgresql:
+  enabled: true
+  postgresqlDatabase: mxtg
+  postgresqlPassword: SET TO RANDOM STRING
+  persistence:
+    size: 2Gi
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 100m
+
+# Homeserver details
+homeserver:
+  # The address that this appservice can use to connect to the homeserver.
+  address: https://example.com
+  # The domain of the homeserver (for MXIDs, etc).
+  domain: example.com
+  # Whether or not to verify the SSL certificate of the homeserver.
+  # Only applies if address starts with https://
+  verifySSL: true
+
+# Application service host/registration related details
+# Changing these values requires regeneration of the registration.
+appservice:
+  # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+  # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+  maxBodySize: 1
+
+  # Public part of web server for out-of-Matrix interaction with the bridge.
+  # Used for things like login if the user wants to make sure the 2FA password isn't stored in
+  # the HS database.
+  public:
+    # Whether or not the public-facing endpoints should be enabled.
+    enabled: true
+    # The prefix to use in the public-facing endpoints.
+    prefix: /public
+    # The base URL where the public-facing endpoints are available. The prefix is not added
+    # implicitly.
+    external: https://example.com/public
+
+  # Provisioning API part of the web server for automated portal creation and fetching information.
+  # Used by things like Dimension (https://dimension.t2bot.io/).
+  provisioning:
+    # Whether or not the provisioning API should be enabled.
+    enabled: true
+    # The prefix to use in the provisioning API endpoints.
+    prefix: /_matrix/provision/v1
+    # The shared secret to authorize users of the API.
+    shared_secret: SET TO RANDOM STRING
+
+  id: telegram
+  botUsername: telegrambot
+  # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+  # to leave display name/avatar as-is.
+  botDisplayname: Telegram bridge bot
+  botAvatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
+
+  # Community ID for bridged users (changes registration file) and rooms.
+  # Must be created manually.
+  communityID: false
+
+  # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
+  asToken: SET TO RANDOM STRING
+  hsToken: SET TO RANDOM STRING
+
+# The keys below can be used to override the configs in the base config:
+# https://github.com/tulir/mautrix-telegram/blob/master/example-config.yaml
+# Note that the "appservice" and "homeserver" sections are above and slightly different than the base.
+
+# Bridge config
+bridge:
+  # Localpart template of MXIDs for Telegram users.
+  # {userid} is replaced with the user ID of the Telegram user.
+  username_template: "telegram_{userid}"
+  # Localpart template of room aliases for Telegram portal rooms.
+  # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} )
+  alias_template: "telegram_{groupname}"
+
+  # Permissions for using the bridge.
+  # Permitted values:
+  #   relaybot - Only use the bridge via the relaybot, no access to commands.
+  #       user - Relaybot level + access to commands to create bridges.
+  #  puppeting - User level + logging in with a Telegram account.
+  #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
+  #      admin - Full access to use the bridge and some extra administration commands.
+  # Permitted keys:
+  #        * - All Matrix users
+  #   domain - All users on that homeserver
+  #     mxid - Specific user
+  permissions:
+    "*": "relaybot"
+    "public.example.com": "user"
+    "example.com": "full"
+    "@admin:example.com": "admin"
+
+# Prometheus telemetry config.
+metrics:
+  enabled: false
+  listen_port: 8000
+
+# Telegram config
+telegram:
+  # Get your own API keys at https://my.telegram.org/apps
+  api_id: 12345
+  api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
+  # (Optional) Create your own bot at https://t.me/BotFather
+  # bot_token: 123456789:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.11.0"
+__version__ = "0.7.2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,129 +13,101 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Optional
+from itertools import chain
 
-from typing import Any
-
-from telethon import __version__ as __telethon_version__
+from alchemysession import AlchemySessionContainer
 
 from mautrix.bridge import Bridge
-from mautrix.types import RoomID, UserID
+from mautrix.util.db import Base
 
-from .bot import Bot
-from .config import Config
-from .db import init as init_db, upgrade_table
-from .matrix import MatrixHandler
-from .portal import Portal
-from .puppet import Puppet
-from .user import User
-from .version import linkified_version, version
 from .web.provisioning import ProvisioningAPI
 from .web.public import PublicBridgeWebsite
+from .commands.manhole import ManholeState
+from .abstract_user import init as init_abstract_user
+from .bot import Bot, init as init_bot
+from .config import Config
+from .context import Context
+from .db import init as init_db
+from .formatter import init as init_formatter
+from .matrix import MatrixHandler
+from .portal import init as init_portal
+from .puppet import Puppet, init as init_puppet
+from .sqlstatestore import SQLStateStore
+from .user import User, init as init_user
+from .version import version, linkified_version
 
-from .abstract_user import AbstractUser  # isort: skip
+try:
+    import prometheus_client as prometheus
+except ImportError:
+    prometheus = None
 
 
 class TelegramBridge(Bridge):
-    module = "mautrix_telegram"
     name = "mautrix-telegram"
     command = "python -m mautrix-telegram"
     description = "A Matrix-Telegram puppeting bridge."
-    repo_url = "https://github.com/mautrix/telegram"
+    repo_url = "https://github.com/tulir/mautrix-telegram"
+    real_user_content_key = "net.maunium.telegram.puppet"
     version = version
     markdown_version = linkified_version
     config_class = Config
     matrix_class = MatrixHandler
-    upgrade_table = upgrade_table
+    state_store_class = SQLStateStore
 
     config: Config
-    bot: Bot | None
-    public_website: PublicBridgeWebsite | None
-    provisioning_api: ProvisioningAPI | None
+    session_container: AlchemySessionContainer
+    bot: Bot
+    manhole: Optional[ManholeState]
 
     def prepare_db(self) -> None:
         super().prepare_db()
         init_db(self.db)
+        self.session_container = AlchemySessionContainer(
+            engine=self.db, table_base=Base, session=False,
+            table_prefix="telethon_", manage_tables=False)
 
-    def _prepare_website(self) -> None:
-        if self.config["appservice.provisioning.enabled"]:
-            self.provisioning_api = ProvisioningAPI(self)
-            self.az.app.add_subapp(
-                self.config["appservice.provisioning.prefix"], self.provisioning_api.app
-            )
-        else:
-            self.provisioning_api = None
-
+    def _prepare_website(self, context: Context) -> None:
         if self.config["appservice.public.enabled"]:
-            self.public_website = PublicBridgeWebsite(self.loop)
-            self.az.app.add_subapp(
-                self.config["appservice.public.prefix"], self.public_website.app
-            )
-        else:
-            self.public_website = None
+            public_website = PublicBridgeWebsite(self.loop)
+            self.az.app.add_subapp(self.config["appservice.public.prefix"], public_website.app)
+            context.public_website = public_website
+
+        if self.config["appservice.provisioning.enabled"]:
+            provisioning_api = ProvisioningAPI(context)
+            self.az.app.add_subapp(self.config["appservice.provisioning.prefix"],
+                                   provisioning_api.app)
+            context.provisioning_api = provisioning_api
+
+        if self.config["metrics.enabled"]:
+            if prometheus:
+                prometheus.start_http_server(self.config["metrics.listen_port"])
+            else:
+                self.log.warning("Metrics are enabled in the config, "
+                                 "but prometheus_client is not installed.")
 
     def prepare_bridge(self) -> None:
-        self._prepare_website()
-        AbstractUser.init_cls(self)
-        bot_token: str = self.config["telegram.bot_token"]
-        if bot_token and not bot_token.lower().startswith("disable"):
-            self.bot = AbstractUser.relaybot = Bot(bot_token)
-        else:
-            self.bot = AbstractUser.relaybot = None
-        self.matrix = MatrixHandler(self)
-        Portal.init_cls(self)
-        self.add_startup_actions(Puppet.init_cls(self))
-        self.add_startup_actions(User.init_cls(self))
-        if self.bot:
-            self.add_startup_actions(self.bot.start())
-        if self.config["bridge.resend_bridge_info"]:
-            self.add_startup_actions(self.resend_bridge_info())
+        self.bot = init_bot(self.config)
+        context = Context(self.az, self.config, self.loop, self.session_container, self, self.bot)
+        self._prepare_website(context)
+        self.matrix = context.mx = MatrixHandler(context)
+        self.manhole = None
 
-    async def resend_bridge_info(self) -> None:
-        self.config["bridge.resend_bridge_info"] = False
-        self.config.save()
-        self.log.info("Re-sending bridge info state event to all portals")
-        async for portal in Portal.all():
-            await portal.update_bridge_info()
-        self.log.info("Finished re-sending bridge info state events")
+        init_abstract_user(context)
+        init_formatter(context)
+        init_portal(context)
+        puppet_startup = init_puppet(context)
+        user_startup = init_user(context)
+        bot_startup = [self.bot.start()] if self.bot else []
+        self.startup_actions = chain(puppet_startup, user_startup, bot_startup)
 
     def prepare_stop(self) -> None:
         for puppet in Puppet.by_custom_mxid.values():
             puppet.stop()
         self.shutdown_actions = (user.stop() for user in User.by_tgid.values())
-
-    async def get_user(self, user_id: UserID, create: bool = True) -> User | None:
-        user = await User.get_by_mxid(user_id, create=create)
-        if user:
-            await user.ensure_started()
-        return user
-
-    async def get_portal(self, room_id: RoomID) -> Portal | None:
-        return await Portal.get_by_mxid(room_id)
-
-    async def get_puppet(self, user_id: UserID, create: bool = False) -> Puppet | None:
-        return await Puppet.get_by_mxid(user_id, create=create)
-
-    async def get_double_puppet(self, user_id: UserID) -> Puppet | None:
-        return await Puppet.get_by_custom_mxid(user_id)
-
-    def is_bridge_ghost(self, user_id: UserID) -> bool:
-        return bool(Puppet.get_id_from_mxid(user_id))
-
-    async def count_logged_in_users(self) -> int:
-        return len([user for user in User.by_tgid.values() if user.tgid])
-
-    async def manhole_global_namespace(self, user_id: UserID) -> dict[str, Any]:
-        return {
-            **await super().manhole_global_namespace(user_id),
-            "User": User,
-            "Portal": Portal,
-            "Puppet": Puppet,
-        }
-
-    @property
-    def manhole_banner_program_version(self) -> str:
-        return f"{super().manhole_banner_program_version} and Telethon {__telethon_version__}"
+        if self.manhole:
+            self.manhole.close()
+            self.manhole = None
 
 
 TelegramBridge().run()

mautrix_telegram/abstract_user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,114 +13,71 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any, Type, Union
+from typing import Tuple, Optional, Union, Dict, Type, Any, TYPE_CHECKING
 from abc import ABC, abstractmethod
 import asyncio
 import logging
 import platform
 import time
 
-from telethon.network import (
-    Connection,
-    ConnectionTcpFull,
-    ConnectionTcpMTProxyRandomizedIntermediate,
-)
 from telethon.sessions import Session
-from telethon.tl.patched import Message, MessageService
+from telethon.network import (ConnectionTcpMTProxyRandomizedIntermediate, ConnectionTcpFull,
+                              Connection)
+from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
-    Channel,
-    Chat,
-    MessageActionChannelMigrateFrom,
-    MessageEmpty,
-    PeerChat,
-    PeerUser,
-    TypeUpdate,
-    UpdateChannelUserTyping,
-    UpdateChatParticipantAdmin,
-    UpdateChatParticipants,
-    UpdateChatUserTyping,
-    UpdateDeleteChannelMessages,
-    UpdateDeleteMessages,
-    UpdateEditChannelMessage,
-    UpdateEditMessage,
-    UpdateFolderPeers,
-    UpdateNewChannelMessage,
-    UpdateNewMessage,
-    UpdateNotifySettings,
-    UpdatePinnedChannelMessages,
-    UpdatePinnedDialogs,
-    UpdatePinnedMessages,
-    UpdateReadChannelInbox,
-    UpdateReadHistoryInbox,
-    UpdateReadHistoryOutbox,
-    UpdateShortChatMessage,
-    UpdateShortMessage,
-    UpdateUserName,
-    UpdateUserPhoto,
-    UpdateUserStatus,
-    UpdateUserTyping,
-    User,
-    UserStatusOffline,
-    UserStatusOnline,
-)
+    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdateChatPinnedMessage,
+    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants,
+    UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
+    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
+    UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
+    UpdateUserTyping, User, UserStatusOffline, UserStatusOnline)
 
-from mautrix.appservice import AppService
+from mautrix.types import UserID, PresenceState
 from mautrix.errors import MatrixError
-from mautrix.types import PresenceState, UserID
-from mautrix.util.logging import TraceLogger
-from mautrix.util.opt_prometheus import Counter, Histogram
+from mautrix.appservice import AppService
+from alchemysession import AlchemySessionContainer
 
-from . import __version__, portal as po, puppet as pu
-from .config import Config
-from .db import Message as DBMessage, PgSession
-from .tgclient import MautrixTelegramClient
+from . import portal as po, puppet as pu, __version__
+from .db import Message as DBMessage
 from .types import TelegramID
+from .tgclient import MautrixTelegramClient
 
 if TYPE_CHECKING:
-    from .__main__ import TelegramBridge
+    from .context import Context
+    from .config import Config
     from .bot import Bot
 
-UpdateMessage = Union[
-    UpdateShortChatMessage,
-    UpdateShortMessage,
-    UpdateNewChannelMessage,
-    UpdateNewMessage,
-    UpdateEditMessage,
-    UpdateEditChannelMessage,
-]
-UpdateMessageContent = Union[
-    UpdateShortMessage, UpdateShortChatMessage, Message, MessageService, MessageEmpty
-]
+config: Optional['Config'] = None
+# Value updated from config in init()
+MAX_DELETIONS: int = 10
 
-UPDATE_TIME = Histogram(
-    name="bridge_telegram_update",
-    documentation="Time spent processing Telegram updates",
-    labelnames=("update_type",),
-)
-UPDATE_ERRORS = Counter(
-    name="bridge_telegram_update_error",
-    documentation="Number of fatal errors while handling Telegram updates",
-    labelnames=("update_type",),
-)
+UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                      UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
+UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
+
+try:
+    from prometheus_client import Histogram
+
+    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
+                            ["update_type"])
+except ImportError:
+    Histogram = None
+    UPDATE_TIME = None
 
 
 class AbstractUser(ABC):
+    session_container: AlchemySessionContainer = None
     loop: asyncio.AbstractEventLoop = None
-    log: TraceLogger
+    log: logging.Logger
     az: AppService
-    bridge: "TelegramBridge"
-    config: Config
-    relaybot: "Bot"
+    relaybot: Optional['Bot']
     ignore_incoming_bot_events: bool = True
-    max_deletions: int = 10
 
-    client: MautrixTelegramClient | None
-    mxid: UserID | None
+    client: Optional[MautrixTelegramClient]
+    mxid: Optional[UserID]
 
-    tgid: TelegramID | None
-    username: str | None
+    tgid: Optional[TelegramID]
+    username: Optional['str']
     is_bot: bool
 
     is_relaybot: bool
@@ -146,16 +103,14 @@ class AbstractUser(ABC):
         return self.client and self.client.is_connected()
 
     @property
-    def _proxy_settings(self) -> tuple[Type[Connection], tuple[Any, ...] | None]:
-        proxy_type = self.config["telegram.proxy.type"].lower()
+    def _proxy_settings(self) -> Tuple[Type[Connection], Optional[Tuple[Any, ...]]]:
+        proxy_type = config["telegram.proxy.type"].lower()
         connection = ConnectionTcpFull
-        connection_data = (
-            self.config["telegram.proxy.address"],
-            self.config["telegram.proxy.port"],
-            self.config["telegram.proxy.rdns"],
-            self.config["telegram.proxy.username"],
-            self.config["telegram.proxy.password"],
-        )
+        connection_data = (config["telegram.proxy.address"],
+                           config["telegram.proxy.port"],
+                           config["telegram.proxy.rdns"],
+                           config["telegram.proxy.username"],
+                           config["telegram.proxy.password"])
         if proxy_type == "disabled":
             connection_data = None
         elif proxy_type == "socks4":
@@ -170,59 +125,49 @@ class AbstractUser(ABC):
 
         return connection, connection_data
 
-    @classmethod
-    def init_cls(cls, bridge: "TelegramBridge") -> None:
-        cls.bridge = bridge
-        cls.config = bridge.config
-        cls.loop = bridge.loop
-        cls.az = bridge.az
-        cls.ignore_incoming_bot_events = cls.config["bridge.relaybot.ignore_own_incoming_events"]
-        cls.max_deletions = cls.config["bridge.max_telegram_delete"]
-
-    async def _init_client(self) -> None:
+    def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
 
-        session = await PgSession.get(self.name)
-        if self.config["telegram.server.enabled"]:
-            session.set_dc(
-                self.config["telegram.server.dc"],
-                self.config["telegram.server.ip"],
-                self.config["telegram.server.port"],
-            )
+        self.session = self.session_container.new_session(self.name)
+        if config["telegram.server.enabled"]:
+            self.session.set_dc(config["telegram.server.dc"],
+                                config["telegram.server.ip"],
+                                config["telegram.server.port"])
 
         if self.is_relaybot:
             base_logger = logging.getLogger("telethon.relaybot")
         else:
             base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
 
-        device = self.config["telegram.device_info.device_model"]
-        sysversion = self.config["telegram.device_info.system_version"]
-        appversion = self.config["telegram.device_info.app_version"]
+        device = config["telegram.device_info.device_model"]
+        sysversion = config["telegram.device_info.system_version"]
+        appversion = config["telegram.device_info.app_version"]
         connection, proxy = self._proxy_settings
 
-        assert isinstance(session, Session)
+        assert isinstance(self.session, Session)
 
         self.client = MautrixTelegramClient(
-            session=session,
-            api_id=self.config["telegram.api_id"],
-            api_hash=self.config["telegram.api_hash"],
+            session=self.session,
+
+            api_id=config["telegram.api_id"],
+            api_hash=config["telegram.api_hash"],
+
             app_version=__version__ if appversion == "auto" else appversion,
-            system_version=(
-                MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion
-            ),
-            device_model=(
-                f"{platform.system()} {platform.release()}" if device == "auto" else device
-            ),
-            timeout=self.config["telegram.connection.timeout"],
-            connection_retries=self.config["telegram.connection.retries"],
-            retry_delay=self.config["telegram.connection.retry_delay"],
-            flood_sleep_threshold=self.config["telegram.connection.flood_sleep_threshold"],
-            request_retries=self.config["telegram.connection.request_retries"],
+            system_version=(MautrixTelegramClient.__version__
+                            if sysversion == "auto" else sysversion),
+            device_model=(f"{platform.system()} {platform.release()}"
+                          if device == "auto" else device),
+
+            timeout=config["telegram.connection.timeout"],
+            connection_retries=config["telegram.connection.retries"],
+            retry_delay=config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
+            request_retries=config["telegram.connection.request_retries"],
             connection=connection,
             proxy=proxy,
-            raise_last_call_error=True,
+
             loop=self.loop,
-            base_logger=base_logger,
+            base_logger=base_logger
         )
         self.client.add_event_handler(self._update_catch)
 
@@ -235,23 +180,22 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     @abstractmethod
-    async def register_portal(self, portal: po.Portal) -> None:
+    def register_portal(self, portal: po.Portal) -> None:
         raise NotImplementedError()
 
     @abstractmethod
-    async def unregister_portal(self, tgid: int, tg_receiver: int) -> None:
+    def unregister_portal(self, portal: po.Portal) -> None:
         raise NotImplementedError()
 
     async def _update_catch(self, update: TypeUpdate) -> None:
         start_time = time.time()
-        update_type = type(update).__name__
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
-            self.log.exception("Failed to handle Telegram update")
-            UPDATE_ERRORS.labels(update_type=update_type).inc()
-        UPDATE_TIME.labels(update_type=update_type).observe(time.time() - start_time)
+            self.log.exception(f"Failed to handle Telegram update {update}")
+        if UPDATE_TIME:
+            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
 
     @property
     @abstractmethod
@@ -259,32 +203,27 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     async def is_logged_in(self) -> bool:
-        return (
-            self.client and self.client.is_connected() and await self.client.is_user_authorized()
-        )
+        return (self.client and self.client.is_connected()
+                and await self.client.is_user_authorized())
 
     async def has_full_access(self, allow_bot: bool = False) -> bool:
-        return (
-            self.puppet_whitelisted
-            and (not self.is_bot or allow_bot)
-            and await self.is_logged_in()
-        )
+        return (self.puppet_whitelisted
+                and (not self.is_bot or allow_bot)
+                and await self.is_logged_in())
 
-    async def start(self, delete_unless_authenticated: bool = False) -> AbstractUser:
+    async def start(self, delete_unless_authenticated: bool = False) -> 'AbstractUser':
         if not self.client:
-            await self._init_client()
+            self._init_client()
         await self.client.connect()
         self.log.debug(f"{'Bot' if self.is_relaybot else self.mxid} connected: {self.connected}")
         return self
 
-    async def ensure_started(self, even_if_no_session=False) -> AbstractUser:
+    async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
         if self.connected:
             return self
-        if even_if_no_session or await PgSession.has(self.mxid):
-            self.log.debug(
-                "Starting client due to ensure_started"
-                f"(even_if_no_session={even_if_no_session})"
-            )
+        if even_if_no_session or self.session_container.has_session(self.mxid):
+            self.log.debug("Starting client due to ensure_started"
+                           f"(even_if_no_session={even_if_no_session})")
             await self.start(delete_unless_authenticated=not even_if_no_session)
         return self
 
@@ -295,24 +234,16 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
-        asyncio.create_task(self._handle_entity_updates(getattr(update, "_entities", {})))
-        if isinstance(
-            update,
-            (
-                UpdateShortChatMessage,
-                UpdateShortMessage,
-                UpdateNewChannelMessage,
-                UpdateNewMessage,
-                UpdateEditMessage,
-                UpdateEditChannelMessage,
-            ),
-        ):
+        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+                              loop=self.loop)
+        if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                               UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
         elif isinstance(update, UpdateDeleteMessages):
             await self.delete_message(update)
         elif isinstance(update, UpdateDeleteChannelMessages):
             await self.delete_channel_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateChannelUserTyping, UpdateUserTyping)):
+        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
@@ -320,212 +251,129 @@ class AbstractUser(ABC):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, (UpdatePinnedMessages, UpdatePinnedChannelMessages)):
+        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
         elif isinstance(update, UpdateReadHistoryOutbox):
             await self.update_read_receipt(update)
-        elif isinstance(update, (UpdateReadHistoryInbox, UpdateReadChannelInbox)):
-            await self.update_own_read_receipt(update)
-        elif isinstance(update, UpdateFolderPeers):
-            await self.update_folder_peers(update)
-        elif isinstance(update, UpdatePinnedDialogs):
-            await self.update_pinned_dialogs(update)
-        elif isinstance(update, UpdateNotifySettings):
-            await self.update_notify_settings(update)
         else:
-            self.log.trace("Unhandled update: %s", update)
+            self.log.debug("Unhandled update: %s", update)
 
-    async def update_folder_peers(self, update: UpdateFolderPeers) -> None:
-        pass
-
-    async def update_pinned_dialogs(self, update: UpdatePinnedDialogs) -> None:
-        pass
-
-    async def update_notify_settings(self, update: UpdateNotifySettings) -> None:
-        pass
-
-    async def update_pinned_messages(
-        self, update: UpdatePinnedMessages | UpdatePinnedChannelMessages
-    ) -> None:
-        if isinstance(update, UpdatePinnedMessages):
-            portal = await po.Portal.get_by_entity(update.peer, tg_receiver=self.tgid)
+    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
+                                                         UpdateChatPinnedMessage]) -> None:
+        if isinstance(update, UpdateChatPinnedMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
         else:
-            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_ids(
-                update.messages, self.tgid, remove=not update.pinned
-            )
+            await portal.receive_telegram_pin_id(update.id, self.tgid)
 
     @staticmethod
     async def update_participants(update: UpdateChatParticipants) -> None:
-        portal = await po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
+        portal = po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
         if portal and portal.mxid:
-            await portal.update_power_levels(update.participants.participants)
+            await portal.update_telegram_participants(update.participants.participants)
 
     async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
         if not isinstance(update.peer, PeerUser):
             self.log.debug("Unexpected read receipt peer: %s", update.peer)
             return
 
-        portal = await po.Portal.get_by_tgid(
-            TelegramID(update.peer.user_id), tg_receiver=self.tgid
-        )
+        portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
         if not portal or not portal.mxid:
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = await DBMessage.get_one_by_tgid(
-            TelegramID(update.max_id), self.tgid, edit_index=-1
-        )
-        if not message:
-            return
-
-        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.peer.user_id))
-        await puppet.intent.mark_read(portal.mxid, message.mxid)
-
-    async def update_own_read_receipt(
-        self, update: UpdateReadHistoryInbox | UpdateReadChannelInbox
-    ) -> None:
-        puppet = await pu.Puppet.get_by_tgid(self.tgid)
-        if not puppet.is_real_user:
-            return
-
-        if isinstance(update, UpdateReadChannelInbox):
-            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
-        elif isinstance(update.peer, PeerChat):
-            portal = await po.Portal.get_by_tgid(TelegramID(update.peer.chat_id))
-        elif isinstance(update.peer, PeerUser):
-            portal = await po.Portal.get_by_tgid(
-                TelegramID(update.peer.user_id), tg_receiver=self.tgid
-            )
-        else:
-            self.log.debug("Unexpected own read receipt peer: %s", update.peer)
-            return
-
-        if not portal or not portal.mxid:
-            return
-
-        tg_space = portal.tgid if portal.peer_type == "channel" else self.tgid
-        message = await DBMessage.get_one_by_tgid(
-            TelegramID(update.max_id), tg_space, edit_index=-1
-        )
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
         if not message:
             return
 
+        puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
     async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
         # TODO duplication not checked
-        portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
         if not portal or not portal.mxid:
             return
 
         await portal.set_telegram_admin(TelegramID(update.user_id))
 
-    async def update_typing(
-        self, update: UpdateUserTyping | UpdateChatUserTyping | UpdateChannelUserTyping
-    ) -> None:
-        sender = None
+    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
         if isinstance(update, UpdateUserTyping):
-            portal = await po.Portal.get_by_tgid(
-                TelegramID(update.user_id), tg_receiver=self.tgid, peer_type="user"
-            )
-            sender = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
-        elif isinstance(update, UpdateChannelUserTyping):
-            portal = await po.Portal.get_by_tgid(TelegramID(update.channel_id))
-        elif isinstance(update, UpdateChatUserTyping):
-            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
         else:
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+
+        if not portal or not portal.mxid:
             return
 
-        if isinstance(update, (UpdateChannelUserTyping, UpdateChatUserTyping)):
-            # Can typing notifications come from non-user peers?
-            if not update.from_id.user_id:
-                return
-            sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id.user_id))
-
-        if not sender or not portal or not portal.mxid:
-            return
-
+        sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
-    async def _handle_entity_updates(self, entities: dict[int, User | Chat | Channel]) -> None:
+    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
+                                     ) -> None:
         try:
             users = (entity for entity in entities.values() if isinstance(entity, User))
-            puppets = ((await pu.Puppet.get_by_tgid(TelegramID(user.id)), user) for user in users)
-            await asyncio.gather(
-                *[puppet.try_update_info(self, info) async for puppet, info in puppets if puppet]
-            )
+            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
+            await asyncio.gather(*[puppet.try_update_info(self, info)
+                                   for puppet, info in puppets if puppet])
         except Exception:
             self.log.exception("Failed to handle entity updates")
 
-    async def update_others_info(self, update: UpdateUserName | UpdateUserPhoto) -> None:
+    async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
         # TODO duplication not checked
-        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
         if isinstance(update, UpdateUserName):
             puppet.username = update.username
             if await puppet.update_displayname(self, update):
-                await puppet.save()
+                puppet.save()
         elif isinstance(update, UpdateUserPhoto):
             if await puppet.update_avatar(self, update.photo):
-                await puppet.save()
+                puppet.save()
         else:
-            self.log.warning(f"Unexpected other user info update: {type(update)}")
+            self.log.warning("Unexpected other user info update: %s", update)
 
     async def update_status(self, update: UpdateUserStatus) -> None:
-        puppet = await pu.Puppet.get_by_tgid(TelegramID(update.user_id))
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
         if isinstance(update.status, UserStatusOnline):
             await puppet.default_mxid_intent.set_presence(PresenceState.ONLINE)
         elif isinstance(update.status, UserStatusOffline):
             await puppet.default_mxid_intent.set_presence(PresenceState.OFFLINE)
         else:
-            self.log.warning(f"Unexpected user status update: type({update})")
+            self.log.warning("Unexpected user status update: %s", update)
         return
 
-    async def get_message_details(
-        self, update: UpdateMessage
-    ) -> tuple[UpdateMessageContent, pu.Puppet | None, po.Portal | None]:
+    def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
+                                                                  Optional[pu.Puppet],
+                                                                  Optional[po.Portal]]:
         if isinstance(update, UpdateShortChatMessage):
-            portal = await po.Portal.get_by_tgid(TelegramID(update.chat_id))
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
             if not portal:
                 self.log.warning(f"Received message in chat with unknown type {update.chat_id}")
-            sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id))
+            sender = pu.Puppet.get(TelegramID(update.from_id))
         elif isinstance(update, UpdateShortMessage):
-            portal = await po.Portal.get_by_tgid(
-                TelegramID(update.user_id), tg_receiver=self.tgid, peer_type="user"
-            )
-            sender = await pu.Puppet.get_by_tgid(self.tgid if update.out else update.user_id)
-        elif isinstance(
-            update,
-            (
-                UpdateNewMessage,
-                UpdateNewChannelMessage,
-                UpdateEditMessage,
-                UpdateEditChannelMessage,
-            ),
-        ):
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
+            sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
+        elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
+                                 UpdateEditMessage, UpdateEditChannelMessage)):
             update = update.message
-            if isinstance(update, MessageEmpty):
-                return update, None, None
-            portal = await po.Portal.get_by_entity(update.peer_id, tg_receiver=self.tgid)
-            if update.out:
-                sender = await pu.Puppet.get_by_tgid(self.tgid)
-            elif isinstance(update.from_id, PeerUser):
-                sender = await pu.Puppet.get_by_tgid(TelegramID(update.from_id.user_id))
+            if isinstance(update.to_id, PeerUser) and not update.out:
+                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
             else:
-                sender = None
+                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
+            sender = pu.Puppet.get(update.from_id) if update.from_id else None
         else:
             self.log.warning(
-                f"Unexpected message type in User#get_message_details: {type(update)}"
-            )
+                f"Unexpected message type in User#get_message_details: {type(update)}")
             return update, None, None
         return update, sender, portal
 
     @staticmethod
     async def _try_redact(message: DBMessage) -> None:
-        portal = await po.Portal.get_by_mxid(message.mx_room)
+        portal = po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
@@ -534,33 +382,29 @@ class AbstractUser(ABC):
             pass
 
     async def delete_message(self, update: UpdateDeleteMessages) -> None:
-        if len(update.messages) > self.max_deletions:
+        if len(update.messages) > MAX_DELETIONS:
             return
 
         for message_id in update.messages:
-            for message in await DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
-                if message.redacted:
-                    continue
-                await message.delete()
-                number_left = await DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+            for message in DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid):
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
                 if number_left == 0:
                     await self._try_redact(message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
-        if len(update.messages) > self.max_deletions:
+        if len(update.messages) > MAX_DELETIONS:
             return
 
         channel_id = TelegramID(update.channel_id)
 
         for message_id in update.messages:
-            for message in await DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
-                if message.redacted:
-                    continue
-                await message.delete()
+            for message in DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id):
+                message.delete()
                 await self._try_redact(message)
 
     async def update_message(self, original_update: UpdateMessage) -> None:
-        update, sender, portal = await self.get_message_details(original_update)
+        update, sender, portal = self.get_message_details(original_update)
         if not portal:
             return
         elif portal and not portal.allow_bridging:
@@ -569,44 +413,27 @@ class AbstractUser(ABC):
 
         if self.is_relaybot:
             if update.is_private:
-                if not self.config["bridge.relaybot.private_chat.invite"]:
-                    if sender:
-                        self.log.debug(f"Ignoring private message to bot from {sender.id}")
+                if not config["bridge.relaybot.private_chat.invite"]:
+                    self.log.debug(f"Ignoring private message to bot from {sender.id}")
                     return
-            elif not portal.mxid and self.config["bridge.relaybot.ignore_unbridged_group_chat"]:
-                self.log.debug(
-                    f"Ignoring message received by bot in unbridged chat {portal.tgid_log}"
-                )
+            elif not portal.mxid and config["bridge.relaybot.ignore_unbridged_group_chat"]:
+                self.log.debug("Ignoring message received by bot"
+                               f" in unbridged chat {portal.tgid_log}")
                 return
 
-        if (
-            self.ignore_incoming_bot_events
-            and self.relaybot
-            and sender
-            and sender.id == self.relaybot.tgid
-        ):
-            self.log.debug("Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
+        if ((self.ignore_incoming_bot_events and self.relaybot
+             and sender and sender.id == self.relaybot.tgid)):
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update.id, portal.tgid_log)
             return
 
-        await portal.backfill_lock.wait(f"update {update.id}")
-
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.trace(
-                    "Received %s in %s by %d, unregistering portal...",
-                    update.action,
-                    portal.tgid_log,
-                    sender.id,
-                )
-                await self.unregister_portal(update.action.chat_id, update.action.chat_id)
-                await self.register_portal(portal)
+                self.log.debug(f"Ignoring action %s to %s by %d", update.action,
+                               portal.tgid_log,
+                               sender.id)
                 return
-            self.log.trace(
-                "Handling action %s to %s by %d",
-                update.action,
-                portal.tgid_log,
-                (sender.id if sender else 0),
-            )
+            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
+                           sender.id)
             return await portal.handle_telegram_action(self, sender, update)
 
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
@@ -614,3 +441,11 @@ class AbstractUser(ABC):
         return await portal.handle_telegram_message(self, sender, update)
 
     # endregion
+
+
+def init(context: 'Context') -> None:
+    global config, MAX_DELETIONS
+    AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
+    AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
+    AbstractUser.session_container = context.session_container
+    MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/bot.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,40 +13,30 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Tuple
+from typing import Awaitable, Callable, Dict, List, Optional, Tuple, TYPE_CHECKING
 import logging
 
-from telethon.errors import ChannelInvalidError, ChannelPrivateError
-from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
-from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
-    ChannelParticipantAdmin,
-    ChannelParticipantCreator,
-    ChatForbidden,
-    ChatParticipantAdmin,
-    ChatParticipantCreator,
-    InputChannel,
-    InputUser,
-    MessageActionChatAddUser,
-    MessageActionChatDeleteUser,
-    MessageActionChatMigrateTo,
-    MessageEntityBotCommand,
-    PeerChannel,
-    PeerChat,
-    PeerUser,
-    TypePeer,
-    UpdateNewChannelMessage,
-    UpdateNewMessage,
-    User,
-)
+    ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
+    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser, PeerUser,
+    MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
+from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
+from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
+from telethon.errors import ChannelInvalidError, ChannelPrivateError
 
 from mautrix.types import UserID
 
-from . import portal as po, puppet as pu, user as u
 from .abstract_user import AbstractUser
 from .db import BotChat
 from .types import TelegramID
+from . import puppet as pu, portal as po, user as u
+
+if TYPE_CHECKING:
+    from .config import Config
+
+config: Optional['Config'] = None
 
 ReplyFunc = Callable[[str], Awaitable[Message]]
 
@@ -69,14 +59,13 @@ class Bot(AbstractUser):
         self.puppet_whitelisted = True
         self.whitelisted = True
         self.relaybot_whitelisted = True
-        self.tg_username = None
+        self.username = None
         self.is_relaybot = True
         self.is_bot = True
         self.chats = {}
         self.tg_whitelist = []
-        self.whitelist_group_admins = (
-            self.config["bridge.relaybot.whitelist_group_admins"] or False
-        )
+        self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
+                                       or False)
         self._me_info = None
         self._me_mxid = None
 
@@ -87,7 +76,7 @@ class Bot(AbstractUser):
         return self._me_info, self._me_mxid
 
     async def init_permissions(self) -> None:
-        whitelist = self.config["bridge.relaybot.whitelist"] or []
+        whitelist = config["bridge.relaybot.whitelist"] or []
         for user_id in whitelist:
             if isinstance(user_id, str):
                 entity = await self.client.get_input_entity(user_id)
@@ -98,8 +87,8 @@ class Bot(AbstractUser):
             if isinstance(user_id, int):
                 self.tg_whitelist.append(user_id)
 
-    async def start(self, delete_unless_authenticated: bool = False) -> "Bot":
-        self.chats = {chat.id: chat.type for chat in await BotChat.all()}
+    async def start(self, delete_unless_authenticated: bool = False) -> 'Bot':
+        self.chats = {chat.id: chat.type for chat in BotChat.all()}
         await super().start(delete_unless_authenticated)
         if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
@@ -110,49 +99,47 @@ class Bot(AbstractUser):
         await self.init_permissions()
         info = await self.client.get_me()
         self.tgid = TelegramID(info.id)
-        self.tg_username = info.username
+        self.username = info.username
         self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
 
         chat_ids = [chat_id for chat_id, chat_type in self.chats.items() if chat_type == "chat"]
         response = await self.client(GetChatsRequest(chat_ids))
         for chat in response.chats:
             if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
-                await self.remove_chat(TelegramID(chat.id))
+                self.remove_chat(TelegramID(chat.id))
 
-        channel_ids = [
-            InputChannel(chat_id, 0)
-            for chat_id, chat_type in self.chats.items()
-            if chat_type == "channel"
-        ]
+        channel_ids = [InputChannel(chat_id, 0)
+                       for chat_id, chat_type in self.chats.items()
+                       if chat_type == "channel"]
         for channel_id in channel_ids:
             try:
                 await self.client(GetChannelsRequest([channel_id]))
             except (ChannelPrivateError, ChannelInvalidError):
-                await self.remove_chat(TelegramID(channel_id.channel_id))
+                self.remove_chat(TelegramID(channel_id.channel_id))
 
-    async def register_portal(self, portal: po.Portal) -> None:
-        await self.add_chat(portal.tgid, portal.peer_type)
+    def register_portal(self, portal: po.Portal) -> None:
+        self.add_chat(portal.tgid, portal.peer_type)
 
-    async def unregister_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
-        await self.remove_chat(tgid)
+    def unregister_portal(self, portal: po.Portal) -> None:
+        self.remove_chat(portal.tgid)
 
-    async def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
+    def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
         if chat_id not in self.chats:
             self.chats[chat_id] = chat_type
-            await BotChat(id=chat_id, type=chat_type).insert()
+            BotChat(id=TelegramID(chat_id), type=chat_type).insert()
 
-    async def remove_chat(self, chat_id: TelegramID) -> None:
+    def remove_chat(self, chat_id: TelegramID) -> None:
         try:
             del self.chats[chat_id]
         except KeyError:
             pass
-        await BotChat.delete_by_id(chat_id)
+        BotChat.delete_by_id(chat_id)
 
     async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
-        user = await u.User.get_by_tgid(tgid)
+        user = u.User.get_by_tgid(tgid)
         if user and user.is_admin:
             self.tg_whitelist.append(user.tgid)
             return True
@@ -160,9 +147,7 @@ class Bot(AbstractUser):
         if self.whitelist_group_admins:
             if isinstance(chat, PeerChannel):
                 p = await self.client(GetParticipantRequest(chat, tgid))
-                return isinstance(
-                    p.participant, (ChannelParticipantCreator, ChannelParticipantAdmin)
-                )
+                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
             elif isinstance(chat, PeerChat):
                 chat = await self.client(GetFullChatRequest(chat.chat_id))
                 participants = chat.full_chat.participants.participants
@@ -172,14 +157,13 @@ class Bot(AbstractUser):
         return False
 
     async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
-        # FIXME event.from_id is not int
         if not await self._can_use_commands(event.to_id, TelegramID(event.from_id)):
             await reply("You do not have the permission to use that command.")
             return False
         return True
 
     async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> Message:
-        if not self.config["bridge.relaybot.authless_portals"]:
+        if not config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
         if not portal.allow_bridging:
@@ -189,31 +173,29 @@ class Bot(AbstractUser):
         if portal.mxid:
             if portal.username:
                 return await reply(
-                    f"Portal is public: [{portal.alias}](https://matrix.to/#/{portal.alias})"
-                )
+                    f"Portal is public: [{portal.alias}](https://matrix.to/#/{portal.alias})")
             else:
-                return await reply("Portal is not public. Use `/invite <mxid>` to get an invite.")
+                return await reply(
+                    "Portal is not public. Use `/invite <mxid>` to get an invite.")
 
-    async def handle_command_invite(
-        self, portal: po.Portal, reply: ReplyFunc, mxid_input: UserID
-    ) -> Message:
+    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc,
+                                    mxid_input: UserID) -> Message:
         if len(mxid_input) == 0:
             return await reply("Usage: `/invite <mxid>`")
         elif not portal.mxid:
-            return await reply("Portal does not have Matrix room. Create one with /portal first.")
-        if mxid_input[0] != "@" or mxid_input.find(":") < 2:
+            return await reply("Portal does not have Matrix room. "
+                               "Create one with /portal first.")
+        if mxid_input[0] != '@' or mxid_input.find(':') < 2:
             return await reply("That doesn't look like a Matrix ID.")
-        user = await u.User.get_and_start_by_mxid(mxid_input)
+        user = await u.User.get_by_mxid(mxid_input).ensure_started()
         if not user.relaybot_whitelisted:
             return await reply("That user is not whitelisted to use the bridge.")
         elif await user.is_logged_in():
-            displayname = f"@{user.tg_username}" if user.tg_username else user.displayname
-            return await reply(
-                "That user seems to be logged in. "
-                f"Just invite [{displayname}](tg://user?id={user.tgid})"
-            )
+            displayname = f"@{user.username}" if user.username else user.displayname
+            return await reply("That user seems to be logged in. "
+                               f"Just invite [{displayname}](tg://user?id={user.tgid})")
         else:
-            await portal.invite_to_matrix(user.mxid)
+            await portal.main_intent.invite_user(portal.mxid, user.mxid)
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
     @staticmethod
@@ -225,14 +207,14 @@ class Bot(AbstractUser):
         elif isinstance(message.to_id, PeerChat):
             return reply(str(-message.to_id.chat_id))
         elif isinstance(message.to_id, PeerUser):
-            return reply(f"Your user ID is {message.to_id.user_id}.")
+            return reply(f"Your user ID is {message.from_id}.")
         else:
             return reply("Failed to find chat ID.")
 
     def match_command(self, text: str, command: str) -> bool:
         text = text.lower()
         command = f"/{command.lower()}"
-        command_targeted = f"{command}@{self.tg_username.lower()}"
+        command_targeted = f"{command}@{self.username.lower()}"
 
         is_plain_command = text == command or text == command_targeted
         if is_plain_command:
@@ -244,16 +226,17 @@ class Bot(AbstractUser):
 
         return False
 
-    async def handle_command(self, message: Message) -> None:
+    async def handle_command(self, message: Message) -> Optional[bool]:
         def reply(reply_text: str) -> Awaitable[Message]:
             return self.client.send_message(message.chat_id, reply_text, reply_to=message.id)
 
         text = message.message
 
         if self.match_command(text, "start"):
-            pcm = self.config["bridge.relaybot.private_chat.message"]
-            if pcm:
-                await reply(pcm)
+            pcm = config["bridge.relaybot.private_chat.message"]
+            if not pcm:
+                return True
+            await reply(pcm)
             return
         elif self.match_command(text, "id"):
             await self.handle_command_id(message, reply)
@@ -261,23 +244,22 @@ class Bot(AbstractUser):
         elif message.is_private:
             return
 
-        portal = await po.Portal.get_by_entity(message.to_id)
+        portal = po.Portal.get_by_entity(message.to_id)
 
-        is_portal_cmd = self.match_command(text, "portal")
-        is_invite_cmd = self.match_command(text, "invite")
-        if is_portal_cmd or is_invite_cmd:
+        if self.match_command(text, "portal"):
             if not await self.check_can_use_commands(message, reply):
                 return
-            if is_portal_cmd:
-                await self.handle_command_portal(portal, reply)
-            elif is_invite_cmd:
-                try:
-                    mxid = text[text.index(" ") + 1 :]
-                except ValueError:
-                    mxid = ""
-                await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
+            await self.handle_command_portal(portal, reply)
+        elif self.match_command(text, "invite"):
+            if not await self.check_can_use_commands(message, reply):
+                return
+            try:
+                mxid = text[text.index(" ") + 1:]
+            except ValueError:
+                mxid = ""
+            await self.handle_command_invite(portal, reply, mxid_input=UserID(mxid))
 
-    async def handle_service_message(self, message: MessageService) -> None:
+    def handle_service_message(self, message: MessageService) -> None:
         to_peer = message.to_id
         if isinstance(to_peer, PeerChannel):
             to_id = TelegramID(to_peer.channel_id)
@@ -290,29 +272,25 @@ class Bot(AbstractUser):
 
         action = message.action
         if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
-            await self.add_chat(to_id, chat_type)
+            self.add_chat(to_id, chat_type)
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
-            await self.remove_chat(to_id)
+            self.remove_chat(to_id)
         elif isinstance(action, MessageActionChatMigrateTo):
-            await self.remove_chat(to_id)
-            await self.add_chat(TelegramID(action.channel_id), "channel")
+            self.remove_chat(to_id)
+            self.add_chat(TelegramID(action.channel_id), "channel")
 
     async def update(self, update) -> bool:
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
             return False
         if isinstance(update.message, MessageService):
-            await self.handle_service_message(update.message)
+            self.handle_service_message(update.message)
             return False
 
-        is_command = (
-            isinstance(update.message, Message)
-            and update.message.entities
-            and len(update.message.entities) > 0
-            and isinstance(update.message.entities[0], MessageEntityBotCommand)
-            and update.message.entities[0].offset == 0
-        )
+        is_command = (isinstance(update.message, Message)
+                      and update.message.entities and len(update.message.entities) > 0
+                      and isinstance(update.message.entities[0], MessageEntityBotCommand))
         if is_command:
-            await self.handle_command(update.message)
+            return not await self.handle_command(update.message)
         return False
 
     def is_in_chat(self, peer_id) -> bool:
@@ -321,3 +299,12 @@ class Bot(AbstractUser):
     @property
     def name(self) -> str:
         return "bot"
+
+
+def init(cfg: 'Config') -> Optional[Bot]:
+    global config
+    config = cfg
+    token = config["telegram.bot_token"]
+    if token and not token.lower().startswith("disable"):
+        return Bot(token)
+    return None

mautrix_telegram/commands/__init__.py

@@ -1,26 +1,8 @@
-from .handler import (
-    SECTION_ADMIN,
-    SECTION_AUTH,
-    SECTION_CREATING_PORTALS,
-    SECTION_MISC,
-    SECTION_PORTAL_MANAGEMENT,
-    CommandEvent,
-    CommandHandler,
-    CommandProcessor,
-    command_handler,
-)
+from .handler import (command_handler, CommandHandler, CommandProcessor, CommandEvent,
+                      SECTION_AUTH, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT,
+                      SECTION_MISC, SECTION_ADMIN)
+from . import portal, telegram, clean_rooms, matrix_auth, manhole
 
-# This has to happen after the handler imports
-from . import matrix_auth, portal, telegram  # isort: skip
-
-__all__ = [
-    "command_handler",
-    "CommandHandler",
-    "CommandProcessor",
-    "CommandEvent",
-    "SECTION_AUTH",
-    "SECTION_MISC",
-    "SECTION_ADMIN",
-    "SECTION_CREATING_PORTALS",
-    "SECTION_PORTAL_MANAGEMENT",
-]
+__all__ = ["command_handler", "CommandHandler", "CommandProcessor", "CommandEvent",
+           "SECTION_AUTH", "SECTION_MISC", "SECTION_ADMIN", "SECTION_CREATING_PORTALS",
+           "SECTION_PORTAL_MANAGEMENT"]

mautrix_telegram/commands/clean_rooms.py

@@ -0,0 +1,177 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, NamedTuple, Tuple, Union
+
+from mautrix.appservice import IntentAPI
+from mautrix.errors import MatrixRequestError
+from mautrix.types import RoomID, UserID, EventID
+
+from . import command_handler, CommandEvent, SECTION_ADMIN
+from .. import puppet as pu, portal as po
+
+ManagementRoom = NamedTuple('ManagementRoom', room_id=RoomID, user_id=UserID)
+
+
+async def _find_rooms(intent: IntentAPI) -> Tuple[List[ManagementRoom], List[RoomID],
+                                                  List['po.Portal'], List['po.Portal']]:
+    management_rooms: List[ManagementRoom] = []
+    unidentified_rooms: List[RoomID] = []
+    portals: List[po.Portal] = []
+    empty_portals: List[po.Portal] = []
+
+    rooms = await intent.get_joined_rooms()
+    for room_id in rooms:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            try:
+                members = await intent.get_room_members(room_id)
+            except MatrixRequestError:
+                members = []
+            if len(members) == 2:
+                other_member = members[0] if members[0] != intent.mxid else members[1]
+                if pu.Puppet.get_id_from_mxid(other_member):
+                    unidentified_rooms.append(room_id)
+                else:
+                    management_rooms.append(ManagementRoom(room_id, other_member))
+            else:
+                unidentified_rooms.append(room_id)
+        else:
+            members = await portal.get_authenticated_matrix_users()
+            if len(members) == 0:
+                empty_portals.append(portal)
+            else:
+                portals.append(portal)
+
+    return management_rooms, unidentified_rooms, portals, empty_portals
+
+
+@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
+                 help_section=SECTION_ADMIN,
+                 help_text="Clean up unused portal/management rooms.")
+async def clean_rooms(evt: CommandEvent) -> EventID:
+    management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
+
+    reply = ["#### Management rooms (M)"]
+    reply += ([f"{n+1}. [M{n+1}](https://matrix.to/#/{room}) (with {other_member}"
+               for n, (room, other_member) in enumerate(management_rooms)]
+              or ["No management rooms found."])
+    reply.append("#### Active portal rooms (A)")
+    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
+               f"(to Telegram chat \"{portal.title}\")"
+               for n, portal in enumerate(portals)]
+              or ["No active portal rooms found."])
+    reply.append("#### Unidentified rooms (U)")
+    reply += ([f"{n+1}. [U{n+1}](https://matrix.to/#/{room})"
+               for n, room in enumerate(unidentified_rooms)]
+              or ["No unidentified rooms found."])
+    reply.append("#### Inactive portal rooms (I)")
+    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
+               f"(to Telegram chat \"{portal.title}\")"
+               for n, portal in enumerate(empty_portals)]
+              or ["No inactive portal rooms found."])
+
+    reply += ["#### Usage",
+              ("To clean the recommended set of rooms (unidentified & inactive portals), "
+               "type `$cmdprefix+sp clean-recommended`"),
+              "",
+              ("To clean other groups of rooms, type `$cmdprefix+sp clean-groups <letters>` "
+               "where `letters` are the first letters of the group names (M, A, U, I)"),
+              "",
+              ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
+               "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
+               "the group name. (e.g. `I2-6`)"),
+              "",
+              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
+               "between each use of the commands above.")]
+
+    evt.sender.command_status = {
+        "next": lambda clean_evt: set_rooms_to_clean(clean_evt, management_rooms,
+                                                     unidentified_rooms, portals, empty_portals),
+        "action": "Room cleaning",
+    }
+
+    return await evt.reply("\n".join(reply))
+
+
+async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
+                             unidentified_rooms: List[RoomID], portals: List["po.Portal"],
+                             empty_portals: List["po.Portal"]) -> None:
+    command = evt.args[0]
+    rooms_to_clean: List[Union[po.Portal, RoomID]] = []
+    if command == "clean-recommended":
+        rooms_to_clean += empty_portals
+        rooms_to_clean += unidentified_rooms
+    elif command == "clean-groups":
+        if len(evt.args) < 2:
+            return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
+        groups_to_clean = evt.args[1].upper()
+        if "M" in groups_to_clean:
+            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
+        if "A" in groups_to_clean:
+            rooms_to_clean += portals
+        if "U" in groups_to_clean:
+            rooms_to_clean += unidentified_rooms
+        if "I" in groups_to_clean:
+            rooms_to_clean += empty_portals
+    elif command == "clean-range":
+        try:
+            clean_range = evt.args[1]
+            group, clean_range = clean_range[0], clean_range[1:]
+            start, end = clean_range.split("-")
+            start, end = int(start), int(end)
+            if group == "M":
+                group = [room_id for (room_id, user_id) in management_rooms]
+            elif group == "A":
+                group = portals
+            elif group == "U":
+                group = unidentified_rooms
+            elif group == "I":
+                group = empty_portals
+            else:
+                raise ValueError("Unknown group")
+            rooms_to_clean = group[start - 1:end]
+        except (KeyError, ValueError):
+            return await evt.reply(
+                "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
+    else:
+        return await evt.reply(f"Unknown room cleaning action `{command}`. "
+                               "Use `$cmdprefix+sp cancel` to cancel room "
+                               "cleaning.")
+
+    evt.sender.command_status = {
+        "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
+        "action": "Room cleaning",
+    }
+    await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type "
+                    "`$cmdprefix+sp confirm-clean`.")
+
+
+async def execute_room_cleanup(evt, rooms_to_clean: List[Union[po.Portal, RoomID]]) -> None:
+    if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
+        await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
+                        "This might take a while.")
+        cleaned = 0
+        for room in rooms_to_clean:
+            if isinstance(room, po.Portal):
+                await room.cleanup_and_delete()
+                cleaned += 1
+            else:
+                await po.Portal.cleanup_room(evt.az.intent, room, "Room deleted")
+                cleaned += 1
+        evt.sender.command_status = None
+        await evt.reply(f"{cleaned} rooms cleaned up successfully.")
+    else:
+        await evt.reply("Room cleaning cancelled.")

mautrix_telegram/commands/handler.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,37 +13,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, NamedTuple
+"""This module contains classes handling commands issued by Matrix users."""
+from typing import Awaitable, Callable, List, Optional, NamedTuple, Any
 
 from telethon.errors import FloodWaitError
 
-from mautrix.bridge.commands import (
-    CommandEvent as BaseCommandEvent,
-    CommandHandler as BaseCommandHandler,
-    CommandHandlerFunc,
-    CommandProcessor as BaseCommandProcessor,
-    HelpSection,
-    command_handler as base_command_handler,
-)
-from mautrix.types import EventID, MessageEventContent, RoomID
-from mautrix.util.format_duration import format_duration
+from mautrix.types import RoomID, EventID, MessageEventContent
+from mautrix.bridge.commands import (HelpSection, CommandEvent as BaseCommandEvent,
+                                     CommandHandler as BaseCommandHandler,
+                                     CommandProcessor as BaseCommandProcessor,
+                                     CommandHandlerFunc, command_handler as base_command_handler)
 
-from .. import portal as po, user as u
-
-if TYPE_CHECKING:
-    from ..__main__ import TelegramBridge
-
-
-class HelpCacheKey(NamedTuple):
-    is_management: bool
-    is_portal: bool
-    puppet_whitelisted: bool
-    matrix_puppet_whitelisted: bool
-    is_admin: bool
-    is_logged_in: bool
+from ..util import format_duration
+from .. import user as u, context as c
 
+HelpCacheKey = NamedTuple('HelpCacheKey',
+                          is_management=bool, is_portal=bool, puppet_whitelisted=bool,
+                          matrix_puppet_whitelisted=bool, is_admin=bool, is_logged_in=bool)
 
 SECTION_AUTH = HelpSection("Authentication", 10, "")
 SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
@@ -54,33 +40,12 @@ SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 class CommandEvent(BaseCommandEvent):
     sender: u.User
-    portal: po.Portal
 
-    def __init__(
-        self,
-        processor: CommandProcessor,
-        room_id: RoomID,
-        event_id: EventID,
-        sender: u.User,
-        command: str,
-        args: list[str],
-        content: MessageEventContent,
-        portal: po.Portal | None,
-        is_management: bool,
-        has_bridge_bot: bool,
-    ) -> None:
-        super().__init__(
-            processor,
-            room_id,
-            event_id,
-            sender,
-            command,
-            args,
-            content,
-            portal,
-            is_management,
-            has_bridge_bot,
-        )
+    def __init__(self, processor: 'CommandProcessor', room_id: RoomID, event_id: EventID,
+                 sender: u.User, command: str, args: List[str], content: MessageEventContent,
+                 is_management: bool, is_portal: bool) -> None:
+        super().__init__(processor, room_id, event_id, sender, command, args, content,
+                         is_management, is_portal)
         self.bridge = processor.bridge
         self.tgbot = processor.tgbot
         self.config = processor.config
@@ -91,103 +56,76 @@ class CommandEvent(BaseCommandEvent):
         return self.sender.is_admin
 
     async def get_help_key(self) -> HelpCacheKey:
-        return HelpCacheKey(
-            self.is_management,
-            self.portal is not None,
-            self.sender.puppet_whitelisted,
-            self.sender.matrix_puppet_whitelisted,
-            self.sender.is_admin,
-            await self.sender.is_logged_in(),
-        )
+        return HelpCacheKey(self.is_management, self.is_portal, self.sender.puppet_whitelisted,
+                            self.sender.matrix_puppet_whitelisted, self.sender.is_admin,
+                            await self.sender.is_logged_in())
 
 
 class CommandHandler(BaseCommandHandler):
     name: str
 
+    management_only: bool
+    needs_auth: bool
     needs_puppeting: bool
     needs_matrix_puppeting: bool
+    needs_admin: bool
 
-    def __init__(
-        self,
-        handler: Callable[[CommandEvent], Awaitable[EventID]],
-        management_only: bool,
-        name: str,
-        help_text: str,
-        help_args: str,
-        help_section: HelpSection,
-        needs_auth: bool,
-        needs_puppeting: bool,
-        needs_matrix_puppeting: bool,
-        needs_admin: bool,
-        **kwargs,
-    ) -> None:
-        super().__init__(
-            handler,
-            management_only,
-            name,
-            help_text,
-            help_args,
-            help_section,
-            needs_auth=needs_auth,
-            needs_puppeting=needs_puppeting,
-            needs_matrix_puppeting=needs_matrix_puppeting,
-            needs_admin=needs_admin,
-            **kwargs,
-        )
+    def __init__(self, handler: Callable[[CommandEvent], Awaitable[EventID]],
+                 management_only: bool, name: str, help_text: str, help_args: str,
+                 help_section: HelpSection, needs_auth: bool, needs_puppeting: bool,
+                 needs_matrix_puppeting: bool, needs_admin: bool) -> None:
+        super().__init__(handler, management_only, name, help_text, help_args, help_section,
+                         needs_auth=needs_auth, needs_puppeting=needs_puppeting,
+                         needs_matrix_puppeting=needs_matrix_puppeting, needs_admin=needs_admin)
 
-    async def get_permission_error(self, evt: CommandEvent) -> str | None:
-        if self.needs_puppeting and not evt.sender.puppet_whitelisted:
+    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        if self.management_only and not evt.is_management:
+            return (f"`{evt.command}` is a restricted command: "
+                    "you may only run it in management rooms.")
+        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
             return "This command requires puppeting privileges."
         elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
             return "This command requires Matrix puppeting privileges."
-        return await super().get_permission_error(evt)
+        elif self.needs_admin and not evt.sender.is_admin:
+            return "This command requires administrator privileges."
+        elif self.needs_auth and not await evt.sender.is_logged_in():
+            return "This command requires you to be logged in."
+        return None
 
     def has_permission(self, key: HelpCacheKey) -> bool:
-        return (
-            super().has_permission(key)
-            and (not self.needs_puppeting or key.puppet_whitelisted)
-            and (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted)
-        )
+        return ((not self.management_only or key.is_management) and
+                (not self.needs_puppeting or key.puppet_whitelisted) and
+                (not self.needs_matrix_puppeting or key.matrix_puppet_whitelisted) and
+                (not self.needs_admin or key.is_admin) and
+                (not self.needs_auth or key.is_logged_in))
 
 
-def command_handler(
-    _func: CommandHandlerFunc | None = None,
-    *,
-    needs_auth: bool = True,
-    needs_puppeting: bool = True,
-    needs_matrix_puppeting: bool = False,
-    needs_admin: bool = False,
-    management_only: bool = False,
-    name: str | None = None,
-    help_text: str = "",
-    help_args: str = "",
-    help_section: HelpSection = None,
-) -> Callable[[CommandHandlerFunc], CommandHandler]:
+def command_handler(_func: Optional[CommandHandlerFunc] = None, *, needs_auth: bool = True,
+                    needs_puppeting: bool = True, needs_matrix_puppeting: bool = False,
+                    needs_admin: bool = False, management_only: bool = False,
+                    name: Optional[str] = None, help_text: str = "", help_args: str = "",
+                    help_section: HelpSection = None) -> Callable[[CommandHandlerFunc],
+                                                                  CommandHandler]:
     return base_command_handler(
-        _func,
-        _handler_class=CommandHandler,
-        name=name,
-        help_text=help_text,
-        help_args=help_args,
-        help_section=help_section,
-        management_only=management_only,
-        needs_auth=needs_auth,
-        needs_admin=needs_admin,
-        needs_puppeting=needs_puppeting,
-        needs_matrix_puppeting=needs_matrix_puppeting,
-    )
+        _func, _handler_class=CommandHandler, name=name, help_text=help_text, help_args=help_args,
+        help_section=help_section, management_only=management_only, needs_auth=needs_auth,
+        needs_admin=needs_admin, needs_puppeting=needs_puppeting,
+        needs_matrix_puppeting=needs_matrix_puppeting)
 
 
 class CommandProcessor(BaseCommandProcessor):
-    def __init__(self, bridge: "TelegramBridge") -> None:
-        super().__init__(event_class=CommandEvent, bridge=bridge)
-        self.tgbot = bridge.bot
-        self.public_website = bridge.public_website
+    def __init__(self, context: c.Context) -> None:
+        super().__init__(az=context.az, config=context.config, event_class=CommandEvent,
+                         loop=context.loop, bridge=context.bridge)
+        self.tgbot = context.bot
+        self.bridge = context.bridge
+        self.az, self.config, self.loop, self.tgbot = context.core
+        self.public_website = context.public_website
+        self.command_prefix = self.config["bridge.command_prefix"]
 
     @staticmethod
-    async def _run_handler(
-        handler: Callable[[CommandEvent], Awaitable[Any]], evt: CommandEvent
-    ) -> Any:
+    async def _run_handler(handler: Callable[[CommandEvent], Awaitable[Any]], evt: CommandEvent
+                           ) -> Any:
         try:
             return await handler(evt)
         except FloodWaitError as e:

mautrix_telegram/commands/manhole.py

@@ -0,0 +1,128 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Set, Callable
+import asyncio
+import sys
+import os
+
+from attr import dataclass
+
+from telethon import __version__ as __telethon_version__
+
+from mautrix import __version__ as __mautrix_version__
+from mautrix.types import UserID
+from mautrix.errors import MatrixConnectionError
+from mautrix.util.manhole import start_manhole
+
+from .. import __version__
+from . import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@dataclass
+class ManholeState:
+    server: asyncio.AbstractServer
+    opened_by: UserID
+    close: Callable[[], None]
+    whitelist: Set[int]
+
+
+@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
+                 help_text="Open a manhole into the bridge.", help_args="<_uid..._>")
+async def open_manhole(evt: CommandEvent) -> None:
+    if not evt.config["manhole.enabled"]:
+        await evt.reply("The manhole has been disabled in the config.")
+        return
+    elif len(evt.args) == 0:
+        await evt.reply("**Usage:** `$cmdprefix+sp open-manhole <uid...>`")
+        return
+
+    whitelist = set()
+    whitelist_whitelist = evt.config["manhole.whitelist"]
+    for arg in evt.args:
+        try:
+            uid = int(arg)
+        except ValueError:
+            await evt.reply(f"{arg} is not an integer.")
+            return
+        if whitelist_whitelist and uid not in whitelist_whitelist:
+            await evt.reply(f"{uid} is not in the list of allowed UIDs.")
+            return
+        whitelist.add(uid)
+
+    if evt.bridge.manhole:
+        added = [uid for uid in whitelist
+                 if uid not in evt.bridge.manhole.whitelist]
+        evt.bridge.manhole.whitelist |= set(added)
+        if len(added) == 0:
+            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
+                            " and all the given UIDs are already whitelisted.")
+        else:
+            added_str = (f"{', '.join(str(uid) for uid in added[:-1])} and {added[-1]}"
+                         if len(added) > 1 else added[0])
+            await evt.reply(f"There's an existing manhole opened by {evt.bridge.manhole.opened_by}"
+                            f". Added {added_str} to the whitelist.")
+            evt.log.info(f"{evt.sender.mxid} added {added_str} to the manhole whitelist.")
+        return
+
+    from ..portal import Portal
+    from ..puppet import Puppet
+    from ..user import User
+    namespace = {
+        "bridge": evt.bridge,
+        "User": User,
+        "Portal": Portal,
+        "Puppet": Puppet,
+    }
+    banner = (f"Python {sys.version} on {sys.platform}\n"
+              f"mautrix-telegram {__version__} with mautrix-python {__mautrix_version__} "
+              f"and Telethon {__telethon_version__}\n\nManhole opened by {evt.sender.mxid}\n")
+    path = evt.config["manhole.path"]
+
+    wl_list = list(whitelist)
+    whitelist_str = (f"{', '.join(str(uid) for uid in wl_list[:-1])} and {wl_list[-1]}"
+                     if len(wl_list) > 1 else wl_list[0])
+    evt.log.info(f"{evt.sender.mxid} opened a manhole with {whitelist_str} whitelisted.")
+    server, close = await start_manhole(path=path, banner=banner, namespace=namespace,
+                                        loop=evt.loop, whitelist=whitelist)
+    evt.bridge.manhole = ManholeState(server=server, opened_by=evt.sender.mxid, close=close,
+                                      whitelist=whitelist)
+    plrl = "s" if len(whitelist) != 1 else ""
+    await evt.reply(f"Opened manhole at unix://{path} with UID{plrl} {whitelist_str} whitelisted")
+    await server.wait_closed()
+    evt.bridge.manhole = None
+    try:
+        os.unlink(path)
+    except FileNotFoundError:
+        pass
+    evt.log.info(f"{evt.sender.mxid}'s manhole was closed.")
+    try:
+        await evt.reply("Your manhole was closed.")
+    except (AttributeError, MatrixConnectionError) as e:
+        evt.log.warning(f"Failed to send manhole close notification: {e}")
+
+
+@command_handler(needs_auth=False, needs_admin=True, help_section=SECTION_ADMIN,
+                 help_text="Close an open manhole.")
+async def close_manhole(evt: CommandEvent) -> None:
+    if not evt.bridge.manhole:
+        await evt.reply("There is no open manhole.")
+        return
+
+    opened_by = evt.bridge.manhole.opened_by
+    evt.bridge.manhole.close()
+    evt.bridge.manhole = None
+    if opened_by != evt.sender.mxid:
+        await evt.reply(f"Closed manhole opened by {opened_by}")

mautrix_telegram/commands/matrix_auth.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,27 +13,33 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.bridge import InvalidAccessToken, OnlyLoginSelf
 from mautrix.types import EventID
+from mautrix.bridge import InvalidAccessToken, OnlyLoginSelf
 
+from . import command_handler, CommandEvent, SECTION_AUTH
 from .. import puppet as pu
-from . import SECTION_AUTH, CommandEvent, command_handler
 
 
-@command_handler(
-    needs_auth=True,
-    management_only=True,
-    needs_matrix_puppeting=True,
-    help_section=SECTION_AUTH,
-    help_text="Replace your Telegram account's Matrix puppet with your own Matrix account.",
-)
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH, help_text="Revert your Telegram account's Matrix "
+                                                      "puppet to use the default Matrix account.")
+async def logout_matrix(evt: CommandEvent) -> EventID:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
+
+
+@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account.")
 async def login_matrix(evt: CommandEvent) -> EventID:
-    puppet = await pu.Puppet.get_by_tgid(evt.sender.tgid)
+    puppet = pu.Puppet.get(evt.sender.tgid)
     if puppet.is_real_user:
-        return await evt.reply(
-            "You have already logged in with your Matrix account. "
-            "Log out with `$cmdprefix+sp logout-matrix` first."
-        )
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
     allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
     if allow_matrix_login:
         evt.sender.command_status = {
@@ -51,35 +57,57 @@ async def login_matrix(evt: CommandEvent) -> EventID:
                 "here.\n"
                 f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
                 "Logging in outside of Matrix is recommended, because in-Matrix login would save "
-                "your access token in the message history."
-            )
-        return await evt.reply(
-            "This bridge instance does not allow logging in inside Matrix.\n\n"
-            f"Please visit [the login page]({url}) to log in."
-        )
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
     elif allow_matrix_login:
         return await evt.reply(
             "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your Matrix access token here to log in."
-        )
+            "Please send your Matrix access token here to log in.")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Pings the server with the stored matrix authentication.")
+async def ping_matrix(evt: CommandEvent) -> EventID:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    try:
+        await puppet.start()
+    except InvalidAccessToken:
+        return await evt.reply("Your access token is invalid.")
+    return await evt.reply("Your Matrix login is working.")
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True, help_section=SECTION_AUTH,
+                 help_text="Clear the Matrix sync token stored for your custom puppet.")
+async def clear_cache_matrix(evt: CommandEvent) -> EventID:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    try:
+        puppet.stop()
+        puppet.next_batch = None
+        await puppet.start()
+    except InvalidAccessToken:
+        return await evt.reply("Your access token is invalid.")
+    return await evt.reply("Cleared cache successfully.")
+
+
 async def enter_matrix_token(evt: CommandEvent) -> EventID:
     evt.sender.command_status = None
 
-    puppet = await pu.Puppet.get_by_tgid(evt.sender.tgid)
+    puppet = pu.Puppet.get(evt.sender.tgid)
     if puppet.is_real_user:
-        return await evt.reply(
-            "You have already logged in with your Matrix account. "
-            "Log out with `$cmdprefix+sp logout-matrix` first."
-        )
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
     try:
         await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
     except OnlyLoginSelf:
         return await evt.reply("You can only log in as your own Matrix user.")
     except InvalidAccessToken:
         return await evt.reply("Failed to verify access token.")
-    return await evt.reply(
-        "Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}."
-    )
+    return await evt.reply("Replaced your Telegram account's Matrix puppet "
+                           f"with {puppet.custom_mxid}.")

mautrix_telegram/commands/portal/admin.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,19 +15,38 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import asyncio
 
+from mautrix.errors import MatrixRequestError
 from mautrix.types import EventID
 
 from ... import portal as po, puppet as pu, user as u
-from .. import SECTION_ADMIN, CommandEvent, command_handler
+from .. import command_handler, CommandEvent, SECTION_ADMIN
 
 
-@command_handler(
-    needs_admin=True,
-    needs_auth=False,
-    help_section=SECTION_ADMIN,
-    help_args="<`portal`|`puppet`|`user`>",
-    help_text="Clear internal bridge caches",
-)
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
+                 help_section=SECTION_ADMIN,
+                 help_args="<_level_> [_mxid_]",
+                 help_text="Set a temporary power level without affecting Telegram.")
+async def set_power_level(evt: CommandEvent) -> EventID:
+    try:
+        level = int(evt.args[0])
+    except (KeyError, IndexError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp set-pl <level> [mxid]`")
+    except ValueError:
+        return await evt.reply("The level must be an integer.")
+    levels = await evt.az.intent.get_power_levels(evt.room_id)
+    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
+    levels.users[mxid] = level
+    try:
+        return await evt.az.intent.set_power_levels(evt.room_id, levels)
+    except MatrixRequestError:
+        evt.log.exception("Failed to set power level.")
+        return await evt.reply("Failed to set power level.")
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`portal`|`puppet`|`user`>",
+                 help_text="Clear internal bridge caches")
 async def clear_db_cache(evt: CommandEvent) -> EventID:
     try:
         section = evt.args[0].lower()
@@ -38,43 +57,41 @@ async def clear_db_cache(evt: CommandEvent) -> EventID:
         po.Portal.by_mxid = {}
         await evt.reply("Cleared portal cache")
     elif section == "puppet":
-        pu.Puppet.by_tgid = {}
+        pu.Puppet.cache = {}
         for puppet in pu.Puppet.by_custom_mxid.values():
-            puppet.stop()
+            puppet.sync_task.cancel()
         pu.Puppet.by_custom_mxid = {}
-        await asyncio.gather(
-            *[puppet.try_start() async for puppet in pu.Puppet.all_with_custom_mxid()]
-        )
+        await asyncio.gather(*[puppet.try_start() for puppet in pu.Puppet.all_with_custom_mxid()],
+                             loop=evt.loop)
         await evt.reply("Cleared puppet cache and restarted custom puppet syncers")
     elif section == "user":
-        u.User.by_mxid = {user.mxid: user for user in u.User.by_tgid.values()}
+        u.User.by_mxid = {
+            user.mxid: user
+            for user in u.User.by_tgid.values()
+        }
         await evt.reply("Cleared non-logged-in user cache")
     else:
         return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
 
 
-@command_handler(
-    needs_admin=True,
-    needs_auth=False,
-    help_section=SECTION_ADMIN,
-    help_args="[_mxid_]",
-    help_text="Reload and reconnect a user",
-)
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="[_mxid_]",
+                 help_text="Reload and reconnect a user")
 async def reload_user(evt: CommandEvent) -> EventID:
     if len(evt.args) > 0:
         mxid = evt.args[0]
     else:
         mxid = evt.sender.mxid
-    user = await u.User.get_by_mxid(mxid, create=False)
+    user = u.User.get_by_mxid(mxid, create=False)
     if not user:
         return await evt.reply("User not found")
-    puppet = await pu.Puppet.get_by_custom_mxid(mxid)
+    puppet = pu.Puppet.get_by_custom_mxid(mxid)
     if puppet:
-        puppet.stop()
+        puppet.sync_task.cancel()
     await user.stop()
-    del u.User.by_tgid[user.tgid]
-    del u.User.by_mxid[user.mxid]
-    user = await u.User.get_by_mxid(mxid)
+    user.delete(delete_db=False)
+    user = u.User.get_by_mxid(mxid)
     await user.ensure_started()
     if puppet:
         await puppet.start()

mautrix_telegram/commands/portal/bridge.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,36 +13,29 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import Awaitable
+from typing import Optional, Tuple, Coroutine
 import asyncio
 
-from telethon.tl.types import ChannelForbidden, ChatForbidden
+from telethon.tl.types import ChatForbidden, ChannelForbidden
 
 from mautrix.types import EventID, RoomID
 
-from ... import portal as po
 from ...types import TelegramID
-from .. import SECTION_CREATING_PORTALS, CommandEvent, command_handler
-from .util import get_initial_state, user_has_power_level, warn_missing_power
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
 
 
-@command_handler(
-    needs_auth=False,
-    needs_puppeting=False,
-    help_section=SECTION_CREATING_PORTALS,
-    help_args="[_id_]",
-    help_text=(
-        "Bridge the current Matrix room to the Telegram chat with the given ID. The ID must be "
-        "the prefixed version that you get with the `/id` command of the Telegram-side bot."
-    ),
-)
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_id_]",
+                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
+                           "ID. The ID must be the prefixed version that you get with the `/id` "
+                           "command of the Telegram-side bot.")
 async def bridge(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`"
-        )
+        return await evt.reply("**Usage:** "
+                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
     force_use_bot = False
     if evt.args[0] == "--usebot" and evt.sender.is_admin:
         force_use_bot = True
@@ -50,7 +43,7 @@ async def bridge(evt: CommandEvent) -> EventID:
     room_id = RoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
     that_this = "This" if room_id == evt.room_id else "That"
 
-    portal = await po.Portal.get_by_mxid(room_id)
+    portal = po.Portal.get_by_mxid(room_id)
     if portal:
         return await evt.reply(f"{that_this} room is already a portal room.")
 
@@ -66,30 +59,24 @@ async def bridge(evt: CommandEvent) -> EventID:
         tgid = TelegramID(-int(tgid_str))
         peer_type = "chat"
     else:
-        return await evt.reply(
-            "That doesn't seem like a prefixed Telegram chat ID.\n\n"
-            "If you did not get the ID using the `/id` bot command, please "
-            "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
-            "Bridging private chats to existing rooms is not allowed."
-        )
+        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
+                               "If you did not get the ID using the `/id` bot command, please "
+                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+                               "Bridging private chats to existing rooms is not allowed.")
 
-    portal = await po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
     if not portal.allow_bridging:
-        return await evt.reply(
-            "This bridge doesn't allow bridging that Telegram chat.\n"
-            "If you're the bridge admin, try "
-            "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first."
-        )
+        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
+                               "If you're the bridge admin, try "
+                               "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
     if portal.mxid:
         has_portal_message = (
             "That Telegram chat already has a portal at "
-            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). "
-        )
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
         if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-            return await evt.reply(
-                f"{has_portal_message}"
-                "Additionally, you do not have the permissions to unbridge that room."
-            )
+            return await evt.reply(f"{has_portal_message}"
+                                   "Additionally, you do not have the permissions to unbridge "
+                                   "that room.")
         evt.sender.command_status = {
             "next": confirm_bridge,
             "action": "Room bridging",
@@ -99,14 +86,12 @@ async def bridge(evt: CommandEvent) -> EventID:
             "peer_type": portal.peer_type,
             "force_use_bot": force_use_bot,
         }
-        return await evt.reply(
-            f"{has_portal_message}"
-            "However, you have the permissions to unbridge that room.\n\n"
-            "To delete that portal completely and continue bridging, use "
-            "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
-            "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
-            "continue`. To cancel, use `$cmdprefix+sp cancel`"
-        )
+        return await evt.reply(f"{has_portal_message}"
+                               "However, you have the permissions to unbridge that room.\n\n"
+                               "To delete that portal completely and continue bridging, use "
+                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
     evt.sender.command_status = {
         "next": confirm_bridge,
         "action": "Room bridging",
@@ -115,114 +100,86 @@ async def bridge(evt: CommandEvent) -> EventID:
         "peer_type": portal.peer_type,
         "force_use_bot": force_use_bot,
     }
-    return await evt.reply(
-        "That Telegram chat has no existing portal. To confirm bridging the "
-        "chat to this room, use `$cmdprefix+sp continue`"
-    )
+    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
+                           "chat to this room, use `$cmdprefix+sp continue`")
 
 
-async def cleanup_old_portal_while_bridging(
-    evt: CommandEvent, portal: po.Portal
-) -> tuple[bool, Awaitable[None] | None]:
+async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
+                                            ) -> Tuple[
+    bool, Optional[Coroutine[None, None, None]]]:
     if not portal.mxid:
-        await evt.reply(
-            "The portal seems to have lost its Matrix room between you"
-            "calling `$cmdprefix+sp bridge` and this command.\n\n"
-            "Continuing without touching previous Matrix room..."
-        )
+        await evt.reply("The portal seems to have lost its Matrix room between you"
+                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                        "Continuing without touching previous Matrix room...")
         return True, None
     elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_portal("Portal deleted (moving to another room)", delete=False)
+        return True, portal.cleanup_portal("Portal deleted (moving to another room)")
     elif evt.args[0] == "unbridge-and-continue":
-        return True, portal.cleanup_portal(
-            "Room unbridged (portal moving to another room)", puppets_only=True, delete=False
-        )
+        return True, portal.cleanup_portal("Room unbridged (portal moving to another room)",
+                                           puppets_only=True)
     else:
         await evt.reply(
             "The chat you were trying to bridge already has a Matrix portal room.\n\n"
             "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
             "continue` to either delete or unbridge the existing room (respectively) and "
             "continue with the bridging.\n\n"
-            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel."
-        )
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
         return False, None
 
 
-async def confirm_bridge(evt: CommandEvent) -> EventID | None:
+async def confirm_bridge(evt: CommandEvent) -> Optional[EventID]:
     status = evt.sender.command_status
     try:
-        portal = await po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
         bridge_to_mxid = status["bridge_to_mxid"]
     except KeyError:
         evt.sender.command_status = None
-        return await evt.reply(
-            "Fatal error: tgid or peer_type missing from command_status. "
-            "This shouldn't happen unless you're messing with the command handler code."
-        )
-
-    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
-
+        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
+                               "This shouldn't happen unless you're messing with the command "
+                               "handler code.")
     if "mxid" in status:
         ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
         if not ok:
             return None
         elif coro:
-            asyncio.create_task(coro)
+            asyncio.ensure_future(coro, loop=evt.loop)
             await evt.reply("Cleaning up previous portal room...")
     elif portal.mxid:
         evt.sender.command_status = None
-        return await evt.reply(
-            "The portal seems to have created a Matrix room between you "
-            "calling `$cmdprefix+sp bridge` and this command.\n\n"
-            "Please start over by calling the bridge command again."
-        )
+        return await evt.reply("The portal seems to have created a Matrix room between you "
+                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                               "Please start over by calling the bridge command again.")
     elif evt.args[0] != "continue":
-        return await evt.reply(
-            "Please use `$cmdprefix+sp continue` to confirm the bridging or "
-            "`$cmdprefix+sp cancel` to cancel."
-        )
+        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
+                               "`$cmdprefix+sp cancel` to cancel.")
 
     evt.sender.command_status = None
-    async with portal._room_create_lock:
-        await _locked_confirm_bridge(
-            evt, portal=portal, room_id=bridge_to_mxid, is_logged_in=is_logged_in
-        )
-
-
-async def _locked_confirm_bridge(
-    evt: CommandEvent, portal: po.Portal, room_id: RoomID, is_logged_in: bool
-) -> EventID | None:
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)
     except Exception:
         evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
         if is_logged_in:
-            return await evt.reply(
-                "Failed to get info of telegram chat. You are logged in, are you in that chat?"
-            )
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You are logged in, are you in that chat?")
         else:
-            return await evt.reply(
-                "Failed to get info of telegram chat. "
-                "You're not logged in, is the relay bot in the chat?"
-            )
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You're not logged in, is the relay bot in the chat?")
     if isinstance(entity, (ChatForbidden, ChannelForbidden)):
         if is_logged_in:
             return await evt.reply("You don't seem to be in that chat.")
         else:
             return await evt.reply("The bot doesn't seem to be in that chat.")
 
-    portal.mxid = room_id
-    portal.by_mxid[portal.mxid] = portal
-    (portal.title, portal.about, levels, portal.encrypted) = await get_initial_state(
-        evt.az.intent, evt.room_id
-    )
+    direct = False
+
+    portal.mxid = bridge_to_mxid
+    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
     portal.photo_id = ""
-    await portal.save()
-    await portal.update_bridge_info()
+    portal.save()
 
-    asyncio.create_task(portal.update_matrix_room(user, entity, direct=False, levels=levels))
-
-    await warn_missing_power(levels, evt)
+    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
+                          loop=evt.loop)
 
     return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,26 +13,19 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import Any, Awaitable
+from typing import Awaitable
 from io import StringIO
 
-from ruamel.yaml import YAMLError
-
-from mautrix.types import EventID
 from mautrix.util.config import yaml
+from mautrix.types import EventID
 
 from ... import portal as po, util
-from .. import SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
 
 
-@command_handler(
-    needs_auth=False,
-    help_section=SECTION_PORTAL_MANAGEMENT,
-    help_text="View or change per-portal settings.",
-    help_args="<`help`|_subcommand_> [...]",
-)
+@command_handler(needs_auth=False, help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="View or change per-portal settings.",
+                 help_args="<`help`|_subcommand_> [...]")
 async def config(evt: CommandEvent) -> None:
     cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
     if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
@@ -42,7 +35,7 @@ async def config(evt: CommandEvent) -> None:
         await config_defaults(evt)
         return
 
-    portal = await po.Portal.get_by_mxid(evt.room_id)
+    portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         await evt.reply("This is not a portal room.")
         return
@@ -55,11 +48,7 @@ async def config(evt: CommandEvent) -> None:
         return
 
     key = evt.args[1] if len(evt.args) > 1 else None
-    try:
-        value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
-    except YAMLError as e:
-        await evt.reply(f"Invalid value provided. Values must be valid YAML.\n{e}")
-        return
+    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
     if cmd == "set":
         await config_set(evt, portal, key, value)
     elif cmd == "unset":
@@ -68,12 +57,11 @@ async def config(evt: CommandEvent) -> None:
         await config_add_del(evt, portal, key, value, cmd)
     else:
         return
-    await portal.save()
+    portal.save()
 
 
 def config_help(evt: CommandEvent) -> Awaitable[EventID]:
-    return evt.reply(
-        """**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
+    return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
 
 * **help** - View this help text.
 * **view** - View the current config data.
@@ -82,49 +70,40 @@ def config_help(evt: CommandEvent) -> Awaitable[EventID]:
 * **unset** <_key_> - Remove a config value.
 * **add** <_key_> <_value_> - Add a value to an array.
 * **del** <_key_> <_value_> - Remove a value from an array.
-"""
-    )
+""")
 
 
 def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[EventID]:
-    return evt.reply(f"Room-specific config:\n{_str_value(portal.local_config).rstrip()}")
+    stream = StringIO()
+    yaml.dump(portal.local_config, stream)
+    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
 
 
 def config_defaults(evt: CommandEvent) -> Awaitable[EventID]:
-    value = _str_value(
-        {
-            "bridge_notices": {
-                "default": evt.config["bridge.bridge_notices.default"],
-                "exceptions": evt.config["bridge.bridge_notices.exceptions"],
-            },
-            "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
-            "inline_images": evt.config["bridge.inline_images"],
-            "message_formats": evt.config["bridge.message_formats"],
-            "emote_format": evt.config["bridge.emote_format"],
-            "state_event_formats": evt.config["bridge.state_event_formats"],
-            "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
-        }
-    )
-    return evt.reply(f"Bridge instance wide config:\n{value.rstrip()}")
-
-
-def _str_value(value: Any) -> str:
     stream = StringIO()
-    yaml.dump(value, stream)
-    value_str = stream.getvalue()
-    if "\n" in value_str:
-        return f"\n```yaml\n{value_str}\n```\n"
-    else:
-        return f"`{value_str}`"
+    yaml.dump({
+        "bridge_notices": {
+            "default": evt.config["bridge.bridge_notices.default"],
+            "exceptions": evt.config["bridge.bridge_notices.exceptions"],
+        },
+        "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
+        "inline_images": evt.config["bridge.inline_images"],
+        "message_formats": evt.config["bridge.message_formats"],
+        "emote_format": evt.config["bridge.emote_format"],
+        "state_event_formats": evt.config["bridge.state_event_formats"],
+        "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
+    }, stream)
+    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
 
 
-def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: Any) -> Awaitable[EventID]:
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[EventID]:
     if not key or value is None:
         return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
     elif util.recursive_set(portal.local_config, key, value):
-        return evt.reply(f"Successfully set the value of `{key}` to {_str_value(value)}".rstrip())
+        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
     else:
-        return evt.reply(f"Failed to set value of `{key}`. Does the path contain non-map types?")
+        return evt.reply(f"Failed to set value of `{key}`. "
+                         "Does the path contain non-map types?")
 
 
 def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[EventID]:
@@ -136,26 +115,24 @@ def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Ev
         return evt.reply(f"`{key}` not found in config.")
 
 
-def config_add_del(
-    evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
-) -> Awaitable[EventID]:
+def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
+                   ) -> Awaitable[EventID]:
     if not key or value is None:
         return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
 
     arr = util.recursive_get(portal.local_config, key)
     if not arr:
-        return evt.reply(
-            f"`{key}` not found in config. Maybe do `$cmdprefix+sp config set {key} []` first?"
-        )
+        return evt.reply(f"`{key}` not found in config. "
+                         f"Maybe do `$cmdprefix+sp config set {key} []` first?")
     elif not isinstance(arr, list):
         return evt.reply("`{key}` does not seem to be an array.")
     elif cmd == "add":
         if value in arr:
-            return evt.reply(f"The array at `{key}` already contains {_str_value(value)}".rstrip())
+            return evt.reply(f"The array at `{key}` already contains `{value}`.")
         arr.append(value)
-        return evt.reply(f"Successfully added {_str_value(value)} to the array at `{key}`")
+        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
     else:
         if value not in arr:
-            return evt.reply(f"The array at `{key}` does not contain {_str_value(value)}")
+            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
         arr.remove(value)
-        return evt.reply(f"Successfully removed {_str_value(value)} from the array at `{key}`")
+        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,38 +13,32 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
 from mautrix.types import EventID
 
 from ... import portal as po
 from ...types import TelegramID
-from .. import SECTION_CREATING_PORTALS, CommandEvent, command_handler
-from .util import get_initial_state, user_has_power_level, warn_missing_power
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
 
 
-@command_handler(
-    help_section=SECTION_CREATING_PORTALS,
-    help_args="[_type_]",
-    help_text=(
-        "Create a Telegram chat of the given type for the current Matrix room. "
-        "The type is either `group`, `supergroup` or `channel` (defaults to `supergroup`)."
-    ),
-)
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_type_]",
+                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
+                           "The type is either `group`, `supergroup` or `channel` (defaults to "
+                           "`group`).")
 async def create(evt: CommandEvent) -> EventID:
-    type = evt.args[0] if len(evt.args) > 0 else "supergroup"
-    if type not in ("chat", "group", "supergroup", "channel"):
+    type = evt.args[0] if len(evt.args) > 0 else "group"
+    if type not in {"chat", "group", "supergroup", "channel"}:
         return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`"
-        )
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
 
-    if await po.Portal.get_by_mxid(evt.room_id):
+    if po.Portal.get_by_mxid(evt.room_id):
         return await evt.reply("This is already a portal room.")
 
     if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
         return await evt.reply("You do not have the permissions to bridge this room.")
 
-    title, about, levels, encrypted = await get_initial_state(evt.az.intent, evt.room_id)
+    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
     if not title:
         return await evt.reply("Please set a title before creating a Telegram chat.")
 
@@ -56,29 +50,11 @@ async def create(evt: CommandEvent) -> EventID:
         "group": "chat",
     }[type]
 
-    portal = po.Portal(
-        tgid=TelegramID(0),
-        tg_receiver=TelegramID(0),
-        peer_type=type,
-        mxid=evt.room_id,
-        title=title,
-        about=about,
-        encrypted=encrypted,
-    )
-    invites, errors = await portal.get_telegram_users_in_matrix_room(evt.sender)
-    if len(errors) > 0:
-        error_list = "\n".join(f"* [{mxid}](https://matrix.to/#/{mxid})" for mxid in errors)
-        await evt.reply(
-            f"Failed to add the following users to the chat:\n\n{error_list}\n\n"
-            "You can try `$cmdprefix+sp search -r <username>` to help the bridge find "
-            "those users."
-        )
-
-    await warn_missing_power(levels, evt)
-
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+                       mxid=evt.room_id, title=title, about=about)
     try:
-        await portal.create_telegram_chat(evt.sender, invites=invites, supergroup=supergroup)
+        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
     except ValueError as e:
-        await portal.delete()
+        portal.delete()
         return await evt.reply(e.args[0])
     return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/filter.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,20 +13,17 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
 from mautrix.types import EventID
 
 from ... import portal as po
-from .. import SECTION_ADMIN, CommandEvent, command_handler
+from .. import command_handler, CommandEvent, SECTION_ADMIN
 
 
-@command_handler(
-    needs_admin=True,
-    help_section=SECTION_ADMIN,
-    help_args="<`whitelist`|`blacklist`>",
-    help_text="Change whether the bridge will allow or disallow bridging rooms by default.",
-)
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`>",
+                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
+                           "default.")
 async def filter_mode(evt: CommandEvent) -> EventID:
     try:
         mode = evt.args[0]
@@ -39,26 +36,19 @@ async def filter_mode(evt: CommandEvent) -> EventID:
     evt.config.save()
     po.Portal.filter_mode = mode
     if mode == "whitelist":
-        return await evt.reply(
-            "The bridge will now disallow bridging chats by default.\n"
-            "To allow bridging a specific chat, use"
-            "`!filter whitelist <chat ID>`."
-        )
+        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
+                               "To allow bridging a specific chat, use"
+                               "`!filter whitelist <chat ID>`.")
     else:
-        return await evt.reply(
-            "The bridge will now allow bridging chats by default.\n"
-            "To disallow bridging a specific chat, use"
-            "`!filter blacklist <chat ID>`."
-        )
+        return await evt.reply("The bridge will now allow bridging chats by default.\n"
+                               "To disallow bridging a specific chat, use"
+                               "`!filter blacklist <chat ID>`.")
 
 
-@command_handler(
-    name="filter",
-    needs_admin=True,
-    help_section=SECTION_ADMIN,
-    help_args="<`whitelist`|`blacklist`> <_chat ID_>",
-    help_text="Allow or disallow bridging a specific chat.",
-)
+@command_handler(name="filter", needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+                 help_text="Allow or disallow bridging a specific chat.")
 async def edit_filter(evt: CommandEvent) -> EventID:
     try:
         action = evt.args[0]
@@ -77,7 +67,7 @@ async def edit_filter(evt: CommandEvent) -> EventID:
 
     mode = evt.config["bridge.filter.mode"]
     if mode not in ("blacklist", "whitelist"):
-        return await evt.reply(f'Unknown filter mode "{mode}". Please fix the bridge config.')
+        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
 
     filter_id_list = evt.config["bridge.filter.list"]
 

mautrix_telegram/commands/portal/misc.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,51 +13,36 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from datetime import datetime, timedelta
-import re
-
-from telethon.errors import (
-    ChatAdminRequiredError,
-    RPCError,
-    UsernameInvalidError,
-    UsernameNotModifiedError,
-    UsernameOccupiedError,
-)
 from telethon.tl.functions.channels import GetFullChannelRequest
 from telethon.tl.functions.messages import GetFullChatRequest
+from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
+                             UsernameNotModifiedError, UsernameOccupiedError, RPCError)
 
 from mautrix.types import EventID
 
 from ... import portal as po
-from .. import SECTION_MISC, SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT, SECTION_MISC
 from .util import user_has_power_level
 
 
-@command_handler(
-    needs_admin=False,
-    needs_puppeting=False,
-    needs_auth=False,
-    help_section=SECTION_MISC,
-    help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.",
-)
+@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.")
 async def sync_state(evt: CommandEvent) -> EventID:
-    portal = await po.Portal.get_by_mxid(evt.room_id)
+    portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
     elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
         return await evt.reply(f"You do not have the permissions to synchronize this room.")
 
-    await portal.main_intent.get_joined_members(portal.mxid)
+    await portal.sync_matrix_members()
     await evt.reply("Synchronization complete")
 
 
-@command_handler(
-    needs_admin=False, needs_puppeting=False, needs_auth=False, help_section=SECTION_MISC
-)
+@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC)
 async def sync_full(evt: CommandEvent) -> EventID:
-    portal = await po.Portal.get_by_mxid(evt.room_id)
+    portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
 
@@ -80,16 +65,11 @@ async def sync_full(evt: CommandEvent) -> EventID:
     return await evt.reply("Portal synced successfully.")
 
 
-@command_handler(
-    name="id",
-    needs_admin=False,
-    needs_puppeting=False,
-    needs_auth=False,
-    help_section=SECTION_MISC,
-    help_text="Get the ID of the Telegram chat where this room is bridged.",
-)
+@command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Get the ID of the Telegram chat where this room is bridged.")
 async def get_id(evt: CommandEvent) -> EventID:
-    portal = await po.Portal.get_by_mxid(evt.room_id)
+    portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
     tgid = portal.tgid
@@ -100,88 +80,10 @@ async def get_id(evt: CommandEvent) -> EventID:
     await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
 
 
-invite_link_usage = (
-    "**Usage:** `$cmdprefix+sp invite-link [--uses=<amount>] [--expire=<delta>]`"
-    "\n\n"
-    "* `--uses`: the number of times the invite link can be used."
-    "            Defaults to unlimited.\n"
-    "* `--expire`: the duration after which the link will expire."
-    "              A number suffixed with d(ay), h(our), m(inute) or s(econd)"
-)
-
-
-def _parse_flag(args: list[str]) -> tuple[str, str]:
-    arg = args.pop(0).lower()
-    if arg.startswith("--"):
-        value_start = arg.index("=")
-        if value_start:
-            flag = arg[2:value_start]
-            value = arg[value_start + 1 :]
-        else:
-            flag = arg[2:]
-            value = args.pop(0).lower()
-    elif arg.startswith("-"):
-        flag = arg[1]
-        if len(arg) > 3 and arg[2] == "=":
-            value = arg[3:]
-        else:
-            value = args.pop(0).lower()
-    else:
-        raise ValueError("invalid flag")
-    return flag, value
-
-
-delta_regex = re.compile(
-    "([0-9]+)(w(?:eek)?|d(?:ay)?|h(?:our)?|m(?:in(?:ute)?)?|s(?:ec(?:ond)?)?)"
-)
-
-
-def _parse_delta(value: str) -> timedelta | None:
-    match = delta_regex.fullmatch(value)
-    if not match:
-        return None
-    number = int(match.group(1))
-    unit = match.group(2)[0]
-    if unit == "w":
-        return timedelta(weeks=number)
-    elif unit == "d":
-        return timedelta(days=number)
-    elif unit == "h":
-        return timedelta(hours=number)
-    elif unit == "m":
-        return timedelta(minutes=number)
-    elif unit == "s":
-        return timedelta(seconds=number)
-    else:
-        return None
-
-
-@command_handler(
-    help_section=SECTION_PORTAL_MANAGEMENT,
-    help_text="Get a Telegram invite link to the current chat.",
-    help_args="[--uses=<amount>] [--expire=<time delta, e.g. 1d>]",
-)
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Get a Telegram invite link to the current chat.")
 async def invite_link(evt: CommandEvent) -> EventID:
-    # TODO once we switch to Python 3.9 minimum, use argparse with exit_on_error=False
-    uses = None
-    expire = None
-    while evt.args:
-        try:
-            flag, value = _parse_flag(evt.args)
-        except (ValueError, IndexError):
-            return await evt.reply(invite_link_usage)
-        if flag in ("uses", "u"):
-            try:
-                uses = int(value)
-            except ValueError:
-                await evt.reply("The number of uses must be an integer")
-        elif flag in ("expire", "e"):
-            expire_delta = _parse_delta(value)
-            if not expire_delta:
-                await evt.reply("Invalid format for expiry time delta")
-            expire = datetime.now() + expire_delta
-
-    portal = await po.Portal.get_by_mxid(evt.room_id)
+    portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
 
@@ -189,7 +91,7 @@ async def invite_link(evt: CommandEvent) -> EventID:
         return await evt.reply("You can't invite users to private chats.")
 
     try:
-        link = await portal.get_invite_link(evt.sender, uses=uses, expire=expire)
+        link = await portal.get_invite_link(evt.sender)
         return await evt.reply(f"Invite link to {portal.title}: {link}")
     except ValueError as e:
         return await evt.reply(e.args[0])
@@ -197,12 +99,10 @@ async def invite_link(evt: CommandEvent) -> EventID:
         return await evt.reply("You don't have the permission to create an invite link.")
 
 
-@command_handler(
-    help_section=SECTION_PORTAL_MANAGEMENT,
-    help_text="Upgrade a normal Telegram group to a supergroup.",
-)
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Upgrade a normal Telegram group to a supergroup.")
 async def upgrade(evt: CommandEvent) -> EventID:
-    portal = await po.Portal.get_by_mxid(evt.room_id)
+    portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
     elif portal.peer_type == "channel":
@@ -219,33 +119,30 @@ async def upgrade(evt: CommandEvent) -> EventID:
         return await evt.reply(e.args[0])
 
 
-@command_handler(
-    help_section=SECTION_PORTAL_MANAGEMENT,
-    help_args="<_name_|`-`>",
-    help_text=(
-        "Change the username of a supergroup/channel. To disable, use a dash (`-`) as the name."
-    ),
-)
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="<_name_|`-`>",
+                 help_text="Change the username of a supergroup/channel. "
+                           "To disable, use a dash (`-`) as the name.")
 async def group_name(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
 
-    portal = await po.Portal.get_by_mxid(evt.room_id)
+    portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")
     elif portal.peer_type != "channel":
         return await evt.reply("Only channels and supergroups have usernames.")
 
     try:
-        await portal.set_telegram_username(evt.sender, evt.args[0] if evt.args[0] != "-" else "")
+        await portal.set_telegram_username(evt.sender,
+                                           evt.args[0] if evt.args[0] != "-" else "")
         if portal.username:
             return await evt.reply(f"Username of channel changed to {portal.username}.")
         else:
             return await evt.reply(f"Channel is now private.")
     except ChatAdminRequiredError:
         return await evt.reply(
-            "You don't have the permission to set the username of this channel."
-        )
+            "You don't have the permission to set the username of this channel.")
     except UsernameNotModifiedError:
         if portal.username:
             return await evt.reply("That is already the username of this channel.")

mautrix_telegram/commands/portal/unbridge.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,42 +13,36 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Dict, Callable, Optional
 
-from typing import Callable
-
-from mautrix.types import EventID, RoomID
+from mautrix.types import RoomID, EventID
 
 from ... import portal as po
-from .. import SECTION_PORTAL_MANAGEMENT, CommandEvent, command_handler
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
 from .util import user_has_power_level
 
 
-async def _get_portal_and_check_permission(evt: CommandEvent) -> po.Portal | None:
+async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
+                                           action: Optional[str] = None
+                                           ) -> Optional[po.Portal]:
     room_id = RoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
 
-    portal = await po.Portal.get_by_mxid(room_id)
+    portal = po.Portal.get_by_mxid(room_id)
     if not portal:
         that_this = "This" if room_id == evt.room_id else "That"
         await evt.reply(f"{that_this} is not a portal room.")
         return None
 
-    if portal.peer_type == "user":
-        if portal.tg_receiver != evt.sender.tgid:
-            await evt.reply("You do not have the permissions to unbridge that portal.")
-            return None
-        return portal
-
-    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-        await evt.reply("You do not have the permissions to unbridge that portal.")
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
+        action = action or f"{permission.replace('_', ' ')}s"
+        await evt.reply(f"You do not have the permissions to {action} that portal.")
         return None
     return portal
 
 
-def _get_portal_murder_function(
-    action: str, room_id: str, function: Callable, command: str, completed_message: str
-) -> dict:
-    async def post_confirm(confirm) -> EventID | None:
+def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
+                                completed_message: str) -> Dict:
+    async def post_confirm(confirm) -> Optional[EventID]:
         confirm.sender.command_status = None
         if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
             await function()
@@ -64,55 +58,40 @@ def _get_portal_murder_function(
     }
 
 
-@command_handler(
-    needs_auth=False,
-    needs_puppeting=False,
-    help_section=SECTION_PORTAL_MANAGEMENT,
-    help_text=(
-        "Remove all users from the current portal room and forget the portal. "
-        "Only works for group chats; to delete a private chat portal, simply leave the room."
-    ),
-)
-async def delete_portal(evt: CommandEvent) -> EventID | None:
-    portal = await _get_portal_and_check_permission(evt)
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove all users from the current portal room and forget the portal. "
+                           "Only works for group chats; to delete a private chat portal, simply "
+                           "leave the room.")
+async def delete_portal(evt: CommandEvent) -> Optional[EventID]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
     if not portal:
         return None
 
-    evt.sender.command_status = _get_portal_murder_function(
-        "Portal deletion",
-        portal.mxid,
-        portal.cleanup_and_delete,
-        "delete",
-        "Portal successfully deleted.",
-    )
-    return await evt.reply(
-        "Please confirm deletion of portal "
-        f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-        f'to Telegram chat "{portal.title}" '
-        "by typing `$cmdprefix+sp confirm-delete`"
-        "\n\n"
-        "**WARNING:** If the bridge bot has the power level to do so, **this "
-        "will kick ALL users** in the room. If you just want to remove the "
-        "bridge, use `$cmdprefix+sp unbridge` instead."
-    )
+    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
+                                                            portal.cleanup_and_delete, "delete",
+                                                            "Portal successfully deleted.")
+    return await evt.reply("Please confirm deletion of portal "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           f"to Telegram chat \"{portal.title}\" "
+                           "by typing `$cmdprefix+sp confirm-delete`"
+                           "\n\n"
+                           "**WARNING:** If the bridge bot has the power level to do so, **this "
+                           "will kick ALL users** in the room. If you just want to remove the "
+                           "bridge, use `$cmdprefix+sp unbridge` instead.")
 
 
-@command_handler(
-    needs_auth=False,
-    needs_puppeting=False,
-    help_section=SECTION_PORTAL_MANAGEMENT,
-    help_text="Remove puppets from the current portal room and forget the portal.",
-)
-async def unbridge(evt: CommandEvent) -> EventID | None:
-    portal = await _get_portal_and_check_permission(evt)
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove puppets from the current portal room and forget the portal.")
+async def unbridge(evt: CommandEvent) -> Optional[EventID]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
     if not portal:
         return None
 
-    evt.sender.command_status = _get_portal_murder_function(
-        "Room unbridging", portal.mxid, portal.unbridge, "unbridge", "Room successfully unbridged."
-    )
-    return await evt.reply(
-        f'Please confirm unbridging chat "{portal.title}" from room '
-        f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-        "by typing `$cmdprefix+sp confirm-unbridge`"
-    )
+    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
+                                                            portal.unbridge, "unbridge",
+                                                            "Room successfully unbridged.")
+    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           "by typing `$cmdprefix+sp confirm-unbridge`")

mautrix_telegram/commands/portal/util.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,24 +13,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Tuple, Optional
 
-from mautrix.appservice import IntentAPI
 from mautrix.errors import MatrixRequestError
-from mautrix.types import EventType, PowerLevelStateEventContent, RoomID
+from mautrix.appservice import IntentAPI
+from mautrix.types import RoomID, EventType, PowerLevelStateEventContent
 
 from ... import user as u
-from .. import CommandEvent
+
+OptStr = Optional[str]
 
 
-async def get_initial_state(
-    intent: IntentAPI, room_id: RoomID
-) -> tuple[str | None, str | None, PowerLevelStateEventContent | None, bool]:
+async def get_initial_state(intent: IntentAPI, room_id: RoomID
+                            ) -> Tuple[OptStr, OptStr, Optional[PowerLevelStateEventContent]]:
     state = await intent.get_state(room_id)
-    title: str | None = None
-    about: str | None = None
-    levels: PowerLevelStateEventContent | None = None
-    encrypted: bool = False
+    title: OptStr = None
+    about: OptStr = None
+    levels: Optional[PowerLevelStateEventContent] = None
     for event in state:
         try:
             if event.type == EventType.ROOM_NAME:
@@ -41,26 +40,14 @@ async def get_initial_state(
                 levels = event.content
             elif event.type == EventType.ROOM_CANONICAL_ALIAS:
                 title = title or event.content.canonical_alias
-            elif event.type == EventType.ROOM_ENCRYPTION:
-                encrypted = True
         except KeyError:
             # Some state event probably has empty content
             pass
-    return title, about, levels, encrypted
+    return title, about, levels
 
 
-async def warn_missing_power(levels: PowerLevelStateEventContent, evt: CommandEvent) -> None:
-    if levels.get_user_level(evt.az.bot_mxid) < levels.redact:
-        await evt.reply(
-            "Warning: The bot does not have privileges to redact messages on Matrix. "
-            "Message deletions from Telegram will not be bridged unless you give "
-            f"redaction permissions to [{evt.az.bot_mxid}](https://matrix.to/#/{evt.az.bot_mxid})"
-        )
-
-
-async def user_has_power_level(
-    room_id: RoomID, intent: IntentAPI, sender: u.User, event: str
-) -> bool:
+async def user_has_power_level(room_id: RoomID, intent: IntentAPI, sender: u.User,
+                               event: str) -> bool:
     if sender.is_admin:
         return True
     # Make sure the state store contains the power levels.
@@ -68,5 +55,6 @@ async def user_has_power_level(
         await intent.get_power_levels(room_id)
     except MatrixRequestError:
         return False
-    event_type = EventType.find(f"net.maunium.telegram.{event}", t_class=EventType.Class.STATE)
-    return await intent.state_store.has_power_level(room_id, sender.mxid, event_type)
+    event_type = EventType.find(f"net.maunium.telegram.{event}")
+    event_type.t_class = EventType.Class.STATE
+    return intent.state_store.has_power_level(room_id, sender.mxid, event_type)

mautrix_telegram/commands/telegram/account.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,36 +13,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Optional
 
-from telethon.errors import (
-    AboutTooLongError,
-    AuthKeyError,
-    FirstNameInvalidError,
-    HashInvalidError,
-    UsernameInvalidError,
-    UsernameNotModifiedError,
-    UsernameOccupiedError,
-)
-from telethon.tl.functions.account import (
-    GetAuthorizationsRequest,
-    ResetAuthorizationRequest,
-    UpdateProfileRequest,
-    UpdateUsernameRequest,
-)
+from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
 from telethon.tl.types import Authorization
+from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
+                                           ResetAuthorizationRequest, UpdateProfileRequest)
 
 from mautrix.types import EventID
 
-from .. import SECTION_AUTH, CommandEvent, command_handler
+from .. import command_handler, CommandEvent, SECTION_AUTH
 
 
-@command_handler(
-    needs_auth=True,
-    help_section=SECTION_AUTH,
-    help_args="<_new username_>",
-    help_text="Change your Telegram username.",
-)
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new username_>",
+                 help_text="Change your Telegram username.")
 async def username(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp username <new username>`")
@@ -54,56 +41,30 @@ async def username(evt: CommandEvent) -> EventID:
     try:
         await evt.sender.client(UpdateUsernameRequest(username=new_name))
     except UsernameInvalidError:
-        return await evt.reply(
-            "Invalid username. Usernames must be between 5 and 30 alphanumeric characters."
-        )
+        return await evt.reply("Invalid username. Usernames must be between 5 and 30 alphanumeric "
+                               "characters.")
     except UsernameNotModifiedError:
         return await evt.reply("That is your current username.")
     except UsernameOccupiedError:
         return await evt.reply("That username is already in use.")
     await evt.sender.update_info()
-    if not evt.sender.tg_username:
+    if not evt.sender.username:
         await evt.reply("Username removed")
     else:
-        await evt.reply(f"Username changed to {evt.sender.tg_username}")
+        await evt.reply(f"Username changed to {evt.sender.username}")
 
 
-@command_handler(
-    needs_auth=True,
-    help_section=SECTION_AUTH,
-    help_args="<_new about_>",
-    help_text="Change your Telegram about section.",
-)
-async def about(evt: CommandEvent) -> EventID:
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp about <new about>`")
-    if evt.sender.is_bot:
-        return await evt.reply("Bots can't set their own about section.")
-    new_about = " ".join(evt.args)
-    if new_about == "-":
-        new_about = ""
-    try:
-        await evt.sender.client(UpdateProfileRequest(about=new_about))
-    except AboutTooLongError:
-        return await evt.reply("The provided about section is too long")
-    return await evt.reply("About section updated")
-
-
-@command_handler(
-    needs_auth=True,
-    help_section=SECTION_AUTH,
-    help_args="<_new displayname_>",
-    help_text="Change your Telegram displayname.",
-)
+@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
+                 help_text="Change your Telegram displayname.")
 async def displayname(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
     if evt.sender.is_bot:
         return await evt.reply("Bots can't set their own displayname.")
 
-    first_name, last_name = (
-        (evt.args[0], "") if len(evt.args) == 1 else (" ".join(evt.args[:-1]), evt.args[-1])
-    )
+    first_name, last_name = ((evt.args[0], "")
+                             if len(evt.args) == 1
+                             else (" ".join(evt.args[:-1]), evt.args[-1]))
     try:
         await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
     except FirstNameInvalidError:
@@ -113,20 +74,16 @@ async def displayname(evt: CommandEvent) -> EventID:
 
 
 def _format_session(sess: Authorization) -> str:
-    return (
-        f"**{sess.app_name} {sess.app_version}**  \n"
-        f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
-        f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
-        f"  **From:** {sess.ip} - {sess.region}, {sess.country}"
-    )
+    return (f"**{sess.app_name} {sess.app_version}**  \n"
+            f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
+            f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
+            f"  **From:** {sess.ip} - {sess.region}, {sess.country}")
 
 
-@command_handler(
-    needs_auth=True,
-    help_section=SECTION_AUTH,
-    help_args="<`list`|`terminate`> [_hash_]",
-    help_text="View or delete other Telegram sessions.",
-)
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<`list`|`terminate`> [_hash_]",
+                 help_text="View or delete other Telegram sessions.")
 async def session(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")
@@ -138,18 +95,14 @@ async def session(evt: CommandEvent) -> EventID:
         session_list = res.authorizations
         current = [s for s in session_list if s.current][0]
         current_text = _format_session(current)
-        other_text = "\n".join(
-            f"* {_format_session(sess)}  \n  **Hash:** {sess.hash}"
-            for sess in session_list
-            if not sess.current
-        )
-        return await evt.reply(
-            f"### Current session\n"
-            f"{current_text}\n"
-            f"\n"
-            f"### Other active sessions\n"
-            f"{other_text}"
-        )
+        other_text = "\n".join(f"* {_format_session(sess)}  \n"
+                               f"  **Hash:** {sess.hash}"
+                               for sess in session_list if not sess.current)
+        return await evt.reply(f"### Current session\n"
+                               f"{current_text}\n"
+                               f"\n"
+                               f"### Other active sessions\n"
+                               f"{other_text}")
     elif cmd == "terminate" and len(evt.args) > 1:
         try:
             session_hash = int(evt.args[1])
@@ -161,9 +114,8 @@ async def session(evt: CommandEvent) -> EventID:
             return await evt.reply("Invalid session hash.")
         except AuthKeyError as e:
             if e.message == "FRESH_RESET_AUTHORISATION_FORBIDDEN":
-                return await evt.reply(
-                    "New sessions can't terminate other sessions. Please wait a while."
-                )
+                return await evt.reply("New sessions can't terminate other sessions. "
+                                       "Please wait a while.")
             raise
         if ok:
             return await evt.reply("Session terminated successfully.")

mautrix_telegram/commands/telegram/auth.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,93 +13,52 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import Any
+from typing import Any, Dict, Optional
 import asyncio
-import io
 
-from telethon.errors import (
-    AccessTokenExpiredError,
-    AccessTokenInvalidError,
-    FirstNameInvalidError,
-    FloodWaitError,
-    PasswordHashInvalidError,
-    PhoneCodeExpiredError,
-    PhoneCodeInvalidError,
-    PhoneNumberAppSignupForbiddenError,
-    PhoneNumberBannedError,
-    PhoneNumberFloodError,
-    PhoneNumberInvalidError,
-    PhoneNumberOccupiedError,
-    PhoneNumberUnoccupiedError,
-    SessionPasswordNeededError,
-)
-from telethon.tl.types import User
+from telethon.errors import (  # isort: skip
+    AccessTokenExpiredError, AccessTokenInvalidError, FirstNameInvalidError, FloodWaitError,
+    PasswordHashInvalidError, PhoneCodeExpiredError, PhoneCodeInvalidError,
+    PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
+    PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError,
+    PhoneNumberInvalidError)
 
-from mautrix.types import (
-    EventID,
-    ImageInfo,
-    MediaMessageEventContent,
-    MessageType,
-    TextMessageEventContent,
-    UserID,
-)
-from mautrix.util.format_duration import format_duration as fmt_duration
+from mautrix.types import EventID
 
 from ... import user as u
-from ...commands import SECTION_AUTH, CommandEvent, command_handler
-from ...types import TelegramID
-
-try:
-    from telethon.tl.custom import QRLogin
-    import PIL as _
-    import qrcode
-except ImportError:
-    qrcode = None
-    QRLogin = None
+from ...commands import command_handler, CommandEvent, SECTION_AUTH
+from ...util import format_duration
 
 
-@command_handler(
-    needs_auth=False, help_section=SECTION_AUTH, help_text="Check if you're logged into Telegram."
-)
+@command_handler(needs_auth=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> EventID:
-    if await evt.sender.is_logged_in():
-        me = await evt.sender.get_me()
-        if me:
-            human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
-            return await evt.reply(f"You're logged in as {human_tg_id}")
-        else:
-            return await evt.reply("You were logged in, but there appears to have been an error.")
+    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
+    if me:
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
+        return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")
 
 
-@command_handler(
-    needs_auth=False,
-    needs_puppeting=False,
-    help_section=SECTION_AUTH,
-    help_text="Get the info of the message relay Telegram bot.",
-)
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Get the info of the message relay Telegram bot.")
 async def ping_bot(evt: CommandEvent) -> EventID:
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
     info, mxid = await evt.tgbot.get_me(use_cache=False)
-    return await evt.reply(
-        "Telegram message relay bot is active: "
-        f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
-        "To use the bot, simply invite it to a portal room."
-    )
+    return await evt.reply("Telegram message relay bot is active: "
+                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
+                           "To use the bot, simply invite it to a portal room.")
 
 
-@command_handler(
-    needs_auth=False,
-    management_only=True,
-    help_section=SECTION_AUTH,
-    help_args="<_phone_> <_full name_>",
-    help_text="Register to Telegram",
-)
-async def register(evt: CommandEvent) -> EventID:
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_phone_> <_full name_>",
+                 help_text="Register to Telegram")
+async def register(evt: CommandEvent) -> Optional[EventID]:
     if await evt.sender.is_logged_in():
         return await evt.reply("You are already logged in.")
     elif len(evt.args) < 1:
@@ -111,19 +70,12 @@ async def register(evt: CommandEvent) -> EventID:
     else:
         full_name = " ".join(evt.args[1:-1]), evt.args[-1]
 
-    await _request_code(
-        evt,
-        phone_number,
-        {
-            "next": enter_code_register,
-            "action": "Register",
-            "full_name": full_name,
-        },
-    )
-    return await evt.reply(
-        "By signing up for Telegram, you agree to "
-        "the terms of service: https://telegram.org/tos"
-    )
+    await _request_code(evt, phone_number, {
+        "next": enter_code_register,
+        "action": "Register",
+        "full_name": full_name,
+    })
+    return None
 
 
 async def enter_code_register(evt: CommandEvent) -> EventID:
@@ -133,110 +85,37 @@ async def enter_code_register(evt: CommandEvent) -> EventID:
         await evt.sender.ensure_started(even_if_no_session=True)
         first_name, last_name = evt.sender.command_status["full_name"]
         user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
-        asyncio.create_task(evt.sender.post_login(user, first_login=True))
+        asyncio.ensure_future(evt.sender.post_login(user, first_login=True), loop=evt.loop)
         evt.sender.command_status = None
         return await evt.reply(f"Successfully registered to Telegram.")
     except PhoneNumberOccupiedError:
-        return await evt.reply(
-            "That phone number has already been registered. "
-            "You can log in with `$cmdprefix+sp login`."
-        )
+        return await evt.reply("That phone number has already been registered. "
+                               "You can log in with `$cmdprefix+sp login`.")
     except FirstNameInvalidError:
         return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
     except PhoneCodeExpiredError:
         return await evt.reply(
-            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`."
-        )
+            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`.")
     except PhoneCodeInvalidError:
         return await evt.reply("Invalid phone code.")
     except Exception:
         evt.log.exception("Error sending phone code")
-        return await evt.reply(
-            "Unhandled exception while sending code. Check console for more details."
-        )
+        return await evt.reply("Unhandled exception while sending code. "
+                               "Check console for more details.")
 
 
-@command_handler(
-    needs_auth=False,
-    management_only=True,
-    help_section=SECTION_AUTH,
-    help_text="Log in by scanning a QR code.",
-)
-async def login_qr(evt: CommandEvent) -> EventID:
-    login_as = evt.sender
-    if len(evt.args) > 0 and evt.sender.is_admin:
-        login_as = await u.User.get_by_mxid(UserID(evt.args[0]))
-    if not qrcode or not QRLogin:
-        return await evt.reply("This bridge instance does not support logging in with a QR code.")
-    if await login_as.is_logged_in():
-        return await evt.reply(f"You are already logged in as {login_as.human_tg_id}.")
-
-    await login_as.ensure_started(even_if_no_session=True)
-    qr_login = QRLogin(login_as.client, ignored_ids=[])
-    qr_event_id: EventID | None = None
-
-    async def upload_qr() -> None:
-        nonlocal qr_event_id
-        buffer = io.BytesIO()
-        image = qrcode.make(qr_login.url)
-        size = image.pixel_size
-        image.save(buffer, "PNG")
-        qr = buffer.getvalue()
-        mxc = await evt.az.intent.upload_media(qr, "image/png", "login-qr.png", len(qr))
-        content = MediaMessageEventContent(
-            body=qr_login.url,
-            url=mxc,
-            msgtype=MessageType.IMAGE,
-            info=ImageInfo(mimetype="image/png", size=len(qr), width=size, height=size),
-        )
-        if qr_event_id:
-            content.set_edit(qr_event_id)
-            await evt.az.intent.send_message(evt.room_id, content)
-        else:
-            content.set_reply(evt.event_id)
-            qr_event_id = await evt.az.intent.send_message(evt.room_id, content)
-
-    retries = 4
-    while retries > 0:
-        await qr_login.recreate()
-        await upload_qr()
-        try:
-            user = await qr_login.wait()
-            break
-        except asyncio.TimeoutError:
-            retries -= 1
-        except SessionPasswordNeededError:
-            evt.sender.command_status = {
-                "next": enter_password,
-                "login_as": login_as if login_as != evt.sender else None,
-                "action": "Login (password entry)",
-            }
-            return await evt.reply(
-                "Your account has two-factor authentication. Please send your password here."
-            )
-    else:
-        timeout = TextMessageEventContent(body="Login timed out", msgtype=MessageType.TEXT)
-        timeout.set_edit(qr_event_id)
-        return await evt.az.intent.send_message(evt.room_id, timeout)
-
-    return await _finish_sign_in(evt, user, login_as=login_as)
-
-
-@command_handler(
-    needs_auth=False,
-    management_only=True,
-    help_section=SECTION_AUTH,
-    help_text="Get instructions on how to log in.",
-)
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Get instructions on how to log in.")
 async def login(evt: CommandEvent) -> EventID:
     override_sender = False
     if len(evt.args) > 0 and evt.sender.is_admin:
-        evt.sender = await u.User.get_and_start_by_mxid(UserID(evt.args[0]))
+        evt.sender = await u.User.get_by_mxid(evt.args[0]).ensure_started()
         override_sender = True
     if await evt.sender.is_logged_in():
         return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 
-    allow_matrix_login = evt.config["bridge.allow_matrix_login"]
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
     if allow_matrix_login and not override_sender:
         evt.sender.command_status = {
             "next": enter_phone_or_token,
@@ -248,32 +127,24 @@ async def login(evt: CommandEvent) -> EventID:
         prefix = evt.config["appservice.public.external"]
         url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
         if override_sender:
-            return await evt.reply(
-                f"[Click here to log in]({url}) as "
-                f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid})."
-            )
+            return await evt.reply(f"[Click here to log in]({url}) as "
+                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
         elif allow_matrix_login:
-            return await evt.reply(
-                f"[Click here to log in]({url}). Alternatively, send your phone"
-                f" number (or bot auth token) here to log in.\n\n{nb}"
-            )
+            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
+                                   f" number (or bot auth token) here to log in.\n\n{nb}")
         return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
     elif allow_matrix_login:
         if override_sender:
             return await evt.reply(
                 "This bridge instance does not allow you to log in outside of Matrix. "
-                "Logging in as another user inside Matrix is not currently possible."
-            )
-        return await evt.reply(
-            "Please send your phone number (or bot auth token) here to start "
-            f"the login process.\n\n{nb}"
-        )
+                "Logging in as another user inside Matrix is not currently possible.")
+        return await evt.reply("Please send your phone number (or bot auth token) here to start "
+                               f"the login process.\n\n{nb}")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 
-async def _request_code(
-    evt: CommandEvent, phone_number: str, next_status: dict[str, Any]
-) -> EventID:
+async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
+                        ) -> EventID:
     ok = False
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
@@ -281,44 +152,38 @@ async def _request_code(
         ok = True
         return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
     except PhoneNumberAppSignupForbiddenError:
-        return await evt.reply("Your phone number does not allow 3rd party apps to sign in.")
+        return await evt.reply(
+            "Your phone number does not allow 3rd party apps to sign in.")
     except PhoneNumberFloodError:
         return await evt.reply(
             "Your phone number has been temporarily blocked for flooding. "
-            "The ban is usually applied for around a day."
-        )
+            "The ban is usually applied for around a day.")
     except FloodWaitError as e:
         return await evt.reply(
             "Your phone number has been temporarily blocked for flooding. "
-            f"Please wait for {fmt_duration(e.seconds)} before trying again."
-        )
+            f"Please wait for {format_duration(e.seconds)} before trying again.")
     except PhoneNumberBannedError:
-        return await evt.reply("Your phone number has been banned from Telegram.")
+        return await  evt.reply("Your phone number has been banned from Telegram.")
     except PhoneNumberUnoccupiedError:
-        return await evt.reply(
-            "That phone number has not been registered. "
-            "Please register with `$cmdprefix+sp register <phone>`."
-        )
+        return await  evt.reply("That phone number has not been registered. "
+                                "Please register with `$cmdprefix+sp register <phone>`.")
     except PhoneNumberInvalidError:
         return await evt.reply("That phone number is not valid.")
     except Exception:
         evt.log.exception("Error requesting phone code")
-        return await evt.reply(
-            "Unhandled exception while requesting code. Check console for more details."
-        )
+        return await evt.reply("Unhandled exception while requesting code. "
+                               "Check console for more details.")
     finally:
         evt.sender.command_status = next_status if ok else None
 
 
 @command_handler(needs_auth=False)
-async def enter_phone_or_token(evt: CommandEvent) -> EventID | None:
+async def enter_phone_or_token(evt: CommandEvent) -> Optional[EventID]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply(
-            "This bridge instance does not allow in-Matrix login. "
-            "Please use `$cmdprefix+sp login` to get login instructions"
-        )
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
 
     # phone numbers don't contain colons but telegram bot auth tokens do
     if evt.args[0].find(":") > 0:
@@ -326,66 +191,66 @@ async def enter_phone_or_token(evt: CommandEvent) -> EventID | None:
             await _sign_in(evt, bot_token=evt.args[0])
         except Exception:
             evt.log.exception("Error sending auth token")
-            return await evt.reply(
-                "Unhandled exception while sending auth token. Check console for more details."
-            )
+            return await evt.reply("Unhandled exception while sending auth token. "
+                                   "Check console for more details.")
     else:
-        await _request_code(evt, evt.args[0], {"next": enter_code, "action": "Login"})
+        await _request_code(evt, evt.args[0], {
+            "next": enter_code,
+            "action": "Login",
+        })
     return None
 
 
 @command_handler(needs_auth=False)
-async def enter_code(evt: CommandEvent) -> EventID | None:
+async def enter_code(evt: CommandEvent) -> Optional[EventID]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply(
-            "This bridge instance does not allow in-Matrix login. "
-            "Please use `$cmdprefix+sp login` to get login instructions"
-        )
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
     try:
         await _sign_in(evt, code=evt.args[0])
     except Exception:
         evt.log.exception("Error sending phone code")
-        return await evt.reply(
-            "Unhandled exception while sending code. Check console for more details."
-        )
+        return await evt.reply("Unhandled exception while sending code. "
+                               "Check console for more details.")
     return None
 
 
 @command_handler(needs_auth=False)
-async def enter_password(evt: CommandEvent) -> EventID | None:
+async def enter_password(evt: CommandEvent) -> Optional[EventID]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply(
-            "This bridge instance does not allow in-Matrix login. "
-            "Please use `$cmdprefix+sp login` to get login instructions"
-        )
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await _sign_in(
-            evt,
-            login_as=evt.sender.command_status.get("login_as", None),
-            password=" ".join(evt.args),
-        )
+        await _sign_in(evt, password=" ".join(evt.args))
     except AccessTokenInvalidError:
         return await evt.reply("That bot token is not valid.")
     except AccessTokenExpiredError:
         return await evt.reply("That bot token has expired.")
     except Exception:
         evt.log.exception("Error sending password")
-        return await evt.reply(
-            "Unhandled exception while sending password. Check console for more details."
-        )
+        return await evt.reply("Unhandled exception while sending password. "
+                               "Check console for more details.")
     return None
 
 
-async def _sign_in(evt: CommandEvent, login_as: u.User = None, **sign_in_info) -> EventID:
-    login_as = login_as or evt.sender
+async def _sign_in(evt: CommandEvent, **sign_in_info) -> EventID:
     try:
-        await login_as.ensure_started(even_if_no_session=True)
-        user = await login_as.client.sign_in(**sign_in_info)
-        await _finish_sign_in(evt, user)
+        await evt.sender.ensure_started(even_if_no_session=True)
+        user = await evt.sender.client.sign_in(**sign_in_info)
+        existing_user = u.User.get_by_tgid(user.id)
+        if existing_user and existing_user != evt.sender:
+            await existing_user.log_out()
+            await evt.reply(f"[{existing_user.displayname}]"
+                            f"(https://matrix.to/#/{existing_user.mxid})"
+                            " was logged out from the account.")
+        asyncio.ensure_future(evt.sender.post_login(user, first_login=True), loop=evt.loop)
+        evt.sender.command_status = None
+        name = f"@{user.username}" if user.username else f"+{user.phone}"
+        return await evt.reply(f"Successfully logged in as {name}")
     except PhoneCodeExpiredError:
         return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
     except PhoneCodeInvalidError:
@@ -397,37 +262,14 @@ async def _sign_in(evt: CommandEvent, login_as: u.User = None, **sign_in_info) -
             "next": enter_password,
             "action": "Login (password entry)",
         }
-        return await evt.reply(
-            "Your account has two-factor authentication. Please send your password here."
-        )
+        return await evt.reply("Your account has two-factor authentication. "
+                               "Please send your password here.")
 
 
-async def _finish_sign_in(evt: CommandEvent, user: User, login_as: u.User = None) -> EventID:
-    login_as = login_as or evt.sender
-    existing_user = await u.User.get_by_tgid(TelegramID(user.id))
-    if existing_user and existing_user != login_as:
-        await existing_user.log_out()
-        await evt.reply(
-            f"[{existing_user.displayname}] (https://matrix.to/#/{existing_user.mxid})"
-            " was logged out from the account."
-        )
-    asyncio.create_task(login_as.post_login(user, first_login=True))
-    evt.sender.command_status = None
-    name = f"@{user.username}" if user.username else f"+{user.phone}"
-    if login_as != evt.sender:
-        msg = (
-            f"Successfully logged in [{login_as.mxid}](https://matrix.to/#/{login_as.mxid})"
-            f" as {name}"
-        )
-    else:
-        msg = f"Successfully logged in as {name}"
-    return await evt.reply(msg)
-
-
-@command_handler(needs_auth=False, help_section=SECTION_AUTH, help_text="Log out from Telegram.")
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Log out from Telegram.")
 async def logout(evt: CommandEvent) -> EventID:
-    if not evt.sender.tgid:
-        return await evt.reply("You're not logged in")
     if await evt.sender.log_out():
         return await evt.reply("Logged out successfully.")
     return await evt.reply("Failed to log out.")

mautrix_telegram/commands/telegram/misc.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,62 +13,34 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import cast
-import base64
+from typing import List, Optional, Tuple, cast
+import logging
 import codecs
+import base64
 import re
 
-from aiohttp import ClientSession, InvalidURL
-from telethon.errors import (
-    ChatIdInvalidError,
-    EmoticonInvalidError,
-    InviteHashExpiredError,
-    InviteHashInvalidError,
-    OptionsTooMuchError,
-    TakeoutInitDelayError,
-    UserAlreadyParticipantError,
-)
-from telethon.tl.functions.channels import JoinChannelRequest
-from telethon.tl.functions.messages import (
-    CheckChatInviteRequest,
-    GetBotCallbackAnswerRequest,
-    ImportChatInviteRequest,
-    SendVoteRequest,
-)
+from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
+                             UserAlreadyParticipantError, ChatIdInvalidError)
 from telethon.tl.patched import Message
-from telethon.tl.types import (
-    InputMediaDice,
-    MessageMediaGame,
-    MessageMediaPoll,
-    TypeInputPeer,
-    TypeUpdates,
-    User as TLUser,
-)
+from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
+                               TypeInputPeer)
 from telethon.tl.types.messages import BotCallbackAnswer
+from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
+                                            GetBotCallbackAnswerRequest, SendVoteRequest)
+from telethon.tl.functions.channels import JoinChannelRequest
 
 from mautrix.types import EventID, Format
 
-from ... import portal as po, puppet as pu
+from ... import puppet as pu, portal as po
 from ...abstract_user import AbstractUser
-from ...commands import (
-    SECTION_CREATING_PORTALS,
-    SECTION_MISC,
-    SECTION_PORTAL_MANAGEMENT,
-    CommandEvent,
-    command_handler,
-)
 from ...db import Message as DBMessage
 from ...types import TelegramID
+from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
 
 
-@command_handler(
-    needs_auth=False,
-    help_section=SECTION_MISC,
-    help_args="<_caption_>",
-    help_text="Set a caption for the next image you send",
-)
+@command_handler(needs_auth=False,
+                 help_section=SECTION_MISC, help_args="<_caption_>",
+                 help_text="Set a caption for the next image you send")
 async def caption(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp caption <caption>`")
@@ -78,17 +50,13 @@ async def caption(evt: CommandEvent) -> EventID:
         evt.content.formatted_body = evt.content.formatted_body.replace(prefix, "", 1)
     evt.content.body = evt.content.body.replace(prefix, "", 1)
     evt.sender.command_status = {"caption": evt.content, "action": "Caption"}
-    return await evt.reply(
-        "Your next image or file will be sent with that caption. "
-        "Use `$cmdprefix+sp cancel` to cancel the caption."
-    )
+    return await evt.reply("Your next image or file will be sent with that caption. "
+                           "Use `$cmdprefix+sp cancel` to cancel the caption.")
 
 
-@command_handler(
-    help_section=SECTION_MISC,
-    help_args="[_-r|--remote_] <_query_>",
-    help_text="Search your contacts or the Telegram servers for users.",
-)
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[_-r|--remote_] <_query_>",
+                 help_text="Search your contacts or the Telegram servers for users.")
 async def search(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
@@ -106,44 +74,35 @@ async def search(evt: CommandEvent) -> EventID:
 
     if not results:
         if len(query) < 5 and remote:
-            return await evt.reply(
-                "No local results. Minimum length of remote query is 5 characters."
-            )
+            return await evt.reply("No local results. "
+                                   "Minimum length of remote query is 5 characters.")
         return await evt.reply("No results 3:")
 
-    reply: list[str] = []
+    reply: List[str] = []
     if remote:
         reply += ["**Results from Telegram server:**", ""]
     else:
         reply += ["**Results in contacts:**", ""]
-    reply += [
-        (
-            f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
-            f"{puppet.id} ({similarity}% match)"
-        )
-        for puppet, similarity in results
-    ]
+    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
+               f"{puppet.id} ({similarity}% match)")
+              for puppet, similarity in results]
 
     # TODO somehow show remote channel results when joining by alias is possible?
 
     return await evt.reply("\n".join(reply))
 
 
-@command_handler(
-    help_section=SECTION_CREATING_PORTALS,
-    help_args="<_identifier_>",
-    help_text="Open a private chat with the given Telegram user. The identifier is "
-    "either the internal user ID, the username or the phone number. "
-    "**N.B.** The phone numbers you start chats with must already be in "
-    "your contacts.",
-)
+@command_handler(help_section=SECTION_CREATING_PORTALS, help_args="<_identifier_>",
+                 help_text="Open a private chat with the given Telegram user. The identifier is "
+                           "either the internal user ID, the username or the phone number. "
+                           "**N.B.** The phone numbers you start chats with must already be in "
+                           "your contacts.")
 async def pm(evt: CommandEvent) -> EventID:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
 
     try:
-        id = "".join(evt.args).translate({ord(c): None for c in "+()- "})
-        user = await evt.sender.client.get_entity(id)
+        user = await evt.sender.client.get_entity(evt.args[0])
     except ValueError:
         return await evt.reply("Invalid user identifier or user not found.")
 
@@ -151,69 +110,50 @@ async def pm(evt: CommandEvent) -> EventID:
         return await evt.reply("User not found.")
     elif not isinstance(user, TLUser):
         return await evt.reply("That doesn't seem to be a user.")
-    portal = await po.Portal.get_by_entity(user, tg_receiver=evt.sender.tgid)
+    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
     await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    displayname, _ = pu.Puppet.get_displayname(user, False)
-    return await evt.reply(f"Created private chat room with {displayname}")
+    return await evt.reply("Created private chat room with "
+                           f"{pu.Puppet.get_displayname(user, False)}")
 
 
-async def _join(
-    evt: CommandEvent, identifier: str, link_type: str
-) -> tuple[TypeUpdates | None, EventID | None]:
-    if link_type == "joinchat":
+async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[EventID]]:
+    if arg.startswith("joinchat/"):
+        invite_hash = arg[len("joinchat/"):]
         try:
-            await evt.sender.client(CheckChatInviteRequest(identifier))
+            await evt.sender.client(CheckChatInviteRequest(invite_hash))
         except InviteHashInvalidError:
             return None, await evt.reply("Invalid invite link.")
         except InviteHashExpiredError:
             return None, await evt.reply("Invite link expired.")
         try:
-            return (await evt.sender.client(ImportChatInviteRequest(identifier))), None
+            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
         except UserAlreadyParticipantError:
             return None, await evt.reply("You are already in that chat.")
     else:
-        channel = await evt.sender.client.get_entity(identifier)
+        channel = await evt.sender.client.get_entity(arg)
         if not channel:
             return None, await evt.reply("Channel/supergroup not found.")
         return await evt.sender.client(JoinChannelRequest(channel)), None
 
 
-@command_handler(
-    help_section=SECTION_CREATING_PORTALS,
-    help_args="<_link_>",
-    help_text="Join a chat with an invite link.",
-)
-async def join(evt: CommandEvent) -> EventID | None:
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_link_>",
+                 help_text="Join a chat with an invite link.")
+async def join(evt: CommandEvent) -> Optional[EventID]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
 
-    url = evt.args[0]
-    if evt.config["bridge.invite_link_resolve"]:
-        try:
-            async with ClientSession() as sess, sess.get(url) as resp:
-                url = str(resp.url)
-        except InvalidURL:
-            return await evt.reply("That doesn't look like a Telegram invite link.")
-
-    regex = re.compile(
-        r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:/(?P<type>joinchat|s))?/(?P<id>[^/]+)/?",
-        flags=re.IGNORECASE,
-    )
-    arg = regex.match(url)
+    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
+    arg = regex.match(evt.args[0])
     if not arg:
         return await evt.reply("That doesn't look like a Telegram invite link.")
 
-    data = arg.groupdict()
-    identifier = data["id"]
-    link_type = data["type"]
-    if link_type:
-        link_type = link_type.lower()
-    updates, _ = await _join(evt, identifier, link_type)
+    updates, _ = await _join(evt, arg.group(1))
     if not updates:
         return None
 
     for chat in updates.chats:
-        portal = await po.Portal.get_by_entity(chat)
+        portal = po.Portal.get_by_entity(chat)
         if portal.mxid:
             await portal.invite_to_matrix([evt.sender.mxid])
             return await evt.reply(f"Invited you to portal of {portal.title}")
@@ -222,20 +162,15 @@ async def join(evt: CommandEvent) -> EventID | None:
             try:
                 await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
             except ChatIdInvalidError as e:
-                evt.log.trace(
-                    "ChatIdInvalidError while creating portal from !tg join command: %s",
-                    updates.stringify(),
-                )
+                logging.getLogger("mau.commands").info(updates.stringify())
                 raise e
             return await evt.reply(f"Created room for {portal.title}")
     return None
 
 
-@command_handler(
-    help_section=SECTION_MISC,
-    help_args="[`chats`|`contacts`|`me`]",
-    help_text="Synchronize your chat portals, contacts and/or own info.",
-)
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[`chats`|`contacts`|`me`]",
+                 help_text="Synchronize your chat portals, contacts and/or own info.")
 async def sync(evt: CommandEvent) -> EventID:
     if len(evt.args) > 0:
         sync_only = evt.args[0]
@@ -245,10 +180,8 @@ async def sync(evt: CommandEvent) -> EventID:
         sync_only = None
 
     if not sync_only or sync_only == "chats":
-        await evt.reply("Synchronizing chats...")
-        await evt.sender.sync_dialogs()
+        await evt.sender.sync_dialogs(synchronous_create=True)
     if not sync_only or sync_only == "contacts":
-        await evt.reply("Synchronizing contacts...")
         await evt.sender.sync_contacts()
     if not sync_only or sync_only == "me":
         await evt.sender.update_info()
@@ -264,9 +197,8 @@ class MessageIDError(ValueError):
         self.message = message
 
 
-async def _parse_encoded_msgid(
-    user: AbstractUser, enc_id: str, type_name: str
-) -> tuple[TypeInputPeer, Message]:
+async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
+                               ) -> Tuple[TypeInputPeer, Message]:
     try:
         enc_id += (4 - len(enc_id) % 4) * "="
         enc_id = base64.b64decode(enc_id)
@@ -280,10 +212,10 @@ async def _parse_encoded_msgid(
         raise MessageIDError(f"Invalid {type_name} ID (format)") from e
 
     if peer_type == PEER_TYPE_CHAT:
-        orig_msg = await DBMessage.get_one_by_tgid(msg_id, space)
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
         if not orig_msg:
             raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
-        new_msg = await DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
+        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
         if not new_msg:
             raise MessageIDError(f"Invalid {type_name} ID (your copy of message not found in db)")
         msg_id = new_msg.tgid
@@ -298,9 +230,9 @@ async def _parse_encoded_msgid(
     return peer, cast(Message, msg)
 
 
-@command_handler(
-    help_section=SECTION_MISC, help_args="<_play ID_>", help_text="Play a Telegram game."
-)
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_play ID_>",
+                 help_text="Play a Telegram game.")
 async def play(evt: CommandEvent) -> EventID:
     if len(evt.args) < 1:
         return await evt.reply("**Usage:** `$cmdprefix+sp play <play ID>`")
@@ -318,23 +250,18 @@ async def play(evt: CommandEvent) -> EventID:
         return await evt.reply("Invalid play ID (message doesn't look like a game)")
 
     game = await evt.sender.client(
-        GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True)
-    )
+        GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
     if not isinstance(game, BotCallbackAnswer):
         return await evt.reply("Game request response invalid")
 
-    return await evt.reply(
-        f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
-        f"{msg.media.game.description}"
-    )
+    return await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
+                           f"{msg.media.game.description}")
 
 
-@command_handler(
-    help_section=SECTION_MISC,
-    help_args="<_poll ID_> <_choice number_>",
-    help_text="Vote in a Telegram poll.",
-)
-async def vote(evt: CommandEvent) -> EventID | None:
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_poll ID_> <_choice number_>",
+                 help_text="Vote in a Telegram poll.")
+async def vote(evt: CommandEvent) -> EventID:
     if len(evt.args) < 1:
         return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
     elif not await evt.sender.is_logged_in():
@@ -359,82 +286,20 @@ async def vote(evt: CommandEvent) -> EventID | None:
         except ValueError:
             option_index = None
         if option_index is None:
-            return await evt.reply(
-                f'Invalid option number "{option}"', render_markdown=False, allow_html=False
-            )
+            return await evt.reply(f"Invalid option number \"{option}\"",
+                                   render_markdown=False, allow_html=False)
         elif option_index < 0:
-            return await evt.reply(
-                f"Invalid option number {option}. Option numbers must be positive."
-            )
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"Option numbers must be positive.")
         elif option_index >= len(msg.media.poll.answers):
-            return await evt.reply(
-                f"Invalid option number {option}. "
-                f"The poll only has {len(msg.media.poll.answers)} options."
-            )
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"The poll only has {len(msg.media.poll.answers)} options.")
         options.append(msg.media.poll.answers[option_index].option)
-    options = [msg.media.poll.answers[int(option) - 1].option for option in evt.args[1:]]
+    options = [msg.media.poll.answers[int(option) - 1].option
+               for option in evt.args[1:]]
     try:
-        await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
+        resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
     except OptionsTooMuchError:
         return await evt.reply("You passed too many options.")
     # TODO use response
     return await evt.mark_read()
-
-
-@command_handler(
-    help_section=SECTION_MISC,
-    help_args="<_emoji_>",
-    help_text="Roll a dice (\U0001F3B2), kick a football (\u26BD\uFE0F) or throw a "
-    "dart (\U0001F3AF) or basketball (\U0001F3C0) on the Telegram servers.",
-)
-async def random(evt: CommandEvent) -> EventID:
-    if not evt.is_portal:
-        return await evt.reply("You can only randomize values in portal rooms")
-    portal = await po.Portal.get_by_mxid(evt.room_id)
-    arg = evt.args[0] if len(evt.args) > 0 else "dice"
-    emoticon = {
-        "dart": "\U0001F3AF",
-        "dice": "\U0001F3B2",
-        "ball": "\U0001F3C0",
-        "basketball": "\U0001F3C0",
-        "football": "\u26BD",
-        "soccer": "\u26BD",
-    }.get(arg, arg)
-    try:
-        await evt.sender.client.send_media(
-            await portal.get_input_entity(evt.sender), InputMediaDice(emoticon)
-        )
-    except EmoticonInvalidError:
-        return await evt.reply("Invalid emoji for randomization")
-
-
-@command_handler(
-    help_section=SECTION_PORTAL_MANAGEMENT,
-    help_args="[_limit_]",
-    help_text="Backfill messages from Telegram history.",
-)
-async def backfill(evt: CommandEvent) -> None:
-    if not evt.is_portal:
-        await evt.reply("You can only use backfill in portal rooms")
-        return
-    try:
-        limit = int(evt.args[0])
-    except (ValueError, IndexError):
-        limit = -1
-    portal = await po.Portal.get_by_mxid(evt.room_id)
-    if not evt.config["bridge.backfill.normal_groups"] and portal.peer_type == "chat":
-        await evt.reply("Backfilling normal groups is disabled in the bridge config")
-        return
-    try:
-        await portal.backfill(evt.sender, limit=limit)
-    except TakeoutInitDelayError:
-        msg = (
-            "Please accept the data export request from a mobile device, "
-            "then re-run the backfill command."
-        )
-        if portal.peer_type == "user":
-            from mautrix.appservice import IntentAPI
-
-            await portal.main_intent.send_notice(evt.room_id, msg)
-        else:
-            await evt.reply(msg)

mautrix_telegram/config.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,25 +13,17 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Any, List, NamedTuple
+from typing import Any, Dict, List, NamedTuple
+from ruamel.yaml.comments import CommentedMap
 import os
 
-from ruamel.yaml.comments import CommentedMap
-
-from mautrix.bridge.config import BaseBridgeConfig
-from mautrix.client import Client
 from mautrix.types import UserID
-from mautrix.util.config import ConfigUpdateHelper, ForbiddenDefault, ForbiddenKey
+from mautrix.client import Client
+from mautrix.bridge.config import (BaseBridgeConfig, ConfigUpdateHelper, ForbiddenKey,
+                                   ForbiddenDefault)
 
-Permissions = NamedTuple(
-    "Permissions",
-    relaybot=bool,
-    user=bool,
-    puppeting=bool,
-    matrix_puppeting=bool,
-    admin=bool,
-    level=str,
-)
+Permissions = NamedTuple("Permissions", relaybot=bool, user=bool, puppeting=bool,
+                         matrix_puppeting=bool, admin=bool, level=str)
 
 
 class Config(BaseBridgeConfig):
@@ -45,38 +37,31 @@ class Config(BaseBridgeConfig):
     def forbidden_defaults(self) -> List[ForbiddenDefault]:
         return [
             *super().forbidden_defaults,
-            ForbiddenDefault(
-                "appservice.database",
-                "postgres://username:password@hostname/dbname",
-            ),
-            ForbiddenDefault(
-                "appservice.public.external",
-                "https://example.com/public",
-                condition="appservice.public.enabled",
-            ),
+            ForbiddenDefault("appservice.public.external", "https://example.com/public",
+                             condition="appservice.public.enabled"),
             ForbiddenDefault("bridge.permissions", ForbiddenKey("example.com")),
             ForbiddenDefault("telegram.api_id", 12345),
             ForbiddenDefault("telegram.api_hash", "tjyd5yge35lbodk1xwzw2jstp90k55qz"),
         ]
 
     def do_update(self, helper: ConfigUpdateHelper) -> None:
-        super().do_update(helper)
         copy, copy_dict, base = helper
 
-        copy("homeserver.asmux")
+        copy("homeserver.address")
+        copy("homeserver.domain")
+        copy("homeserver.verify_ssl")
 
         if "appservice.protocol" in self and "appservice.address" not in self:
-            protocol, hostname, port = (
-                self["appservice.protocol"],
-                self["appservice.hostname"],
-                self["appservice.port"],
-            )
+            protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
+                                        self["appservice.port"])
             base["appservice.address"] = f"{protocol}://{hostname}:{port}"
-        if "appservice.debug" in self and "logging" not in self:
-            level = "DEBUG" if self["appservice.debug"] else "INFO"
-            base["logging.root.level"] = level
-            base["logging.loggers.mau.level"] = level
-            base["logging.loggers.telethon.level"] = level
+        else:
+            copy("appservice.address")
+        copy("appservice.hostname")
+        copy("appservice.port")
+        copy("appservice.max_body_size")
+
+        copy("appservice.database")
 
         copy("appservice.public.enabled")
         copy("appservice.public.prefix")
@@ -88,36 +73,35 @@ class Config(BaseBridgeConfig):
         if base["appservice.provisioning.shared_secret"] == "generate":
             base["appservice.provisioning.shared_secret"] = self._new_token()
 
-        if "pool_size" in base["appservice.database_opts"]:
-            pool_size = base["appservice.database_opts"].pop("pool_size")
-            base["appservice.database_opts.min_size"] = pool_size
-            base["appservice.database_opts.max_size"] = pool_size
-        if "pool_pre_ping" in base["appservice.database_opts"]:
-            del base["appservice.database_opts.pool_pre_ping"]
+        copy("appservice.id")
+        copy("appservice.bot_username")
+        copy("appservice.bot_displayname")
+        copy("appservice.bot_avatar")
 
         copy("appservice.community_id")
 
+        copy("appservice.as_token")
+        copy("appservice.hs_token")
+
         copy("metrics.enabled")
         copy("metrics.listen_port")
 
+        copy("manhole.enabled")
+        copy("manhole.path")
+        copy("manhole.whitelist")
+
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
 
         copy("bridge.displayname_preference")
         copy("bridge.displayname_max_length")
-        copy("bridge.allow_avatar_remove")
 
         copy("bridge.max_initial_member_sync")
         copy("bridge.sync_channel_members")
         copy("bridge.skip_deleted_members")
         copy("bridge.startup_sync")
-        if "bridge.sync_dialog_limit" in self:
-            base["bridge.sync_create_limit"] = self["bridge.sync_dialog_limit"]
-            base["bridge.sync_update_limit"] = self["bridge.sync_dialog_limit"]
-        else:
-            copy("bridge.sync_update_limit")
-            copy("bridge.sync_create_limit")
+        copy("bridge.sync_dialog_limit")
         copy("bridge.sync_direct_chats")
         copy("bridge.max_telegram_delete")
         copy("bridge.sync_matrix_state")
@@ -125,17 +109,8 @@ class Config(BaseBridgeConfig):
         copy("bridge.plaintext_highlights")
         copy("bridge.public_portals")
         copy("bridge.sync_with_custom_puppets")
-        copy("bridge.sync_direct_chat_list")
-        copy("bridge.double_puppet_server_map")
-        copy("bridge.double_puppet_allow_discovery")
-        if "bridge.login_shared_secret" in self:
-            base["bridge.login_shared_secret_map"] = {
-                base["homeserver.domain"]: self["bridge.login_shared_secret"]
-            }
-        else:
-            copy("bridge.login_shared_secret_map")
+        copy("bridge.login_shared_secret")
         copy("bridge.telegram_link_preview")
-        copy("bridge.invite_link_resolve")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
         copy("bridge.max_document_size")
@@ -143,28 +118,6 @@ class Config(BaseBridgeConfig):
         copy("bridge.federate_rooms")
         copy("bridge.animated_sticker.target")
         copy("bridge.animated_sticker.args")
-        copy("bridge.encryption.allow")
-        copy("bridge.encryption.default")
-        copy("bridge.encryption.database")
-        copy("bridge.encryption.key_sharing.allow")
-        copy("bridge.encryption.key_sharing.require_cross_signing")
-        copy("bridge.encryption.key_sharing.require_verification")
-        copy("bridge.private_chat_portal_meta")
-        copy("bridge.delivery_receipts")
-        copy("bridge.delivery_error_reports")
-        copy("bridge.resend_bridge_info")
-        copy("bridge.mute_bridging")
-        copy("bridge.pinned_tag")
-        copy("bridge.archive_tag")
-        copy("bridge.tag_only_on_create")
-        copy("bridge.bridge_matrix_leave")
-        copy("bridge.kick_on_logout")
-        copy("bridge.backfill.invite_own_puppet")
-        copy("bridge.backfill.takeout_limit")
-        copy("bridge.backfill.initial_limit")
-        copy("bridge.backfill.missed_limit")
-        copy("bridge.backfill.disable_notifications")
-        copy("bridge.backfill.normal_groups")
 
         copy("bridge.initial_power_level_overrides.group")
         copy("bridge.initial_power_level_overrides.user")
@@ -195,11 +148,9 @@ class Config(BaseBridgeConfig):
 
         copy("bridge.command_prefix")
 
-        migrate_permissions = (
-            "bridge.permissions" not in self
-            or "bridge.whitelist" in self
-            or "bridge.admins" in self
-        )
+        migrate_permissions = ("bridge.permissions" not in self
+                               or "bridge.whitelist" in self
+                               or "bridge.admins" in self)
         if migrate_permissions:
             permissions = self["bridge.permissions"] or CommentedMap()
             for entry in self["bridge.whitelist"] or []:
@@ -251,6 +202,14 @@ class Config(BaseBridgeConfig):
         copy("telegram.proxy.username")
         copy("telegram.proxy.password")
 
+        if "appservice.debug" in self and "logging" not in self:
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
+            base["logging.root.level"] = level
+            base["logging.loggers.mau.level"] = level
+            base["logging.loggers.telethon.level"] = level
+        else:
+            copy("logging")
+
     def _get_permissions(self, key: str) -> Permissions:
         level = self["bridge.permissions"].get(key, "")
         admin = level == "admin"
@@ -270,3 +229,24 @@ class Config(BaseBridgeConfig):
             return self._get_permissions(homeserver)
 
         return self._get_permissions("*")
+
+    @property
+    def namespaces(self) -> Dict[str, List[Dict[str, Any]]]:
+        homeserver = self["homeserver.domain"]
+
+        username_format = self["bridge.username_template"].format(userid=".+")
+        alias_format = self["bridge.alias_template"].format(groupname=".+")
+        group_id = ({"group_id": self["appservice.community_id"]}
+                    if self["appservice.community_id"] else {})
+
+        return {
+            "users": [{
+                "exclusive": True,
+                "regex": f"@{username_format}:{homeserver}",
+                **group_id,
+            }],
+            "aliases": [{
+                "exclusive": True,
+                "regex": f"#{alias_format}:{homeserver}",
+            }]
+        }

mautrix_telegram/context.py

@@ -0,0 +1,57 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Tuple, TYPE_CHECKING
+import asyncio
+
+from alchemysession import AlchemySessionContainer
+
+from mautrix.appservice import AppService
+
+if TYPE_CHECKING:
+    from .web import PublicBridgeWebsite, ProvisioningAPI
+    from .config import Config
+    from .bot import Bot
+    from .matrix import MatrixHandler
+    from .__main__ import TelegramBridge
+
+
+class Context:
+    az: AppService
+    config: 'Config'
+    loop: asyncio.AbstractEventLoop
+    bridge: 'TelegramBridge'
+    bot: Optional['Bot']
+    mx: Optional['MatrixHandler']
+    session_container: AlchemySessionContainer
+    public_website: Optional['PublicBridgeWebsite']
+    provisioning_api: Optional['ProvisioningAPI']
+
+    def __init__(self, az: AppService, config: 'Config', loop: asyncio.AbstractEventLoop,
+                 session_container: AlchemySessionContainer, bridge: 'TelegramBridge',
+                 bot: Optional['Bot']) -> None:
+        self.az = az
+        self.config = config
+        self.loop = loop
+        self.bridge = bridge
+        self.bot = bot
+        self.mx = None
+        self.session_container = session_container
+        self.public_website = None
+        self.provisioning_api = None
+
+    @property
+    def core(self) -> Tuple[AppService, 'Config', asyncio.AbstractEventLoop, Optional['Bot']]:
+        return self.az, self.config, self.loop, self.bot

mautrix_telegram/db/__init__.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,31 +13,22 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix.util.async_db import Database
+from sqlalchemy.engine.base import Engine
+
+from mautrix.bridge.db import UserProfile, RoomState
 
 from .bot_chat import BotChat
 from .message import Message
 from .portal import Portal
 from .puppet import Puppet
 from .telegram_file import TelegramFile
-from .telethon_session import PgSession
-from .upgrade import upgrade_table
-from .user import User
+from .user import User, UserPortal, Contact
 
 
-def init(db: Database) -> None:
-    for table in (Portal, Message, User, Puppet, TelegramFile, BotChat, PgSession):
-        table.db = db
-
-
-__all__ = [
-    "upgrade_table",
-    "init",
-    "Portal",
-    "Message",
-    "User",
-    "Puppet",
-    "TelegramFile",
-    "BotChat",
-    "PgSession",
-]
+def init(db_engine: Engine) -> None:
+    for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
+                  RoomState, BotChat):
+        table.db = db_engine
+        table.t = table.__table__
+        table.c = table.t.c
+        table.column_names = table.c.keys()

mautrix_telegram/db/bot_chat.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,43 +13,26 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Iterable
 
-from typing import TYPE_CHECKING, ClassVar
+from sqlalchemy import Column, Integer, String
 
-from asyncpg import Record
-from attr import dataclass
-
-from mautrix.util.async_db import Database
+from mautrix.util.db import Base
 
 from ..types import TelegramID
 
-fake_db = Database.create("") if TYPE_CHECKING else None
-
 
 # Fucking Telegram not telling bots what chats they are in 3:<
-@dataclass
-class BotChat:
-    db: ClassVar[Database] = fake_db
-
-    id: TelegramID
-    type: str
+class BotChat(Base):
+    __tablename__ = "bot_chat"
+    id: TelegramID = Column(Integer, primary_key=True)
+    type: str = Column(String, nullable=False)
 
     @classmethod
-    def _from_row(cls, row: Record | None) -> BotChat | None:
-        if row is None:
-            return None
-        return cls(**row)
+    def delete_by_id(cls, chat_id: TelegramID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.id == chat_id))
 
     @classmethod
-    async def delete_by_id(cls, chat_id: TelegramID) -> None:
-        await cls.db.execute("DELETE FROM bot_chat WHERE id=$1", chat_id)
-
-    @classmethod
-    async def all(cls) -> list[BotChat]:
-        rows = await cls.db.fetch("SELECT id, type FROM bot_chat")
-        return [cls._from_row(row) for row in rows]
-
-    async def insert(self) -> None:
-        q = "INSERT INTO bot_chat (id, type) VALUES ($1, $2)"
-        await self.db.execute(q, self.id, self.type)
+    def all(cls) -> Iterable['BotChat']:
+        return cls._select_all()

mautrix_telegram/db/message.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,149 +13,72 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Optional, Iterator
 
-from typing import TYPE_CHECKING, ClassVar
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
 
-from asyncpg import Record
-from attr import dataclass
-
-from mautrix.types import EventID, RoomID
-from mautrix.util.async_db import Database
+from mautrix.types import RoomID, EventID
+from mautrix.util.db import Base
 
 from ..types import TelegramID
 
-fake_db = Database.create("") if TYPE_CHECKING else None
 
+class Message(Base):
+    __tablename__ = "message"
 
-@dataclass
-class Message:
-    db: ClassVar[Database] = fake_db
+    mxid: EventID = Column(String)
+    mx_room: RoomID = Column(String)
+    tgid: TelegramID = Column(Integer, primary_key=True)
+    tg_space: TelegramID = Column(Integer, primary_key=True)
+    edit_index: int = Column(Integer, primary_key=True)
 
-    mxid: EventID
-    mx_room: RoomID
-    tgid: TelegramID
-    tg_space: TelegramID
-    edit_index: int
-    redacted: bool = False
+    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"),)
 
     @classmethod
-    def _from_row(cls, row: Record | None) -> Message | None:
-        if row is None:
-            return None
-        return cls(**row)
-
-    columns: ClassVar[str] = "mxid, mx_room, tgid, tg_space, edit_index, redacted"
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Iterator['Message']:
+        return cls._select_all(cls.c.tgid == tgid, cls.c.tg_space == tg_space)
 
     @classmethod
-    async def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> list[Message]:
-        q = f"SELECT {cls.columns} FROM message WHERE tgid=$1 AND tg_space=$2"
-        rows = await cls.db.fetch(q, tgid, tg_space)
-        return [cls._from_row(row) for row in rows]
-
-    @classmethod
-    async def get_one_by_tgid(
-        cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
-    ) -> Message | None:
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
         if edit_index < 0:
-            q = (
-                f"SELECT {cls.columns} FROM message WHERE tgid=$1 AND tg_space=$2 "
-                f"ORDER BY edit_index DESC LIMIT 1 OFFSET {-edit_index - 1}"
-            )
-            row = await cls.db.fetchrow(q, tgid, tg_space)
+            return cls._one_or_none(cls.db.execute(
+                cls.t.select()
+                    .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                    .order_by(desc(cls.c.edit_index))
+                    .limit(1).offset(-edit_index - 1)))
         else:
-            q = (
-                f"SELECT {cls.columns} FROM message"
-                " WHERE tgid=$1 AND tg_space=$2 AND edit_index=$3"
-            )
-            row = await cls.db.fetchrow(q, tgid, tg_space, edit_index)
-        return cls._from_row(row)
+            return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                           cls.c.edit_index == edit_index)
 
     @classmethod
-    async def get_first_by_tgids(
-        cls, tgids: list[TelegramID], tg_space: TelegramID
-    ) -> list[Message]:
-        if cls.db.scheme == "postgres":
-            q = (
-                f"SELECT {cls.columns} FROM message"
-                " WHERE tgid=ANY($1) AND tg_space=$2 AND edit_index=0"
-            )
-            rows = await cls.db.fetch(q, tgids, tg_space)
-        else:
-            tgid_placeholders = ("?," * len(tgids)).rstrip(",")
-            q = (
-                f"SELECT {cls.columns} FROM message "
-                f"WHERE tg_space=? AND edit_index=0 AND tgid IN ({tgid_placeholders})"
-            )
-            rows = await cls.db.fetch(q, tg_space, *tgids)
-        return [cls._from_row(row) for row in rows]
+    def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
+        rows = cls.db.execute(select([func.count(cls.c.tg_space)])
+                              .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
+        try:
+            count, = next(rows)
+            return count
+        except StopIteration:
+            return 0
 
     @classmethod
-    async def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
-        return (
-            await cls.db.fetchval(
-                "SELECT COUNT(tg_space) FROM message WHERE mxid=$1 AND mx_room=$2", mxid, mx_room
-            )
-            or 0
-        )
+    def get_by_mxid(cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
+                    ) -> Optional['Message']:
+        return cls._select_one_or_none(cls.c.mxid == mxid, cls.c.mx_room == mx_room,
+                                       cls.c.tg_space == tg_space)
 
     @classmethod
-    async def find_last(cls, mx_room: RoomID, tg_space: TelegramID) -> Message | None:
-        q = (
-            f"SELECT {cls.columns} FROM message WHERE mx_room=$1 AND tg_space=$2 "
-            f"ORDER BY tgid DESC LIMIT 1"
-        )
-        return cls._from_row(await cls.db.fetchrow(q, mx_room, tg_space))
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
+                         .values(**values))
 
     @classmethod
-    async def delete_all(cls, mx_room: RoomID) -> None:
-        await cls.db.execute("DELETE FROM message WHERE mx_room=$1", mx_room)
-
-    @classmethod
-    async def get_by_mxid(
-        cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
-    ) -> Message | None:
-        q = f"SELECT {cls.columns} FROM message WHERE mxid=$1 AND mx_room=$2 AND tg_space=$3"
-        return cls._from_row(await cls.db.fetchrow(q, mxid, mx_room, tg_space))
-
-    @classmethod
-    async def get_by_mxids(
-        cls, mxids: list[EventID], mx_room: RoomID, tg_space: TelegramID
-    ) -> list[Message]:
-        if cls.db.scheme == "postgres":
-            q = (
-                f"SELECT {cls.columns} FROM message"
-                " WHERE mxid=ANY($1) AND mx_room=$2 AND tg_space=$3"
-            )
-            rows = await cls.db.fetch(q, mxids, mx_room, tg_space)
-        else:
-            mxid_placeholders = ("?," * len(mxids)).rstrip(",")
-            q = (
-                f"SELECT {cls.columns} FROM message "
-                f"WHERE mx_room=? AND tg_space=? AND mxid IN ({mxid_placeholders})"
-            )
-            rows = await cls.db.fetch(q, mx_room, tg_space, *mxids)
-        return [cls._from_row(row) for row in rows]
-
-    @classmethod
-    async def replace_temp_mxid(cls, temp_mxid: str, mx_room: RoomID, real_mxid: EventID) -> None:
-        q = "UPDATE message SET mxid=$1 WHERE mxid=$2 AND mx_room=$3"
-        await cls.db.execute(q, real_mxid, temp_mxid, mx_room)
-
-    async def insert(self) -> None:
-        q = (
-            "INSERT INTO message (mxid, mx_room, tgid, tg_space, edit_index, redacted) "
-            "VALUES ($1, $2, $3, $4, $5, $6)"
-        )
-        await self.db.execute(
-            q, self.mxid, self.mx_room, self.tgid, self.tg_space, self.edit_index, self.redacted
-        )
-
-    async def delete(self) -> None:
-        q = "DELETE FROM message WHERE mxid=$1 AND mx_room=$2 AND tg_space=$3"
-        await self.db.execute(q, self.mxid, self.mx_room, self.tg_space)
-
-    async def mark_redacted(self) -> None:
-        self.redacted = True
-        q = "UPDATE message SET redacted=true WHERE mxid=$1 AND mx_room=$2"
-        await self.db.execute(q, self.mxid, self.mx_room)
+    def update_by_mxid(cls, s_mxid: EventID, s_mx_room: RoomID, **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
+                         .values(**values))

mautrix_telegram/db/portal.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,136 +13,44 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Optional
 
-from typing import TYPE_CHECKING, Any, ClassVar
-import json
+from sqlalchemy import Column, Integer, String, Boolean, Text, func
 
-from asyncpg import Record
-from attr import dataclass
-import attr
-
-from mautrix.types import ContentURI, EventID, RoomID
-from mautrix.util.async_db import Database
+from mautrix.types import RoomID
+from mautrix.util.db import Base
 
 from ..types import TelegramID
 
-fake_db = Database.create("") if TYPE_CHECKING else None
 
-
-@dataclass
-class Portal:
-    db: ClassVar[Database] = fake_db
+class Portal(Base):
+    __tablename__ = "portal"
 
     # Telegram chat information
-    tgid: TelegramID
-    tg_receiver: TelegramID
-    peer_type: str
-    megagroup: bool
+    tgid: TelegramID = Column(Integer, primary_key=True)
+    tg_receiver: TelegramID = Column(Integer, primary_key=True)
+    peer_type: str = Column(String, nullable=False)
+    megagroup: bool = Column(Boolean)
 
     # Matrix portal information
-    mxid: RoomID | None
-    avatar_url: ContentURI | None
-    encrypted: bool
+    mxid: RoomID = Column(String, unique=True, nullable=True)
 
-    sponsored_event_id: EventID | None
-    sponsored_event_ts: int | None
-    sponsored_msg_random_id: bytes | None
+    config: str = Column(Text, nullable=True)
 
     # Telegram chat metadata
-    username: str | None
-    title: str | None
-    about: str | None
-    photo_id: str | None
-
-    local_config: dict[str, Any] = attr.ib(factory=lambda: {})
+    username: str = Column(String, nullable=True)
+    title: str = Column(String, nullable=True)
+    about: str = Column(String, nullable=True)
+    photo_id: str = Column(String, nullable=True)
 
     @classmethod
-    def _from_row(cls, row: Record | None) -> Portal | None:
-        if row is None:
-            return None
-        data = {**row}
-        data["local_config"] = json.loads(data.pop("config", None) or "{}")
-        return cls(**data)
-
-    columns: ClassVar[str] = (
-        "tgid, tg_receiver, peer_type, megagroup, mxid, avatar_url, encrypted, sponsored_event_id,"
-        "sponsored_event_ts, sponsored_msg_random_id, username, title, about, photo_id, config"
-    )
+    def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver)
 
     @classmethod
-    async def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Portal | None:
-        q = f"SELECT {cls.columns} FROM portal WHERE tgid=$1 AND tg_receiver=$2"
-        return cls._from_row(await cls.db.fetchrow(q, tgid, tg_receiver))
+    def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
 
     @classmethod
-    async def get_by_mxid(cls, mxid: RoomID) -> Portal | None:
-        q = f"SELECT {cls.columns} FROM portal WHERE mxid=$1"
-        return cls._from_row(await cls.db.fetchrow(q, mxid))
-
-    @classmethod
-    async def find_by_username(cls, username: str) -> Portal | None:
-        q = f"SELECT {cls.columns} FROM portal WHERE lower(username)=$1"
-        return cls._from_row(await cls.db.fetchrow(q, username.lower()))
-
-    @classmethod
-    async def find_private_chats(cls, tg_receiver: TelegramID) -> list[Portal]:
-        q = f"SELECT {cls.columns} FROM portal WHERE tg_receiver=$1 AND peer_type='user'"
-        return [cls._from_row(row) for row in await cls.db.fetch(q, tg_receiver)]
-
-    @classmethod
-    async def all(cls) -> list[Portal]:
-        rows = await cls.db.fetch(f"SELECT {cls.columns} FROM portal")
-        return [cls._from_row(row) for row in rows]
-
-    @property
-    def _values(self):
-        return (
-            self.tgid,
-            self.tg_receiver,
-            self.peer_type,
-            self.mxid,
-            self.avatar_url,
-            self.encrypted,
-            self.sponsored_event_id,
-            self.sponsored_event_ts,
-            self.sponsored_msg_random_id,
-            self.username,
-            self.title,
-            self.about,
-            self.photo_id,
-            self.megagroup,
-            json.dumps(self.local_config) if self.local_config else None,
-        )
-
-    async def save(self) -> None:
-        q = (
-            "UPDATE portal SET mxid=$4, avatar_url=$5, encrypted=$6, sponsored_event_id=$7,"
-            "                  sponsored_event_ts=$8, sponsored_msg_random_id=$9, username=$10,"
-            "                  title=$11, about=$12, photo_id=$13, megagroup=$14, config=$15 "
-            "WHERE tgid=$1 AND tg_receiver=$2 AND (peer_type=$3 OR true)"
-        )
-        await self.db.execute(q, *self._values)
-
-    async def update_id(self, id: TelegramID, peer_type: str) -> None:
-        q = (
-            "UPDATE portal SET tgid=$1, tg_receiver=$1, peer_type=$2 "
-            "WHERE tgid=$3 AND tg_receiver=$3"
-        )
-        await self.db.execute(q, id, peer_type, self.tgid)
-        self.tgid = id
-        self.tg_receiver = id
-        self.peer_type = peer_type
-
-    async def insert(self) -> None:
-        q = (
-            "INSERT INTO portal (tgid, tg_receiver, peer_type, mxid, avatar_url, encrypted,"
-            "                    sponsored_event_id, sponsored_event_ts, sponsored_msg_random_id,"
-            "                    username, title, about, photo_id, megagroup, config) "
-            "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"
-        )
-        await self.db.execute(q, *self._values)
-
-    async def delete(self) -> None:
-        q = "DELETE FROM portal WHERE tgid=$1 AND tg_receiver=$2"
-        await self.db.execute(q, self.tgid, self.tg_receiver)
+    def get_by_username(cls, username: str) -> Optional['Portal']:
+        return cls._select_one_or_none(func.lower(cls.c.username) == username)

mautrix_telegram/db/puppet.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,118 +13,48 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Optional, Iterable
 
-from typing import TYPE_CHECKING, ClassVar
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.sql import expression, func
 
-from asyncpg import Record
-from attr import dataclass
-from yarl import URL
-
-from mautrix.types import SyncToken, UserID
-from mautrix.util.async_db import Database
+from mautrix.types import UserID, SyncToken
+from mautrix.util.db import Base
 
 from ..types import TelegramID
 
-fake_db = Database.create("") if TYPE_CHECKING else None
 
+class Puppet(Base):
+    __tablename__ = "puppet"
 
-@dataclass
-class Puppet:
-    db: ClassVar[Database] = fake_db
-
-    id: TelegramID
-
-    is_registered: bool
-
-    displayname: str | None
-    displayname_source: TelegramID | None
-    displayname_contact: bool
-    displayname_quality: int
-    disable_updates: bool
-    username: str | None
-    photo_id: str | None
-    is_bot: bool | None
-
-    custom_mxid: UserID | None
-    access_token: str | None
-    next_batch: SyncToken | None
-    base_url: URL | None
+    id: TelegramID = Column(Integer, primary_key=True)
+    custom_mxid: UserID = Column(String, nullable=True)
+    access_token: str = Column(String, nullable=True)
+    next_batch: SyncToken = Column(String, nullable=True)
+    displayname: str = Column(String, nullable=True)
+    displayname_source: TelegramID = Column(Integer, nullable=True)
+    username: str = Column(String, nullable=True)
+    photo_id: str = Column(String, nullable=True)
+    is_bot: bool = Column(Boolean, nullable=True)
+    matrix_registered: bool = Column(Boolean, nullable=False, server_default=expression.false())
+    disable_updates: bool = Column(Boolean, nullable=False, server_default=expression.false())
 
     @classmethod
-    def _from_row(cls, row: Record | None) -> Puppet | None:
-        if row is None:
-            return None
-        data = {**row}
-        base_url = data.pop("base_url", None)
-        return cls(**data, base_url=URL(base_url) if base_url else None)
-
-    columns: ClassVar[str] = (
-        "id, is_registered, displayname, displayname_source, displayname_contact, "
-        "displayname_quality, disable_updates, username, photo_id, is_bot, "
-        "custom_mxid, access_token, next_batch, base_url"
-    )
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        yield from cls._select_all(cls.c.custom_mxid != None)
 
     @classmethod
-    async def all_with_custom_mxid(cls) -> list[Puppet]:
-        q = f"SELECT {cls.columns} FROM puppet WHERE custom_mxid<>''"
-        return [cls._from_row(row) for row in await cls.db.fetch(q)]
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.id == tgid)
 
     @classmethod
-    async def get_by_tgid(cls, tgid: TelegramID) -> Puppet | None:
-        q = f"SELECT {cls.columns} FROM puppet WHERE id=$1"
-        return cls._from_row(await cls.db.fetchrow(q, tgid))
+    def get_by_custom_mxid(cls, mxid: UserID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.custom_mxid == mxid)
 
     @classmethod
-    async def get_by_custom_mxid(cls, mxid: UserID) -> Puppet | None:
-        q = f"SELECT {cls.columns} FROM puppet WHERE custom_mxid=$1"
-        return cls._from_row(await cls.db.fetchrow(q, mxid))
+    def get_by_username(cls, username: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(func.lower(cls.c.username) == username)
 
     @classmethod
-    async def find_by_username(cls, username: str) -> Puppet | None:
-        q = f"SELECT {cls.columns} FROM puppet WHERE lower(username)=$1"
-        return cls._from_row(await cls.db.fetchrow(q, username.lower()))
-
-    @classmethod
-    async def find_by_displayname(cls, displayname: str) -> Puppet | None:
-        q = f"SELECT {cls.columns} FROM puppet WHERE displayname=$1"
-        return cls._from_row(await cls.db.fetchrow(q, displayname))
-
-    @property
-    def _values(self):
-        return (
-            self.id,
-            self.is_registered,
-            self.displayname,
-            self.displayname_source,
-            self.displayname_contact,
-            self.displayname_quality,
-            self.disable_updates,
-            self.username,
-            self.photo_id,
-            self.is_bot,
-            self.custom_mxid,
-            self.access_token,
-            self.next_batch,
-            str(self.base_url) if self.base_url else None,
-        )
-
-    async def save(self) -> None:
-        q = (
-            "UPDATE puppet "
-            "SET is_registered=$2, displayname=$3, displayname_source=$4, displayname_contact=$5,"
-            "    displayname_quality=$6, disable_updates=$7, username=$8, photo_id=$9, is_bot=$10,"
-            "    custom_mxid=$11, access_token=$12, next_batch=$13, base_url=$14 "
-            "WHERE id=$1"
-        )
-        await self.db.execute(q, *self._values)
-
-    async def insert(self) -> None:
-        q = (
-            "INSERT INTO puppet ("
-            "    id, is_registered, displayname, displayname_source, displayname_contact,"
-            "    displayname_quality, disable_updates, username, photo_id, is_bot,"
-            "    custom_mxid, access_token, next_batch, base_url"
-            ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)"
-        )
-        await self.db.execute(q, *self._values)
+    def get_by_displayname(cls, displayname: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.displayname == displayname)

mautrix_telegram/db/telegram_file.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,70 +13,44 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Optional
 
-from typing import TYPE_CHECKING, ClassVar
+from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
+from sqlalchemy.engine.result import RowProxy
 
-from attr import dataclass
-
-from mautrix.types import ContentURI, EncryptedFile
-from mautrix.util.async_db import Database
-
-fake_db = Database.create("") if TYPE_CHECKING else None
+from mautrix.types import ContentURI
+from mautrix.util.db import Base
 
 
-@dataclass
-class TelegramFile:
-    db: ClassVar[Database] = fake_db
+class TelegramFile(Base):
+    __tablename__ = "telegram_file"
 
-    id: str
-    mxc: ContentURI
-    mime_type: str
-    was_converted: bool
-    timestamp: int
-    size: int | None
-    width: int | None
-    height: int | None
-    decryption_info: EncryptedFile | None
-    thumbnail: TelegramFile | None = None
+    id: str = Column(String, primary_key=True)
+    mxc: ContentURI = Column(String)
+    mime_type: str = Column(String)
+    was_converted: bool = Column(Boolean)
+    timestamp: int = Column(BigInteger)
+    size: Optional[int] = Column(Integer, nullable=True)
+    width: Optional[int] = Column(Integer, nullable=True)
+    height: Optional[int] = Column(Integer, nullable=True)
+    thumbnail_id: str = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+    thumbnail: Optional['TelegramFile'] = None
 
     @classmethod
-    async def get(cls, loc_id: str, *, _thumbnail: bool = False) -> TelegramFile | None:
-        q = (
-            "SELECT id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail,"
-            "       decryption_info "
-            "FROM telegram_file WHERE id=$1"
-        )
-        row = await cls.db.fetchrow(q, loc_id)
-        if row is None:
-            return None
-        data = {**row}
-        thumbnail_id = data.pop("thumbnail", None)
-        if _thumbnail:
-            # Don't allow more than one level of recursion
-            thumbnail_id = None
-        decryption_info = data.pop("decryption_info", None)
-        return cls(
-            **data,
-            thumbnail=(await cls.get(thumbnail_id, _thumbnail=True)) if thumbnail_id else None,
-            decryption_info=EncryptedFile.parse_json(decryption_info) if decryption_info else None,
-        )
+    def scan(cls, row: RowProxy) -> 'TelegramFile':
+        telegram_file: TelegramFile = super().scan(row)
+        if isinstance(telegram_file.thumbnail, str):
+            telegram_file.thumbnail = cls.get(telegram_file.thumbnail)
+        return telegram_file
 
-    async def insert(self) -> None:
-        q = (
-            "INSERT INTO telegram_file (id, mxc, mime_type, was_converted, size, width, height, "
-            "                           thumbnail, decryption_info) "
-            "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"
-        )
-        await self.db.execute(
-            q,
-            self.id,
-            self.mxc,
-            self.mime_type,
-            self.was_converted,
-            self.size,
-            self.width,
-            self.height,
-            self.thumbnail.id if self.thumbnail else None,
-            self.decryption_info.json() if self.decryption_info else None,
-        )
+    @classmethod
+    def get(cls, loc_id: str) -> Optional['TelegramFile']:
+        return cls._select_one_or_none(cls.c.id == loc_id)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, mxc=self.mxc, mime_type=self.mime_type,
+                was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
+                width=self.width, height=self.height,
+                thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/telethon_session.py

@@ -1,207 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, ClassVar
-import asyncio
-import datetime
-
-from telethon import utils
-from telethon.crypto import AuthKey
-from telethon.sessions import MemorySession
-from telethon.tl.types import PeerChannel, PeerChat, PeerUser, updates
-
-from mautrix.util.async_db import Database
-
-fake_db = Database.create("") if TYPE_CHECKING else None
-
-
-class PgSession(MemorySession):
-    db: ClassVar[Database] = fake_db
-
-    session_id: str
-    _dc_id: int
-    _server_address: str | None
-    _port: int | None
-    _auth_key: AuthKey | None
-    _takeout_id: int | None
-    _process_entities_lock: asyncio.Lock
-
-    def __init__(
-        self,
-        session_id: str,
-        dc_id: int = 0,
-        server_address: str | None = None,
-        port: int | None = None,
-        auth_key: AuthKey | None = None,
-        takeout_id: int | None = None,
-    ) -> None:
-        super().__init__()
-        self.session_id = session_id
-        self._dc_id = dc_id
-        self._server_address = server_address
-        self._port = port
-        self._auth_key = auth_key
-        self._takeout_id = takeout_id
-        self._process_entities_lock = asyncio.Lock()
-
-    def clone(self, to_instance=None) -> MemorySession:
-        # We don't want to store data of clones
-        # (which are used for temporarily connecting to different DCs)
-        return super().clone(MemorySession())
-
-    @property
-    def auth_key_bytes(self) -> bytes | None:
-        return self._auth_key.key if self._auth_key else None
-
-    @classmethod
-    async def get(cls, session_id: str) -> PgSession:
-        q = (
-            "SELECT session_id, dc_id, server_address, port, auth_key FROM telethon_sessions "
-            "WHERE session_id=$1"
-        )
-        row = await cls.db.fetchrow(q, session_id)
-        if row is None:
-            return cls(session_id)
-        data = {**row}
-        auth_key = AuthKey(data.pop("auth_key", None))
-        return cls(**data, auth_key=auth_key)
-
-    @classmethod
-    async def has(cls, session_id: str) -> bool:
-        q = "SELECT COUNT(*) FROM telethon_sessions WHERE session_id=$1"
-        count = await cls.db.fetchval(q, session_id)
-        return count > 0
-
-    async def save(self) -> None:
-        q = (
-            "INSERT INTO telethon_sessions (session_id, dc_id, server_address, port, auth_key) "
-            "VALUES ($1, $2, $3, $4, $5) ON CONFLICT (session_id) "
-            "DO UPDATE SET dc_id=$2, server_address=$3, port=$4, auth_key=$5"
-        )
-        await self.db.execute(
-            q, self.session_id, self.dc_id, self.server_address, self.port, self.auth_key_bytes
-        )
-
-    _tables: ClassVar[tuple[str, ...]] = (
-        "telethon_sessions",
-        "telethon_entities",
-        "telethon_sent_files",
-        "telethon_update_state",
-    )
-
-    async def delete(self) -> None:
-        async with self.db.acquire() as conn, conn.transaction():
-            for table in self._tables:
-                await conn.execute(f"DELETE FROM {table} WHERE session_id=$1", self.session_id)
-
-    async def close(self) -> None:
-        # Nothing to do here, DB connection is global
-        pass
-
-    async def get_update_state(self, entity_id: int) -> updates.State | None:
-        q = (
-            "SELECT pts, qts, date, seq, unread_count FROM telethon_update_state "
-            "WHERE session_id=$1 AND entity_id=$2"
-        )
-        row = await self.db.fetchrow(q, self.session_id, entity_id)
-        if row is None:
-            return None
-        date = datetime.datetime.utcfromtimestamp(row["date"])
-        return updates.State(row["pts"], row["qts"], date, row["seq"], row["unread_count"])
-
-    async def set_update_state(self, entity_id: int, row: updates.State) -> None:
-        q = (
-            "INSERT INTO telethon_update_state"
-            "    (session_id, entity_id, pts, qts, date, seq, unread_count) "
-            "VALUES ($1, $2, $3, $4, $5, $6, $7)"
-            "ON CONFLICT (session_id, entity_id) DO UPDATE"
-            "    SET pts=$3, qts=$4, date=$5, seq=$6, unread_count=$7"
-        )
-        ts = row.date.timestamp()
-        await self.db.execute(
-            q, self.session_id, entity_id, row.pts, row.qts, ts, row.seq, row.unread_count
-        )
-
-    def _entity_values_to_row(
-        self, id: int, hash: int, username: str | None, phone: str | int | None, name: str | None
-    ) -> tuple[str, int, int, str | None, str | None, str | None]:
-        return self.session_id, id, hash, username, str(phone) if phone else None, name
-
-    async def process_entities(self, tlo) -> None:
-        # Postgres likes to deadlock on simultaneous upserts, so just lock the whole thing here
-        # TODO: make sure postgres doesn't deadlock on upserts when session_id is different
-        async with self._process_entities_lock:
-            await self._locked_process_entities(tlo)
-
-    async def _locked_process_entities(self, tlo) -> None:
-        rows: list[
-            tuple[str, int, int, str | None, str | None, str | None]
-        ] = self._entities_to_rows(tlo)
-        if not rows:
-            return
-        if self.db.scheme == "postgres":
-            q = (
-                "INSERT INTO telethon_entities (session_id, id, hash, username, phone, name) "
-                "VALUES ($1, unnest($2::bigint[]), unnest($3::bigint[]), "
-                "        unnest($4::text[]), unnest($5::text[]), unnest($6::text[])) "
-                "ON CONFLICT (session_id, id) DO UPDATE"
-                "  SET hash=excluded.hash, username=excluded.username,"
-                "      phone=excluded.phone, name=excluded.name"
-            )
-            _, ids, hashes, usernames, phones, names = zip(*rows)
-            await self.db.execute(q, self.session_id, ids, hashes, usernames, phones, names)
-        else:
-            q = (
-                "INSERT INTO telethon_entities (session_id, id, hash, username, phone, name) "
-                "VALUES ($1, $2, $3, $4, $5, $6) "
-                "ON CONFLICT (session_id, id) DO UPDATE "
-                "    SET hash=$3, username=$4, phone=$5, name=$6"
-            )
-            await self.db.executemany(q, rows)
-
-    async def _select_entity(
-        self, constraint: str, *args: str | int | tuple[int, ...]
-    ) -> tuple[int, int] | None:
-        row = await self.db.fetchrow(
-            f"SELECT id, hash FROM telethon_entities WHERE {constraint}", *args
-        )
-        if row is None:
-            return None
-        return row["id"], row["hash"]
-
-    async def get_entity_rows_by_phone(self, key: str | int) -> tuple[int, int] | None:
-        return await self._select_entity("phone=$1", str(key))
-
-    async def get_entity_rows_by_username(self, key: str) -> tuple[int, int] | None:
-        return await self._select_entity("username=$1", key)
-
-    async def get_entity_rows_by_name(self, key: str) -> tuple[int, int] | None:
-        return await self._select_entity("name=$1", key)
-
-    async def get_entity_rows_by_id(self, key: int, exact: bool = True) -> tuple[int, int] | None:
-        if exact:
-            return await self._select_entity("id=$1", key)
-
-        ids = (
-            utils.get_peer_id(PeerUser(key)),
-            utils.get_peer_id(PeerChat(key)),
-            utils.get_peer_id(PeerChannel(key)),
-        )
-        if self.db.scheme == "postgres":
-            return await self._select_entity("id=ANY($1)", ids)
-        else:
-            return await self._select_entity(f"id IN ($1, $2, $3)", *ids)

mautrix_telegram/db/upgrade/__init__.py

@@ -1,5 +0,0 @@
-from mautrix.util.async_db import UpgradeTable
-
-upgrade_table = UpgradeTable()
-
-from . import v01_initial_revision, v02_sponsored_events

mautrix_telegram/db/upgrade/v01_initial_revision.py

@@ -1,315 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from asyncpg import Connection
-
-from . import upgrade_table
-
-legacy_version_query = "SELECT version_num FROM alembic_version"
-last_legacy_version = "bfc0a39bfe02"
-
-
-def table_exists(scheme: str, name: str) -> str:
-    if scheme == "sqlite":
-        return f"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='{name}')"
-    elif scheme == "postgres":
-        return f"SELECT EXISTS(SELECT FROM information_schema.tables WHERE table_name='{name}')"
-    raise RuntimeError("unsupported database scheme")
-
-
-@upgrade_table.register(description="Initial asyncpg revision")
-async def upgrade_v1(conn: Connection, scheme: str) -> None:
-    is_legacy = await conn.fetchval(table_exists(scheme, "alembic_version"))
-    if is_legacy:
-        await migrate_legacy_to_v1(conn, scheme)
-    else:
-        await create_v1_tables(conn)
-
-
-async def migrate_legacy_to_v1(conn: Connection, scheme: str) -> None:
-    legacy_version = await conn.fetchval(legacy_version_query)
-    if legacy_version != last_legacy_version:
-        raise RuntimeError(
-            "Legacy database is not on last version. "
-            "Please upgrade the old database with alembic or drop it completely first."
-        )
-    if scheme != "sqlite":
-        await conn.execute(
-            """
-            ALTER TABLE contact
-              DROP CONSTRAINT contact_user_fkey,
-              DROP CONSTRAINT contact_contact_fkey,
-              ADD CONSTRAINT contact_user_fkey FOREIGN KEY (contact) REFERENCES puppet(id)
-                ON DELETE CASCADE ON UPDATE CASCADE,
-              ADD CONSTRAINT contact_contact_fkey FOREIGN KEY ("user") REFERENCES "user"(tgid)
-                ON DELETE CASCADE ON UPDATE CASCADE
-            """
-        )
-        await conn.execute(
-            """
-            ALTER TABLE telethon_sessions
-              DROP CONSTRAINT telethon_sessions_pkey,
-              ADD CONSTRAINT telethon_sessions_pkey PRIMARY KEY (session_id)
-            """
-        )
-        await conn.execute(
-            """
-            ALTER TABLE telegram_file
-              DROP CONSTRAINT fk_file_thumbnail,
-              ADD CONSTRAINT fk_file_thumbnail
-                FOREIGN KEY (thumbnail) REFERENCES telegram_file(id)
-                ON UPDATE CASCADE ON DELETE SET NULL
-            """
-        )
-        await conn.execute("ALTER TABLE puppet ALTER COLUMN id DROP DEFAULT")
-        await conn.execute("DROP SEQUENCE puppet_id_seq")
-        await conn.execute("ALTER TABLE bot_chat ALTER COLUMN id DROP DEFAULT")
-        await conn.execute("DROP SEQUENCE bot_chat_id_seq")
-        await conn.execute("ALTER TABLE portal ALTER COLUMN config TYPE jsonb USING config::jsonb")
-        await conn.execute(
-            "ALTER TABLE telegram_file ALTER COLUMN decryption_info TYPE jsonb "
-            "USING decryption_info::jsonb"
-        )
-        await varchar_to_text(conn)
-    else:
-        await conn.execute(
-            """CREATE TABLE telethon_sessions_new (
-                session_id     TEXT PRIMARY KEY,
-                dc_id          INTEGER,
-                server_address TEXT,
-                port           INTEGER,
-                auth_key       bytea
-            )"""
-        )
-        await conn.execute(
-            """
-            INSERT INTO telethon_sessions_new (session_id, dc_id, server_address, port, auth_key)
-            SELECT session_id, dc_id, server_address, port, auth_key FROM telethon_sessions
-            """
-        )
-        await conn.execute("DROP TABLE telethon_sessions")
-        await conn.execute("ALTER TABLE telethon_sessions_new RENAME TO telethon_sessions")
-
-    await update_state_store(conn, scheme)
-    await conn.execute('ALTER TABLE "user" ADD COLUMN is_bot BOOLEAN NOT NULL DEFAULT false')
-    await conn.execute("ALTER TABLE puppet RENAME COLUMN matrix_registered TO is_registered")
-    await conn.execute("DROP TABLE telethon_version")
-    await conn.execute("DROP TABLE alembic_version")
-
-
-async def update_state_store(conn: Connection, scheme: str) -> None:
-    # The Matrix state store already has more or less the correct schema, so set the version
-    await conn.execute("CREATE TABLE mx_version (version INTEGER PRIMARY KEY)")
-    await conn.execute("INSERT INTO mx_version (version) VALUES (2)")
-    await conn.execute("UPDATE mx_user_profile SET membership='LEAVE' WHERE membership='LEFT'")
-    if scheme != "sqlite":
-        # Also add the membership type on postgres
-        await conn.execute(
-            "CREATE TYPE membership AS ENUM ('join', 'leave', 'invite', 'ban', 'knock')"
-        )
-        await conn.execute(
-            "ALTER TABLE mx_user_profile ALTER COLUMN membership TYPE membership "
-            "USING LOWER(membership)::membership"
-        )
-    else:
-        # On SQLite there's no custom type, but we still want to lowercase everything
-        await conn.execute("UPDATE mx_user_profile SET membership=LOWER(membership)")
-
-
-async def varchar_to_text(conn: Connection) -> None:
-    columns_to_adjust = {
-        "user": ("mxid", "tg_username", "tg_phone"),
-        "portal": (
-            "peer_type",
-            "mxid",
-            "username",
-            "title",
-            "about",
-            "photo_id",
-            "avatar_url",
-            "config",
-        ),
-        "message": ("mxid", "mx_room"),
-        "puppet": (
-            "displayname",
-            "username",
-            "photo_id",
-            "access_token",
-            "custom_mxid",
-            "next_batch",
-            "base_url",
-        ),
-        "bot_chat": ("type",),
-        "telegram_file": ("id", "mxc", "mime_type", "thumbnail"),
-        # Phone is a bigint in the old schema, which is safe, but we don't do math on it,
-        # so let's change it to a string
-        "telethon_entities": ("session_id", "username", "name", "phone"),
-        "telethon_sent_files": ("session_id",),
-        "telethon_sessions": ("session_id", "server_address"),
-        "telethon_update_state": ("session_id",),
-        "mx_room_state": ("room_id",),
-        "mx_user_profile": ("room_id", "user_id", "displayname", "avatar_url"),
-    }
-    for table, columns in columns_to_adjust.items():
-        for column in columns:
-            await conn.execute(f'ALTER TABLE "{table}" ALTER COLUMN {column} TYPE TEXT')
-
-
-async def create_v1_tables(conn: Connection) -> None:
-    await conn.execute(
-        """CREATE TABLE "user" (
-            mxid TEXT   PRIMARY KEY,
-            tgid BIGINT UNIQUE,
-            tg_username    TEXT,
-            tg_phone       TEXT,
-            is_bot         BOOLEAN NOT NULL DEFAULT false,
-            saved_contacts INTEGER NOT NULL DEFAULT 0
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE portal (
-            tgid        BIGINT,
-            tg_receiver BIGINT,
-            peer_type   TEXT NOT NULL,
-            mxid        TEXT UNIQUE,
-            avatar_url  TEXT,
-            encrypted   BOOLEAN NOT NULL DEFAULT false,
-            username    TEXT,
-            title       TEXT,
-            about       TEXT,
-            photo_id    TEXT,
-            megagroup   BOOLEAN,
-            config      jsonb,
-            PRIMARY KEY (tgid, tg_receiver)
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE message (
-            mxid       TEXT,
-            mx_room    TEXT,
-            tgid       BIGINT NOT NULL,
-            tg_space   BIGINT NOT NULL,
-            edit_index INTEGER NOT NULL,
-            redacted   BOOLEAN NOT NULL DEFAULT false,
-            PRIMARY KEY (tgid, tg_space, edit_index),
-            UNIQUE (mxid, mx_room, tg_space)
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE puppet (
-            id BIGINT PRIMARY KEY,
-
-            is_registered BOOLEAN NOT NULL DEFAULT false,
-
-            displayname         TEXT,
-            displayname_source  BIGINT,
-            displayname_contact BOOLEAN NOT NULL DEFAULT true,
-            displayname_quality INTEGER NOT NULL DEFAULT 0,
-            disable_updates     BOOLEAN NOT NULL DEFAULT false,
-            username            TEXT,
-            photo_id            TEXT,
-            is_bot              BOOLEAN,
-
-            access_token TEXT,
-            custom_mxid  TEXT,
-            next_batch   TEXT,
-            base_url     TEXT
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telegram_file (
-            id              TEXT PRIMARY KEY,
-            mxc             TEXT NOT NULL,
-            mime_type       TEXT,
-            was_converted   BOOLEAN NOT NULL DEFAULT false,
-            timestamp       BIGINT  NOT NULL DEFAULT 0,
-            size            BIGINT,
-            width           INTEGER,
-            height          INTEGER,
-            thumbnail       TEXT,
-            decryption_info jsonb,
-            FOREIGN KEY (thumbnail) REFERENCES telegram_file(id)
-                ON UPDATE CASCADE ON DELETE SET NULL
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE bot_chat (
-            id   BIGINT PRIMARY KEY,
-            type TEXT NOT NULL
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE user_portal (
-            "user"          BIGINT,
-            portal          BIGINT,
-            portal_receiver BIGINT,
-            PRIMARY KEY ("user", portal, portal_receiver),
-            FOREIGN KEY ("user") REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
-            FOREIGN KEY (portal, portal_receiver) REFERENCES portal(tgid, tg_receiver)
-                 ON DELETE CASCADE ON UPDATE CASCADE
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE contact (
-            "user"  BIGINT,
-            contact BIGINT,
-            PRIMARY KEY ("user", contact),
-            FOREIGN KEY ("user")  REFERENCES "user"(tgid) ON DELETE CASCADE ON UPDATE CASCADE,
-            FOREIGN KEY (contact) REFERENCES puppet(id)   ON DELETE CASCADE ON UPDATE CASCADE
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telethon_sessions (
-            session_id     TEXT PRIMARY KEY,
-            dc_id          INTEGER,
-            server_address TEXT,
-            port           INTEGER,
-            auth_key       bytea
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telethon_entities (
-            session_id TEXT,
-            id         BIGINT,
-            hash       BIGINT NOT NULL,
-            username   TEXT,
-            phone      TEXT,
-            name       TEXT,
-            PRIMARY KEY (session_id, id)
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telethon_sent_files (
-            session_id TEXT,
-            md5_digest bytea,
-            file_size  INTEGER,
-            type       INTEGER,
-            id         BIGINT,
-            hash       BIGINT,
-            PRIMARY KEY (session_id, md5_digest, file_size, type)
-        )"""
-    )
-    await conn.execute(
-        """CREATE TABLE telethon_update_state (
-            session_id   TEXT,
-            entity_id    BIGINT,
-            pts          BIGINT,
-            qts          BIGINT,
-            date         BIGINT,
-            seq          BIGINT,
-            unread_count INTEGER,
-            PRIMARY KEY (session_id, entity_id)
-        )"""
-    )

mautrix_telegram/db/user.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,126 +13,96 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Optional, Iterable, Tuple
 
-from typing import TYPE_CHECKING, ClassVar, Iterable
-
-from asyncpg import Record
-from attr import dataclass
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String, func
 
 from mautrix.types import UserID
-from mautrix.util.async_db import Database
+from mautrix.util.db import Base
 
 from ..types import TelegramID
 
-fake_db = Database.create("") if TYPE_CHECKING else None
 
+class User(Base):
+    __tablename__ = "user"
 
-@dataclass
-class User:
-    db: ClassVar[Database] = fake_db
-
-    mxid: UserID
-    tgid: TelegramID | None
-    tg_username: str | None
-    tg_phone: str | None
-    is_bot: bool
-    saved_contacts: int
+    mxid: UserID = Column(String, primary_key=True)
+    tgid: Optional[TelegramID] = Column(Integer, nullable=True, unique=True)
+    tg_username: str = Column(String, nullable=True)
+    tg_phone: str = Column(String, nullable=True)
+    saved_contacts: int = Column(Integer, default=0, nullable=False)
 
     @classmethod
-    def _from_row(cls, row: Record | None) -> User | None:
-        if row is None:
-            return None
-        return cls(**row)
-
-    columns: ClassVar[str] = "mxid, tgid, tg_username, tg_phone, is_bot, saved_contacts"
+    def all_with_tgid(cls) -> Iterable['User']:
+        return cls._select_all(cls.c.tgid != None)
 
     @classmethod
-    async def get_by_tgid(cls, tgid: TelegramID) -> User | None:
-        q = f'SELECT {cls.columns} FROM "user" WHERE tgid=$1'
-        return cls._from_row(await cls.db.fetchrow(q, tgid))
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tgid == tgid)
 
     @classmethod
-    async def get_by_mxid(cls, mxid: UserID) -> User | None:
-        q = f'SELECT {cls.columns} FROM "user" WHERE mxid=$1'
-        return cls._from_row(await cls.db.fetchrow(q, mxid))
+    def get_by_mxid(cls, mxid: UserID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
 
     @classmethod
-    async def find_by_username(cls, username: str) -> User | None:
-        q = f'SELECT {cls.columns} FROM "user" WHERE lower(tg_username)=$1'
-        return cls._from_row(await cls.db.fetchrow(q, username.lower()))
-
-    @classmethod
-    async def all_with_tgid(cls) -> list[User]:
-        q = f'SELECT {cls.columns} FROM "user" WHERE tgid IS NOT NULL'
-        return [cls._from_row(row) for row in await cls.db.fetch(q)]
-
-    async def delete(self) -> None:
-        await self.db.execute('DELETE FROM "user" WHERE mxid=$1', self.mxid)
+    def get_by_username(cls, username: str) -> Optional['User']:
+        return cls._select_one_or_none(func.lower(cls.c.tg_username) == username)
 
     @property
-    def _values(self):
-        return (
-            self.mxid,
-            self.tgid,
-            self.tg_username,
-            self.tg_phone,
-            self.is_bot,
-            self.saved_contacts,
-        )
+    def contacts(self) -> Iterable[TelegramID]:
+        rows = self.db.execute(Contact.t.select().where(Contact.c.user == self.tgid))
+        for row in rows:
+            user, contact = row
+            yield contact
 
-    async def save(self) -> None:
-        q = (
-            'UPDATE "user" SET tgid=$2, tg_username=$3, tg_phone=$4, is_bot=$5, saved_contacts=$6 '
-            "WHERE mxid=$1"
-        )
-        await self.db.execute(q, *self._values)
+    @contacts.setter
+    def contacts(self, puppets: Iterable[TelegramID]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
+            insert_puppets = [{"user": self.tgid, "contact": tgid} for tgid in puppets]
+            if insert_puppets:
+                conn.execute(Contact.t.insert(), insert_puppets)
 
-    async def insert(self) -> None:
-        q = (
-            'INSERT INTO "user" (mxid, tgid, tg_username, tg_phone, is_bot, saved_contacts) '
-            "VALUES ($1, $2, $3, $4, $5, $6)"
-        )
-        await self.db.execute(q, *self._values)
+    @property
+    def portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        rows = self.db.execute(UserPortal.t.select().where(UserPortal.c.user == self.tgid))
+        for row in rows:
+            user, portal, portal_receiver = row
+            yield (portal, portal_receiver)
 
-    async def get_contacts(self) -> list[TelegramID]:
-        rows = await self.db.fetch('SELECT contact FROM contact WHERE "user"=$1', self.tgid)
-        return [TelegramID(row["contact"]) for row in rows]
+    @portals.setter
+    def portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
+            insert_portals = [{
+                "user": self.tgid,
+                "portal": tgid,
+                "portal_receiver": tg_receiver
+            } for tgid, tg_receiver in portals]
+            if insert_portals:
+                conn.execute(UserPortal.t.insert(), insert_portals)
 
-    async def set_contacts(self, puppets: Iterable[TelegramID]) -> None:
-        columns = ["user", "contact"]
-        records = [(self.tgid, puppet_id) for puppet_id in puppets]
-        async with self.db.acquire() as conn, conn.transaction():
-            await conn.execute('DELETE FROM contact WHERE "user"=$1', self.tgid)
-            if self.db.scheme == "postgres":
-                await conn.copy_records_to_table("contact", records=records, columns=columns)
-            else:
-                q = 'INSERT INTO contact ("user", contact) VALUES ($1, $2)'
-                await conn.executemany(q, records)
+    def delete(self) -> None:
+        super().delete()
+        self.portals = []
+        self.contacts = []
 
-    async def get_portals(self) -> list[tuple[TelegramID, TelegramID]]:
-        q = 'SELECT portal, portal_receiver FROM user_portal WHERE "user"=$1'
-        rows = await self.db.fetch(q, self.tgid)
-        return [(TelegramID(row["portal"]), TelegramID(row["portal_receiver"])) for row in rows]
 
-    async def set_portals(self, portals: Iterable[tuple[TelegramID, TelegramID]]) -> None:
-        columns = ["user", "portal", "portal_receiver"]
-        records = [(self.tgid, tgid, tg_receiver) for tgid, tg_receiver in portals]
-        async with self.db.acquire() as conn, conn.transaction():
-            await conn.execute('DELETE FROM user_portal WHERE "user"=$1', self.tgid)
-            if self.db.scheme == "postgres":
-                await conn.copy_records_to_table("user_portal", records=records, columns=columns)
-            else:
-                q = 'INSERT INTO user_portal ("user", portal, portal_receiver) VALUES ($1, $2, $3)'
-                await conn.executemany(q, records)
+class UserPortal(Base):
+    __tablename__ = "user_portal"
 
-    async def register_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
-        q = (
-            'INSERT INTO user_portal ("user", portal, portal_receiver) VALUES ($1, $2, $3) '
-            'ON CONFLICT ("user", portal, portal_receiver) DO NOTHING'
-        )
-        await self.db.execute(q, self.tgid, tgid, tg_receiver)
+    user: TelegramID = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE",
+                                                  ondelete="CASCADE"), primary_key=True)
+    portal: TelegramID = Column(Integer, primary_key=True)
+    portal_receiver: TelegramID = Column(Integer, primary_key=True)
 
-    async def unregister_portal(self, tgid: TelegramID, tg_receiver: TelegramID) -> None:
-        q = 'DELETE FROM user_portal WHERE "user"=$1 AND portal=$2 AND portal_receiver=$3'
-        await self.db.execute(q, self.tgid, tgid, tg_receiver)
+    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
+                                           ("portal.tgid", "portal.tg_receiver"),
+                                           onupdate="CASCADE", ondelete="CASCADE"),)
+
+
+class Contact(Base):
+    __tablename__ = "contact"
+
+    user: TelegramID = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
+    contact: TelegramID = Column(Integer, ForeignKey("puppet.id"), primary_key=True)

mautrix_telegram/formatter/__init__.py

@@ -1,2 +1,8 @@
-from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram
+from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
+                          init_mx)
 from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
+from .. import context as c
+
+
+def init(context: c.Context) -> None:
+    init_mx(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,77 +13,48 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
+from typing import Optional, List, Tuple, Callable, Pattern, Match, TYPE_CHECKING
 import re
+import logging
 
-from telethon import TelegramClient
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
+                               TypeMessageEntity)
 from telethon.helpers import add_surrogate, del_surrogate
-from telethon.tl.types import MessageEntityItalic, TypeMessageEntity
 
-from mautrix.types import MessageEventContent, RoomID
+from mautrix.types import RoomID, MessageEventContent
 
-from ...db import Message as DBMessage
+from ... import puppet as pu
 from ...types import TelegramID
-from .parser import MatrixParser
+from ...db import Message as DBMessage
+from .parser import ParsedMessage, parse_html
+
+if TYPE_CHECKING:
+    from ...context import Context
+
+log: logging.Logger = logging.getLogger("mau.fmt.mx")
+should_bridge_plaintext_highlights: bool = False
+
+command_regex: Pattern = re.compile(r"^!([A-Za-z0-9@]+)")
+not_command_regex: Pattern = re.compile(r"^\\(![A-Za-z0-9@]+)")
+plain_mention_regex: Optional[Pattern] = None
+
+
+def plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
 
-command_regex = re.compile(r"^!([A-Za-z0-9@]+)")
-not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")
 
 MAX_LENGTH = 4096
 CUTOFF_TEXT = " [message cut]"
 CUT_MAX_LENGTH = MAX_LENGTH - len(CUTOFF_TEXT)
 
 
-class FormatError(Exception):
-    pass
-
-
-async def matrix_reply_to_telegram(
-    content: MessageEventContent, tg_space: TelegramID, room_id: RoomID | None = None
-) -> TelegramID | None:
-    event_id = content.get_reply_to()
-    if not event_id:
-        return
-    content.trim_reply_fallback()
-
-    message = await DBMessage.get_by_mxid(event_id, room_id, tg_space)
-    if message:
-        return message.tgid
-    return None
-
-
-async def matrix_to_telegram(
-    client: TelegramClient, *, text: str | None = None, html: str | None = None
-) -> tuple[str, list[TypeMessageEntity]]:
-    if html is not None:
-        return await _matrix_html_to_telegram(client, html)
-    elif text is not None:
-        return _matrix_text_to_telegram(text), []
-    else:
-        raise ValueError("text or html must be provided to convert formatting")
-
-
-async def _matrix_html_to_telegram(
-    client: TelegramClient, html: str
-) -> tuple[str, list[TypeMessageEntity]]:
-    try:
-        html = command_regex.sub(r"<command>\1</command>", html)
-        html = html.replace("\t", " " * 4)
-        html = not_command_regex.sub(r"\1", html)
-
-        parsed = await MatrixParser(client).parse(add_surrogate(html))
-        text = del_surrogate(parsed.text.strip())
-        text, entities = _cut_long_message(text, parsed.telegram_entities)
-
-        return text, entities
-    except Exception as e:
-        raise FormatError(f"Failed to convert Matrix format: {html}") from e
-
-
-def _cut_long_message(
-    message: str, entities: list[TypeMessageEntity]
-) -> tuple[str, list[TypeMessageEntity]]:
+def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
     if len(message) > MAX_LENGTH:
         message = message[0:CUT_MAX_LENGTH] + CUTOFF_TEXT
         new_entities = []
@@ -98,8 +69,77 @@ def _cut_long_message(
     return message, entities
 
 
-def _matrix_text_to_telegram(text: str) -> str:
+class FormatError(Exception):
+    pass
+
+
+def matrix_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(plain_mention_to_html, html)
+
+        text, entities = parse_html(add_surrogate(html))
+        text = del_surrogate(text.strip())
+        text, entities = cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def matrix_reply_to_telegram(content: MessageEventContent, tg_space: TelegramID,
+                             room_id: Optional[RoomID] = None) -> Optional[TelegramID]:
+    event_id = content.get_reply_to()
+    if not event_id:
+        return
+    content.trim_reply_fallback()
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
+    return None
+
+
+def matrix_text_to_telegram(text: str) -> ParsedMessage:
     text = command_regex.sub(r"/\1", text)
     text = text.replace("\t", " " * 4)
     text = not_command_regex.sub(r"\1", text)
-    return text
+    if should_bridge_plaintext_highlights:
+        entities, pmr_replacer = plain_mention_to_text()
+        text = plain_mention_regex.sub(pmr_replacer, text)
+    else:
+        entities = []
+    return text, entities
+
+
+def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[Match], str]]:
+    entities = []
+
+    def replacer(match: Match) -> str:
+        puppet = pu.Puppet.find_by_displayname(match.group(2))
+        if puppet:
+            offset = match.start()
+            length = match.end() - offset
+            if puppet.username:
+                entity = MessageEntityMention(offset, length)
+                text = f"@{puppet.username}"
+            else:
+                entity = MessageEntityMentionName(offset, length, user_id=puppet.tgid)
+                text = puppet.displayname
+            entities.append(entity)
+            return text
+        return "".join(match.groups())
+
+    return entities, replacer
+
+
+def init_mx(context: "Context") -> None:
+    global plain_mention_regex, should_bridge_plaintext_highlights
+    config = context.config
+    dn_template = config["bridge.displayname_template"]
+    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
+    plain_mention_regex = re.compile(f"^({dn_template})")
+    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"]

mautrix_telegram/formatter/from_matrix/parser.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,87 +13,77 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import List, Tuple, Optional
 
-import logging
+from telethon.tl.types import TypeMessageEntity
 
-from telethon import TelegramClient
-
-from mautrix.types import RoomID, UserID
+from mautrix.types import UserID, RoomID
 from mautrix.util.formatter import MatrixParser as BaseMatrixParser, RecursionContext
-from mautrix.util.formatter.html_reader_htmlparser import HTMLNode, read_html
-from mautrix.util.logging import TraceLogger
+from mautrix.util.formatter.html_reader_htmlparser import read_html, HTMLNode
 
-from ... import portal as po, puppet as pu, user as u
-from .telegram_message import TelegramEntityType, TelegramMessage
+from ... import user as u, puppet as pu, portal as po
+from .telegram_message import TelegramMessage, TelegramEntityType
 
-log: TraceLogger = logging.getLogger("mau.fmt.mx")
+
+ParsedMessage = Tuple[str, List[TypeMessageEntity]]
+
+
+def parse_html(input_html: str) -> ParsedMessage:
+    msg = MatrixParser.parse(input_html)
+    return msg.text, msg.telegram_entities
 
 
 class MatrixParser(BaseMatrixParser[TelegramMessage]):
     e = TelegramEntityType
     fs = TelegramMessage
-    read_html = staticmethod(read_html)
-    client: TelegramClient
+    read_html = read_html
 
-    def __init__(self, client: TelegramClient) -> None:
-        self.client = client
-
-    async def custom_node_to_fstring(
-        self, node: HTMLNode, ctx: RecursionContext
-    ) -> TelegramMessage | None:
-        msg = await self.tag_aware_parse_node(node, ctx)
+    @classmethod
+    def custom_node_to_fstring(cls, node: HTMLNode, ctx: RecursionContext
+                               ) -> Optional[TelegramMessage]:
+        msg = cls.tag_aware_parse_node(node, ctx)
         if node.tag == "command":
-            return msg.prepend("/").format(TelegramEntityType.COMMAND)
+            msg.format(TelegramEntityType.COMMAND)
         return None
 
-    async def user_pill_to_fstring(self, msg: TelegramMessage, user_id: UserID) -> TelegramMessage:
-        user = await pu.Puppet.get_by_mxid(user_id) or await u.User.get_by_mxid(
-            user_id, create=False
-        )
+    @classmethod
+    def user_pill_to_fstring(cls, msg: TelegramMessage, user_id: UserID) -> TelegramMessage:
+        user = (pu.Puppet.get_by_mxid(user_id)
+                or u.User.get_by_mxid(user_id, create=False))
         if not user:
             return msg
-        if user.tg_username:
-            return TelegramMessage(f"@{user.tg_username}").format(TelegramEntityType.MENTION)
+        if user.username:
+            return TelegramMessage(f"@{user.username}").format(TelegramEntityType.MENTION)
         elif user.tgid:
             displayname = user.plain_displayname or msg.text
-            msg = TelegramMessage(displayname)
-            try:
-                input_entity = self.client.get_input_entity(user.tgid)
-            except (ValueError, TypeError) as e:
-                log.trace(f"Dropping mention of {user.tgid}: {e}")
-            else:
-                msg = msg.format(TelegramEntityType.MENTION_NAME, user_id=input_entity)
+            return TelegramMessage(displayname).format(TelegramEntityType.MENTION_NAME,
+                                                       user_id=user.tgid)
         return msg
 
-    async def url_to_fstring(self, msg: TelegramMessage, url: str) -> TelegramMessage:
+    @classmethod
+    def url_to_fstring(cls, msg: TelegramMessage, url: str) -> TelegramMessage:
         if url == msg.text:
-            return msg.format(self.e.URL)
+            return msg.format(cls.e.URL)
         else:
-            return msg.format(self.e.INLINE_URL, url=url)
+            return msg.format(cls.e.INLINE_URL, url=url)
 
-    async def room_pill_to_fstring(self, msg: TelegramMessage, room_id: RoomID) -> TelegramMessage:
+    @classmethod
+    def room_pill_to_fstring(cls, msg: TelegramMessage, room_id: RoomID) -> TelegramMessage:
         username = po.Portal.get_username_from_mx_alias(room_id)
-        portal = await po.Portal.find_by_username(username)
+        portal = po.Portal.find_by_username(username)
         if portal and portal.username:
             return TelegramMessage(f"@{portal.username}").format(TelegramEntityType.MENTION)
 
-    async def header_to_fstring(self, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        children = await self.node_to_fstrings(node, ctx)
+    @classmethod
+    def header_to_fstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        children = cls.node_to_fstrings(node, ctx)
         length = int(node.tag[1])
         prefix = "#" * length + " "
         return TelegramMessage.join(children, "").prepend(prefix).format(TelegramEntityType.BOLD)
 
-    async def blockquote_to_fstring(
-        self, node: HTMLNode, ctx: RecursionContext
-    ) -> TelegramMessage:
-        msg = await self.tag_aware_parse_node(node, ctx)
+    @classmethod
+    def blockquote_to_fstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
         children = msg.trim().split("\n")
         children = [child.prepend("> ") for child in children]
         return TelegramMessage.join(children, "\n")
-
-    async def color_to_fstring(self, msg: TelegramMessage, color: str) -> TelegramMessage:
-        return msg
-
-    async def spoiler_to_fstring(self, msg: TelegramMessage, spoiler: str) -> TelegramMessage:
-        return msg

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,35 +13,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import Any, Type
+from typing import Optional, Union, Any, List, Type, Dict
 from enum import Enum
 
-from telethon.tl.types import (
-    InputMessageEntityMentionName as InputMentionName,
-    MessageEntityBlockquote as Blockquote,
-    MessageEntityBold as Bold,
-    MessageEntityBotCommand as Command,
-    MessageEntityCode as Code,
-    MessageEntityEmail as Email,
-    MessageEntityItalic as Italic,
-    MessageEntityMention as Mention,
-    MessageEntityMentionName as MentionName,
-    MessageEntityPre as Pre,
-    MessageEntityStrike as Strike,
-    MessageEntityTextUrl as TextURL,
-    MessageEntityUnderline as Underline,
-    MessageEntityUrl as URL,
-    TypeMessageEntity,
-)
+from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
+                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
+                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
+                               MessageEntityBold as Bold, MessageEntityItalic as Italic,
+                               MessageEntityCode as Code, MessageEntityPre as Pre,
+                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
+                               MessageEntityBlockquote as Blockquote, TypeMessageEntity,
+                               InputMessageEntityMentionName as InputMentionName)
 
 from mautrix.util.formatter import EntityString, SemiAbstractEntity
 
 
 class TelegramEntityType(Enum):
     """EntityType is a Matrix formatting entity type."""
-
     BOLD = Bold
     ITALIC = Italic
     STRIKETHROUGH = Strike
@@ -53,7 +41,7 @@ class TelegramEntityType(Enum):
     INLINE_CODE = Code
     BLOCKQUOTE = Blockquote
     MENTION = Mention
-    MENTION_NAME = InputMentionName
+    MENTION_NAME = MentionName
     COMMAND = Command
 
     USER_MENTION = 1
@@ -64,20 +52,15 @@ class TelegramEntityType(Enum):
 class TelegramEntity(SemiAbstractEntity):
     internal: TypeMessageEntity
 
-    def __init__(
-        self,
-        type: TelegramEntityType | Type[TypeMessageEntity],
-        offset: int,
-        length: int,
-        extra_info: dict[str, Any],
-    ) -> None:
+    def __init__(self, type: Union[TelegramEntityType, Type[TypeMessageEntity]],
+                 offset: int, length: int, extra_info: Dict[str, Any]) -> None:
         if isinstance(type, TelegramEntityType):
             if isinstance(type.value, int):
                 raise ValueError(f"Can't create Entity with non-Telegram EntityType {type}")
             type = type.value
         self.internal = type(offset=offset, length=length, **extra_info)
 
-    def copy(self) -> TelegramEntity:
+    def copy(self) -> Optional['TelegramEntity']:
         extra_info = {}
         if isinstance(self.internal, Pre):
             extra_info["language"] = self.internal.language
@@ -85,12 +68,8 @@ class TelegramEntity(SemiAbstractEntity):
             extra_info["url"] = self.internal.url
         elif isinstance(self.internal, (MentionName, InputMentionName)):
             extra_info["user_id"] = self.internal.user_id
-        return TelegramEntity(
-            type(self.internal),
-            offset=self.internal.offset,
-            length=self.internal.length,
-            extra_info=extra_info,
-        )
+        return TelegramEntity(type(self.internal), offset=self.internal.offset,
+                              length=self.internal.length, extra_info=extra_info)
 
     def __repr__(self) -> str:
         return str(self.internal)
@@ -116,5 +95,5 @@ class TelegramMessage(EntityString[TelegramEntity, TelegramEntityType]):
     entity_class = TelegramEntity
 
     @property
-    def telegram_entities(self) -> list[TypeMessageEntity]:
+    def telegram_entities(self) -> List[TypeMessageEntity]:
         return [entity.internal for entity in self.entities]

mautrix_telegram/formatter/from_telegram.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,121 +13,88 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
+from typing import List, Optional, TYPE_CHECKING
 from html import escape
 import logging
 import re
 
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityUrl,
+                               MessageEntityEmail, MessageEntityTextUrl, MessageEntityBold,
+                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
+                               MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
+                               MessageEntityPhone, TypeMessageEntity, PeerChannel,
+                               MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
+                               MessageEntityUnderline, PeerUser)
+from telethon.tl.custom import Message
 from telethon.errors import RPCError
 from telethon.helpers import add_surrogate, del_surrogate
-from telethon.tl.custom import Message
-from telethon.tl.types import (
-    MessageEntityBlockquote,
-    MessageEntityBold,
-    MessageEntityBotCommand,
-    MessageEntityCashtag,
-    MessageEntityCode,
-    MessageEntityEmail,
-    MessageEntityHashtag,
-    MessageEntityItalic,
-    MessageEntityMention,
-    MessageEntityMentionName,
-    MessageEntityPhone,
-    MessageEntityPre,
-    MessageEntityStrike,
-    MessageEntityTextUrl,
-    MessageEntityUnderline,
-    MessageEntityUrl,
-    MessageFwdHeader,
-    PeerChannel,
-    PeerChat,
-    PeerUser,
-    SponsoredMessage,
-    TypeMessageEntity,
-)
 
+from mautrix.errors import MatrixRequestError
 from mautrix.appservice import IntentAPI
-from mautrix.types import (
-    EventType,
-    Format,
-    MessageType,
-    RelatesTo,
-    RelationType,
-    TextMessageEventContent,
-)
+from mautrix.types import (TextMessageEventContent, RelatesTo, RelationType, Format, MessageType,
+                           MessageEvent)
 
-from .. import abstract_user as au, portal as po, puppet as pu, user as u
-from ..db import Message as DBMessage
+from .. import user as u, puppet as pu, portal as po
 from ..types import TelegramID
+from ..db import Message as DBMessage
+
+if TYPE_CHECKING:
+    from ..abstract_user import AbstractUser
 
 log: logging.Logger = logging.getLogger("mau.fmt.tg")
 
 
-async def telegram_reply_to_matrix(evt: Message, source: au.AbstractUser) -> RelatesTo | None:
-    if evt.reply_to:
-        space = (
-            evt.peer_id.channel_id
-            if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
-            else source.tgid
-        )
-        msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
+def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Optional[RelatesTo]:
+    if evt.reply_to_msg_id:
+        space = (evt.to_id.channel_id
+                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+                 else source.tgid)
+        msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to_msg_id), space)
         if msg:
-            return RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
+            return RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
     return None
 
 
-async def _add_forward_header(
-    source: au.AbstractUser, content: TextMessageEventContent, fwd_from: MessageFwdHeader
-) -> None:
+async def _add_forward_header(source: 'AbstractUser', content: TextMessageEventContent,
+                              fwd_from: MessageFwdHeader) -> None:
     if not content.formatted_body or content.format != Format.HTML:
         content.format = Format.HTML
         content.formatted_body = escape(content.body)
     fwd_from_html, fwd_from_text = None, None
-    if isinstance(fwd_from.from_id, PeerUser):
-        user = await u.User.get_by_tgid(TelegramID(fwd_from.from_id.user_id))
+    if fwd_from.from_id:
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
-            fwd_from_html = (
-                f"<a href='https://matrix.to/#/{user.mxid}'>{escape(fwd_from_text)}</a>"
-            )
+            fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
+                             f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
-            puppet = await pu.Puppet.get_by_tgid(
-                TelegramID(fwd_from.from_id.user_id), create=False
-            )
+            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
-                fwd_from_html = (
-                    f"<a href='https://matrix.to/#/{puppet.mxid}'>{escape(fwd_from_text)}</a>"
-                )
+                fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                                 f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             try:
-                user = await source.client.get_entity(fwd_from.from_id)
+                user = await source.client.get_entity(PeerUser(fwd_from.from_id))
                 if user:
-                    fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
+                    fwd_from_text = pu.Puppet.get_displayname(user, False)
                     fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
             except (ValueError, RPCError):
                 fwd_from_text = fwd_from_html = "unknown user"
-    elif isinstance(fwd_from.from_id, (PeerChannel, PeerChat)):
-        from_id = (
-            fwd_from.from_id.chat_id
-            if isinstance(fwd_from.from_id, PeerChat)
-            else fwd_from.from_id.channel_id
-        )
-        portal = await po.Portal.get_by_tgid(TelegramID(from_id))
-        if portal and portal.title:
+    elif fwd_from.channel_id:
+        portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
+        if portal:
             fwd_from_text = portal.title
             if portal.alias:
-                fwd_from_html = (
-                    f"<a href='https://matrix.to/#/{portal.alias}'>{escape(fwd_from_text)}</a>"
-                )
+                fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
+                                 f"{escape(fwd_from_text)}</a>")
             else:
                 fwd_from_html = f"channel <b>{escape(fwd_from_text)}</b>"
         else:
             try:
-                channel = await source.client.get_entity(fwd_from.from_id)
+                channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
                 if channel:
                     fwd_from_text = f"channel {channel.title}"
                     fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
@@ -144,48 +111,37 @@ async def _add_forward_header(
     content.body = f"Forwarded from {fwd_from_text}:\n{content.body}"
     content.formatted_body = (
         f"Forwarded message from {fwd_from_html}<br/>"
-        f"<tg-forward><blockquote>{content.formatted_body}</blockquote></tg-forward>"
-    )
+        f"<tg-forward><blockquote>{content.formatted_body}</blockquote></tg-forward>")
 
 
-async def _add_reply_header(
-    source: au.AbstractUser, content: TextMessageEventContent, evt: Message, main_intent: IntentAPI
-) -> None:
-    space = (
-        evt.peer_id.channel_id
-        if isinstance(evt, Message) and isinstance(evt.peer_id, PeerChannel)
-        else source.tgid
-    )
+async def _add_reply_header(source: 'AbstractUser', content: TextMessageEventContent, evt: Message,
+                            main_intent: IntentAPI):
+    space = (evt.to_id.channel_id
+             if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+             else source.tgid)
 
-    msg = await DBMessage.get_one_by_tgid(TelegramID(evt.reply_to.reply_to_msg_id), space)
+    msg = DBMessage.get_one_by_tgid(TelegramID(evt.reply_to_msg_id), space)
     if not msg:
         return
 
-    content.relates_to = RelatesTo(rel_type=RelationType.REPLY, event_id=msg.mxid)
+    content.relates_to = RelatesTo(rel_type=RelationType.REFERENCE, event_id=msg.mxid)
 
     try:
-        event = await main_intent.get_event(msg.mx_room, msg.mxid)
-        if event.type == EventType.ROOM_ENCRYPTED and source.bridge.matrix.e2ee:
-            event = await source.bridge.matrix.e2ee.decrypt(event)
+        event: MessageEvent = await main_intent.get_event(msg.mx_room, msg.mxid)
         if isinstance(event.content, TextMessageEventContent):
             event.content.trim_reply_fallback()
-        puppet = await pu.Puppet.get_by_mxid(event.sender, create=False)
+        puppet = pu.Puppet.get_by_mxid(event.sender, create=False)
         content.set_reply(event, displayname=puppet.displayname if puppet else event.sender)
-    except Exception:
+    except MatrixRequestError:
         log.exception("Failed to get event to add reply fallback")
 
 
-async def telegram_to_matrix(
-    evt: Message | SponsoredMessage,
-    source: au.AbstractUser,
-    main_intent: IntentAPI | None = None,
-    prefix_text: str | None = None,
-    prefix_html: str | None = None,
-    override_text: str = None,
-    override_entities: list[TypeMessageEntity] = None,
-    no_reply_fallback: bool = False,
-    require_html: bool = False,
-) -> TextMessageEventContent:
+async def telegram_to_matrix(evt: Message, source: "AbstractUser",
+                             main_intent: Optional[IntentAPI] = None,
+                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
+                             override_text: str = None,
+                             override_entities: List[TypeMessageEntity] = None,
+                             no_reply_fallback: bool = False) -> TextMessageEventContent:
     content = TextMessageEventContent(
         msgtype=MessageType.TEXT,
         body=add_surrogate(override_text or evt.message),
@@ -193,50 +149,49 @@ async def telegram_to_matrix(
     entities = override_entities or evt.entities
     if entities:
         content.format = Format.HTML
-        html = await _telegram_entities_to_matrix_catch(add_surrogate(content.body), entities)
-        content.formatted_body = del_surrogate(html).replace("\n", "<br/>")
+        content.formatted_body = _telegram_entities_to_matrix_catch(content.body, entities)
 
-    def force_html():
+    if prefix_html:
         if not content.formatted_body:
             content.format = Format.HTML
             content.formatted_body = escape(content.body)
-
-    if require_html:
-        force_html()
-
-    if prefix_html:
-        force_html()
         content.formatted_body = prefix_html + content.formatted_body
     if prefix_text:
         content.body = prefix_text + content.body
 
-    if getattr(evt, "fwd_from", None):
+    if evt.fwd_from:
         await _add_forward_header(source, content, evt.fwd_from)
 
-    if getattr(evt, "reply_to", None) and not no_reply_fallback:
+    if evt.reply_to_msg_id and not no_reply_fallback:
         await _add_reply_header(source, content, evt, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
-        force_html()
+        if not content.formatted_body:
+            content.formatted_body = escape(content.body)
         content.body += f"\n- {evt.post_author}"
         content.formatted_body += f"<br/><i>- <u>{evt.post_author}</u></i>"
 
+    content.body = del_surrogate(content.body)
+
+    if content.formatted_body:
+        content.formatted_body = del_surrogate(content.formatted_body.replace("\n", "<br/>"))
+
     return content
 
 
-async def _telegram_entities_to_matrix_catch(text: str, entities: list[TypeMessageEntity]) -> str:
+def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEntity]) -> str:
     try:
-        return await _telegram_entities_to_matrix(text, entities)
+        return _telegram_entities_to_matrix(text, entities)
     except Exception:
-        log.exception(
-            "Failed to convert Telegram format:\nmessage=%s\nentities=%s", text, entities
-        )
+        log.exception("Failed to convert Telegram format:\n"
+                      "message=%s\n"
+                      "entities=%s",
+                      text, entities)
     return "[failed conversion in _telegram_entities_to_matrix]"
 
 
-async def _telegram_entities_to_matrix(
-    text: str, entities: list[TypeMessageEntity], offset: int = 0, length: int = None
-) -> str:
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity],
+                                 offset: int = 0, length: int = None) -> str:
     if not entities:
         return escape(text)
     if length is None:
@@ -253,12 +208,9 @@ async def _telegram_entities_to_matrix(
             continue
 
         skip_entity = False
-        entity_text = await _telegram_entities_to_matrix(
-            text=text[relative_offset : relative_offset + entity.length],
-            entities=entities[i + 1 :],
-            offset=entity.offset,
-            length=entity.length,
-        )
+        entity_text = _telegram_entities_to_matrix(
+            text=text[relative_offset:relative_offset + entity.length],
+            entities=entities[i + 1:], offset=entity.offset, length=entity.length)
         entity_type = type(entity)
 
         if entity_type == MessageEntityBold:
@@ -272,25 +224,22 @@ async def _telegram_entities_to_matrix(
         elif entity_type == MessageEntityBlockquote:
             html.append(f"<blockquote>{entity_text}</blockquote>")
         elif entity_type == MessageEntityCode:
-            html.append(
-                f"<pre><code>{entity_text}</code></pre>"
-                if "\n" in entity_text
-                else f"<code>{entity_text}</code>"
-            )
+            html.append(f"<pre><code>{entity_text}</code></pre>"
+                        if "\n" in entity_text
+                        else f"<code>{entity_text}</code>")
         elif entity_type == MessageEntityPre:
             skip_entity = _parse_pre(html, entity_text, entity.language)
         elif entity_type == MessageEntityMention:
-            skip_entity = await _parse_mention(html, entity_text)
+            skip_entity = _parse_mention(html, entity_text)
         elif entity_type == MessageEntityMentionName:
-            skip_entity = await _parse_name_mention(html, entity_text, TelegramID(entity.user_id))
+            skip_entity = _parse_name_mention(html, entity_text, TelegramID(entity.user_id))
         elif entity_type == MessageEntityEmail:
             html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
         elif entity_type in (MessageEntityTextUrl, MessageEntityUrl):
-            skip_entity = await _parse_url(
-                html, entity_text, entity.url if entity_type == MessageEntityTextUrl else None
-            )
+            skip_entity = _parse_url(html, entity_text,
+                                     entity.url if entity_type == MessageEntityTextUrl else None)
         elif entity_type == MessageEntityBotCommand:
-            html.append(f"<font color='blue'>{entity_text}</font>")
+            html.append(f"<font color='blue'>!{entity_text[1:]}</font>")
         elif entity_type in (MessageEntityHashtag, MessageEntityCashtag, MessageEntityPhone):
             html.append(f"<font color='blue'>{entity_text}</font>")
         else:
@@ -301,22 +250,24 @@ async def _telegram_entities_to_matrix(
     return "".join(html)
 
 
-def _parse_pre(html: list[str], entity_text: str, language: str) -> bool:
+def _parse_pre(html: List[str], entity_text: str, language: str) -> bool:
     if language:
-        html.append(f"<pre><code class='language-{language}'>{entity_text}</code></pre>")
+        html.append("<pre>"
+                    f"<code class='language-{language}'>{entity_text}</code>"
+                    "</pre>")
     else:
         html.append(f"<pre><code>{entity_text}</code></pre>")
     return False
 
 
-async def _parse_mention(html: list[str], entity_text: str) -> bool:
+def _parse_mention(html: List[str], entity_text: str) -> bool:
     username = entity_text[1:]
 
-    user = await u.User.find_by_username(username) or await pu.Puppet.find_by_username(username)
+    user = u.User.find_by_username(username) or pu.Puppet.find_by_username(username)
     if user:
         mxid = user.mxid
     else:
-        portal = await po.Portal.find_by_username(username)
+        portal = po.Portal.find_by_username(username)
         mxid = portal.alias or portal.mxid if portal else None
 
     if mxid:
@@ -326,12 +277,12 @@ async def _parse_mention(html: list[str], entity_text: str) -> bool:
     return False
 
 
-async def _parse_name_mention(html: list[str], entity_text: str, user_id: TelegramID) -> bool:
-    user = await u.User.get_by_tgid(user_id)
+def _parse_name_mention(html: List[str], entity_text: str, user_id: TelegramID) -> bool:
+    user = u.User.get_by_tgid(user_id)
     if user:
         mxid = user.mxid
     else:
-        puppet = await pu.Puppet.get_by_tgid(user_id, create=False)
+        puppet = pu.Puppet.get(user_id, create=False)
         mxid = puppet.mxid if puppet else None
     if mxid:
         html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
@@ -340,12 +291,11 @@ async def _parse_name_mention(html: list[str], entity_text: str, user_id: Telegr
     return False
 
 
-message_link_regex = re.compile(
-    r"https?://t(?:elegram)?\.(?:me|dog)/([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})"
-)
+message_link_regex = re.compile(r"https?://t(?:elegram)?\.(?:me|dog)/"
+                                r"([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})")
 
 
-async def _parse_url(html: list[str], entity_text: str, url: str) -> bool:
+def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
     url = escape(url) if url else entity_text
     if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
         url = "http://" + url
@@ -355,9 +305,9 @@ async def _parse_url(html: list[str], entity_text: str, url: str) -> bool:
         group, msgid_str = message_link_match.groups()
         msgid = int(msgid_str)
 
-        portal = await po.Portal.find_by_username(group)
+        portal = po.Portal.find_by_username(group)
         if portal:
-            message = await DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
+            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 

mautrix_telegram/get_version.py

@@ -1,6 +1,6 @@
-import os
-import shutil
 import subprocess
+import shutil
+import os
 
 from . import __version__
 
@@ -15,11 +15,10 @@ cmd_env = {
 def run(cmd):
     return subprocess.check_output(cmd, stderr=subprocess.DEVNULL, env=cmd_env)
 
-
 if os.path.exists(".git") and shutil.which("git"):
     try:
         git_revision = run(["git", "rev-parse", "HEAD"]).strip().decode("ascii")
-        git_revision_url = f"https://github.com/mautrix/telegram/commit/{git_revision}"
+        git_revision_url = f"https://github.com/tulir/mautrix-telegram/commit/{git_revision}"
         git_revision = git_revision[:8]
     except (subprocess.SubprocessError, OSError):
         git_revision = "unknown"
@@ -34,7 +33,8 @@ else:
     git_revision_url = None
     git_tag = None
 
-git_tag_url = f"https://github.com/mautrix/telegram/releases/tag/{git_tag}" if git_tag else None
+git_tag_url = (f"https://github.com/tulir/mautrix-telegram/releases/tag/{git_tag}"
+               if git_tag else None)
 
 if git_tag and __version__ == git_tag[1:].replace("-", ""):
     version = __version__

mautrix_telegram/matrix.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,203 +13,174 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
+from typing import Dict, Set, Tuple, Union, Iterable, TYPE_CHECKING
 
-from typing import TYPE_CHECKING, Iterable
-
-from mautrix.appservice import DOUBLE_PUPPET_SOURCE_KEY
 from mautrix.bridge import BaseMatrixHandler
+from mautrix.types import (Event, EventType, RoomID, UserID, EventID, ReceiptEvent, ReceiptType,
+                           ReceiptEventContent, PresenceEvent, PresenceState, TypingEvent,
+                           MessageEvent, StateEvent, RedactionEvent, RoomNameStateEventContent,
+                           RoomAvatarStateEventContent, RoomTopicStateEventContent,
+                           MemberStateEventContent)
 from mautrix.errors import MatrixError
-from mautrix.types import (
-    Event,
-    EventID,
-    EventType,
-    MemberStateEventContent,
-    MessageType,
-    PresenceEvent,
-    PresenceState,
-    ReceiptEvent,
-    ReceiptEventContent,
-    ReceiptType,
-    RedactionEvent,
-    RoomAvatarStateEventContent as AvatarContent,
-    RoomID,
-    RoomNameStateEventContent as NameContent,
-    RoomTopicStateEventContent as TopicContent,
-    SingleReceiptEventContent,
-    StateEvent,
-    TextMessageEventContent,
-    TypingEvent,
-    UserID,
-)
 
-from . import commands as com, portal as po, puppet as pu, user as u
+from . import user as u, portal as po, puppet as pu, commands as com
 
 if TYPE_CHECKING:
-    from .__main__ import TelegramBridge
+    from .context import Context
+    from .bot import Bot
+
+try:
+    from prometheus_client import Histogram
+
+    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events", ["event_type"])
+except ImportError:
+    Histogram = None
+    EVENT_TIME = None
+
+RoomMetaStateEventContent = Union[RoomNameStateEventContent, RoomAvatarStateEventContent,
+                                  RoomTopicStateEventContent]
 
 
 class MatrixHandler(BaseMatrixHandler):
-    commands: com.CommandProcessor
-    _previously_typing: dict[RoomID, set[UserID]]
+    bot: 'Bot'
+    commands: 'com.CommandProcessor'
+    previously_typing: Dict[RoomID, Set[UserID]]
 
-    def __init__(self, bridge: "TelegramBridge") -> None:
-        prefix, suffix = bridge.config["bridge.username_template"].format(userid=":").split(":")
-        homeserver = bridge.config["homeserver.domain"]
-        self.user_id_prefix = f"@{prefix}"
-        self.user_id_suffix = f"{suffix}:{homeserver}"
+    def __init__(self, context: 'Context') -> None:
+        super(MatrixHandler, self).__init__(context.az, context.config, loop=context.loop,
+                                            command_processor=com.CommandProcessor(context))
+        self.bot = context.bot
+        self.previously_typing = {}
 
-        super().__init__(command_processor=com.CommandProcessor(bridge), bridge=bridge)
+    async def get_user(self, user_id: UserID) -> 'u.User':
+        return await u.User.get_by_mxid(user_id).ensure_started()
 
-        self._previously_typing = {}
+    async def get_portal(self, room_id: RoomID) -> 'po.Portal':
+        return po.Portal.get_by_mxid(room_id)
 
-    async def handle_puppet_invite(
-        self, room_id: RoomID, puppet: pu.Puppet, inviter: u.User, event_id: EventID
-    ) -> None:
+    async def get_puppet(self, user_id: UserID) -> 'pu.Puppet':
+        return pu.Puppet.get_by_mxid(user_id)
+
+    async def handle_puppet_invite(self, room_id: RoomID, puppet: pu.Puppet, inviter: u.User,
+                                   event_id: EventID) -> None:
         intent = puppet.default_mxid_intent
-        self.log.debug(f"{inviter.mxid} invited puppet for {puppet.tgid} to {room_id}")
+        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}")
         if not await inviter.is_logged_in():
             await intent.error_and_leave(
-                room_id, text="Please log in before inviting Telegram puppets."
-            )
+                room_id, text="Please log in before inviting Telegram puppets.")
             return
-        portal = await po.Portal.get_by_mxid(room_id)
+        portal = po.Portal.get_by_mxid(room_id)
         if portal:
             if portal.peer_type == "user":
                 await intent.error_and_leave(
-                    room_id, text="You can not invite additional users to private chats."
-                )
+                    room_id, text="You can not invite additional users to private chats.")
                 return
             await portal.invite_telegram(inviter, puppet)
             await intent.join_room(room_id)
             return
         try:
-            members = await intent.get_room_members(room_id)
+            members = await self.az.intent.get_room_members(room_id)
         except MatrixError:
-            self.log.exception(f"Failed to get members after joining {room_id} as {intent.mxid}")
-            return
+            members = []
         if self.az.bot_mxid not in members:
-            if len(members) > 2:
-                await intent.error_and_leave(
-                    room_id,
-                    text=None,
-                    html=(
-                        f"Please invite "
-                        f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
-                        f"first if you want to create a Telegram chat."
-                    ),
-                )
+            if len(members) > 1:
+                await intent.error_and_leave(room_id, text=None, html=(
+                    f"Please invite "
+                    f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
+                    f"first if you want to create a Telegram chat."))
                 return
 
             await intent.join_room(room_id)
-            portal = await po.Portal.get_by_tgid(
-                puppet.tgid, tg_receiver=inviter.tgid, peer_type="user"
-            )
+            portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
             if portal.mxid:
                 try:
-                    await portal.invite_to_matrix(inviter.mxid)
+                    await intent.invite_user(portal.mxid, inviter.mxid)
                     await intent.send_notice(
-                        room_id,
-                        text=f"You already have a private chat with me: {portal.mxid}",
-                        html=(
-                            "You already have a private chat with me: "
-                            f"<a href='https://matrix.to/#/{portal.mxid}'>Link to room</a>"
-                        ),
-                    )
+                        room_id, text=f"You already have a private chat with me: {portal.mxid}",
+                        html=("You already have a private chat with me: "
+                              f"<a href='https://matrix.to/#/{portal.mxid}'>Link to room</a>"))
                     await intent.leave_room(room_id)
                     return
                 except MatrixError:
                     pass
             portal.mxid = room_id
-            e2be_ok = await portal.check_dm_encryption()
-            await portal.save()
-            await inviter.register_portal(portal)
-            if e2be_ok is True:
-                evt_type, content = await self.e2ee.encrypt(
-                    room_id,
-                    EventType.ROOM_MESSAGE,
-                    TextMessageEventContent(
-                        msgtype=MessageType.NOTICE,
-                        body=(
-                            "Portal to private chat created and end-to-bridge encryption enabled."
-                        ),
-                    ),
-                )
-                await intent.send_message_event(room_id, evt_type, content)
-            else:
-                message = "Portal to private chat created."
-                if e2be_ok is False:
-                    message += "\n\nWarning: Failed to enable end-to-bridge encryption"
-                await intent.send_notice(room_id, message)
-            await portal.update_bridge_info()
+            portal.save()
+            inviter.register_portal(portal)
+            await intent.send_notice(room_id, "Portal to private chat created.")
         else:
             await intent.join_room(room_id)
-            await intent.send_notice(
-                room_id,
-                "This puppet will remain inactive until a Telegram chat is created for this room.",
-            )
+            await intent.send_notice(room_id, "This puppet will remain inactive until a "
+                                              "Telegram chat is created for this room.")
 
-    async def handle_invite(
-        self, room_id: RoomID, user_id: UserID, inviter: u.User, event_id: EventID
-    ) -> None:
-        user = await u.User.get_by_mxid(user_id, create=False)
+    async def send_welcome_message(self, room_id: RoomID, inviter: 'u.User') -> None:
+        try:
+            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
+        except MatrixError:
+            # The AS bot is not in the room.
+            return
+        cmd_prefix = self.commands.command_prefix
+        text = html = "Hello, I'm a Telegram bridge bot. "
+        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
+            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
+            html += (f"Use <code>{cmd_prefix} help</code> for help"
+                     f" or <code>{cmd_prefix} login</code> to log in.")
+        else:
+            text += f"Use `{cmd_prefix} help` for help."
+            html += f"Use <code>{cmd_prefix} help</code> for help."
+        await self.az.intent.send_notice(room_id, text=text, html=html)
+
+    async def handle_invite(self, room_id: RoomID, user_id: UserID, inviter: 'u.User',
+                            event_id: EventID) -> None:
+        user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
-        portal = await po.Portal.get_by_mxid(room_id)
-        if user and await user.has_full_access(allow_bot=True):
-            if portal and portal.allow_bridging:
-                await portal.invite_telegram(inviter, user)
+        portal = po.Portal.get_by_mxid(room_id)
+        if user and await user.has_full_access(allow_bot=True) and portal:
+            await portal.invite_telegram(inviter, user)
 
     async def handle_join(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None:
-        user = await u.User.get_and_start_by_mxid(user_id)
+        user = await u.User.get_by_mxid(user_id).ensure_started()
 
-        portal = await po.Portal.get_by_mxid(room_id)
-        if not portal or not portal.allow_bridging:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
             return
 
         if not user.relaybot_whitelisted:
-            await portal.main_intent.kick_user(
-                room_id, user.mxid, "You are not whitelisted on this Telegram bridge."
-            )
+            await portal.main_intent.kick_user(room_id, user.mxid,
+                                               "You are not whitelisted on this Telegram bridge.")
             return
         elif not await user.is_logged_in() and not portal.has_bot:
-            await portal.main_intent.kick_user(
-                room_id,
-                user.mxid,
-                "This chat does not have a bot relaying messages for unauthenticated users.",
-            )
+            await portal.main_intent.kick_user(room_id, user.mxid,
+                                               "This chat does not have a bot relaying "
+                                               "messages for unauthenticated users.")
             return
 
-        self.log.debug(f"{user.mxid} joined {room_id}")
+        self.log.debug(f"{user} joined {room_id}")
         if await user.is_logged_in() or portal.has_bot:
             await portal.join_matrix(user, event_id)
 
+    async def get_leave_handle_info(self) -> Tuple[po.Portal, u.User]:
+        pass
+
     async def handle_leave(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None:
         self.log.debug(f"{user_id} left {room_id}")
-        portal = await po.Portal.get_by_mxid(room_id)
-        if not portal or not portal.allow_bridging:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
             return
 
-        user = await u.User.get_by_mxid(user_id, create=False)
+        user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
         await portal.leave_matrix(user, event_id)
 
-    async def handle_kick_ban(
-        self,
-        ban: bool,
-        room_id: RoomID,
-        user_id: UserID,
-        sender: UserID,
-        reason: str,
-        event_id: EventID,
-    ) -> None:
+    async def handle_kick_ban(self, ban: bool, room_id: RoomID, user_id: UserID, sender: UserID,
+                              reason: str, event_id: EventID) -> None:
         action = "banned" if ban else "kicked"
         self.log.debug(f"{user_id} was {action} from {room_id} by {sender} for {reason}")
-        portal = await po.Portal.get_by_mxid(room_id)
-        if not portal or not portal.allow_bridging:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
             return
 
         if user_id == self.az.bot_mxid:
@@ -218,12 +189,12 @@ class MatrixHandler(BaseMatrixHandler):
                 await portal.unbridge()
             return
 
-        sender = await u.User.get_by_mxid(sender, create=False)
+        sender = u.User.get_by_mxid(sender, create=False)
         if not sender:
             return
         await sender.ensure_started()
 
-        puppet = await pu.Puppet.get_by_mxid(user_id)
+        puppet = pu.Puppet.get_by_mxid(user_id)
         if puppet:
             if ban:
                 await portal.ban_matrix(puppet, sender)
@@ -231,7 +202,7 @@ class MatrixHandler(BaseMatrixHandler):
                 await portal.kick_matrix(puppet, sender)
             return
 
-        user = await u.User.get_by_mxid(user_id, create=False)
+        user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
@@ -240,140 +211,134 @@ class MatrixHandler(BaseMatrixHandler):
         else:
             await portal.kick_matrix(user, sender)
 
-    async def handle_kick(
-        self, room_id: RoomID, user_id: UserID, kicked_by: UserID, reason: str, event_id: EventID
-    ) -> None:
+    async def handle_kick(self, room_id: RoomID, user_id: UserID, kicked_by: UserID, reason: str,
+                          event_id: EventID) -> None:
         await self.handle_kick_ban(False, room_id, user_id, kicked_by, reason, event_id)
 
-    async def handle_unban(
-        self, room_id: RoomID, user_id: UserID, unbanned_by: UserID, reason: str, event_id: EventID
-    ) -> None:
+    async def handle_unban(self, room_id: RoomID, user_id: UserID, unbanned_by: UserID,
+                           reason: str, event_id: EventID) -> None:
         # TODO handle unbans properly instead of handling it as a kick
         await self.handle_kick_ban(False, room_id, user_id, unbanned_by, reason, event_id)
 
-    async def handle_ban(
-        self, room_id: RoomID, user_id: UserID, banned_by: UserID, reason: str, event_id: EventID
-    ) -> None:
+    async def handle_ban(self, room_id: RoomID, user_id: UserID, banned_by: UserID, reason: str,
+                         event_id: EventID) -> None:
         await self.handle_kick_ban(True, room_id, user_id, banned_by, reason, event_id)
 
-    async def allow_message(self, user: u.User) -> bool:
+    @staticmethod
+    async def allow_message(user: 'u.User') -> bool:
         return user.relaybot_whitelisted
 
-    async def allow_command(self, user: u.User) -> bool:
+    @staticmethod
+    async def allow_command(user: 'u.User') -> bool:
         return user.whitelisted
 
     @staticmethod
-    async def allow_bridging_message(user: u.User, portal: po.Portal) -> bool:
+    async def allow_bridging_message(user: 'u.User', portal: 'po.Portal') -> bool:
         return await user.is_logged_in() or portal.has_bot
 
     @staticmethod
     async def handle_redaction(evt: RedactionEvent) -> None:
-        sender = await u.User.get_and_start_by_mxid(evt.sender)
+        sender = await u.User.get_by_mxid(evt.sender).ensure_started()
         if not sender.relaybot_whitelisted:
             return
 
-        portal = await po.Portal.get_by_mxid(evt.room_id)
-        if not portal or not portal.allow_bridging:
+        portal = po.Portal.get_by_mxid(evt.room_id)
+        if not portal:
             return
 
-        await portal.handle_matrix_deletion(sender, evt.redacts, evt.event_id)
+        await portal.handle_matrix_deletion(sender, evt.redacts)
 
     @staticmethod
     async def handle_power_levels(evt: StateEvent) -> None:
-        portal = await po.Portal.get_by_mxid(evt.room_id)
-        sender = await u.User.get_and_start_by_mxid(evt.sender)
-        if await sender.has_full_access(allow_bot=True) and portal and portal.allow_bridging:
-            await portal.handle_matrix_power_levels(
-                sender, evt.content.users, evt.unsigned.prev_content.users, evt.event_id
-            )
+        portal = po.Portal.get_by_mxid(evt.room_id)
+        sender = await u.User.get_by_mxid(evt.sender).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
+            await portal.handle_matrix_power_levels(sender, evt.content.users,
+                                                    evt.unsigned.prev_content.users)
 
     @staticmethod
-    async def handle_room_meta(
-        evt_type: EventType,
-        room_id: RoomID,
-        sender_mxid: UserID,
-        content: NameContent | AvatarContent | TopicContent,
-        event_id: EventID,
-    ) -> None:
-        portal = await po.Portal.get_by_mxid(room_id)
-        sender = await u.User.get_and_start_by_mxid(sender_mxid)
-        if await sender.has_full_access(allow_bot=True) and portal and portal.allow_bridging:
+    async def handle_room_meta(evt_type: EventType, room_id: RoomID, sender_mxid: UserID,
+                               content: RoomMetaStateEventContent) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             handler, content_type, content_key = {
-                EventType.ROOM_NAME: (portal.handle_matrix_title, NameContent, "name"),
-                EventType.ROOM_TOPIC: (portal.handle_matrix_about, TopicContent, "topic"),
-                EventType.ROOM_AVATAR: (portal.handle_matrix_avatar, AvatarContent, "url"),
+                EventType.ROOM_NAME: (portal.handle_matrix_title, RoomNameStateEventContent, "name"),
+                EventType.ROOM_TOPIC: (portal.handle_matrix_about, RoomTopicStateEventContent, "topic"),
+                EventType.ROOM_AVATAR: (portal.handle_matrix_avatar, RoomAvatarStateEventContent, "url"),
             }[evt_type]
             if not isinstance(content, content_type):
                 return
-            await handler(sender, content[content_key], event_id)
+            await handler(sender, content[content_key])
 
     @staticmethod
-    async def handle_room_pin(
-        room_id: RoomID,
-        sender_mxid: UserID,
-        new_events: set[str],
-        old_events: set[str],
-        event_id: EventID,
-    ) -> None:
-        portal = await po.Portal.get_by_mxid(room_id)
-        sender = await u.User.get_and_start_by_mxid(sender_mxid)
-        if await sender.has_full_access(allow_bot=True) and portal and portal.allow_bridging:
-            if not new_events:
-                await portal.handle_matrix_unpin_all(sender, event_id)
-            else:
-                changes = {
-                    event_id: event_id in new_events for event_id in new_events ^ old_events
-                }
-                await portal.handle_matrix_pin(sender, changes, event_id)
+    async def handle_room_pin(room_id: RoomID, sender_mxid: UserID,
+                              new_events: Set[str], old_events: Set[str]) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
+            events = new_events - old_events
+            if len(events) > 0:
+                # New event pinned, set that as pinned in Telegram.
+                await portal.handle_matrix_pin(sender, EventID(events.pop()))
+            elif len(new_events) == 0:
+                # All pinned events removed, remove pinned event in Telegram.
+                await portal.handle_matrix_pin(sender, None)
 
     @staticmethod
-    async def handle_room_upgrade(
-        room_id: RoomID, sender: UserID, new_room_id: RoomID, event_id: EventID
-    ) -> None:
-        portal = await po.Portal.get_by_mxid(room_id)
-        if portal and portal.allow_bridging:
-            await portal.handle_matrix_upgrade(sender, new_room_id, event_id)
+    async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if portal:
+            await portal.handle_matrix_upgrade(sender, new_room_id)
 
-    async def handle_member_info_change(
-        self,
-        room_id: RoomID,
-        user_id: UserID,
-        profile: MemberStateEventContent,
-        prev_profile: MemberStateEventContent,
-        event_id: EventID,
-    ) -> None:
+    async def handle_member_info_change(self, room_id: RoomID, user_id: UserID,
+                                        profile: MemberStateEventContent,
+                                        prev_profile: MemberStateEventContent,
+                                        event_id: EventID) -> None:
         if profile.displayname == prev_profile.displayname:
             return
 
-        portal = await po.Portal.get_by_mxid(room_id)
-        if not portal or not portal.has_bot or not portal.allow_bridging:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.has_bot:
             return
 
-        user = await u.User.get_and_start_by_mxid(user_id)
+        user = await u.User.get_by_mxid(user_id).ensure_started()
         if await user.needs_relaybot(portal):
-            await portal.name_change_matrix(
-                user, profile.displayname, prev_profile.displayname, event_id
-            )
+            await portal.name_change_matrix(user, profile.displayname, prev_profile.displayname,
+                                            event_id)
 
-    async def handle_read_receipt(
-        self, user: u.User, portal: po.Portal, event_id: EventID, data: SingleReceiptEventContent
-    ) -> None:
-        if not portal.allow_bridging:
+    @staticmethod
+    def parse_read_receipts(content: ReceiptEventContent) -> Iterable[Tuple[UserID, EventID]]:
+        return ((user_id, event_id)
+                for event_id, receipts in content.items()
+                for user_id in receipts.get(ReceiptType.READ, {}))
+
+    @staticmethod
+    async def handle_read_receipts(room_id: RoomID, receipts: Iterable[Tuple[UserID, EventID]]
+                                   ) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
             return
-        await portal.mark_read(user, event_id, data.get("ts", 0))
+
+        for user_id, event_id in receipts:
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+            await portal.mark_read(user, event_id)
 
     @staticmethod
     async def handle_presence(user_id: UserID, presence: PresenceState) -> None:
-        user = await u.User.get_by_mxid(user_id, check_db=False, create=False)
-        if user and await user.is_logged_in():
-            await user.set_presence(presence == PresenceState.ONLINE)
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if not await user.is_logged_in():
+            return
+        await user.set_presence(presence == PresenceState.ONLINE)
 
-    async def handle_typing(self, room_id: RoomID, now_typing: set[UserID]) -> None:
-        portal = await po.Portal.get_by_mxid(room_id)
-        if not portal or not portal.allow_bridging:
+    async def handle_typing(self, room_id: RoomID, now_typing: Set[UserID]) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
             return
 
-        previously_typing = self._previously_typing.get(room_id, set())
+        previously_typing = self.previously_typing.get(room_id, set())
 
         for user_id in set(previously_typing | now_typing):
             is_typing = user_id in now_typing
@@ -381,17 +346,24 @@ class MatrixHandler(BaseMatrixHandler):
             if is_typing and was_typing:
                 continue
 
-            user = await u.User.get_by_mxid(user_id, check_db=False, create=False)
-            if user and await user.is_logged_in():
-                await portal.set_typing(user, is_typing)
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
 
-        self._previously_typing[room_id] = now_typing
+            await portal.set_typing(user, is_typing)
 
-    async def handle_ephemeral_event(
-        self, evt: ReceiptEvent | PresenceEvent | TypingEvent
-    ) -> None:
+        self.previously_typing[room_id] = now_typing
+
+    def filter_matrix_event(self, evt: Event) -> bool:
+        if not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent)):
+            return True
+        return evt.sender and (evt.sender == self.az.bot_mxid
+                               or pu.Puppet.get_id_from_mxid(evt.sender) is not None)
+
+    async def handle_ephemeral_event(self, evt: Union[ReceiptEvent, PresenceEvent, TypingEvent]
+                                     ) -> None:
         if evt.type == EventType.RECEIPT:
-            await self.handle_receipt(evt)
+            await self.handle_read_receipts(evt.room_id, self.parse_read_receipts(evt.content))
         elif evt.type == EventType.PRESENCE:
             await self.handle_presence(evt.sender, evt.content.presence)
         elif evt.type == EventType.TYPING:
@@ -405,19 +377,17 @@ class MatrixHandler(BaseMatrixHandler):
         if evt.type == EventType.ROOM_POWER_LEVELS:
             await self.handle_power_levels(evt)
         elif evt.type in (EventType.ROOM_NAME, EventType.ROOM_AVATAR, EventType.ROOM_TOPIC):
-            await self.handle_room_meta(
-                evt.type, evt.room_id, evt.sender, evt.content, evt.event_id
-            )
+            await self.handle_room_meta(evt.type, evt.room_id, evt.sender, evt.content)
         elif evt.type == EventType.ROOM_PINNED_EVENTS:
             new_events = set(evt.content.pinned)
             try:
                 old_events = set(evt.unsigned.prev_content.pinned)
             except (KeyError, ValueError, TypeError, AttributeError):
                 old_events = set()
-            await self.handle_room_pin(
-                evt.room_id, evt.sender, new_events, old_events, evt.event_id
-            )
+            await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events)
         elif evt.type == EventType.ROOM_TOMBSTONE:
-            await self.handle_room_upgrade(
-                evt.room_id, evt.sender, evt.content.replacement_room, evt.event_id
-            )
+            await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room)
+
+    async def log_event_handle_duration(self, evt: Event, duration: float) -> None:
+        if EVENT_TIME:
+            EVENT_TIME.labels(event_type=str(evt.type)).observe(duration)

mautrix_telegram/portal/__init__.py

@@ -0,0 +1,21 @@
+from .base import BasePortal, init as init_base
+from .matrix import PortalMatrix, init as init_matrix
+from .metadata import PortalMetadata, init as init_metadata
+from .telegram import PortalTelegram, init as init_telegram
+from .deduplication import init as init_dedup
+from ..context import Context
+
+
+class Portal(PortalMatrix, PortalTelegram, PortalMetadata):
+    pass
+
+
+def init(context: Context) -> None:
+    init_base(context)
+    init_dedup(context)
+    init_metadata(context)
+    init_telegram(context)
+    init_matrix(context)
+
+
+__all__ = ["Portal", "init"]

mautrix_telegram/portal/__init__.pyi

@@ -0,0 +1,15 @@
+from typing import Union
+from .base import BasePortal
+from .portal_matrix import PortalMatrix
+from .portal_metadata import PortalMetadata
+from .portal_telegram import PortalTelegram
+from ..context import Context
+
+Portal = Union[BasePortal, PortalMatrix, PortalMetadata, PortalTelegram]
+
+
+def init(context: Context) -> None:
+	pass
+
+
+__all__ = ["Portal", "init"]

mautrix_telegram/portal/base.py

@@ -0,0 +1,508 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
+from abc import ABC, abstractmethod
+import asyncio
+import logging
+import json
+
+from telethon.tl.functions.messages import ExportChatInviteRequest
+from telethon.tl.types import (Channel, ChannelFull, Chat, ChatFull, ChatInviteEmpty, InputChannel,
+                               InputPeerChannel, InputPeerChat, InputPeerUser, InputUser,
+                               PeerChannel, PeerChat, PeerUser, TypeChat, TypeInputPeer, TypePeer,
+                               TypeUser, TypeUserFull, User, UserFull, TypeInputChannel, Photo,
+                               Document, TypePhotoSize, PhotoSize, InputPhotoFileLocation,
+                               TypeChatParticipant, TypeChannelParticipant, PhotoEmpty, ChatPhoto,
+                               ChatPhotoEmpty)
+
+from mautrix.errors import MatrixRequestError, IntentError
+from mautrix.appservice import AppService, IntentAPI
+from mautrix.types import RoomID, RoomAlias, UserID, EventType, PowerLevelStateEventContent
+from mautrix.util.simple_template import SimpleTemplate
+
+from ..types import TelegramID
+from ..context import Context
+from ..db import Portal as DBPortal
+from .. import puppet as p, user as u, util
+from .deduplication import PortalDedup
+from .send_lock import PortalSendLock
+
+if TYPE_CHECKING:
+    from ..bot import Bot
+    from ..abstract_user import AbstractUser
+    from ..config import Config
+    from . import Portal
+
+TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
+TypeChatPhoto = Union[ChatPhoto, ChatPhotoEmpty, Photo, PhotoEmpty]
+InviteList = Union[UserID, List[UserID]]
+
+config: Optional['Config'] = None
+
+
+class BasePortal(ABC):
+    base_log: logging.Logger = logging.getLogger("mau.portal")
+    az: AppService = None
+    bot: 'Bot' = None
+    loop: asyncio.AbstractEventLoop = None
+
+    # Config cache
+    filter_mode: str = None
+    filter_list: List[str] = None
+
+    max_initial_member_sync: int = -1
+    sync_channel_members: bool = True
+    sync_matrix_state: bool = True
+    public_portals: bool = False
+
+    alias_template: SimpleTemplate[str]
+    hs_domain: str
+
+    # Instance cache
+    by_mxid: Dict[RoomID, 'Portal'] = {}
+    by_tgid: Dict[Tuple[TelegramID, TelegramID], 'Portal'] = {}
+
+    mxid: Optional[RoomID]
+    tgid: TelegramID
+    tg_receiver: TelegramID
+    peer_type: str
+    username: str
+    megagroup: bool
+    title: Optional[str]
+    about: Optional[str]
+    photo_id: Optional[str]
+    local_config: Dict[str, Any]
+    deleted: bool
+    log: logging.Logger
+
+    alias: Optional[RoomAlias]
+
+    dedup: PortalDedup
+    send_lock: PortalSendLock
+
+    _db_instance: DBPortal
+    _main_intent: Optional[IntentAPI]
+
+    def __init__(self, tgid: TelegramID, peer_type: str, tg_receiver: Optional[TelegramID] = None,
+                 mxid: Optional[RoomID] = None, username: Optional[str] = None,
+                 megagroup: Optional[bool] = False, title: Optional[str] = None,
+                 about: Optional[str] = None, photo_id: Optional[str] = None,
+                 local_config: Optional[str] = None, db_instance: DBPortal = None) -> None:
+        self.mxid = mxid
+        self.tgid = tgid
+        self.tg_receiver = tg_receiver or tgid
+        self.peer_type = peer_type
+        self.username = username
+        self.megagroup = megagroup
+        self.title = title
+        self.about = about
+        self.photo_id = photo_id
+        self.local_config = json.loads(local_config or "{}")
+        self._db_instance = db_instance
+        self._main_intent = None
+        self.deleted = False
+        self.log = self.base_log.getChild(self.tgid_log if self.tgid else self.mxid)
+
+        self.dedup = PortalDedup(self)
+        self.send_lock = PortalSendLock()
+
+        if tgid:
+            self.by_tgid[self.tgid_full] = self
+        if mxid:
+            self.by_mxid[mxid] = self
+
+    # region Propegrties
+
+    @property
+    def tgid_full(self) -> Tuple[TelegramID, TelegramID]:
+        return self.tgid, self.tg_receiver
+
+    @property
+    def tgid_log(self) -> str:
+        if self.tgid == self.tg_receiver:
+            return str(self.tgid)
+        return f"{self.tg_receiver}<->{self.tgid}"
+
+    @property
+    def alias(self) -> Optional[RoomAlias]:
+        if not self.username:
+            return None
+        return RoomAlias(f"#{self.alias_localpart}:{self.hs_domain}")
+
+    @property
+    def alias_localpart(self) -> Optional[str]:
+        if not self.username:
+            return None
+        return self.alias_template.format(self.username)
+
+    @property
+    def peer(self) -> Union[TypePeer, TypeInputPeer]:
+        if self.peer_type == "user":
+            return PeerUser(user_id=self.tgid)
+        elif self.peer_type == "chat":
+            return PeerChat(chat_id=self.tgid)
+        elif self.peer_type == "channel":
+            return PeerChannel(channel_id=self.tgid)
+
+    @property
+    def has_bot(self) -> bool:
+        return (bool(self.bot)
+                and (self.bot.is_in_chat(self.tgid)
+                     or (self.peer_type == "user" and self.tg_receiver == self.bot.tgid)))
+
+    @property
+    def main_intent(self) -> IntentAPI:
+        if not self._main_intent:
+            direct = self.peer_type == "user"
+            puppet = p.Puppet.get(self.tgid) if direct else None
+            self._main_intent = puppet.intent_for(self) if direct else self.az.intent
+        return self._main_intent
+
+    @property
+    def allow_bridging(self) -> bool:
+        if self.peer_type == "user":
+            return True
+        elif self.filter_mode == "whitelist":
+            return self.tgid in self.filter_list
+        elif self.filter_mode == "blacklist":
+            return self.tgid not in self.filter_list
+        return True
+
+    # endregion
+    # region Miscellaneous getters
+
+    def get_config(self, key: str) -> Any:
+        local = util.recursive_get(self.local_config, key)
+        if local is not None:
+            return local
+        return config[f"bridge.{key}"]
+
+    @staticmethod
+    def _get_largest_photo_size(photo: Union[Photo, Document]
+                                ) -> Tuple[Optional[InputPhotoFileLocation],
+                                           Optional[TypePhotoSize]]:
+        if not photo:
+            return None, None
+        if isinstance(photo, Document) and not photo.thumbs:
+            return None, None
+
+        largest = max(photo.thumbs if isinstance(photo, Document) else photo.sizes,
+                      key=(lambda photo2: (len(photo2.bytes)
+                                           if not isinstance(photo2, PhotoSize)
+                                           else photo2.size)))
+        return InputPhotoFileLocation(
+            id=photo.id,
+            access_hash=photo.access_hash,
+            file_reference=photo.file_reference,
+            thumb_size=largest.type,
+        ), largest
+
+    async def can_user_perform(self, user: 'u.User', event: str) -> bool:
+        if user.is_admin:
+            return True
+        if not self.mxid:
+            # No room for anybody to perform actions in
+            return False
+        try:
+            await self.main_intent.get_power_levels(self.mxid)
+        except MatrixRequestError:
+            return False
+        evt_type = EventType.find(f"net.maunium.telegram.{event}")
+        evt_type.t_class = EventType.Class.STATE
+        return self.main_intent.state_store.has_power_level(self.mxid, user.mxid, evt_type)
+
+    def get_input_entity(self, user: 'AbstractUser'
+                         ) -> Awaitable[Union[TypeInputPeer, TypeInputChannel]]:
+        return user.client.get_input_entity(self.peer)
+
+    async def get_entity(self, user: 'AbstractUser') -> TypeChat:
+        try:
+            return await user.client.get_entity(self.peer)
+        except ValueError:
+            if user.is_bot:
+                self.log.warning(f"Could not find entity with bot {user.tgid}. "
+                                 "Failing...")
+                raise
+            self.log.warning(f"Could not find entity with user {user.tgid}. "
+                             "falling back to get_dialogs.")
+            async for dialog in user.client.iter_dialogs():
+                if dialog.entity.id == self.tgid:
+                    return dialog.entity
+            raise
+
+    async def get_invite_link(self, user: 'u.User') -> str:
+        if self.peer_type == "user":
+            raise ValueError("You can't invite users to private chats.")
+        if self.username:
+            return f"https://t.me/{self.username}"
+        link = await user.client(ExportChatInviteRequest(peer=await self.get_input_entity(user)))
+        if isinstance(link, ChatInviteEmpty):
+            raise ValueError("Failed to get invite link.")
+        return link.link
+
+    # endregion
+    # region Matrix room cleanup
+
+    async def get_authenticated_matrix_users(self) -> List['u.User']:
+        try:
+            members = await self.main_intent.get_room_members(self.mxid)
+        except MatrixRequestError:
+            return []
+        authenticated: List[u.User] = []
+        has_bot = self.has_bot
+        for member_str in members:
+            member = UserID(member_str)
+            if p.Puppet.get_id_from_mxid(member) or member == self.main_intent.mxid:
+                continue
+            user = await u.User.get_by_mxid(member).ensure_started()
+            authenticated_through_bot = has_bot and user.relaybot_whitelisted
+            if authenticated_through_bot or await user.has_full_access(allow_bot=True):
+                authenticated.append(user)
+        return authenticated
+
+    @staticmethod
+    async def cleanup_room(intent: IntentAPI, room_id: RoomID, message: str,
+                           puppets_only: bool = False) -> None:
+        try:
+            members = await intent.get_room_members(room_id)
+        except MatrixRequestError:
+            members = []
+        for user in members:
+            puppet = p.Puppet.get_by_mxid(UserID(user), create=False)
+            if user != intent.mxid and (not puppets_only or puppet):
+                try:
+                    if puppet:
+                        await puppet.default_mxid_intent.leave_room(room_id)
+                    else:
+                        await intent.kick_user(room_id, user, message)
+                except (MatrixRequestError, IntentError):
+                    pass
+        try:
+            await intent.leave_room(room_id)
+        except (MatrixRequestError, IntentError):
+            self.log.warning("Failed to leave room when cleaning up room", exc_info=True)
+
+    async def cleanup_portal(self, message: str, puppets_only: bool = False) -> None:
+        if self.username:
+            try:
+                await self.main_intent.remove_room_alias(self.alias_localpart)
+            except (MatrixRequestError, IntentError):
+                self.log.warning("Failed to remove alias when cleaning up room", exc_info=True)
+        await self.cleanup_room(self.main_intent, self.mxid, message, puppets_only)
+
+    async def unbridge(self) -> None:
+        await self.cleanup_portal("Room unbridged", puppets_only=True)
+        self.delete()
+
+    async def cleanup_and_delete(self) -> None:
+        await self.cleanup_portal("Portal deleted")
+        self.delete()
+
+    # endregion
+    # region Database conversion
+
+    @property
+    def db_instance(self) -> DBPortal:
+        if not self._db_instance:
+            self._db_instance = self.new_db_instance()
+        return self._db_instance
+
+    def new_db_instance(self) -> DBPortal:
+        return DBPortal(tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
+                        mxid=self.mxid, username=self.username, megagroup=self.megagroup,
+                        title=self.title, about=self.about, photo_id=self.photo_id,
+                        config=json.dumps(self.local_config))
+
+    def save(self) -> None:
+        self.db_instance.edit(mxid=self.mxid, username=self.username, title=self.title,
+                              about=self.about, photo_id=self.photo_id, megagroup=self.megagroup,
+                              config=json.dumps(self.local_config))
+
+    def delete(self) -> None:
+        try:
+            del self.by_tgid[self.tgid_full]
+        except KeyError:
+            pass
+        try:
+            del self.by_mxid[self.mxid]
+        except KeyError:
+            pass
+        if self._db_instance:
+            self._db_instance.delete()
+        self.deleted = True
+
+    @classmethod
+    def from_db(cls, db_portal: DBPortal) -> 'Portal':
+        return cls(tgid=db_portal.tgid, tg_receiver=db_portal.tg_receiver,
+                   peer_type=db_portal.peer_type, mxid=db_portal.mxid,
+                   username=db_portal.username, megagroup=db_portal.megagroup,
+                   title=db_portal.title, about=db_portal.about, photo_id=db_portal.photo_id,
+                   local_config=db_portal.config, db_instance=db_portal)
+
+    # endregion
+    # region Class instance lookup
+
+    @classmethod
+    def get_by_mxid(cls, mxid: RoomID) -> Optional['Portal']:
+        try:
+            return cls.by_mxid[mxid]
+        except KeyError:
+            pass
+
+        portal = DBPortal.get_by_mxid(mxid)
+        if portal:
+            return cls.from_db(portal)
+
+        return None
+
+    @classmethod
+    def get_username_from_mx_alias(cls, alias: str) -> Optional[str]:
+        return cls.alias_template.parse(alias)
+
+    @classmethod
+    def find_by_username(cls, username: str) -> Optional['Portal']:
+        if not username:
+            return None
+
+        username = username.lower()
+
+        for _, portal in cls.by_tgid.items():
+            if portal.username and portal.username.lower() == username:
+                return portal
+
+        dbportal = DBPortal.get_by_username(username)
+        if dbportal:
+            return cls.from_db(dbportal)
+
+        return None
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID, tg_receiver: Optional[TelegramID] = None,
+                    peer_type: str = None) -> Optional['Portal']:
+        tg_receiver = tg_receiver or tgid
+        tgid_full = (tgid, tg_receiver)
+        try:
+            return cls.by_tgid[tgid_full]
+        except KeyError:
+            pass
+
+        db_portal = DBPortal.get_by_tgid(tgid, tg_receiver)
+        if db_portal:
+            return cls.from_db(db_portal)
+
+        if peer_type:
+            cls.log.info(f"Creating portal for {peer_type} {tgid} (receiver {tg_receiver})")
+            # TODO enable this for non-release builds
+            #      (or add better wrong peer type error handling)
+            # if peer_type == "chat":
+            #     import traceback
+            #     cls.log.info("Chat portal stack trace:\n" + "".join(traceback.format_stack()))
+            portal = cls(tgid, peer_type=peer_type, tg_receiver=tg_receiver)
+            portal.db_instance.insert()
+            return portal
+
+        return None
+
+    @classmethod
+    def get_by_entity(cls, entity: Union[TypeChat, TypePeer, TypeUser, TypeUserFull,
+                                         TypeInputPeer],
+                      receiver_id: Optional[TelegramID] = None, create: bool = True
+                      ) -> Optional['Portal']:
+        entity_type = type(entity)
+        if entity_type in (Chat, ChatFull):
+            type_name = "chat"
+            entity_id = entity.id
+        elif entity_type in (PeerChat, InputPeerChat):
+            type_name = "chat"
+            entity_id = entity.chat_id
+        elif entity_type in (Channel, ChannelFull):
+            type_name = "channel"
+            entity_id = entity.id
+        elif entity_type in (PeerChannel, InputPeerChannel, InputChannel):
+            type_name = "channel"
+            entity_id = entity.channel_id
+        elif entity_type in (User, UserFull):
+            type_name = "user"
+            entity_id = entity.id
+        elif entity_type in (PeerUser, InputPeerUser, InputUser):
+            type_name = "user"
+            entity_id = entity.user_id
+        else:
+            raise ValueError(f"Unknown entity type {entity_type.__name__}")
+        return cls.get_by_tgid(TelegramID(entity_id),
+                               receiver_id if type_name == "user" else entity_id,
+                               type_name if create else None)
+
+    # endregion
+    # region Abstract methods (cross-called in matrix/metadata/telegram classes)
+
+    @abstractmethod
+    async def update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
+                                 direct: bool, puppet: p.Puppet = None,
+                                 levels: PowerLevelStateEventContent = None,
+                                 users: List[User] = None,
+                                 participants: List[TypeParticipant] = None) -> None:
+        pass
+
+    @abstractmethod
+    async def create_matrix_room(self, user: 'AbstractUser', entity: TypeChat = None,
+                                 invites: InviteList = None, update_if_exists: bool = True,
+                                 synchronous: bool = False) -> Optional[str]:
+        pass
+
+    @abstractmethod
+    async def _add_telegram_user(self, user_id: TelegramID, source: Optional['AbstractUser'] = None
+                                 ) -> None:
+        pass
+
+    @abstractmethod
+    async def _delete_telegram_user(self, user_id: TelegramID, sender: p.Puppet) -> None:
+        pass
+
+    @abstractmethod
+    async def _update_title(self, title: str, sender: Optional['p.Puppet'] = None,
+                            save: bool = False) -> bool:
+        pass
+
+    @abstractmethod
+    async def _update_avatar(self, user: 'AbstractUser', photo: Union[TypeChatPhoto],
+                             sender: Optional['p.Puppet'] = None, save: bool = False) -> bool:
+        pass
+
+    @abstractmethod
+    def _migrate_and_save_telegram(self, new_id: TelegramID) -> None:
+        pass
+
+    @abstractmethod
+    def handle_matrix_power_levels(self, sender: 'u.User', new_levels: Dict[UserID, int],
+                                   old_levels: Dict[UserID, int]) -> Awaitable[None]:
+        pass
+
+    # endregion
+
+
+def init(context: Context) -> None:
+    global config
+    BasePortal.az, config, BasePortal.loop, BasePortal.bot = context.core
+    BasePortal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
+    BasePortal.sync_channel_members = config["bridge.sync_channel_members"]
+    BasePortal.sync_matrix_state = config["bridge.sync_matrix_state"]
+    BasePortal.public_portals = config["bridge.public_portals"]
+    BasePortal.filter_mode = config["bridge.filter.mode"]
+    BasePortal.filter_list = config["bridge.filter.list"]
+    BasePortal.hs_domain = config["homeserver.domain"]
+    BasePortal.alias_template = SimpleTemplate(config["bridge.alias_template"], "groupname",
+                                               prefix="#", suffix=f":{BasePortal.hs_domain}")

mautrix_telegram/portal/deduplication.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,29 +13,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import Tuple
+from typing import Optional, Deque, Dict, Tuple, TYPE_CHECKING
 from collections import deque
 import hashlib
 
 from telethon.tl.patched import Message, MessageService
-from telethon.tl.types import (
-    MessageMediaContact,
-    MessageMediaDocument,
-    MessageMediaGeo,
-    MessageMediaPhoto,
-    TypeMessage,
-    TypeUpdates,
-    UpdateNewChannelMessage,
-    UpdateNewMessage,
-)
+from telethon.tl.types import (MessageMediaContact, MessageMediaDocument, MessageMediaGeo,
+                               MessageMediaPhoto, TypeMessage, TypeUpdates, UpdateNewMessage,
+                               UpdateNewChannelMessage)
 
 from mautrix.types import EventID
 
-from .. import portal as po
+from ..context import Context
 from ..types import TelegramID
 
+if TYPE_CHECKING:
+    from .base import BasePortal
+
 DedupMXID = Tuple[EventID, TelegramID]
 
 
@@ -43,12 +37,12 @@ class PortalDedup:
     pre_db_check: bool = False
     cache_queue_length: int = 20
 
-    _dedup: deque[str]
-    _dedup_mxid: dict[str, DedupMXID]
-    _dedup_action: deque[str]
-    _portal: po.Portal
+    _dedup: Deque[str]
+    _dedup_mxid: Dict[str, DedupMXID]
+    _dedup_action: Deque[str]
+    _portal: 'BasePortal'
 
-    def __init__(self, portal: po.Portal) -> None:
+    def __init__(self, portal: 'BasePortal') -> None:
         self._dedup = deque()
         self._dedup_mxid = {}
         self._dedup_action = deque()
@@ -56,7 +50,7 @@ class PortalDedup:
 
     @property
     def _always_force_hash(self) -> bool:
-        return self._portal.peer_type == "chat"
+        return self._portal.peer_type != 'channel'
 
     @staticmethod
     def _hash_event(event: TypeMessage) -> str:
@@ -67,20 +61,23 @@ class PortalDedup:
         if isinstance(event, MessageService):
             hash_content = [event.date.timestamp(), event.from_id, event.action]
         else:
-            hash_content = [event.date.timestamp(), event.message.strip()]
+            hash_content = [event.date.timestamp(), event.message]
             if event.fwd_from:
-                hash_content += [event.fwd_from.from_id]
+                hash_content += [event.fwd_from.from_id, event.fwd_from.channel_id]
             elif isinstance(event, Message) and event.media:
                 try:
                     hash_content += {
                         MessageMediaContact: lambda media: [media.user_id],
                         MessageMediaDocument: lambda media: [media.document.id],
-                        MessageMediaPhoto: lambda media: [media.photo.id if media.photo else 0],
+                        MessageMediaPhoto: lambda media: [media.photo.id],
                         MessageMediaGeo: lambda media: [media.geo.long, media.geo.lat],
                     }[type(event.media)](event.media)
                 except KeyError:
                     pass
-        return hashlib.md5("-".join(str(a) for a in hash_content).encode("utf-8")).hexdigest()
+        return hashlib.md5("-"
+                           .join(str(a) for a in hash_content)
+                           .encode("utf-8")
+                           ).hexdigest()
 
     def check_action(self, event: TypeMessage) -> bool:
         evt_hash = self._hash_event(event) if self._always_force_hash else event.id
@@ -93,13 +90,9 @@ class PortalDedup:
             self._dedup_action.popleft()
         return False
 
-    def update(
-        self,
-        event: TypeMessage,
-        mxid: DedupMXID = None,
-        expected_mxid: DedupMXID | None = None,
-        force_hash: bool = False,
-    ) -> DedupMXID | None:
+    def update(self, event: TypeMessage, mxid: DedupMXID = None,
+               expected_mxid: Optional[DedupMXID] = None, force_hash: bool = False
+               ) -> Optional[DedupMXID]:
         evt_hash = self._hash_event(event) if self._always_force_hash or force_hash else event.id
         try:
             found_mxid = self._dedup_mxid[evt_hash]
@@ -111,10 +104,11 @@ class PortalDedup:
         self._dedup_mxid[evt_hash] = mxid
         return None
 
-    def check(
-        self, event: TypeMessage, mxid: DedupMXID = None, force_hash: bool = False
-    ) -> DedupMXID | None:
-        evt_hash = self._hash_event(event) if self._always_force_hash or force_hash else event.id
+    def check(self, event: TypeMessage, mxid: DedupMXID = None, force_hash: bool = False
+              ) -> Optional[DedupMXID]:
+        evt_hash = (self._hash_event(event)
+                    if self._always_force_hash or force_hash
+                    else event.id)
         if evt_hash in self._dedup:
             return self._dedup_mxid[evt_hash]
 
@@ -127,8 +121,13 @@ class PortalDedup:
 
     def register_outgoing_actions(self, response: TypeUpdates) -> None:
         for update in response.updates:
-            check_dedup = isinstance(
-                update, (UpdateNewMessage, UpdateNewChannelMessage)
-            ) and isinstance(update.message, MessageService)
+            check_dedup = (isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage))
+                           and isinstance(update.message, MessageService))
             if check_dedup:
                 self.check(update.message)
+
+
+def init(context: Context) -> None:
+    cfg = context.config
+    PortalDedup.dedup_pre_db_check = cfg["bridge.deduplication.pre_db_check"]
+    PortalDedup.dedup_cache_queue_length = cfg["bridge.deduplication.cache_queue_length"]

mautrix_telegram/portal/matrix.py

@@ -0,0 +1,537 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Dict, List, Optional, Tuple, Union, Any, TYPE_CHECKING
+from html import escape as escape_html
+from string import Template
+from abc import ABC
+
+import magic
+
+from telethon.tl.functions.messages import (EditChatPhotoRequest, EditChatTitleRequest,
+                                            UpdatePinnedMessageRequest, SetTypingRequest,
+                                            EditChatAboutRequest)
+from telethon.tl.functions.channels import EditPhotoRequest, EditTitleRequest, JoinChannelRequest
+from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError,
+                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError)
+from telethon.tl.patched import Message, MessageService
+from telethon.tl.types import (
+    DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
+    InputChatUploadedPhoto, MessageActionChatEditPhoto, MessageMediaGeo,
+    SendMessageCancelAction, SendMessageTypingAction, TypeInputPeer, TypeMessageEntity,
+    UpdateNewMessage, InputMediaUploadedDocument, InputMediaUploadedPhoto)
+
+from mautrix.types import (EventID, RoomID, UserID, ContentURI, MessageType, MessageEventContent,
+                           TextMessageEventContent, MediaMessageEventContent, Format,
+                           LocationMessageEventContent)
+from mautrix.bridge import BasePortal as MautrixBasePortal
+
+from ..types import TelegramID
+from ..db import Message as DBMessage
+from ..util import sane_mimetypes, parallel_transfer_to_telegram
+from ..context import Context
+from .. import puppet as p, user as u, formatter, util
+from .base import BasePortal
+
+if TYPE_CHECKING:
+    from ..abstract_user import AbstractUser
+    from ..tgclient import MautrixTelegramClient
+    from ..config import Config
+
+TypeMessage = Union[Message, MessageService]
+
+config: Optional['Config'] = None
+
+
+class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
+    async def _get_state_change_message(self, event: str, user: 'u.User', **kwargs: Any
+                                        ) -> Optional[str]:
+        tpl = self.get_config(f"state_event_formats.{event}")
+        if len(tpl) == 0:
+            # Empty format means they don't want the message
+            return None
+        displayname = await self.get_displayname(user)
+
+        tpl_args = {
+            "mxid": user.mxid,
+            "username": user.mxid_localpart,
+            "displayname": escape_html(displayname),
+            **kwargs,
+        }
+        return Template(tpl).safe_substitute(tpl_args)
+
+    async def _send_state_change_message(self, event: str, user: 'u.User', event_id: EventID,
+                                         **kwargs: Any) -> None:
+        if not self.has_bot:
+            return
+        elif self.peer_type == "user" and not config["bridge.relaybot.private_chat.state_changes"]:
+            return
+        async with self.send_lock(self.bot.tgid):
+            message = await self._get_state_change_message(event, user, **kwargs)
+            if not message:
+                return
+            response = await self.bot.client.send_message(
+                self.peer, message,
+                parse_mode=self._matrix_event_to_entities)
+            space = self.tgid if self.peer_type == "channel" else self.bot.tgid
+            self.dedup.check(response, (event_id, space))
+
+    async def name_change_matrix(self, user: 'u.User', displayname: str, prev_displayname: str,
+                                 event_id: EventID) -> None:
+        await self._send_state_change_message("name_change", user, event_id,
+                                              displayname=displayname,
+                                              prev_displayname=prev_displayname)
+
+    async def get_displayname(self, user: 'u.User') -> str:
+        return await self.main_intent.get_room_displayname(self.mxid, user.mxid) or user.mxid
+
+    def set_typing(self, user: 'u.User', typing: bool = True,
+                   action: type = SendMessageTypingAction) -> Awaitable[bool]:
+        return user.client(SetTypingRequest(
+            self.peer, action() if typing else SendMessageCancelAction()))
+
+    async def mark_read(self, user: 'u.User', event_id: EventID) -> None:
+        if user.is_bot:
+            return
+        space = self.tgid if self.peer_type == "channel" else user.tgid
+        message = DBMessage.get_by_mxid(event_id, self.mxid, space)
+        if not message:
+            return
+        await user.client.send_read_acknowledge(self.peer, max_id=message.tgid,
+                                                clear_mentions=True)
+
+    async def _preproc_kick_ban(self, user: Union['u.User', 'p.Puppet'], source: 'u.User'
+                                ) -> Optional['AbstractUser']:
+        if user.tgid == source.tgid:
+            return None
+        if self.peer_type == "user" and user.tgid == self.tgid:
+            self.delete()
+            return None
+        if isinstance(user, u.User) and await user.needs_relaybot(self):
+            if not self.bot:
+                return None
+            # TODO kick message
+            return None
+        if await source.needs_relaybot(self):
+            if not self.has_bot:
+                return None
+            return self.bot
+        return source
+
+    async def kick_matrix(self, user: Union['u.User', 'p.Puppet'], source: 'u.User') -> None:
+        source = await self._preproc_kick_ban(user, source)
+        if source is not None:
+            await source.client.kick_participant(self.peer, user.peer)
+
+    async def ban_matrix(self, user: Union['u.User', 'p.Puppet'], source: 'u.User'):
+        source = await self._preproc_kick_ban(user, source)
+        if source is not None:
+            await source.client.edit_permissions(self.peer, user.peer, view_messages=False)
+
+    async def leave_matrix(self, user: 'u.User', event_id: EventID) -> None:
+        if await user.needs_relaybot(self):
+            await self._send_state_change_message("leave", user, event_id)
+            return
+
+        if self.peer_type == "user":
+            await self.main_intent.leave_room(self.mxid)
+            self.delete()
+            try:
+                del self.by_tgid[self.tgid_full]
+                del self.by_mxid[self.mxid]
+            except KeyError:
+                pass
+        else:
+            await user.client.delete_dialog(self.peer)
+
+    async def join_matrix(self, user: 'u.User', event_id: EventID) -> None:
+        if await user.needs_relaybot(self):
+            await self._send_state_change_message("join", user, event_id)
+            return
+
+        if self.peer_type == "channel" and not user.is_bot:
+            await user.client(JoinChannelRequest(channel=await self.get_input_entity(user)))
+        else:
+            # We'll just assume the user is already in the chat.
+            pass
+
+    async def _apply_msg_format(self, sender: 'u.User', content: MessageEventContent
+                                ) -> None:
+        if not isinstance(content, TextMessageEventContent) or content.format != Format.HTML:
+            content.format = Format.HTML
+            content.formatted_body = escape_html(content.body).replace("\n", "<br/>")
+
+        tpl = (self.get_config(f"message_formats.[{content.msgtype.value}]")
+               or "<b>$sender_displayname</b>: $message")
+        displayname = await self.get_displayname(sender)
+        tpl_args = dict(sender_mxid=sender.mxid,
+                        sender_username=sender.mxid_localpart,
+                        sender_displayname=escape_html(displayname),
+                        message=content.formatted_body,
+                        body=content.body, formatted_body=content.formatted_body)
+        content.formatted_body = Template(tpl).safe_substitute(tpl_args)
+
+    async def _apply_emote_format(self, sender: 'u.User',
+                                  content: TextMessageEventContent) -> None:
+        if content.format != Format.HTML:
+            content.format = Format.HTML
+            content.formatted_body = escape_html(content.body).replace("\n", "<br/>")
+
+        tpl = self.get_config("emote_format")
+        puppet = p.Puppet.get(sender.tgid)
+        content.formatted_body = Template(tpl).safe_substitute(
+            dict(sender_mxid=sender.mxid,
+                 sender_username=sender.mxid_localpart,
+                 sender_displayname=escape_html(await self.get_displayname(sender)),
+                 mention=f"<a href='https://matrix.to/#/{puppet.mxid}'>{puppet.displayname}</a>",
+                 username=sender.username,
+                 displayname=puppet.displayname,
+                 body=content.body,
+                 formatted_body=content.formatted_body))
+        content.msgtype = MessageType.TEXT
+
+    async def _pre_process_matrix_message(self, sender: 'u.User', use_relaybot: bool,
+                                          content: MessageEventContent) -> None:
+        if use_relaybot:
+            await self._apply_msg_format(sender, content)
+        elif content.msgtype == MessageType.EMOTE:
+            await self._apply_emote_format(sender, content)
+
+    @staticmethod
+    def _matrix_event_to_entities(event: Union[str, MessageEventContent]
+                                  ) -> Tuple[str, Optional[List[TypeMessageEntity]]]:
+        try:
+            if isinstance(event, str):
+                message, entities = formatter.matrix_to_telegram(event)
+            elif isinstance(event, TextMessageEventContent) and event.format == Format.HTML:
+                message, entities = formatter.matrix_to_telegram(event.formatted_body)
+            else:
+                message, entities = formatter.matrix_text_to_telegram(event.body)
+        except KeyError:
+            message, entities = None, None
+        return message, entities
+
+    async def _handle_matrix_text(self, sender_id: TelegramID, event_id: EventID,
+                                  space: TelegramID, client: 'MautrixTelegramClient',
+                                  content: TextMessageEventContent, reply_to: TelegramID) -> None:
+        async with self.send_lock(sender_id):
+            lp = self.get_config("telegram_link_preview")
+            if content.get_edit():
+                orig_msg = DBMessage.get_by_mxid(content.get_edit(), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid, content,
+                                                         parse_mode=self._matrix_event_to_entities,
+                                                         link_preview=lp)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
+            response = await client.send_message(self.peer, content, reply_to=reply_to,
+                                                 parse_mode=self._matrix_event_to_entities,
+                                                 link_preview=lp)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
+
+    async def _handle_matrix_file(self, sender_id: TelegramID, event_id: EventID,
+                                  space: TelegramID, client: 'MautrixTelegramClient',
+                                  content: MediaMessageEventContent, reply_to: TelegramID,
+                                  caption: TextMessageEventContent = None) -> None:
+        mime = content.info.mimetype
+        w, h = content.info.width, content.info.height
+        file_name = content["net.maunium.telegram.internal.filename"]
+        max_image_size = config["bridge.image_as_file_size"] * 1000 ** 2
+
+        if config["bridge.parallel_file_transfer"]:
+            file_handle, file_size = await parallel_transfer_to_telegram(client, self.main_intent,
+                                                                         content.url, sender_id)
+        else:
+            file = await self.main_intent.download_media(content.url)
+
+            if content.msgtype == MessageType.STICKER:
+                if mime != "image/gif":
+                    mime, file, w, h = util.convert_image(file, source_mime=mime,
+                                                          target_type="webp")
+                else:
+                    # Remove sticker description
+                    file_name = "sticker.gif"
+
+            file_handle = await client.upload_file(file)
+            file_size = len(file)
+
+        file_handle.name = file_name
+
+        attributes = [DocumentAttributeFilename(file_name=file_name)]
+        if w and h:
+            attributes.append(DocumentAttributeImageSize(w, h))
+
+        if (mime == "image/png" or mime == "image/jpeg") and file_size < max_image_size:
+            media = InputMediaUploadedPhoto(file_handle)
+        else:
+            media = InputMediaUploadedDocument(file=file_handle, attributes=attributes,
+                                               mime_type=mime or "application/octet-stream")
+
+        caption, entities = self._matrix_event_to_entities(caption) if caption else (None, None)
+
+        async with self.send_lock(sender_id):
+            if await self._matrix_document_edit(client, content, space, caption, media, event_id):
+                return
+            try:
+                response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                                   caption=caption, entities=entities)
+            except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
+                media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
+                                                   attributes=attributes)
+                response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                                   caption=caption, entities=entities)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
+
+    async def _matrix_document_edit(self, client: 'MautrixTelegramClient',
+                                    content: MessageEventContent, space: TelegramID,
+                                    caption: str, media: Any, event_id: EventID) -> bool:
+        if content.get_edit():
+            orig_msg = DBMessage.get_by_mxid(content.get_edit(), self.mxid, space)
+            if orig_msg:
+                response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                     caption, file=media)
+                self._add_telegram_message_to_db(event_id, space, -1, response)
+                return True
+        return False
+
+    async def _handle_matrix_location(self, sender_id: TelegramID, event_id: EventID,
+                                      space: TelegramID, client: 'MautrixTelegramClient',
+                                      content: LocationMessageEventContent, reply_to: TelegramID
+                                      ) -> None:
+        try:
+            lat, long = content.geo_uri[len("geo:"):].split(",")
+            lat, long = float(lat), float(long)
+        except (KeyError, ValueError):
+            self.log.exception("Failed to parse location")
+            return None
+        caption, entities = self._matrix_event_to_entities(content)
+        media = MessageMediaGeo(geo=GeoPoint(lat, long, access_hash=0))
+
+        async with self.send_lock(sender_id):
+            if await self._matrix_document_edit(client, content, space, caption, media, event_id):
+                return
+            response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                               caption=caption, entities=entities)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
+
+    def _add_telegram_message_to_db(self, event_id: EventID, space: TelegramID,
+                                    edit_index: int, response: TypeMessage) -> None:
+        self.log.debug("Handled Matrix message: %s", response)
+        self.dedup.check(response, (event_id, space), force_hash=edit_index != 0)
+        if edit_index < 0:
+            prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1)
+            edit_index = prev_edit.edit_index + 1
+        DBMessage(
+            tgid=TelegramID(response.id),
+            tg_space=space,
+            mx_room=self.mxid,
+            mxid=event_id,
+            edit_index=edit_index).insert()
+
+    async def handle_matrix_message(self, sender: 'u.User', content: MessageEventContent,
+                                    event_id: EventID) -> None:
+        if not content.body or not content.msgtype:
+            self.log.debug(f"Ignoring message {event_id} in {self.mxid} without body or msgtype")
+            return
+
+        puppet = p.Puppet.get_by_custom_mxid(sender.mxid)
+        if puppet and content.get("net.maunium.telegram.puppet", False):
+            self.log.debug("Ignoring puppet-sent message by confirmed puppet user %s", sender.mxid)
+            return
+
+        logged_in = not await sender.needs_relaybot(self)
+        client = sender.client if logged_in else self.bot.client
+        sender_id = sender.tgid if logged_in else self.bot.tgid
+        space = (self.tgid if self.peer_type == "channel"  # Channels have their own ID space
+                 else (sender.tgid if logged_in else self.bot.tgid))
+        reply_to = formatter.matrix_reply_to_telegram(content, space, room_id=self.mxid)
+
+        media = (MessageType.STICKER, MessageType.IMAGE, MessageType.FILE, MessageType.AUDIO,
+                 MessageType.VIDEO)
+
+        if content.msgtype == MessageType.NOTICE:
+            bridge_notices = self.get_config("bridge_notices.default")
+            excepted = sender.mxid in self.get_config("bridge_notices.exceptions")
+            if not bridge_notices and not excepted:
+                return
+
+        if content.msgtype in (MessageType.TEXT, MessageType.EMOTE, MessageType.NOTICE):
+            await self._pre_process_matrix_message(sender, not logged_in, content)
+            await self._handle_matrix_text(sender_id, event_id, space, client, content, reply_to)
+        elif content.msgtype == MessageType.LOCATION:
+            await self._pre_process_matrix_message(sender, not logged_in, content)
+            await self._handle_matrix_location(sender_id, event_id, space, client, content,
+                                               reply_to)
+        elif content.msgtype in media:
+            content["net.maunium.telegram.internal.filename"] = content.body
+            try:
+                caption_content: MessageEventContent = sender.command_status["caption"]
+                reply_to = reply_to or formatter.matrix_reply_to_telegram(caption_content, space,
+                                                                          room_id=self.mxid)
+                sender.command_status = None
+            except (KeyError, TypeError):
+                caption_content = None if logged_in else TextMessageEventContent(body=content.body)
+            if caption_content:
+                caption_content.msgtype = content.msgtype
+                await self._pre_process_matrix_message(sender, not logged_in, caption_content)
+            await self._handle_matrix_file(sender_id, event_id, space, client, content, reply_to,
+                                           caption_content)
+        else:
+            self.log.debug(f"Unhandled Matrix event: {content}")
+
+    async def handle_matrix_pin(self, sender: 'u.User',
+                                pinned_message: Optional[EventID]) -> None:
+        if self.peer_type != "chat" and self.peer_type != "channel":
+            return
+        try:
+            if not pinned_message:
+                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=0))
+            else:
+                tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
+                message = DBMessage.get_by_mxid(pinned_message, self.mxid, tg_space)
+                if message is None:
+                    self.log.warning(f"Could not find pinned {pinned_message} in {self.mxid}")
+                    return
+                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=message.tgid))
+        except ChatNotModifiedError:
+            pass
+
+    async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID) -> None:
+        real_deleter = deleter if not await deleter.needs_relaybot(self) else self.bot
+        space = self.tgid if self.peer_type == "channel" else real_deleter.tgid
+        message = DBMessage.get_by_mxid(event_id, self.mxid, space)
+        if not message:
+            return
+        if message.edit_index == 0:
+            await real_deleter.client.delete_messages(self.peer, [message.tgid])
+        else:
+            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
+
+    async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
+                                           level: int) -> None:
+        moderator = level >= 50
+        admin = level >= 75
+        await sender.client.edit_admin(self.peer, user_id,
+                                       change_info=moderator, post_messages=moderator,
+                                       edit_messages=moderator, delete_messages=moderator,
+                                       ban_users=moderator, invite_users=moderator,
+                                       pin_messages=moderator, add_admins=admin)
+
+    async def handle_matrix_power_levels(self, sender: 'u.User', new_users: Dict[UserID, int],
+                                         old_users: Dict[UserID, int]) -> None:
+        # TODO handle all power level changes and bridge exact admin rights to supergroups/channels
+        for user, level in new_users.items():
+            if not user or user == self.main_intent.mxid or user == sender.mxid:
+                continue
+            user_id = p.Puppet.get_id_from_mxid(user)
+            if not user_id:
+                mx_user = u.User.get_by_mxid(user, create=False)
+                if not mx_user or not mx_user.tgid:
+                    continue
+                user_id = mx_user.tgid
+            if not user_id or user_id == sender.tgid:
+                continue
+            if user not in old_users or level != old_users[user]:
+                await self._update_telegram_power_level(sender, user_id, level)
+
+    async def handle_matrix_about(self, sender: 'u.User', about: str) -> None:
+        if self.peer_type not in ("chat", "channel"):
+            return
+        peer = await self.get_input_entity(sender)
+        await sender.client(EditChatAboutRequest(peer=peer, about=about))
+        self.about = about
+        self.save()
+
+    async def handle_matrix_title(self, sender: 'u.User', title: str) -> None:
+        if self.peer_type not in ("chat", "channel"):
+            return
+
+        if self.peer_type == "chat":
+            response = await sender.client(EditChatTitleRequest(chat_id=self.tgid, title=title))
+        else:
+            channel = await self.get_input_entity(sender)
+            response = await sender.client(EditTitleRequest(channel=channel, title=title))
+        self.dedup.register_outgoing_actions(response)
+        self.title = title
+        self.save()
+
+    async def handle_matrix_avatar(self, sender: 'u.User', url: ContentURI) -> None:
+        if self.peer_type not in ("chat", "channel"):
+            # Invalid peer type
+            return
+
+        file = await self.main_intent.download_media(url)
+        mime = magic.from_buffer(file, mime=True)
+        ext = sane_mimetypes.guess_extension(mime)
+        uploaded = await sender.client.upload_file(file, file_name=f"avatar{ext}")
+        photo = InputChatUploadedPhoto(file=uploaded)
+
+        if self.peer_type == "chat":
+            response = await sender.client(EditChatPhotoRequest(chat_id=self.tgid, photo=photo))
+        else:
+            channel = await self.get_input_entity(sender)
+            response = await sender.client(EditPhotoRequest(channel=channel, photo=photo))
+        self.dedup.register_outgoing_actions(response)
+        for update in response.updates:
+            is_photo_update = (isinstance(update, UpdateNewMessage)
+                               and isinstance(update.message, MessageService)
+                               and isinstance(update.message.action, MessageActionChatEditPhoto))
+            if is_photo_update:
+                loc, size = self._get_largest_photo_size(update.message.action.photo)
+                self.photo_id = f"{size.location.volume_id}-{size.location.local_id}"
+                self.save()
+                break
+
+    async def handle_matrix_upgrade(self, sender: UserID, new_room: RoomID) -> None:
+        _, server = self.main_intent.parse_user_id(sender)
+        old_room = self.mxid
+        self.migrate_and_save_matrix(new_room)
+        await self.main_intent.join_room(new_room, servers=[server])
+        entity: Optional[TypeInputPeer] = None
+        user: Optional[AbstractUser] = None
+        if self.bot and self.has_bot:
+            user = self.bot
+            entity = await self.get_input_entity(self.bot)
+        if not entity:
+            user_mxids = await self.main_intent.get_room_members(self.mxid)
+            for user_str in user_mxids:
+                user_id = UserID(user_str)
+                if user_id == self.az.bot_mxid:
+                    continue
+                user = u.User.get_by_mxid(user_id, create=False)
+                if user and user.tgid:
+                    entity = await self.get_input_entity(user)
+                    if entity:
+                        break
+        if not entity:
+            self.log.error("Failed to fully migrate to upgraded Matrix room: "
+                           "no Telegram user found.")
+            return
+        await self.update_matrix_room(user, entity, direct=self.peer_type == "user")
+        self.log.info(f"{sender} upgraded room from {old_room} to {self.mxid}")
+
+    def migrate_and_save_matrix(self, new_id: RoomID) -> None:
+        try:
+            del self.by_mxid[self.mxid]
+        except KeyError:
+            pass
+        self.mxid = new_id
+        self.db_instance.edit(mxid=self.mxid)
+        self.by_mxid[self.mxid] = self
+
+
+def init(context: Context) -> None:
+    global config
+    config = context.config

mautrix_telegram/portal/metadata.py

@@ -0,0 +1,708 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Optional, Tuple, Union, Callable, TYPE_CHECKING
+from abc import ABC
+import asyncio
+
+from telethon.tl.functions.messages import (AddChatUserRequest, CreateChatRequest,
+                                            GetFullChatRequest, MigrateChatRequest)
+from telethon.tl.functions.channels import (CreateChannelRequest, GetParticipantsRequest,
+                                            InviteToChannelRequest, UpdateUsernameRequest)
+from telethon.errors import ChatAdminRequiredError
+from telethon.tl.types import (
+    Channel, ChatBannedRights, ChannelParticipantsRecent, ChannelParticipantsSearch, ChatPhoto,
+    PhotoEmpty, InputChannel, InputUser, ChatPhotoEmpty, PeerUser, Photo, TypeChat, TypeInputPeer,
+    TypeUser, User, InputPeerPhotoFileLocation, ChatParticipantAdmin, ChannelParticipantAdmin,
+    ChatParticipantCreator, ChannelParticipantCreator)
+
+from mautrix.errors import MForbidden
+from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership, Member,
+                           PowerLevelStateEventContent, RoomAlias)
+from mautrix.appservice import IntentAPI
+
+from ..types import TelegramID
+from ..context import Context
+from .. import puppet as p, user as u, util
+from .base import BasePortal, InviteList, TypeParticipant, TypeChatPhoto
+
+if TYPE_CHECKING:
+    from ..abstract_user import AbstractUser
+    from ..config import Config
+
+config: Optional['Config'] = None
+
+
+class PortalMetadata(BasePortal, ABC):
+    _room_create_lock: asyncio.Lock
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._room_create_lock = asyncio.Lock()
+
+    # region Matrix -> Telegram
+
+    async def _get_telegram_users_in_matrix_room(self) -> List[Union[InputUser, PeerUser]]:
+        user_tgids = set()
+        user_mxids = await self.main_intent.get_room_members(self.mxid, (Membership.JOIN,
+                                                                         Membership.INVITE))
+        for user_str in user_mxids:
+            user = UserID(user_str)
+            if user == self.az.bot_mxid:
+                continue
+            mx_user = u.User.get_by_mxid(user, create=False)
+            if mx_user and mx_user.tgid:
+                user_tgids.add(mx_user.tgid)
+            puppet_id = p.Puppet.get_id_from_mxid(user)
+            if puppet_id:
+                user_tgids.add(puppet_id)
+        return [PeerUser(user_id) for user_id in user_tgids]
+
+    async def upgrade_telegram_chat(self, source: 'u.User') -> None:
+        if self.peer_type != "chat":
+            raise ValueError("Only normal group chats are upgradable to supergroups.")
+
+        response = await source.client(MigrateChatRequest(chat_id=self.tgid))
+        entity = None
+        for chat in response.chats:
+            if isinstance(chat, Channel):
+                entity = chat
+                break
+        if not entity:
+            raise ValueError("Upgrade may have failed: output channel not found.")
+        self.peer_type = "channel"
+        self._migrate_and_save_telegram(TelegramID(entity.id))
+        await self.update_info(source, entity)
+
+    def _migrate_and_save_telegram(self, new_id: TelegramID) -> None:
+        try:
+            del self.by_tgid[self.tgid_full]
+        except KeyError:
+            pass
+        try:
+            existing = self.by_tgid[(new_id, new_id)]
+            existing.delete()
+        except KeyError:
+            pass
+        self.db_instance.edit(tgid=new_id, tg_receiver=new_id, peer_type=self.peer_type)
+        old_id = self.tgid
+        self.tgid = new_id
+        self.tg_receiver = new_id
+        self.by_tgid[self.tgid_full] = self
+        self.log = self.base_log.getChild(self.tgid_log)
+        self.log.info(f"Telegram chat upgraded from {old_id}")
+
+    async def set_telegram_username(self, source: 'u.User', username: str) -> None:
+        if self.peer_type != "channel":
+            raise ValueError("Only channels and supergroups have usernames.")
+        await source.client(
+            UpdateUsernameRequest(await self.get_input_entity(source), username))
+        if await self._update_username(username):
+            self.save()
+
+    async def create_telegram_chat(self, source: 'u.User', supergroup: bool = False) -> None:
+        if not self.mxid:
+            raise ValueError("Can't create Telegram chat for portal without Matrix room.")
+        elif self.tgid:
+            raise ValueError("Can't create Telegram chat for portal with existing Telegram chat.")
+
+        invites = await self._get_telegram_users_in_matrix_room()
+        if len(invites) < 2:
+            if self.bot is not None:
+                info, mxid = await self.bot.get_me()
+                raise ValueError("Not enough Telegram users to create a chat. "
+                                 "Invite more Telegram ghost users to the room, such as the "
+                                 f"relaybot ([{info.first_name}](https://matrix.to/#/{mxid})).")
+            raise ValueError("Not enough Telegram users to create a chat. "
+                             "Invite more Telegram ghost users to the room.")
+        if self.peer_type == "chat":
+            response = await source.client(CreateChatRequest(title=self.title, users=invites))
+            entity = response.chats[0]
+        elif self.peer_type == "channel":
+            response = await source.client(CreateChannelRequest(title=self.title,
+                                                                about=self.about or "",
+                                                                megagroup=supergroup))
+            entity = response.chats[0]
+            await source.client(InviteToChannelRequest(
+                channel=await source.client.get_input_entity(entity),
+                users=invites))
+        else:
+            raise ValueError("Invalid peer type for Telegram chat creation")
+
+        self.tgid = entity.id
+        self.tg_receiver = self.tgid
+        self.by_tgid[self.tgid_full] = self
+        await self.update_info(source, entity)
+        self.db_instance.insert()
+        self.log = self.base_log.getChild(self.tgid_log)
+
+        if self.bot and self.bot.tgid in invites:
+            self.bot.add_chat(self.tgid, self.peer_type)
+
+        levels = await self.main_intent.get_power_levels(self.mxid)
+        if levels.get_user_level(self.main_intent.mxid) == 100:
+            levels = self._get_base_power_levels(levels, entity)
+            await self.main_intent.set_power_levels(self.mxid, levels)
+        await self.handle_matrix_power_levels(source, levels.users, {})
+
+    async def invite_telegram(self, source: 'u.User',
+                              puppet: Union[p.Puppet, 'AbstractUser']) -> None:
+        if self.peer_type == "chat":
+            await source.client(
+                AddChatUserRequest(chat_id=self.tgid, user_id=puppet.tgid, fwd_limit=0))
+        elif self.peer_type == "channel":
+            await source.client(InviteToChannelRequest(channel=self.peer, users=[puppet.tgid]))
+        # We don't care if there are invites for private chat portals with the relaybot.
+        elif not self.bot or self.tg_receiver != self.bot.tgid:
+            raise ValueError("Invalid peer type for Telegram user invite")
+
+    async def sync_matrix_members(self) -> None:
+        resp = await self.main_intent.get_room_joined_memberships(self.mxid)
+        members = resp["joined"]
+        for mxid, info in members.items():
+            member = Member(membership=Membership.JOIN)
+            if "display_name" in info:
+                member.displayname = info["display_name"]
+            if "avatar_url" in info:
+                member.avatar_url = info["avatar_url"]
+            self.az.state_store.set_member(self.mxid, mxid, member)
+
+    # endregion
+    # region Telegram -> Matrix
+
+    async def invite_to_matrix(self, users: InviteList) -> None:
+        if isinstance(users, list):
+            for user in users:
+                await self.main_intent.invite_user(self.mxid, user, check_cache=True)
+        else:
+            await self.main_intent.invite_user(self.mxid, users, check_cache=True)
+
+    async def update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
+                                 direct: bool = None, puppet: p.Puppet = None,
+                                 levels: PowerLevelStateEventContent = None,
+                                 users: List[User] = None,
+                                 participants: List[TypeParticipant] = None) -> None:
+        if direct is None:
+            direct = self.peer_type == "user"
+        try:
+            await self._update_matrix_room(user, entity, direct, puppet, levels, users,
+                                           participants)
+        except Exception:
+            self.log.exception("Fatal error updating Matrix room")
+
+    async def _update_matrix_room(self, user: 'AbstractUser', entity: Union[TypeChat, User],
+                                  direct: bool, puppet: p.Puppet = None,
+                                  levels: PowerLevelStateEventContent = None,
+                                  users: List[User] = None,
+                                  participants: List[TypeParticipant] = None) -> None:
+        if not direct:
+            await self.update_info(user, entity)
+            if not users or not participants:
+                users, participants = await self._get_users(user, entity)
+            await self._sync_telegram_users(user, users)
+            await self.update_telegram_participants(participants, levels)
+        else:
+            if not puppet:
+                puppet = p.Puppet.get(self.tgid)
+            await puppet.update_info(user, entity)
+            await puppet.intent_for(self).join_room(self.mxid)
+        if self.sync_matrix_state:
+            await self.sync_matrix_members()
+
+    async def create_matrix_room(self, user: 'AbstractUser', entity: TypeChat = None,
+                                 invites: InviteList = None, update_if_exists: bool = True,
+                                 synchronous: bool = False) -> Optional[str]:
+        if self.mxid:
+            if update_if_exists:
+                if not entity:
+                    try:
+                        entity = await self.get_entity(user)
+                    except Exception:
+                        self.log.exception(f"Failed to get entity through {user.tgid} for update")
+                        return self.mxid
+                update = self.update_matrix_room(user, entity, self.peer_type == "user")
+                if synchronous:
+                    await update
+                else:
+                    asyncio.ensure_future(update, loop=self.loop)
+                await self.invite_to_matrix(invites or [])
+            return self.mxid
+        async with self._room_create_lock:
+            try:
+                return await self._create_matrix_room(user, entity, invites)
+            except Exception:
+                self.log.exception("Fatal error creating Matrix room")
+
+    async def _create_matrix_room(self, user: 'AbstractUser', entity: TypeChat, invites: InviteList
+                                  ) -> Optional[RoomID]:
+        direct = self.peer_type == "user"
+
+        if self.mxid:
+            return self.mxid
+
+        if not self.allow_bridging:
+            return None
+
+        if not entity:
+            entity = await self.get_entity(user)
+            self.log.debug(f"Fetched data: {entity}")
+
+        self.log.debug("Creating room")
+
+        try:
+            self.title = entity.title
+        except AttributeError:
+            self.title = None
+
+        if direct and self.tgid == user.tgid:
+            self.title = "Telegram Saved Messages"
+            self.about = "Your Telegram cloud storage chat"
+
+        puppet = p.Puppet.get(self.tgid) if direct else None
+        self._main_intent = puppet.intent_for(self) if direct else self.az.intent
+
+        if self.peer_type == "channel":
+            self.megagroup = entity.megagroup
+
+        if self.peer_type == "channel" and entity.username:
+            preset = RoomCreatePreset.PUBLIC
+            self.username = entity.username
+            alias = self.alias_localpart
+        else:
+            preset = RoomCreatePreset.PRIVATE
+            # TODO invite link alias?
+            alias = None
+
+        if alias:
+            # TODO? properly handle existing room aliases
+            await self.main_intent.remove_room_alias(alias)
+
+        power_levels = self._get_base_power_levels(entity=entity)
+        users = participants = None
+        if not direct:
+            users, participants = await self._get_users(user, entity)
+            if self.has_bot:
+                extra_invites = config["bridge.relaybot.group_chat_invite"]
+                invites += extra_invites
+                for invite in extra_invites:
+                    power_levels.users.setdefault(invite, 100)
+            self._participants_to_power_levels(participants, power_levels)
+        elif self.bot and self.tg_receiver == self.bot.tgid:
+            invites = config["bridge.relaybot.private_chat.invite"]
+            for invite in invites:
+                power_levels.users.setdefault(invite, 100)
+            self.title = puppet.displayname
+        initial_state = [{
+            "type": EventType.ROOM_POWER_LEVELS.serialize(),
+            "content": power_levels.serialize(),
+        }]
+        if config["appservice.community_id"]:
+            initial_state.append({
+                "type": "m.room.related_groups",
+                "content": {"groups": [config["appservice.community_id"]]},
+            })
+        creation_content = {}
+        if not config["bridge.federate_rooms"]:
+            creation_content["m.federate"] = False
+
+        room_id = await self.main_intent.create_room(alias_localpart=alias, preset=preset,
+                                                     is_direct=direct, invitees=invites or [],
+                                                     name=self.title, topic=self.about,
+                                                     initial_state=initial_state,
+                                                     creation_content=creation_content)
+        if not room_id:
+            raise Exception(f"Failed to create room")
+
+        self.mxid = RoomID(room_id)
+        self.by_mxid[self.mxid] = self
+        self.save()
+        self.az.state_store.set_power_levels(self.mxid, power_levels)
+        user.register_portal(self)
+        asyncio.ensure_future(self.update_matrix_room(user, entity, direct, puppet,
+                                                      levels=power_levels, users=users,
+                                                      participants=participants), loop=self.loop)
+
+        return self.mxid
+
+    def _get_base_power_levels(self, levels: PowerLevelStateEventContent = None,
+                               entity: TypeChat = None) -> PowerLevelStateEventContent:
+        levels = levels or PowerLevelStateEventContent()
+        if self.peer_type == "user":
+            overrides = config["bridge.initial_power_level_overrides.user"]
+            levels.ban = overrides.get("ban", 100)
+            levels.kick = overrides.get("kick", 100)
+            levels.invite = overrides.get("invite", 100)
+            levels.redact = overrides.get("redact", 0)
+            levels.events[EventType.ROOM_NAME] = 0
+            levels.events[EventType.ROOM_AVATAR] = 0
+            levels.events[EventType.ROOM_TOPIC] = 0
+            levels.state_default = overrides.get("state_default", 0)
+            levels.users_default = overrides.get("users_default", 0)
+            levels.events_default = overrides.get("events_default", 0)
+        else:
+            overrides = config["bridge.initial_power_level_overrides.group"]
+            dbr = entity.default_banned_rights
+            if not dbr:
+                self.log.debug(f"default_banned_rights is None in {entity}")
+                dbr = ChatBannedRights(invite_users=True, change_info=True, pin_messages=True,
+                                       send_stickers=False, send_messages=False, until_date=None)
+            levels.ban = overrides.get("ban", 50)
+            levels.kick = overrides.get("kick", 50)
+            levels.redact = overrides.get("redact", 50)
+            levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
+            levels.events[EventType.ROOM_ENCRYPTED] = 99
+            levels.events[EventType.ROOM_TOMBSTONE] = 99
+            levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
+            levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0
+            levels.events[EventType.ROOM_TOPIC] = 50 if dbr.change_info else 0
+            levels.events[EventType.ROOM_PINNED_EVENTS] = 50 if dbr.pin_messages else 0
+            levels.events[EventType.ROOM_POWER_LEVELS] = 75
+            levels.events[EventType.ROOM_HISTORY_VISIBILITY] = 75
+            levels.events[EventType.STICKER] = 50 if dbr.send_stickers else levels.events_default
+            levels.state_default = overrides.get("state_default", 50)
+            levels.users_default = overrides.get("users_default", 0)
+            levels.events_default = (
+                overrides.get("events_default",
+                              50 if (self.peer_type == "channel" and not entity.megagroup
+                                     or entity.default_banned_rights.send_messages)
+                              else 0))
+        for evt_type, value in overrides.get("events", {}).items():
+            levels.events[EventType.find(evt_type)] = value
+        levels.users = overrides.get("users", {})
+        if self.main_intent.mxid not in levels.users:
+            levels.users[self.main_intent.mxid] = 100
+        return levels
+
+    @staticmethod
+    def _get_level_from_participant(participant: TypeParticipant) -> int:
+        # TODO use the power level requirements to get better precision in channels
+        if isinstance(participant, (ChatParticipantAdmin, ChannelParticipantAdmin)):
+            return 50
+        elif isinstance(participant, (ChatParticipantCreator, ChannelParticipantCreator)):
+            return 95
+        return 0
+
+    @staticmethod
+    def _participant_to_power_levels(levels: PowerLevelStateEventContent,
+                                     user: Union['u.User', p.Puppet], new_level: int,
+                                     bot_level: int) -> bool:
+        new_level = min(new_level, bot_level)
+        user_level = levels.get_user_level(user.mxid)
+        if user_level != new_level and user_level < bot_level:
+            levels.users[user.mxid] = new_level
+            return True
+        return False
+
+    def _participants_to_power_levels(self, participants: List[TypeParticipant],
+                                      levels: PowerLevelStateEventContent) -> bool:
+        bot_level = levels.get_user_level(self.main_intent.mxid)
+        if bot_level < levels.get_event_level(EventType.ROOM_POWER_LEVELS):
+            return False
+        changed = False
+        admin_power_level = min(75 if self.peer_type == "channel" else 50, bot_level)
+        if levels.events[EventType.ROOM_POWER_LEVELS] != admin_power_level:
+            changed = True
+            levels.events[EventType.ROOM_POWER_LEVELS] = admin_power_level
+
+        for participant in participants:
+            puppet = p.Puppet.get(TelegramID(participant.user_id))
+            user = u.User.get_by_tgid(TelegramID(participant.user_id))
+            new_level = self._get_level_from_participant(participant)
+
+            if user:
+                user.register_portal(self)
+                changed = self._participant_to_power_levels(levels, user, new_level,
+                                                            bot_level) or changed
+
+            if puppet:
+                changed = self._participant_to_power_levels(levels, puppet, new_level,
+                                                            bot_level) or changed
+        return changed
+
+    async def update_telegram_participants(self, participants: List[TypeParticipant],
+                                           levels: PowerLevelStateEventContent = None) -> None:
+        if not levels:
+            levels = await self.main_intent.get_power_levels(self.mxid)
+        if self._participants_to_power_levels(participants, levels):
+            await self.main_intent.set_power_levels(self.mxid, levels)
+
+    def _add_bot_chat(self, bot: User) -> None:
+        if self.bot and bot.id == self.bot.tgid:
+            self.bot.add_chat(self.tgid, self.peer_type)
+            return
+
+        user = u.User.get_by_tgid(TelegramID(bot.id))
+        if user and user.is_bot:
+            user.register_portal(self)
+
+    async def _sync_telegram_users(self, source: 'AbstractUser', users: List[User]) -> None:
+        allowed_tgids = set()
+        skip_deleted = config["bridge.skip_deleted_members"]
+        for entity in users:
+            if skip_deleted and entity.deleted:
+                continue
+            puppet = p.Puppet.get(TelegramID(entity.id))
+            if entity.bot:
+                self._add_bot_chat(entity)
+            allowed_tgids.add(entity.id)
+            await puppet.intent_for(self).ensure_joined(self.mxid)
+            await puppet.update_info(source, entity)
+
+            user = u.User.get_by_tgid(TelegramID(entity.id))
+            if user:
+                await self.invite_to_matrix(user.mxid)
+
+        # We can't trust the member list if any of the following cases is true:
+        #  * There are close to 10 000 users, because Telegram might not be sending all members.
+        #  * The member sync count is limited, because then we might ignore some members.
+        #  * It's a channel, because non-admins don't have access to the member list.
+        trust_member_list = (len(allowed_tgids) < 9900
+                             and self.max_initial_member_sync == -1
+                             and (self.megagroup or self.peer_type != "channel"))
+        if trust_member_list:
+            joined_mxids = await self.main_intent.get_room_members(self.mxid)
+            for user_mxid in joined_mxids:
+                if user_mxid == self.az.bot_mxid:
+                    continue
+                puppet_id = p.Puppet.get_id_from_mxid(user_mxid)
+                if puppet_id and puppet_id not in allowed_tgids:
+                    if self.bot and puppet_id == self.bot.tgid:
+                        self.bot.remove_chat(self.tgid)
+                    try:
+                        await self.main_intent.kick_user(self.mxid, user_mxid,
+                                                         "User had left this Telegram chat.")
+                    except MForbidden:
+                        pass
+                    continue
+                mx_user = u.User.get_by_mxid(user_mxid, create=False)
+                if mx_user and mx_user.is_bot and mx_user.tgid not in allowed_tgids:
+                    mx_user.unregister_portal(self)
+
+                if mx_user and not self.has_bot and mx_user.tgid not in allowed_tgids:
+                    try:
+                        await self.main_intent.kick_user(self.mxid, mx_user.mxid,
+                                                         "You had left this Telegram chat.")
+                    except MForbidden:
+                        pass
+                    continue
+
+    async def _add_telegram_user(self, user_id: TelegramID, source: Optional['AbstractUser'] = None
+                                 ) -> None:
+        puppet = p.Puppet.get(user_id)
+        if source:
+            entity: User = await source.client.get_entity(PeerUser(user_id))
+            await puppet.update_info(source, entity)
+            await puppet.intent_for(self).ensure_joined(self.mxid)
+
+        user = u.User.get_by_tgid(user_id)
+        if user:
+            user.register_portal(self)
+            await self.invite_to_matrix(user.mxid)
+
+    async def _delete_telegram_user(self, user_id: TelegramID, sender: p.Puppet) -> None:
+        puppet = p.Puppet.get(user_id)
+        user = u.User.get_by_tgid(user_id)
+        kick_message = (f"Kicked by {sender.displayname}"
+                        if sender and sender.tgid != puppet.tgid
+                        else "Left Telegram chat")
+        if sender.tgid != puppet.tgid:
+            try:
+                await sender.intent_for(self).kick_user(self.mxid, puppet.mxid)
+            except MForbidden:
+                await self.main_intent.kick_user(self.mxid, puppet.mxid, kick_message)
+        else:
+            await puppet.intent_for(self).leave_room(self.mxid)
+        if user:
+            user.unregister_portal(self)
+            if sender.tgid != puppet.tgid:
+                try:
+                    await sender.intent_for(self).kick_user(self.mxid, puppet.mxid)
+                    return
+                except MForbidden:
+                    pass
+            try:
+                await self.main_intent.kick_user(self.mxid, user.mxid, kick_message)
+            except MForbidden as e:
+                self.log.warning(f"Failed to kick {user.mxid}: {e}")
+
+    async def update_info(self, user: 'AbstractUser', entity: TypeChat = None) -> None:
+        if self.peer_type == "user":
+            self.log.warning("Called update_info() for direct chat portal")
+            return
+
+        self.log.debug("Updating info")
+        try:
+            if not entity:
+                entity = await self.get_entity(user)
+                self.log.debug(f"Fetched data: {entity}")
+            changed = False
+
+            if self.peer_type == "channel":
+                changed = self.megagroup != entity.megagroup or changed
+                self.megagroup = entity.megagroup
+                changed = await self._update_username(entity.username) or changed
+
+            if hasattr(entity, "about"):
+                changed = self._update_about(entity.about) or changed
+
+            changed = await self._update_title(entity.title) or changed
+
+            if isinstance(entity.photo, ChatPhoto):
+                changed = await self._update_avatar(user, entity.photo) or changed
+        except Exception:
+            self.log.exception(f"Failed to update info from source {user.tgid}")
+
+        if changed:
+            self.save()
+
+    async def _update_username(self, username: str, save: bool = False) -> bool:
+        if self.username == username:
+            return False
+
+        if self.username:
+            await self.main_intent.remove_room_alias(self.alias_localpart)
+        self.username = username or None
+        if self.username:
+            await self.main_intent.add_room_alias(self.mxid, self.alias_localpart, override=True)
+            if self.public_portals:
+                await self.main_intent.set_join_rule(self.mxid, "public")
+        else:
+            await self.main_intent.set_join_rule(self.mxid, "invite")
+
+        if save:
+            self.save()
+        return True
+
+    async def _try_use_intent(self, sender: Optional['p.Puppet'],
+                              action: Callable[[IntentAPI], None]) -> None:
+        if sender:
+            try:
+                await action(sender.intent_for(self))
+            except MForbidden:
+                await action(self.main_intent)
+        else:
+            await action(self.main_intent)
+
+    async def _update_about(self, about: str, sender: Optional['p.Puppet'] = None,
+                            save: bool = False) -> bool:
+        if self.about == about:
+            return False
+
+        self.about = about
+        await self._try_use_intent(sender,
+                                   lambda intent: intent.set_room_topic(self.mxid, self.about))
+        if save:
+            self.save()
+        return True
+
+    async def _update_title(self, title: str, sender: Optional['p.Puppet'] = None,
+                            save: bool = False) -> bool:
+        if self.title == title:
+            return False
+
+        self.title = title
+        await self._try_use_intent(sender,
+                                   lambda intent: intent.set_room_name(self.mxid, self.title))
+        if save:
+            self.save()
+        return True
+
+    async def _update_avatar(self, user: 'AbstractUser', photo: TypeChatPhoto,
+                             sender: Optional['p.Puppet'] = None, save: bool = False) -> bool:
+        if isinstance(photo, ChatPhoto):
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(user),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            photo_id = f"{loc.volume_id}-{loc.local_id}"
+        elif isinstance(photo, Photo):
+            loc, largest = self._get_largest_photo_size(photo)
+            photo_id = f"{largest.location.volume_id}-{largest.location.local_id}"
+        elif isinstance(photo, (ChatPhotoEmpty, PhotoEmpty)):
+            photo_id = ""
+            loc = None
+        else:
+            raise ValueError(f"Unknown photo type {type(photo)}")
+        if self.photo_id != photo_id:
+            if not photo_id:
+                await self._try_use_intent(sender,
+                                           lambda intent: intent.set_room_avatar(self.mxid, None))
+                self.photo_id = ""
+                if save:
+                    self.save()
+                return True
+            file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
+            if file:
+                await self._try_use_intent(sender, lambda intent: intent.set_room_avatar(self.mxid,
+                                                                                         file.mxc))
+                self.photo_id = photo_id
+                if save:
+                    self.save()
+                return True
+        return False
+
+    async def _get_users(self, user: 'AbstractUser',
+                         entity: Union[TypeInputPeer, InputUser, TypeChat, TypeUser, InputChannel]
+                         ) -> Tuple[List[TypeUser], List[TypeParticipant]]:
+        # TODO replace with client.get_participants
+        if self.peer_type == "chat":
+            chat = await user.client(GetFullChatRequest(chat_id=self.tgid))
+            return chat.users, chat.full_chat.participants.participants
+        elif self.peer_type == "channel":
+            if not self.megagroup and not self.sync_channel_members:
+                return [], []
+
+            limit = self.max_initial_member_sync
+            if limit == 0:
+                return [], []
+
+            try:
+                if 0 < limit <= 200:
+                    response = await user.client(GetParticipantsRequest(
+                        entity, ChannelParticipantsRecent(), offset=0, limit=limit, hash=0))
+                    return response.users, response.participants
+                elif limit > 200 or limit == -1:
+                    users: List[TypeUser] = []
+                    participants: List[TypeParticipant] = []
+                    offset = 0
+                    remaining_quota = limit if limit > 0 else 1000000
+                    query = (ChannelParticipantsSearch("") if limit == -1
+                             else ChannelParticipantsRecent())
+                    while True:
+                        if remaining_quota <= 0:
+                            break
+                        response = await user.client(GetParticipantsRequest(
+                            entity, query, offset=offset, limit=min(remaining_quota, 100), hash=0))
+                        if not response.users:
+                            break
+                        participants += response.participants
+                        users += response.users
+                        offset += len(response.participants)
+                        remaining_quota -= len(response.participants)
+                    return users, participants
+            except ChatAdminRequiredError:
+                return [], []
+        elif self.peer_type == "user":
+            return [entity], []
+        return [], []
+
+    # endregion
+
+
+def init(context: Context) -> None:
+    global config
+    config = context.config

mautrix_telegram/portal/send_lock.py

@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,8 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
+from typing import Dict
 from asyncio import Lock
 
 from ..types import TelegramID
@@ -29,7 +28,7 @@ class FakeLock:
 
 
 class PortalSendLock:
-    _send_locks: dict[int, Lock]
+    _send_locks: Dict[int, Lock]
     _noop_lock: Lock = FakeLock()
 
     def __init__(self) -> None:
@@ -41,4 +40,5 @@ class PortalSendLock:
         try:
             return self._send_locks[user_id]
         except KeyError:
-            return self._send_locks.setdefault(user_id, Lock()) if required else self._noop_lock
+            return (self._send_locks.setdefault(user_id, Lock())
+                    if required else self._noop_lock)

mautrix_telegram/portal/telegram.py

@@ -0,0 +1,523 @@
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Dict, List, Optional, Tuple, Union, NamedTuple, TYPE_CHECKING
+from html import escape as escape_html
+from abc import ABC
+import random
+import mimetypes
+import codecs
+import unicodedata
+import base64
+
+from sqlalchemy.exc import IntegrityError
+
+from telethon.tl.patched import Message, MessageService
+from telethon.tl.types import (
+    Poll, DocumentAttributeFilename, DocumentAttributeSticker, DocumentAttributeVideo,
+    MessageMediaPoll, MessageActionChannelCreate, MessageActionChatAddUser,
+    MessageActionChatCreate, MessageActionChatDeletePhoto, MessageActionChatDeleteUser,
+    MessageActionChatEditPhoto, MessageActionChatEditTitle, MessageActionChatJoinedByLink,
+    MessageActionChatMigrateTo, MessageActionPinMessage, MessageActionGameScore,
+    MessageMediaDocument, MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported,
+    MessageMediaGame, PeerUser, PhotoCachedSize, TypeChannelParticipant, TypeChatParticipant,
+    TypeDocumentAttribute, TypeMessageAction, TypePhotoSize, PhotoSize, UpdateChatUserTyping,
+    UpdateUserTyping, MessageEntityPre, ChatPhotoEmpty)
+
+from mautrix.appservice import IntentAPI
+from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType,
+                           EventType, MediaMessageEventContent, TextMessageEventContent,
+                           LocationMessageEventContent, Format)
+
+from ..types import TelegramID
+from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
+from ..util import sane_mimetypes
+from ..context import Context
+from .. import puppet as p, user as u, formatter, util
+from .base import BasePortal
+
+if TYPE_CHECKING:
+    from ..abstract_user import AbstractUser
+    from ..config import Config
+
+InviteList = Union[UserID, List[UserID]]
+TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
+DocAttrs = NamedTuple("DocAttrs", name=Optional[str], mime_type=Optional[str], is_sticker=bool,
+                      sticker_alt=Optional[str], width=int, height=int)
+
+config: Optional['Config'] = None
+
+
+class PortalTelegram(BasePortal, ABC):
+    async def handle_telegram_typing(self, user: p.Puppet,
+                                     _: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+        await user.intent_for(self).set_typing(self.mxid, is_typing=True)
+
+    def _get_external_url(self, evt: Message) -> Optional[str]:
+        if self.peer_type == "channel" and self.username is not None:
+            return f"https://t.me/{self.username}/{evt.id}"
+        elif self.peer_type != "user":
+            return f"https://t.me/c/{self.tgid}/{evt.id}"
+        return None
+
+    async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
+                                    relates_to: Dict = None) -> Optional[EventID]:
+        loc, largest_size = self._get_largest_photo_size(evt.media.photo)
+        file = await util.transfer_file_to_matrix(source.client, intent, loc)
+        if not file:
+            return None
+        if self.get_config("inline_images") and (evt.message
+                                                 or evt.fwd_from or evt.reply_to_msg_id):
+            content = await formatter.telegram_to_matrix(
+                evt, source, self.main_intent,
+                prefix_html=f"<img src='{file.mxc}' alt='Inline Telegram photo'/><br/>",
+                prefix_text="Inline image: ")
+            content.external_url = self._get_external_url(evt)
+            await intent.set_typing(self.mxid, is_typing=False)
+            return await intent.send_message(self.mxid, content, timestamp=evt.date)
+        info = ImageInfo(
+            height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type,
+            size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize))
+                  else largest_size.size))
+        name = f"image{sane_mimetypes.guess_extension(file.mime_type)}"
+        await intent.set_typing(self.mxid, is_typing=False)
+        content = MediaMessageEventContent(url=file.mxc, msgtype=MessageType.IMAGE, info=info,
+                                           body=name, relates_to=relates_to,
+                                           external_url=self._get_external_url(evt))
+        result = await intent.send_message(self.mxid, content, timestamp=evt.date)
+        if evt.message:
+            caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                                 no_reply_fallback=True)
+            caption_content.external_url = content.external_url
+            result = await intent.send_message(self.mxid, caption_content, timestamp=evt.date)
+        return result
+
+    @staticmethod
+    def _parse_telegram_document_attributes(attributes: List[TypeDocumentAttribute]) -> DocAttrs:
+        name, mime_type, is_sticker, sticker_alt, width, height = None, None, False, None, 0, 0
+        for attr in attributes:
+            if isinstance(attr, DocumentAttributeFilename):
+                name = name or attr.file_name
+                mime_type, _ = mimetypes.guess_type(attr.file_name)
+            elif isinstance(attr, DocumentAttributeSticker):
+                is_sticker = True
+                sticker_alt = attr.alt
+            elif isinstance(attr, DocumentAttributeVideo):
+                width, height = attr.w, attr.h
+        return DocAttrs(name, mime_type, is_sticker, sticker_alt, width, height)
+
+    @staticmethod
+    def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: DocAttrs,
+                                      thumb_size: TypePhotoSize) -> Tuple[ImageInfo, str]:
+        document = evt.media.document
+        name = evt.message or attrs.name
+        if attrs.is_sticker:
+            alt = attrs.sticker_alt
+            if len(alt) > 0:
+                try:
+                    name = f"{alt} ({unicodedata.name(alt[0]).lower()})"
+                except ValueError:
+                    name = alt
+
+        generic_types = ("text/plain", "application/octet-stream")
+        if file.mime_type in generic_types and document.mime_type not in generic_types:
+            mime_type = document.mime_type or file.mime_type
+        else:
+            mime_type = file.mime_type or document.mime_type
+        info = ImageInfo(size=file.size, mimetype=mime_type)
+
+        if attrs.mime_type and not file.was_converted:
+            file.mime_type = attrs.mime_type or file.mime_type
+        if file.width and file.height:
+            info.width, info.height = file.width, file.height
+        elif attrs.width and attrs.height:
+            info.width, info.height = attrs.width, attrs.height
+
+        if file.thumbnail:
+            info.thumbnail_url = file.thumbnail.mxc
+            info.thumbnail_info = ThumbnailInfo(mimetype=file.thumbnail.mime_type,
+                                                height=file.thumbnail.height or thumb_size.h,
+                                                width=file.thumbnail.width or thumb_size.w,
+                                                size=file.thumbnail.size)
+
+        return info, name
+
+    async def handle_telegram_document(self, source: 'AbstractUser', intent: IntentAPI,
+                                       evt: Message, relates_to: RelatesTo = None
+                                       ) -> Optional[EventID]:
+        document = evt.media.document
+
+        attrs = self._parse_telegram_document_attributes(document.attributes)
+
+        if document.size > config["bridge.max_document_size"] * 1000 ** 2:
+            name = attrs.name or ""
+            caption = f"\n{evt.message}" if evt.message else ""
+            return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
+
+        thumb_loc, thumb_size = self._get_largest_photo_size(document)
+        if thumb_size and not isinstance(thumb_size, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb_size)}")
+            thumb_loc = None
+            thumb_size = None
+        parallel_id = source.tgid if config["bridge.parallel_file_transfer"] else None
+        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
+                                                  is_sticker=attrs.is_sticker,
+                                                  tgs_convert=config["bridge.animated_sticker"],
+                                                  filename=attrs.name, parallel_id=parallel_id)
+        if not file:
+            return None
+
+        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb_size)
+
+        await intent.set_typing(self.mxid, is_typing=False)
+
+        event_type = EventType.ROOM_MESSAGE
+        # Riot only supports images as stickers, so send animated webm stickers as m.video
+        if attrs.is_sticker and file.mime_type.startswith("image/"):
+            event_type = EventType.STICKER
+        content = MediaMessageEventContent(
+            body=name or "unnamed file", info=info, url=file.mxc, relates_to=relates_to,
+            external_url=self._get_external_url(evt),
+            msgtype={
+                "video/": MessageType.VIDEO,
+                "audio/": MessageType.AUDIO,
+                "image/": MessageType.IMAGE,
+            }.get(info.mimetype[:6], MessageType.FILE))
+        return await intent.send_message_event(self.mxid, event_type, content, timestamp=evt.date)
+
+    def handle_telegram_location(self, _: 'AbstractUser', intent: IntentAPI, evt: Message,
+                                 relates_to: dict = None) -> Awaitable[EventID]:
+        long = evt.media.geo.long
+        lat = evt.media.geo.lat
+        long_char = "E" if long > 0 else "W"
+        lat_char = "N" if lat > 0 else "S"
+
+        body = f"{round(lat, 5)}° {lat_char}, {round(long, 5)}° {long_char}"
+        url = f"https://maps.google.com/?q={lat},{long}"
+
+        content = LocationMessageEventContent(
+            msgtype=MessageType.LOCATION, geo_uri=f"geo:{lat},{long}",
+            body=f"Location: {body}\n{url}",
+            relates_to=relates_to, external_url=self._get_external_url(evt))
+        content["format"] = str(Format.HTML)
+        content["formatted_body"] = f"Location: <a href='{url}'>{body}</a>"
+
+        return intent.send_message(self.mxid, content, timestamp=evt.date)
+
+    async def handle_telegram_text(self, source: 'AbstractUser', intent: IntentAPI, is_bot: bool,
+                                   evt: Message) -> EventID:
+        self.log.debug(f"Sending {evt.message} to {self.mxid} by {intent.mxid}")
+        content = await formatter.telegram_to_matrix(evt, source, self.main_intent)
+        content.external_url = self._get_external_url(evt)
+        if is_bot and self.get_config("bot_messages_as_notices"):
+            content.msgtype = MessageType.NOTICE
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+
+    async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
+                                          evt: Message, relates_to: dict = None) -> EventID:
+        override_text = ("This message is not supported on your version of Mautrix-Telegram. "
+                         "Please check https://github.com/tulir/mautrix-telegram or ask your "
+                         "bridge administrator about possible updates.")
+        content = await formatter.telegram_to_matrix(
+            evt, source, self.main_intent, override_text=override_text)
+        content.msgtype = MessageType.NOTICE
+        content.external_url = self._get_external_url(evt)
+        content["net.maunium.telegram.unsupported"] = True
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+
+    async def handle_telegram_poll(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
+                                   relates_to: RelatesTo) -> EventID:
+        poll: Poll = evt.media.poll
+        poll_id = self._encode_msgid(source, evt)
+
+        _n = 0
+
+        def n() -> int:
+            nonlocal _n
+            _n += 1
+            return _n
+
+        text_answers = "\n".join(f"{n()}. {answer.text}" for answer in poll.answers)
+        html_answers = "\n".join(f"<li>{answer.text}</li>" for answer in poll.answers)
+        content = TextMessageEventContent(
+            msgtype=MessageType.TEXT, format=Format.HTML,
+            body=f"Poll: {poll.question}\n{text_answers}\n"
+                 f"Vote with !tg vote {poll_id} <choice number>",
+            formatted_body=f"<strong>Poll</strong>: {poll.question}<br/>\n"
+                           f"<ol>{html_answers}</ol>\n"
+                           f"Vote with <code>!tg vote {poll_id} &lt;choice number&gt;</code>",
+            relates_to=relates_to, external_url=self._get_external_url(evt))
+
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+
+    @staticmethod
+    def _int_to_bytes(i: int) -> bytes:
+        hex_value = "{0:010x}".format(i)
+        return codecs.decode(hex_value, "hex_codec")
+
+    def _encode_msgid(self, source: 'AbstractUser', evt: Message) -> str:
+        if self.peer_type == "channel":
+            play_id = (b"c"
+                       + self._int_to_bytes(self.tgid)
+                       + self._int_to_bytes(evt.id))
+        elif self.peer_type == "chat":
+            play_id = (b"g"
+                       + self._int_to_bytes(self.tgid)
+                       + self._int_to_bytes(evt.id)
+                       + self._int_to_bytes(source.tgid))
+        elif self.peer_type == "user":
+            play_id = (b"u"
+                       + self._int_to_bytes(self.tgid)
+                       + self._int_to_bytes(evt.id))
+        else:
+            raise ValueError("Portal has invalid peer type")
+        return base64.b64encode(play_id).decode("utf-8").rstrip("=")
+
+    async def handle_telegram_game(self, source: 'AbstractUser', intent: IntentAPI,
+                                   evt: Message, relates_to: RelatesTo = None) -> EventID:
+        game = evt.media.game
+        play_id = self._encode_msgid(source, evt)
+        command = f"!tg play {play_id}"
+        override_text = f"Run {command} in your bridge management room to play {game.title}"
+        override_entities = [
+            MessageEntityPre(offset=len("Run "), length=len(command), language="")]
+
+        content = await formatter.telegram_to_matrix(
+            evt, source, self.main_intent,
+            override_text=override_text, override_entities=override_entities)
+        content.msgtype = MessageType.NOTICE
+        content.external_url = self._get_external_url(evt)
+        content["net.maunium.telegram.game"] = play_id
+
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await intent.send_message(self.mxid, content, timestamp=evt.date)
+
+    async def handle_telegram_edit(self, source: 'AbstractUser', sender: p.Puppet, evt: Message
+                                   ) -> None:
+        if not self.mxid:
+            return
+        elif hasattr(evt, "media") and isinstance(evt.media, MessageMediaGame):
+            self.log.debug("Ignoring game message edit event")
+            return
+
+        async with self.send_lock(sender.tgid if sender else None, required=False):
+            tg_space = self.tgid if self.peer_type == "channel" else source.tgid
+
+            temporary_identifier = EventID(
+                f"${random.randint(1000000000000, 9999999999999)}TGBRIDGEDITEMP")
+            duplicate_found = self.dedup.check(evt, (temporary_identifier, tg_space),
+                                               force_hash=True)
+            if duplicate_found:
+                mxid, other_tg_space = duplicate_found
+                if tg_space != other_tg_space:
+                    prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1)
+                    if not prev_edit_msg:
+                        return
+                    DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space,
+                              tgid=TelegramID(evt.id), edit_index=prev_edit_msg.edit_index + 1
+                              ).insert()
+                return
+
+        content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                     no_reply_fallback=True)
+        editing_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
+        if not editing_msg:
+            self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) "
+                          "in database.")
+            return
+
+        content.msgtype = (MessageType.NOTICE if (sender and sender.is_bot
+                                                  and self.get_config("bot_messages_as_notices"))
+                           else MessageType.TEXT)
+        content.external_url = self._get_external_url(evt)
+        content.set_edit(editing_msg.mxid)
+
+        intent = sender.intent_for(self) if sender else self.main_intent
+        await intent.set_typing(self.mxid, is_typing=False)
+        event_id = await intent.send_message(self.mxid, content)
+
+        prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
+        DBMessage(mxid=event_id, mx_room=self.mxid, tg_space=tg_space, tgid=TelegramID(evt.id),
+                  edit_index=prev_edit_msg.edit_index + 1).insert()
+        DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=event_id)
+
+    async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
+                                      evt: Message) -> None:
+        if not self.mxid:
+            await self.create_matrix_room(source, invites=[source.mxid], update_if_exists=False)
+
+        if (self.peer_type == "user" and sender.tgid == self.tg_receiver
+            and not sender.is_real_user and not self.az.state_store.is_joined(self.mxid,
+                                                                              sender.mxid)):
+            self.log.debug(f"Ignoring private chat message {evt.id}@{source.tgid} as receiver does"
+                           " not have matrix puppeting and their default puppet isn't in the room")
+            return
+
+        async with self.send_lock(sender.tgid if sender else None, required=False):
+            tg_space = self.tgid if self.peer_type == "channel" else source.tgid
+
+            temporary_identifier = EventID(
+                f"${random.randint(1000000000000, 9999999999999)}TGBRIDGETEMP")
+            duplicate_found = self.dedup.check(evt, (temporary_identifier, tg_space))
+            if duplicate_found:
+                mxid, other_tg_space = duplicate_found
+                self.log.debug(f"Ignoring message {evt.id}@{tg_space} (src {source.tgid}) "
+                               f"as it was already handled (in space {other_tg_space})")
+                if tg_space != other_tg_space:
+                    DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
+                              tg_space=tg_space, edit_index=0).insert()
+                return
+
+        if self.dedup.pre_db_check and self.peer_type == "channel":
+            msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
+            if msg:
+                self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
+                               f"handled into {msg.mxid}. This duplicate was catched in the db "
+                               "check. If you get this message often, consider increasing"
+                               "bridge.deduplication.cache_queue_length in the config.")
+                return
+
+        if sender and not sender.displayname:
+            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a "
+                           "displayname, updating info...")
+            entity = await source.client.get_entity(PeerUser(sender.tgid))
+            await sender.update_info(source, entity)
+
+        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
+                         MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
+        media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
+                                                                  allowed_media) else None
+        intent = sender.intent_for(self) if sender else self.main_intent
+        if not media and evt.message:
+            is_bot = sender.is_bot if sender else False
+            event_id = await self.handle_telegram_text(source, intent, is_bot, evt)
+        elif media:
+            event_id = await {
+                MessageMediaPhoto: self.handle_telegram_photo,
+                MessageMediaDocument: self.handle_telegram_document,
+                MessageMediaGeo: self.handle_telegram_location,
+                MessageMediaPoll: self.handle_telegram_poll,
+                MessageMediaUnsupported: self.handle_telegram_unsupported,
+                MessageMediaGame: self.handle_telegram_game,
+            }[type(media)](source, intent, evt,
+                           relates_to=formatter.telegram_reply_to_matrix(evt, source))
+        else:
+            self.log.debug("Unhandled Telegram message: %s", evt)
+            return
+
+        if not event_id:
+            return
+
+        prev_id = self.dedup.update(evt, (event_id, tg_space), (temporary_identifier, tg_space))
+        if prev_id:
+            self.log.debug(f"Sent message {evt.id}@{tg_space} to Matrix as {event_id}. "
+                           f"Temporary dedup identifier was {temporary_identifier}, "
+                           f"but dedup map contained {prev_id[1]} instead! -- "
+                           "This was probably a race condition caused by Telegram sending updates"
+                           "to other clients before responding to the sender. I'll just redact "
+                           "the likely duplicate message now.")
+            await intent.redact(self.mxid, event_id)
+            return
+
+        self.log.debug("Handled Telegram message: %s", evt)
+        try:
+            DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=event_id,
+                      tg_space=tg_space, edit_index=0).insert()
+            DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=event_id)
+        except IntegrityError as e:
+            self.log.exception(f"{e.__class__.__name__} while saving message mapping. "
+                               "This might mean that an update was handled after it left the "
+                               "dedup cache queue. You can try enabling bridge.deduplication."
+                               "pre_db_check in the config.")
+            await intent.redact(self.mxid, event_id)
+
+    async def _create_room_on_action(self, source: 'AbstractUser',
+                                     action: TypeMessageAction) -> bool:
+        if source.is_relaybot and config["bridge.ignore_unbridged_group_chat"]:
+            return False
+        create_and_exit = (MessageActionChatCreate, MessageActionChannelCreate)
+        create_and_continue = (MessageActionChatAddUser, MessageActionChatJoinedByLink)
+        if isinstance(action, create_and_exit) or isinstance(action, create_and_continue):
+            await self.create_matrix_room(source, invites=[source.mxid],
+                                          update_if_exists=isinstance(action, create_and_exit))
+        if not isinstance(action, create_and_continue):
+            return False
+        return True
+
+    async def handle_telegram_action(self, source: 'AbstractUser', sender: p.Puppet,
+                                     update: MessageService) -> None:
+        action = update.action
+        should_ignore = ((not self.mxid and not await self._create_room_on_action(source, action))
+                         or self.dedup.check_action(update))
+        if should_ignore or not self.mxid:
+            return
+        if isinstance(action, MessageActionChatEditTitle):
+            await self._update_title(action.title, sender=sender, save=True)
+        elif isinstance(action, MessageActionChatEditPhoto):
+            await self._update_avatar(source, action.photo, sender=sender, save=True)
+        elif isinstance(action, MessageActionChatDeletePhoto):
+            await self._update_avatar(source, ChatPhotoEmpty(), sender=sender, save=True)
+        elif isinstance(action, MessageActionChatAddUser):
+            for user_id in action.users:
+                await self._add_telegram_user(TelegramID(user_id), source)
+        elif isinstance(action, MessageActionChatJoinedByLink):
+            await self._add_telegram_user(sender.id, source)
+        elif isinstance(action, MessageActionChatDeleteUser):
+            await self._delete_telegram_user(TelegramID(action.user_id), sender)
+        elif isinstance(action, MessageActionChatMigrateTo):
+            self.peer_type = "channel"
+            self._migrate_and_save_telegram(TelegramID(action.channel_id))
+            await sender.intent_for(self).send_emote(self.mxid,
+                                                     "upgraded this group to a supergroup.")
+        elif isinstance(action, MessageActionGameScore):
+            # TODO handle game score
+            pass
+        else:
+            self.log.debug("Unhandled Telegram action in %s: %s", self.title, action)
+
+    async def set_telegram_admin(self, user_id: TelegramID) -> None:
+        puppet = p.Puppet.get(user_id)
+        user = u.User.get_by_tgid(user_id)
+
+        levels = await self.main_intent.get_power_levels(self.mxid)
+        if user:
+            levels.users[user.mxid] = 50
+        if puppet:
+            levels.users[puppet.mxid] = 50
+        await self.main_intent.set_power_levels(self.mxid, levels)
+
+    async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
+        tg_space = receiver  if self.peer_type != "channel" else self.tgid
+        message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
+        if message:
+            await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
+        else:
+            await self.main_intent.set_pinned_messages(self.mxid, [])
+
+    async def set_telegram_admins_enabled(self, enabled: bool) -> None:
+        level = 50 if enabled else 10
+        levels = await self.main_intent.get_power_levels(self.mxid)
+        levels.invite = level
+        levels.events[EventType.ROOM_NAME] = level
+        levels.events[EventType.ROOM_AVATAR] = level
+        await self.main_intent.set_power_levels(self.mxid, levels)
+
+
+def init(context: Context) -> None:
+    global config
+    config = context.config

mautrix_telegram/portal_util/__init__.py

@@ -1,6 +0,0 @@
-from .deduplication import PortalDedup
-from .media_fallback import make_contact_event_content, make_dice_event_content
-from .participants import get_users
-from .power_levels import get_base_power_levels, participants_to_power_levels
-from .send_lock import PortalSendLock
-from .sponsored_message import get_sponsored_message, make_sponsored_message_content

mautrix_telegram/portal_util/media_fallback.py

@@ -1,133 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-import html
-
-from telethon.tl.types import MessageMediaContact, MessageMediaDice, PeerUser
-
-from mautrix.types import Format, MessageType, TextMessageEventContent
-
-from .. import abstract_user as au, puppet as pu
-from ..types import TelegramID
-
-try:
-    import phonenumbers
-except ImportError:
-    phonenumbers = None
-
-
-def _format_dice(roll: MessageMediaDice) -> str:
-    if roll.emoticon == "\U0001F3B0":
-        emojis = {
-            0: "\U0001F36B",  # "🍫",
-            1: "\U0001F352",  # "🍒",
-            2: "\U0001F34B",  # "🍋",
-            3: "7\ufe0f\u20e3",  # "7️⃣",
-        }
-        res = roll.value - 1
-        slot1, slot2, slot3 = emojis[res % 4], emojis[res // 4 % 4], emojis[res // 16]
-        return f"{slot1} {slot2} {slot3} ({roll.value})"
-    elif roll.emoticon == "\u26BD":
-        results = {
-            1: "miss",
-            2: "hit the woodwork",
-            3: "goal",  # seems to go in through the center
-            4: "goal",
-            5: "goal 🎉",  # seems to go in through the top right corner, includes confetti
-        }
-    elif roll.emoticon == "\U0001F3B3":
-        results = {
-            1: "miss",
-            2: "1 pin down",
-            3: "3 pins down, split",
-            4: "4 pins down, split",
-            5: "5 pins down",
-            6: "strike 🎉",
-        }
-    # elif roll.emoticon == "\U0001F3C0":
-    #     results = {
-    #         2: "rolled off",
-    #         3: "stuck",
-    #     }
-    # elif roll.emoticon == "\U0001F3AF":
-    #     results = {
-    #         1: "bounced off",
-    #         2: "outer rim",
-    #
-    #         6: "bullseye",
-    #     }
-    else:
-        return str(roll.value)
-    return f"{results[roll.value]} ({roll.value})"
-
-
-def make_dice_event_content(roll: MessageMediaDice) -> TextMessageEventContent:
-    emoji_text = {
-        "\U0001F3AF": " Dart throw",
-        "\U0001F3B2": " Dice roll",
-        "\U0001F3C0": " Basketball throw",
-        "\U0001F3B0": " Slot machine",
-        "\U0001F3B3": " Bowling",
-        "\u26BD": " Football kick",
-    }
-    text = f"{roll.emoticon}{emoji_text.get(roll.emoticon, '')} result: {_format_dice(roll)}"
-    content = TextMessageEventContent(
-        msgtype=MessageType.TEXT, format=Format.HTML, body=text, formatted_body=f"<h4>{text}</h4>"
-    )
-    content["net.maunium.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
-    return content
-
-
-async def make_contact_event_content(
-    source: au.AbstractUser, contact: MessageMediaContact
-) -> TextMessageEventContent:
-    name = " ".join(x for x in [contact.first_name, contact.last_name] if x)
-    formatted_phone = f"+{contact.phone_number}"
-    if phonenumbers is not None:
-        try:
-            parsed = phonenumbers.parse(formatted_phone)
-            fmt = phonenumbers.PhoneNumberFormat.INTERNATIONAL
-            formatted_phone = phonenumbers.format_number(parsed, fmt)
-        except phonenumbers.NumberParseException:
-            pass
-    content = TextMessageEventContent(
-        msgtype=MessageType.TEXT,
-        body=f"Shared contact info for {name}: {formatted_phone}",
-    )
-    content["net.maunium.telegram.contact"] = {
-        "user_id": contact.user_id,
-        "first_name": contact.first_name,
-        "last_name": contact.last_name,
-        "phone_number": contact.phone_number,
-        "vcard": contact.vcard,
-    }
-
-    puppet = await pu.Puppet.get_by_tgid(TelegramID(contact.user_id))
-    if not puppet.displayname:
-        try:
-            entity = await source.client.get_entity(PeerUser(contact.user_id))
-            await puppet.update_info(source, entity)
-        except Exception as e:
-            source.log.warning(f"Failed to sync puppet info of received contact: {e}")
-    else:
-        content.format = Format.HTML
-        content.formatted_body = (
-            f"Shared contact info for "
-            f"<a href='https://matrix.to/#/{puppet.mxid}'>{html.escape(name)}</a>: "
-            f"{html.escape(formatted_phone)}"
-        )
-    return content

mautrix_telegram/portal_util/participants.py

@@ -1,106 +0,0 @@
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2021 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from __future__ import annotations
-
-from typing import Iterable
-
-from telethon.errors import ChatAdminRequiredError
-from telethon.tl.functions.channels import GetParticipantsRequest
-from telethon.tl.functions.messages import GetFullChatRequest
-from telethon.tl.types import (
-    ChannelParticipantBanned,
-    ChannelParticipantsRecent,
-    ChannelParticipantsSearch,
-    InputChannel,
-    InputUser,
-    TypeChannelParticipant,
-    TypeChat,
-    TypeChatParticipant,
-    TypeInputPeer,
-    TypeUser,
-)
-
-from ..tgclient import MautrixTelegramClient
-
-
-def _filter_participants(
-    users: list[TypeUser], participants: list[TypeChatParticipant | TypeChannelParticipant]
-) -> Iterable[TypeUser]:
-    participant_map = {
-        part.user_id: part
-        for part in participants
-        if not isinstance(part, ChannelParticipantBanned)
-    }
-    for user in users:
-        try:
-            user.participant = participant_map[user.id]
-        except KeyError:
-            pass
-        else:
-            yield user
-
-
-async def _get_channel_users(
-    client: MautrixTelegramClient, entity: InputChannel, limit: int
-) -> list[TypeUser]:
-    if 0 < limit <= 200:
-        response = await client(
-            GetParticipantsRequest(
-                entity, ChannelParticipantsRecent(), offset=0, limit=limit, hash=0
-            )
-        )
-        return list(_filter_participants(response.users, response.participants))
-    elif limit > 200 or limit == -1:
-        users: list[TypeUser] = []
-        offset = 0
-        remaining_quota = limit if limit > 0 else 1000000
-        query = ChannelParticipantsSearch("") if limit == -1 else ChannelParticipantsRecent()
-        while True:
-            if remaining_quota <= 0:
-                break
-            response = await client(
-                GetParticipantsRequest(
-                    entity, query, offset=offset, limit=min(remaining_quota, 200), hash=0
-                )
-            )
-            if not response.users:
-                break
-            users += _filter_participants(response.users, response.participants)
-            offset += len(response.participants)
-            remaining_quota -= len(response.participants)
-        return users
-
-
-async def get_users(
-    client: MautrixTelegramClient,
-    tgid: int,
-    entity: TypeInputPeer | InputUser | TypeChat | TypeUser | InputChannel,
-    limit: int,
-    peer_type: str,
-) -> list[TypeUser]:
-    if peer_type == "chat":
-        chat = await client(GetFullChatRequest(chat_id=tgid))
-        users = list(_filter_participants(chat.users, chat.full_chat.participants.participants))
-        return users[:limit] if limit > 0 else users
-    elif peer_type == "channel":
-        try:
-            return await _get_channel_users(client, entity, limit)
-        except ChatAdminRequiredError:
-            return []
-    elif peer_type == "user":
-        return [entity]
-    else:
-        raise RuntimeError(f"Unexpected peer type {peer_type}")