Bump version to 0.6.1rc2

Users who the bridge only saw via logged in users with the target user
in their contact lists wouldn't get their displayname updated due to an
invalid condition in the update_displayname function.

.codeclimate.yml

@@ -0,0 +1,8 @@
+engines:
+  sonar-python:
+    enabled: true
+    checks:
+      python:S107:
+        enabled: false
+exclude_patterns:
+- "alembic/"

.dockerignore

@@ -0,0 +1,4 @@
+.editorconfig
+.codeclimate.yml
+*.png
+*.md

.editorconfig

@@ -8,5 +8,8 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 
+[*.py]
+max_line_length = 99
+
 [*.{yaml,yml,py}]
 indent_style = space

.gitignore

@@ -1,12 +1,17 @@
 .idea/
 
 .venv
+env/
 pip-selfcheck.json
 *.pyc
 __pycache__
+build
+dist
+*.egg-info
+.eggs
 
 config.yaml
 registration.yaml
+*.log*
 *.db
-*.session
-*.json
+*.bak

.gitlab-ci.yml

@@ -0,0 +1,38 @@
+image: docker:stable
+
+stages:
+- build
+- push
+
+default:
+  before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+
+build:
+  stage: build
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+
+push latest:
+  stage: push
+  only:
+  - master
+  variables:
+    GIT_STRATEGY: none
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:latest
+
+push tag:
+  stage: push
+  variables:
+    GIT_STRATEGY: none
+  except:
+  - master
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME

Dockerfile

@@ -0,0 +1,44 @@
+FROM docker.io/alpine:3.10
+
+ENV UID=1337 \
+    GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
+
+COPY . /opt/mautrix-telegram
+WORKDIR /opt/mautrix-telegram
+RUN apk add --no-cache \
+      py3-virtualenv \
+      py3-pillow \
+      py3-aiohttp \
+      py3-magic \
+      py3-sqlalchemy \
+      py3-markdown \
+      py3-psycopg2 \
+      py3-ruamel.yaml \
+      # Indirect dependencies
+      #commonmark
+        py3-future \
+      #alembic
+        py3-mako \
+        py3-dateutil \
+        py3-markupsafe \
+      #moviepy
+        py3-decorator \
+        #py3-tqdm \
+        py3-requests \
+        #imageio
+          py3-numpy \
+      #telethon
+        py3-rsa \
+      # Other dependencies
+      python3-dev \
+      libffi-dev \
+      build-base \
+      ffmpeg \
+      ca-certificates \
+      su-exec \
+ && pip3 install .[all]
+
+VOLUME /data
+
+CMD ["/opt/mautrix-telegram/docker-run.sh"]

LICENSE

@@ -1,23 +1,21 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
                             Preamble
 
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
 
   The licenses for most software and other practical works are designed
 to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
+our General Public Licenses are intended to guarantee your freedom to
 share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
+software for all its users.
 
   When we speak of free software, we are referring to freedom, not
 price.  Our General Public Licenses are designed to make sure that you
@@ -26,44 +24,34 @@ them if you wish), that you receive source code or can get it if you
 want it, that you can change the software or use pieces of it in new
 free programs, and that you know you can do these things.
 
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
 
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
 
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
 
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
@@ -72,7 +60,7 @@ modification follow.
 
   0. Definitions.
 
-  "This License" refers to version 3 of the GNU General Public License.
+  "This License" refers to version 3 of the GNU Affero General Public License.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
@@ -549,35 +537,45 @@ to collect a royalty for further conveying from those to whom you convey
 the Program, the only way you could satisfy both those terms and this
 License would be to refrain entirely from conveying the Program.
 
-  13. Use with the GNU Affero General Public License.
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
 
   Notwithstanding any other provision of this License, you have
 permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
+under version 3 of the GNU General Public License into a single
 combined work, and to convey the resulting work.  The terms of this
 License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
 
   14. Revised Versions of this License.
 
   The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
 address new problems or concerns.
 
   Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
+Program specifies that a certain numbered version of the GNU Affero General
 Public License "or any later version" applies to it, you have the
 option of following the terms and conditions either of that numbered
 version or of any later version published by the Free Software
 Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
+GNU Affero General Public License, you may choose any version ever published
 by the Free Software Foundation.
 
   If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
+versions of the GNU Affero General Public License can be used, that proxy's
 public statement of acceptance of a version permanently authorizes you
 to choose that version for the Program.
 
@@ -635,40 +633,29 @@ the "copyright" line and a pointer to where the full notice is found.
     Copyright (C) <year>  <name of author>
 
     This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
+    it under the terms of the GNU Affero General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
     (at your option) any later version.
 
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
+    GNU Affero General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
 
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

README.md

@@ -1,12 +1,14 @@
 # mautrix-telegram
-A Matrix-Telegram puppeting bridge.
+A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
 ### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)
 
-### [Features & Roadmap](ROADMAP.md)
+### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
 
 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)
 
-A Telegram chat bridged to the Matrix room will be created once the bridge supports using a bot
-for unauthenticated users.
+Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
+
+## Preview
+![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)

ROADMAP.md

@@ -1,95 +1,60 @@
 # Features & roadmap
 
 * Matrix → Telegram
-  * [x] Plaintext messages
-  * [x] Formatted messages
-    * [ ] Bot commands (!command -> /command)
-    * [x] Mentions
-  * [x] Rich quotes
-  * [ ] Locations (not implemented in Riot)
-  * [x] Images
-  * [x] Files
+  * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
-  * [ ] † Presence
-  * [ ] † Typing notifications
-  * [ ] † Read receipts
-  * [ ] Pinning messages
+  * [x] Message edits
+  * [ ] ‡ Message history
+  * [x] Presence
+  * [x] Typing notifications
+  * [x] Read receipts
+  * [x] Pinning messages
   * [x] Power level
     * [x] Normal chats
       * [ ] Non-hardcoded PL requirements
 	* [x] Supergroups/channels
 	  * [ ] Precise bridging (non-hardcoded PL requirements, bridge specific permissions, etc..)
-  * [ ] Membership actions
-    * [x] Inviting
-      * [x] Puppets
-      * [x] Matrix users who have logged into Telegram
-    * [x] Kicking
-    * [ ] Joining
-      * [ ] Chat name as alias
-      * [ ] ‡ Chat invite link as alias
-    * [x] Leaving
+  * [x] Membership actions (invite/kick/join/leave)
   * [x] Room metadata changes (name, topic, avatar)
   * [x] Initial room metadata
   * [ ] User metadata
     * [ ] Initial displayname/username/avatar at register
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
-  * [x] Plaintext messages
-  * [x] Formatted messages
-    * [x] Bot commands (/command -> !command)
-    * [x] Mentions
-  * [x] Replies
-  * [x] Forwards
-  * [x] Images
-  * [x] Locations
-  * [x] Stickers
-  * [x] Audio messages
-  * [x] Video messages
-  * [x] Documents
-  * [ ] Message deletions (no way to tell difference between user-specific deletion and global deletion)
-  * [ ] Message edits (not supported in Matrix)
+  * [x] Message content (text, formatting, files, etc..)
+  * [ ] Advanced message content/media
+    * [x] Polls
+	* [x] Games
+	* [ ] Buttons
+  * [x] Message deletions
+  * [x] Message edits
+  * [ ] Message history
   * [x] Avatars
   * [x] Presence
   * [x] Typing notifications
   * [x] Read receipts (private chat only)
   * [x] Pinning messages
   * [x] Admin/chat creator status
-  * [ ] Supergroup/channel permissions (precise per-user not supported in Matrix)
-  * [x] Membership actions
-    * [x] Inviting
-    * [x] Kicking
-    * [x] Joining/leaving
+  * [ ] Supergroup/channel permissions (precise per-user permissions not supported in Matrix)
+  * [x] Membership actions (invite/kick/join/leave)
   * [ ] Chat metadata changes
     * [x] Title
     * [x] Avatar
     * [ ] † About text
     * [ ] † Public channel username
   * [x] Initial chat metadata (about text missing)
-  * [x] User metadata
-    * [x] Initial displayname/avatar
-    * [x] Changes to displayname/avatar
+  * [x] User metadata (displayname/avatar)
   * [x] Supergroup upgrade
 * Misc
   * [x] Automatic portal creation
     * [x] At startup
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [ ] Option to use bot to relay messages for unauthenticated Matrix users
-  * [ ] Option to use own Matrix account for messages sent from other Telegram clients
-* [Commands](https://github.com/tulir/mautrix-telegram/wiki/Management-commands)
-  * [x] Logging in and out (`login` + code entering)
-  * [x] Logging out
-  * [ ] Registering (`register`)
-  * [x] Searching for users (`search`)
-  * [x] Starting private chats (`pm`)
-  * [x] Joining chats with invite links (`join`)
-  * [x] Creating a Telegram chat for an existing Matrix room (`create`)
-  * [x] Upgrading the chat of a portal room into a supergroup (`upgrade`)
-  * [x] Change username of supergroup/channel (`groupname`)
-  * [x] Getting the Telegram invite link to a Matrix room (`invitelink`)
-  * Bridge administration
-    * [x] Clean up and forget a portal room (`deleteportal`)
-    * [ ] Setting Matrix-only power levels (`powerlevel`)
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients
+  * [ ] ‡ Calls (hard, not yet supported by Telethon)
+  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic.ini

@@ -35,9 +35,6 @@ script_location = alembic
 # are written from script.py.mako
 # output_encoding = utf-8
 
-sqlalchemy.url = sqlite:///mautrix-telegram.db
-
-
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -1,19 +1,35 @@
-from __future__ import with_statement
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig
 
 import sys
 from os.path import abspath, dirname
+
 sys.path.insert(0, dirname(dirname(abspath(__file__))))
 
-from mautrix_telegram.base import Base
-import mautrix_telegram.db
+from mautrix_telegram.db import Base
+from mautrix_telegram.config import Config
+from alchemysession import AlchemySessionContainer
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
 
+mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+mxtg_config = Config(mxtg_config_path, None, None)
+mxtg_config.load()
+config.set_main_option("sqlalchemy.url",
+                       mxtg_config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+
+
+class FakeDB:
+    @staticmethod
+    def query_property():
+        return None
+
+
+AlchemySessionContainer.create_table_classes(FakeDB(), "telethon_", Base)
+
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
@@ -24,6 +40,7 @@ fileConfig(config.config_file_name)
 # target_metadata = mymodel.Base.metadata
 target_metadata = Base.metadata
 
+
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
@@ -71,6 +88,7 @@ def run_migrations_online():
         with context.begin_transaction():
             context.run_migrations()
 
+
 if context.is_offline_mode():
     run_migrations_offline()
 else:

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -0,0 +1,27 @@
+"""Add disable_updates field for puppets
+
+Revision ID: 17574c57f3f8
+Revises: a9119be92164
+Create Date: 2019-05-15 00:24:46.967529
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '17574c57f3f8'
+down_revision = 'a9119be92164'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("disable_updates")

alembic/versions/1b241f7e8530_add_telegramfile_table.py

@@ -0,0 +1,28 @@
+"""Add TelegramFile table
+
+Revision ID: 1b241f7e8530
+Revises: 97d2a942bcf8
+Create Date: 2018-02-19 23:52:06.605741
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '1b241f7e8530'
+down_revision = '97d2a942bcf8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('telegram_file',
+                    sa.Column('id', sa.String(), nullable=False),
+                    sa.Column('mxc', sa.String(), nullable=True),
+                    sa.Column('mime_type', sa.String(), nullable=True),
+                    sa.Column('was_converted', sa.Boolean(), nullable=True),
+                    sa.PrimaryKeyConstraint('id'))
+
+
+def downgrade():
+    op.drop_table('telegram_file')

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -0,0 +1,26 @@
+"""Add is_bot field to puppets
+
+Revision ID: 1fa46383a9d3
+Revises: 30eca60587f1
+Create Date: 2018-04-29 23:44:40.102333
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1fa46383a9d3'
+down_revision = '30eca60587f1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('is_bot')

alembic/versions/2228d49c383f_add_cascade_rules_to_userportal.py

@@ -0,0 +1,41 @@
+"""Add cascade rules to UserPortal
+
+Revision ID: 2228d49c383f
+Revises: bcfefa1f1299
+Create Date: 2018-05-31 11:11:59.482112
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '2228d49c383f'
+down_revision = 'bcfefa1f1299'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+            batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+            batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+            batch_op.create_foreign_key("user_portal_user_fkey", "user", ["user"], ["tgid"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+            batch_op.create_foreign_key("user_portal_portal_fkey", "portal",
+                                        ["portal", "portal_receiver"], ["tgid", "tg_receiver"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+    except ValueError:
+        return
+
+
+def downgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+                batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+                batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+                batch_op.create_foreign_key("user_portal_user_fkey", "portal",
+                                            ["portal", "portal_receiver"], ["tgid", "tg_receiver"])
+                batch_op.create_foreign_key("user_portal_portal_fkey", "user", ["user"], ["tgid"])
+    except ValueError:
+        return

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -0,0 +1,25 @@
+"""Add megagroup field to portals
+
+Revision ID: 30eca60587f1
+Revises: cfc972368e50
+Create Date: 2018-04-29 15:51:04.656605
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '30eca60587f1'
+down_revision = 'cfc972368e50'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column('megagroup')

alembic/versions/501dad2868bc_move_sessions_to_main_database.py

@@ -0,0 +1,111 @@
+"""Move sessions to main database
+
+Revision ID: 501dad2868bc
+Revises: 7d47d84380b6
+Create Date: 2018-03-02 19:15:53.826985
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlite3
+import os
+
+# revision identifiers, used by Alembic.
+revision = '501dad2868bc'
+down_revision = '7d47d84380b6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    Session = op.create_table('telethon_sessions',
+                              sa.Column('session_id', sa.String, nullable=False),
+                              sa.Column('dc_id', sa.Integer, nullable=False),
+                              sa.Column('server_address', sa.String, nullable=True),
+                              sa.Column('port', sa.Integer, nullable=True),
+                              sa.Column('auth_key', sa.LargeBinary, nullable=True),
+                              sa.PrimaryKeyConstraint('session_id', 'dc_id'))
+    SentFile = op.create_table('telethon_sent_files',
+                               sa.Column('session_id', sa.String, nullable=False),
+                               sa.Column('md5_digest', sa.LargeBinary, nullable=False),
+                               sa.Column('file_size', sa.Integer, nullable=False),
+                               sa.Column('type', sa.Integer, nullable=False),
+                               sa.Column('id', sa.BigInteger, nullable=True),
+                               sa.Column('hash', sa.BigInteger, nullable=True),
+                               sa.PrimaryKeyConstraint('session_id', 'md5_digest', 'file_size',
+                                                       'type'))
+    Entity = op.create_table('telethon_entities',
+                             sa.Column('session_id', sa.String, nullable=False),
+                             sa.Column('id', sa.Integer, nullable=False),
+                             sa.Column('hash', sa.Integer, nullable=False),
+                             sa.Column('username', sa.String, nullable=True),
+                             sa.Column('phone', sa.Integer, nullable=True),
+                             sa.Column('name', sa.String, nullable=True),
+                             sa.PrimaryKeyConstraint('session_id', 'id'))
+    Version = op.create_table('telethon_version',
+                              sa.Column('version', sa.Integer, nullable=False),
+                              sa.PrimaryKeyConstraint('version'))
+    conn = op.get_bind()
+    sessions = [os.path.basename(f) for f in os.listdir(".") if f.endswith(".session")]
+    for session in sessions:
+        session_to_sqlalchemy(conn, session, Session, SentFile, Entity)
+
+
+def session_to_sqlalchemy(conn, path, Session, SentFile, Entity):
+    session_conn = sqlite3.connect(path)
+    session_id = os.path.splitext(path)[0]
+    c = session_conn.cursor()
+
+    auth_data_tuples = c.execute("SELECT * FROM sessions").fetchall()
+    auth_data_dicts = []
+    for row in auth_data_tuples:
+        dc_id, server_address, port, auth_key = row
+        auth_data_dicts.append({
+            "session_id": session_id,
+            "dc_id": dc_id,
+            "server_address": server_address,
+            "port": port,
+            "auth_key": auth_key,
+        })
+    if auth_data_dicts:
+        conn.execute(Session.insert().values(auth_data_dicts))
+
+    sent_file_tuples = c.execute("SELECT * FROM sent_files").fetchall()
+    sent_file_dicts = []
+    for row in sent_file_tuples:
+        md5_digest, file_size, type, id, hash = row
+        sent_file_dicts.append({
+            "session_id": session_id,
+            "md5_digest": md5_digest,
+            "file_size": file_size,
+            "type": type,
+            "id": id,
+            "hash": hash,
+        })
+    if sent_file_dicts:
+        conn.execute(SentFile.insert().values(sent_file_dicts))
+
+    entity_tuples = c.execute("SELECT * FROM entities").fetchall()
+    entity_dicts = []
+    for row in entity_tuples:
+        id, hash, username, phone, name = row
+        entity_dicts.append({
+            "session_id": session_id,
+            "id": id,
+            "hash": hash,
+            "username": username,
+            "phone": phone,
+            "name": name,
+        })
+    if entity_dicts:
+        conn.execute(Entity.insert().values(entity_dicts))
+
+    c.close()
+    session_conn.close()
+
+
+def downgrade():
+    op.drop_table('telethon_version')
+    op.drop_table('telethon_entities')
+    op.drop_table('telethon_sent_files')
+    op.drop_table('telethon_sessions')

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -0,0 +1,137 @@
+"""Move state store to main database
+
+Revision ID: 6ca3d74d51e4
+Revises: 2228d49c383f
+Create Date: 2018-06-26 21:31:26.911307
+
+"""
+from alembic import context, op
+import sqlalchemy.orm as orm
+import sqlalchemy as sa
+import json
+import re
+
+from mautrix_telegram.config import Config
+from mautrix_telegram.db import Base
+
+# revision identifiers, used by Alembic.
+revision = "6ca3d74d51e4"
+down_revision = "2228d49c383f"
+branch_labels = None
+depends_on = None
+
+
+class RoomState(Base):
+    query = None
+    __tablename__ = "mx_room_state"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    power_levels = sa.Column("power_levels", sa.Text, nullable=True)
+
+
+class UserProfile(Base):
+    query = None
+    __tablename__ = "mx_user_profile"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    user_id = sa.Column(sa.String, primary_key=True)
+    membership = sa.Column(sa.String, nullable=False, default="leave")
+    displayname = sa.Column(sa.String, nullable=True)
+    avatar_url = sa.Column(sa.String, nullable=True)
+
+
+class Puppet(Base):
+    query = None
+    __tablename__ = "puppet"
+    __table_args__ = {"extend_existing": True}
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    displayname = sa.Column(sa.String, nullable=True)
+    displayname_source = sa.Column(sa.Integer, nullable=True)
+    username = sa.Column(sa.String, nullable=True)
+    photo_id = sa.Column(sa.String, nullable=True)
+    is_bot = sa.Column(sa.Boolean, nullable=True)
+    matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False)
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+    op.create_table("mx_room_state",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("power_levels", sa.Text(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id"))
+    op.create_table("mx_user_profile",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("user_id", sa.String(), nullable=False),
+                    sa.Column("membership", sa.String(), nullable=False,
+                              default="leave"),
+                    sa.Column("displayname", sa.String(), nullable=True),
+                    sa.Column("avatar_url", sa.String(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id", "user_id"))
+
+    try:
+        migrate_state_store()
+    except Exception as e:
+        print("Failed to migrate state store:", e)
+        print("Migrating the state store isn't required, but you can retry by alembic downgrading "
+              "to revision 2228d49c383f and upgrading again.")
+
+
+def migrate_state_store():
+    conn = op.get_bind()
+    session = orm.sessionmaker(bind=conn)()  # type: orm.Session
+
+    try:
+        with open("mx-state.json") as file:
+            data = json.load(file)
+    except FileNotFoundError:
+        return
+    if not data:
+        return
+    registrations = data.get("registrations", [])
+
+    mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+    mxtg_config = Config(mxtg_config_path, None, None)
+    mxtg_config.load()
+
+    username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}")
+    hs_domain = mxtg_config["homeserver.domain"]
+    localpart = username_template.format(userid="(.+)")
+    mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain))
+    for user in registrations:
+        match = mxid_regex.match(user)
+        if not match:
+            continue
+
+        puppet = session.query(Puppet).get(match.group(1))
+        if not puppet:
+            continue
+
+        puppet.matrix_registered = True
+        session.merge(puppet)
+    session.commit()
+
+    user_profiles = [UserProfile(room_id=room, user_id=user,
+                                 membership=member.get("membership", "leave"),
+                                 displayname=member.get("displayname", None),
+                                 avatar_url=member.get("avatar_url", None))
+                     for room, members in data.get("members", {}).items()
+                     for user, member in members.items()]
+    session.add_all(user_profiles)
+    session.commit()
+
+    room_state = [RoomState(room_id=room, power_levels=json.dumps(levels))
+                  for room, levels in data.get("power_levels", {}).items()]
+    session.add_all(room_state)
+    session.commit()
+
+
+def downgrade():
+    op.drop_table("mx_user_profile")
+    op.drop_table("mx_room_state")
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("matrix_registered")

alembic/versions/7d47d84380b6_add_timestamp_to_telegramfile.py

@@ -0,0 +1,26 @@
+"""Add timestamp to TelegramFile
+
+Revision ID: 7d47d84380b6
+Revises: 1b241f7e8530
+Create Date: 2018-02-19 23:53:18.050871
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '7d47d84380b6'
+down_revision = '1b241f7e8530'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('telegram_file',
+                  sa.Column('timestamp', sa.BigInteger(), nullable=True, default=0,
+                            server_default="0"))
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column('timestamp')

alembic/versions/97d2a942bcf8_initial_revision.py

@@ -1,14 +1,13 @@
 """initial revision
 
 Revision ID: 97d2a942bcf8
-Revises: 
+Revises:
 Create Date: 2018-02-11 18:40:55.483842
 
 """
 from alembic import op
 import sqlalchemy as sa
 
-
 # revision identifiers, used by Alembic.
 revision = '97d2a942bcf8'
 down_revision = None
@@ -17,12 +16,65 @@ depends_on = None
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    pass
-    # ### end Alembic commands ###
+    op.create_table('portal',
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_receiver', sa.Integer),
+                    sa.Column('peer_type', sa.String, nullable=False, default=""),
+                    sa.Column('mxid', sa.String, nullable=True),
+                    sa.Column('username', sa.String, nullable=True),
+                    sa.Column('title', sa.String, nullable=True),
+                    sa.Column('about', sa.String, nullable=True),
+                    sa.Column('photo_id', sa.String, nullable=True),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_receiver'),
+                    sa.UniqueConstraint('mxid'))
+    op.create_table('user',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('tgid', sa.Integer, nullable=True, unique=True),
+                    sa.Column('tg_username', sa.String, nullable=True),
+                    sa.Column('saved_contacts', sa.Integer, nullable=False, default=0),
+                    sa.PrimaryKeyConstraint('mxid'))
+    op.create_table('puppet',
+                    sa.Column('id', sa.Integer),
+                    sa.Column('displayname', sa.String, nullable=True),
+                    sa.Column('username', sa.String, nullable=True),
+                    sa.Column('photo_id', sa.String, nullable=True),
+                    sa.PrimaryKeyConstraint('id'))
+    op.create_table('contact',
+                    sa.Column('user', sa.Integer),
+                    sa.Column('contact', sa.Integer),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
+                    sa.ForeignKeyConstraint(("contact",), ("puppet.id",)),
+                    sa.PrimaryKeyConstraint('user', 'contact'))
+    op.create_table('user_portal',
+                    sa.Column('user', sa.Integer),
+                    sa.Column('portal', sa.Integer),
+                    sa.Column('portal_receiver', sa.Integer),
+                    sa.PrimaryKeyConstraint('user', 'portal', 'portal_receiver'),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",),
+                                            name="user_portal_user_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"),
+                    sa.ForeignKeyConstraint(("portal", "portal_receiver"),
+                                            ("portal.tgid", "portal.tg_receiver"),
+                                            name="user_portal_portal_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"))
+    op.create_table('message',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    op.create_table('bot_chat',
+                    sa.Column('id', sa.Integer),
+                    sa.Column('type', sa.String, nullable=False, default=""),
+                    sa.PrimaryKeyConstraint('id'))
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    pass
-    # ### end Alembic commands ###
+    op.drop_table('bot_chat')
+    op.drop_table('message')
+    op.drop_table('user_portal')
+    op.drop_table('contact')
+    op.drop_table('puppet')
+    op.drop_table('user')
+    op.drop_table('portal')

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,48 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

alembic/versions/a9119be92164_add_phone_number_field_to_users.py

@@ -0,0 +1,25 @@
+"""Add phone number field to users
+
+Revision ID: a9119be92164
+Revises: b54929c22c86
+Create Date: 2018-09-28 02:38:40.626282
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "a9119be92164"
+down_revision = "b54929c22c86"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("user", sa.Column("tg_phone", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("user") as batch_op:
+        batch_op.drop_column("tg_phone")

alembic/versions/b54929c22c86_add_portal_specific_config.py

@@ -0,0 +1,25 @@
+"""Add portal-specific config
+
+Revision ID: b54929c22c86
+Revises: d5f7b8b4b456
+Create Date: 2018-09-24 23:40:33.528710
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "b54929c22c86"
+down_revision = "d5f7b8b4b456"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("portal", sa.Column("config", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column("config")

alembic/versions/bcfefa1f1299_add_displayname_source_fields_for_.py

@@ -0,0 +1,24 @@
+"""Add displayname source fields for puppets
+
+Revision ID: bcfefa1f1299
+Revises: bdadd173ee02
+Create Date: 2018-05-19 17:00:21.078098
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'bcfefa1f1299'
+down_revision = 'bdadd173ee02'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('puppet', sa.Column('displayname_source', sa.Integer(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('displayname_source')

alembic/versions/bdadd173ee02_update_telethon_update_state_table.py

@@ -0,0 +1,43 @@
+"""Update telethon update state table
+
+Revision ID: bdadd173ee02
+Revises: eeaf0dae87ce
+Create Date: 2018-05-13 10:42:59.395597
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'bdadd173ee02'
+down_revision = 'eeaf0dae87ce'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column("id", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("hash", existing_type=sa.Integer, type_=sa.BigInteger)
+
+    with op.batch_alter_table("telethon_update_state") as batch_op:
+        batch_op.alter_column("entity_id", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("pts", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("qts", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("date", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("seq", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.add_column(sa.Column("unread_count", sa.Integer))
+
+
+def downgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column("id", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("hash", existing_type=sa.BigInteger, type_=sa.Integer)
+
+    with op.batch_alter_table("telethon_update_state") as batch_op:
+        batch_op.alter_column("entity_id", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("pts", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("qts", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("date", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("seq", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.drop_column("unread_count")

alembic/versions/cfc972368e50_add_metadata_to_telegramfile.py

@@ -0,0 +1,35 @@
+"""Add metadata to TelegramFile
+
+Revision ID: cfc972368e50
+Revises: 501dad2868bc
+Create Date: 2018-03-09 16:07:01.236712
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'cfc972368e50'
+down_revision = '501dad2868bc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.add_column(sa.Column('size', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('width', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('height', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('thumbnail', sa.String(), nullable=True))
+        batch_op.create_foreign_key(constraint_name="fk_file_thumbnail",
+                                    referent_table="telegram_file",
+                                    local_cols=['thumbnail'],
+                                    remote_cols=['id'])
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column('size')
+        batch_op.drop_column('width')
+        batch_op.drop_column('height')
+        batch_op.drop_column('thumbnail')

alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py

@@ -0,0 +1,26 @@
+"""Add access_token and custom_mxid fields for puppets
+
+Revision ID: d5f7b8b4b456
+Revises: 6ca3d74d51e4
+Create Date: 2018-07-20 12:09:30.277960
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d5f7b8b4b456"
+down_revision = "6ca3d74d51e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
+    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("custom_mxid")
+        batch_op.drop_column("access_token")

alembic/versions/eeaf0dae87ce_add_telethon_update_state_table.py

@@ -0,0 +1,34 @@
+"""Add telethon update state table
+
+Revision ID: eeaf0dae87ce
+Revises: 1fa46383a9d3
+Create Date: 2018-04-30 17:30:59.610885
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'eeaf0dae87ce'
+down_revision = '1fa46383a9d3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column('phone', existing_type=sa.Integer, type_=sa.BigInteger)
+    op.create_table('telethon_update_state',
+                    sa.Column('session_id', sa.String, nullable=False),
+                    sa.Column('entity_id', sa.Integer, nullable=False),
+                    sa.Column('pts', sa.Integer, nullable=True),
+                    sa.Column('qts', sa.Integer, nullable=True),
+                    sa.Column('date', sa.Integer, nullable=True),
+                    sa.Column('seq', sa.Integer, nullable=True),
+                    sa.PrimaryKeyConstraint('session_id', 'entity_id'))
+
+
+def downgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column('phone', existing_type=sa.BigInteger, type_=sa.Integer)
+    op.drop_table('telethon_update_state')

docker-run.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# Define functions.
+function fixperms {
+	chown -R $UID:$GID /data /opt/mautrix-telegram
+}
+
+cd /opt/mautrix-telegram
+
+# Replace database path in config.
+sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
+
+if [ -f /data/mx-state.json ]; then
+	ln -s /data/mx-state.json
+fi
+# Check that database is in the right state
+alembic -x config=/data/config.yaml upgrade head
+
+if [ ! -f /data/config.yaml ]; then
+	cp example-config.yaml /data/config.yaml
+	echo "Didn't find a config file."
+	echo "Copied default config file to /data/config.yaml"
+	echo "Modify that config file to your liking."
+	echo "Start the container again after that to generate the registration file."
+	fixperms
+	exit
+fi
+
+if [ ! -f /data/registration.yaml ]; then
+	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
+	echo "Didn't find a registration file."
+	echo "Generated one for you."
+	echo "Copy that over to synapses app service directory."
+	fixperms
+	exit
+fi
+
+fixperms
+exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

example-config.yaml

@@ -1,32 +1,78 @@
 # Homeserver details
 homeserver:
+    # The address that this appservice can use to connect to the homeserver.
     address: https://matrix.org
+    # The domain of the homeserver (for MXIDs, etc).
     domain: matrix.org
+    # Whether or not to verify the SSL certificate of the homeserver.
+    # Only applies if address starts with https://
+    verify_ssl: true
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
 appservice:
-    # The protocol the homeserver should use when connecting to this appservice.
-    # Usually "http" or "https".
-    protocol: http
+    # The address that the homeserver can use to connect to this appservice.
+    address: http://localhost:8080
 
-    # The hostname and port where the homeserver can find this appservice.
-    hostname: localhost
+    # The hostname and port where this appservice should listen.
+    hostname: 0.0.0.0
     port: 8080
+    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+    max_body_size: 1
 
-    # Whether or not to enable debug messages in the console.
-    debug: false
+    # The full URI to the database. SQLite and Postgres are fully supported.
+    # Other DBMSes supported by SQLAlchemy may or may not work.
+    # Format examples:
+    #   SQLite:   sqlite:///filename.db
+    #   Postgres: postgres://username:password@hostname/dbname
+    database: sqlite:///mautrix-telegram.db
+
+    # Public part of web server for out-of-Matrix interaction with the bridge.
+    # Used for things like login if the user wants to make sure the 2FA password isn't stored in
+    # the HS database.
+    public:
+        # Whether or not the public-facing endpoints should be enabled.
+        enabled: true
+        # The prefix to use in the public-facing endpoints.
+        prefix: /public
+        # The base URL where the public-facing endpoints are available. The prefix is not added
+        # implicitly.
+        external: https://example.com/public
+
+    # Provisioning API part of the web server for automated portal creation and fetching information.
+    # Used by things like Dimension (https://dimension.t2bot.io/).
+    provisioning:
+        # Whether or not the provisioning API should be enabled.
+        enabled: true
+        # The prefix to use in the provisioning API endpoints.
+        prefix: /_matrix/provision/v1
+        # The shared secret to authorize users of the API.
+        # Set to "generate" to generate and save a new token.
+        shared_secret: generate
 
     # The unique ID of this appservice.
     id: telegram
     # Username of the appservice bot.
     bot_username: telegrambot
+    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+    # to leave display name/avatar as-is.
     bot_displayname: Telegram bridge bot
+    bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
+
+    # Community ID for bridged users (changes registration file) and rooms.
+    # Must be created manually.
+    community_id: false
 
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
 
+# Prometheus telemetry config. Requires prometheus-client to be installed.
+metrics:
+    enabled: false
+    listen_port: 8000
+
 # Bridge config
 bridge:
     # Localpart template of MXIDs for Telegram users.
@@ -58,32 +104,246 @@ bridge:
     - username
     - phone number
 
-    # Whether or not to use native Matrix replies. At the time of writing, only riot-web supports
-    # replies and the format of them is subject to change.
-    native_replies: True
-    # If native replies are disabled, should the custom replies contain a link to the message being
-    # replied to?
-    link_in_reply: False
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: False
+    # Maximum number of members to sync per portal when starting up. Other members will be
+    # synced when they send messages. The maximum is 10000, after which the Telegram server
+    # will not send any more members.
+    # Defaults to no local limit (-> limited to 10000 by server)
+    max_initial_member_sync: -1
+    # Whether or not to sync the member list in channels.
+    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
+    # list regardless of this setting.
+    sync_channel_members: true
+    # Whether or not to skip deleted members when syncing members.
+    skip_deleted_members: true
+    # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
+    # their Telegram account at startup.
+    startup_sync: true
+    # Number of most recently active dialogs to check when syncing chats.
+    # Dialogs include groups and private chats, but only groups are synced.
+    # Set to 0 to remove limit.
+    sync_dialog_limit: 30
+    # The maximum number of simultaneous Telegram deletions to handle.
+    # A large number of simultaneous redactions could put strain on your homeserver.
+    max_telegram_delete: 10
+    # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
+    # at startup and when creating a bridge.
+    sync_matrix_state: true
+    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
+    # login website (see appservice.public config section)
+    allow_matrix_login: true
+    # Whether or not to bridge plaintext highlights.
+    # Only enable this if your displayname_template has some static part that the bridge can use to
+    # reliably identify what is a plaintext highlight.
+    plaintext_highlights: false
+    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
+    public_portals: true
+    # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
+    # Currently only works for private chats and normal groups.
+    # WARNING: This feature seems to be broken in the telegram library.
+    catch_up: false
+    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # your own Matrix account as the Matrix puppet for your Telegram account.
+    sync_with_custom_puppets: true
+    # Set to false to disable link previews in messages sent to Telegram.
+    telegram_link_preview: true
+    # Use inline images instead of a separate message for the caption.
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    inline_images: false
+    # Maximum size of image in megabytes before sending to Telegram as a document.
+    image_as_file_size: 10
+    # Maximum size of Telegram documents in megabytes to bridge.
+    max_document_size: 100
+
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    bridge_notices:
+        # Whether or not Matrix bot messages (type m.notice) should be bridged.
+        default: false
+        # List of user IDs for whom the previous flag is flipped.
+        # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
+        #      notices from users listed here will be bridged.
+        exceptions:
+        - "@importantbot:example.com"
+
+    # Some config options related to Telegram message deduplication.
+    # The default values are usually fine, but some debug messages/warnings might recommend you
+    # change these.
+    deduplication:
+        # Whether or not to check the database if the message about to be sent is a duplicate.
+        pre_db_check: false
+        # The number of latest events to keep when checking for duplicates.
+        # You might need to increase this on high-traffic bridge instances.
+        cache_queue_length: 20
+
+
+    # The formats to use when sending messages to Telegram via the relay bot.
+    #
+    # Telegram doesn't have built-in emotes, so the m.emote format is also used for non-relaybot users.
+    #
+    # Available variables:
+    #   $sender_displayname    - The display name of the sender (e.g. Example User)
+    #   $sender_username       - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
+    #   $sender_mxid           - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $message               - The message content as HTML
+    message_formats:
+        m.text: "<b>$sender_displayname</b>: $message"
+        m.emote: "* <b>$sender_displayname</b> $message"
+        m.file: "<b>$sender_displayname</b> sent a file: $message"
+        m.image: "<b>$sender_displayname</b> sent an image: $message"
+        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "<b>$sender_displayname</b> sent a video: $message"
+        m.location: "<b>$sender_displayname</b> sent a location: $message"
+
+    # The formats to use when sending state events to Telegram via the relay bot.
+    #
+    # Variables from `message_formats` that have the `sender_` prefix are available without the prefix.
+    # In name_change events, `$prev_displayname` is the previous displayname.
+    #
+    # Set format to an empty string to disable the messages for that event.
+    state_event_formats:
+        join: "<b>$displayname</b> joined the room."
+        leave: "<b>$displayname</b> left the room."
+        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+
+    # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
+    # `filter-mode` management commands.
+    #
+    # Filters do not affect direct chats.
+    # An empty blacklist will essentially disable the filter.
+    filter:
+        # Filter mode to use. Either "blacklist" or "whitelist".
+        # If the mode is "blacklist", the listed chats will never be bridged.
+        # If the mode is "whitelist", only the listed chats can be bridged.
+        mode: blacklist
+        # The list of group/channel IDs to filter.
+        list: []
 
     # The prefix for commands. Only required in non-management rooms.
     command_prefix: "!tg"
 
-    # Whitelist of user IDs that are allowed to use this bridge. Leave empty to disable.
-    # You can enter a domain without the localpart to allow all users from that homeserver to use the bridge.
-    whitelist:
-    - "internal.example.com"
-    - "@user:public.example.com"
+    # Permissions for using the bridge.
+    # Permitted values:
+    #   relaybot - Only use the bridge via the relaybot, no access to commands.
+    #       user - Relaybot level + access to commands to create bridges.
+    #  puppeting - User level + logging in with a Telegram account.
+    #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
+    #      admin - Full access to use the bridge and some extra administration commands.
+    # Permitted keys:
+    #        * - All Matrix users
+    #   domain - All users on that homeserver
+    #     mxid - Specific user
+    permissions:
+        "*": "relaybot"
+        "public.example.com": "user"
+        "example.com": "full"
+        "@admin:example.com": "admin"
 
-    # Admins can do things like delete portal rooms. Here you must specify the exact MXID, domains
-    # are not accepted.
-    admins:
-    - "@admin:internal.example.com"
+    # Options related to the message relay Telegram bot.
+    relaybot:
+        # Whether or not to allow creating portals from Telegram.
+        authless_portals: true
+        # Whether or not to allow Telegram group admins to use the bot commands.
+        whitelist_group_admins: true
+        # Whether or not to ignore incoming events sent by the relay bot.
+        ignore_own_incoming_events: true
+        # List of usernames/user IDs who are also allowed to use the bot commands.
+        whitelist:
+        - myusername
+        - 12345678
 
 # Telegram config
 telegram:
     # Get your own API keys at https://my.telegram.org/apps
     api_id: 12345
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
+    # (Optional) Create your own bot at https://t.me/BotFather
+    bot_token: disabled
+
+    # Telethon connection options.
+    connection:
+        # The timeout in seconds to be used when connecting.
+        timeout: 120
+        # How many times the reconnection should retry, either on the initial connection or when
+        # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
+        # this is not recommended, since the program can get stuck in an infinite loop.
+        retries: 5
+        # The delay in seconds to sleep between automatic reconnections.
+        retry_delay: 1
+        # The threshold below which the library should automatically sleep on flood wait errors
+        # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
+        # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
+        # the error instead. Values larger than a day (86400) will be changed to a day.
+        flood_sleep_threshold: 60
+        # How many times a request should be retried. Request are retried when Telegram is having
+        # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
+        # there's a migrate error. May take a negative or null value for infinite retries, but this
+        # is not recommended, since some requests can always trigger a call fail (such as searching
+        # for messages).
+        request_retries: 5
+
+    # Device info sent to Telegram.
+    device_info:
+        # "auto" = OS name+version.
+        device_model: auto
+        # "auto" = Telethon version.
+        system_version: auto
+        # "auto" = mautrix-telegram version.
+        app_version: auto
+        lang_code: en
+        system_lang_code: en
+
+    # Custom server to connect to.
+    server:
+        # Set to true to use these server settings. If false, will automatically
+        # use production server assigned by Telegram. Set to false in production.
+        enabled: false
+        # The DC ID to connect to.
+        dc: 2
+        # The IP to connect to.
+        ip: 149.154.167.40
+        # The port to connect to. 443 may not work, 80 is better and both are equally secure.
+        port: 80
+
+    # Telethon proxy configuration.
+    # You must install PySocks from pip for proxies to work.
+    proxy:
+        # Allowed types: disabled, socks4, socks5, http
+        type: disabled
+        # Proxy IP address and port.
+        address: 127.0.0.1
+        port: 1080
+        # Whether or not to perform DNS resolving remotely.
+        rdns: true
+        # Proxy authentication (optional).
+        username: ""
+        password: ""
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        precise:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: precise
+            filename: ./mautrix-telegram.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: precise
+    loggers:
+        mau:
+            level: DEBUG
+        telethon:
+            level: DEBUG
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]

mautrix_appservice/__init__.py

@@ -1,5 +0,0 @@
-from .appservice import AppService
-from .errors import MatrixError, MatrixRequestError, IntentError
-
-__version__ = "0.1.0"
-__author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_appservice/appservice.py

@@ -1,180 +0,0 @@
-# -*- coding: future_fstrings -*-
-# matrix-appservice-python - A Matrix Application Service framework written in Python.
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Partly based on github.com/Cadair/python-appservice-framework (MIT license)
-from functools import partial
-from contextlib import contextmanager
-from aiohttp import web
-import aiohttp
-import asyncio
-import logging
-
-from .intent_api import HTTPAPI
-from .state_store import StateStore
-
-
-class AppService:
-    def __init__(self, server, domain, as_token, hs_token, bot_localpart, loop=None, log=None,
-                 query_user=None, query_alias=None):
-        self.server = server
-        self.domain = domain
-        self.as_token = as_token
-        self.hs_token = hs_token
-        self.bot_mxid = f"@{bot_localpart}:{domain}"
-        self.state_store = StateStore(autosave_file="mx-state.json")
-        self.state_store.load("mx-state.json")
-
-        self.transactions = []
-
-        self._http_session = None
-        self._intent = None
-
-        self.loop = loop or asyncio.get_event_loop()
-        self.log = (logging.getLogger(log) if isinstance(log, str)
-                    else log or logging.getLogger("mautrix_appservice"))
-
-        def default_query_handler(_):
-            return None
-
-        self.query_user = query_user or default_query_handler
-        self.query_alias = query_alias or default_query_handler
-
-        self.event_handlers = []
-
-        self.app = web.Application(loop=self.loop)
-        self.app.router.add_route("PUT", "/transactions/{transaction_id}",
-                                  self._http_handle_transaction)
-        self.app.router.add_route("GET", "/rooms/{alias}", self._http_query_alias)
-        self.app.router.add_route("GET", "/users/{user_id}", self._http_query_user)
-
-        self.matrix_event_handler(self.update_state_store)
-
-    @property
-    def http_session(self):
-        if self._http_session is None:
-            raise AttributeError("the http_session attribute can only be used "
-                                 "from within the `AppService.run` context manager")
-        else:
-            return self._http_session
-
-    @property
-    def intent(self):
-        if self._intent is None:
-            raise AttributeError("the intent attribute can only be used from "
-                                 "within the `AppService.run` context manager")
-        else:
-            return self._intent
-
-    @contextmanager
-    def run(self, host="127.0.0.1", port=8080):
-        self._http_session = aiohttp.ClientSession(loop=self.loop)
-        self._intent = HTTPAPI(base_url=self.server, domain=self.domain, bot_mxid=self.bot_mxid,
-                               token=self.as_token, log=self.log, state_store=self.state_store,
-                               client_session=self._http_session).bot_intent()
-
-        yield self.loop.create_server(self.app.make_handler(), host, port)
-
-        self._intent = None
-        self._http_session.close()
-        self._http_session = None
-
-    def _check_token(self, request):
-        try:
-            token = request.rel_url.query["access_token"]
-        except KeyError:
-            return False
-
-        if token != self.hs_token:
-            return False
-
-        return True
-
-    async def _http_query_user(self, request):
-        if not self._check_token(request):
-            return web.Response(status=401)
-
-        user_id = request.match_info["userId"]
-
-        try:
-            response = await self.query_user(user_id)
-        except Exception:
-            self.log.exception("Exception in user query handler")
-            return web.Response(status=500)
-
-        if not response:
-            return web.Response(status=404)
-        return web.json_response(response)
-
-    async def _http_query_alias(self, request):
-        if not self._check_token(request):
-            return web.Response(status=401)
-
-        alias = request.match_info["alias"]
-
-        try:
-            response = await self.query_alias(alias)
-        except Exception:
-            self.log.exception("Exception in alias query handler")
-            return web.Response(status=500)
-
-        if not response:
-            return web.Response(status=404)
-        return web.json_response(response)
-
-    async def _http_handle_transaction(self, request):
-        if not self._check_token(request):
-            return web.Response(status=401)
-
-        transaction_id = request.match_info["transaction_id"]
-        if transaction_id in self.transactions:
-            return web.Response(status=200)
-
-        json = await request.json()
-
-        try:
-            events = json["events"]
-        except KeyError:
-            return web.Response(status=400)
-
-        for event in events:
-            self.handle_matrix_event(event)
-
-        self.transactions.append(transaction_id)
-
-        return web.json_response({})
-
-    async def update_state_store(self, event):
-        event_type = event["type"]
-        if event_type == "m.room.power_levels":
-            self.state_store.set_power_levels(event["room_id"], event["content"])
-        elif event_type == "m.room.member":
-            self.state_store.set_membership(event["room_id"], event["state_key"],
-                                            event["content"]["membership"])
-
-    def handle_matrix_event(self, event):
-        async def try_handle(handler):
-            try:
-                await handler(event)
-            except Exception:
-                self.log.exception("Exception in Matrix event handler")
-
-        for handler in self.event_handlers:
-            asyncio.ensure_future(try_handle(handler), loop=self.loop)
-
-    def matrix_event_handler(self, func):
-        self.event_handlers.append(func)
-        return func

mautrix_appservice/errors.py

@@ -1,38 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-
-class MatrixError(Exception):
-    """A generic Matrix error. Specific errors will subclass this."""
-    pass
-
-
-class IntentError(MatrixError):
-    def __init__(self, message, source):
-        super().__init__(message)
-        self.source = source
-
-
-class MatrixRequestError(MatrixError):
-    """ The home server returned an error response. """
-
-    def __init__(self, code=0, text="", errcode=None, message=None):
-        super().__init__("%d: %s" % (code, text))
-        self.code = code
-        self.text = text
-        self.errcode = errcode
-        self.message = message

mautrix_appservice/intent_api.py

@@ -1,521 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from urllib.parse import quote
-from time import time
-from json.decoder import JSONDecodeError
-from aiohttp.client_exceptions import ContentTypeError
-import re
-import json
-import magic
-import asyncio
-
-from .errors import MatrixError, MatrixRequestError, IntentError
-
-
-class HTTPAPI:
-    def __init__(self, base_url, domain=None, bot_mxid=None, token=None, identity=None, log=None,
-                 state_store=None, client_session=None, child=False):
-        self.base_url = base_url
-        self.token = token
-        self.identity = identity
-        self.validate_cert = True
-        self.session = client_session
-
-        self.domain = domain
-        self.bot_mxid = bot_mxid
-        self._bot_intent = None
-        self.state_store = state_store
-
-        if child:
-            self.log = log
-        else:
-            self.intent_log = log.getChild("intent")
-            self.log = log.getChild("api")
-            self.txn_id = 0
-            self.children = {}
-
-    def user(self, user):
-        try:
-            return self.children[user]
-        except KeyError:
-            child = ChildHTTPAPI(user, self)
-            self.children[user] = child
-            return child
-
-    def bot_intent(self):
-        if self._bot_intent:
-            return self._bot_intent
-        return IntentAPI(self.bot_mxid, self, state_store=self.state_store, log=self.intent_log)
-
-    def intent(self, user):
-        return IntentAPI(user, self.user(user), self.bot_intent(), self.state_store,
-                         self.intent_log)
-
-    async def _send(self, method, endpoint, content, query_params, headers):
-        while True:
-            query_params["access_token"] = self.token
-            request = self.session.request(method, endpoint, params=query_params,
-                                           data=content, headers=headers)
-            async with request as response:
-                if response.status < 200 or response.status >= 300:
-                    errcode = message = None
-                    try:
-                        response_data = await response.json()
-                        errcode = response_data["errcode"]
-                        message = response_data["error"]
-                    except (JSONDecodeError, ContentTypeError, KeyError):
-                        pass
-                    raise MatrixRequestError(code=response.status, text=await response.text(),
-                                             errcode=errcode, message=message)
-
-                if response.status == 429:
-                    await asyncio.sleep(response.json()["retry_after_ms"] / 1000)
-                else:
-                    return await response.json()
-
-    def _log_request(self, method, path, content, query_params):
-        log_content = content if not isinstance(content, bytes) else f"<{len(content)} bytes>"
-        log_content = log_content or "(No content)"
-        query_identity = query_params["user_id"] if "user_id" in query_params else "No identity"
-        self.log.debug("%s %s %s as user %s", method, path, log_content, query_identity)
-
-    def request(self, method, path, content=None, query_params=None, headers=None,
-                api_path="/_matrix/client/r0"):
-        content = content or {}
-        query_params = query_params or {}
-        headers = headers or {}
-
-        method = method.upper()
-        if method not in ["GET", "PUT", "DELETE", "POST"]:
-            raise MatrixError("Unsupported HTTP method: %s" % method)
-
-        if "Content-Type" not in headers:
-            headers["Content-Type"] = "application/json"
-        if headers["Content-Type"] == "application/json":
-            content = json.dumps(content)
-
-        if self.identity:
-            query_params["user_id"] = self.identity
-
-        self._log_request(method, path, content, query_params)
-
-        endpoint = self.base_url + api_path + path
-        return self._send(method, endpoint, content, query_params, headers or {})
-
-    def get_download_url(self, mxcurl):
-        if mxcurl.startswith('mxc://'):
-            return f"{self.base_url}/_matrix/media/r0/download/{mxcurl[6:]}"
-        else:
-            raise ValueError("MXC URL did not begin with 'mxc://'")
-
-    async def get_display_name(self, user_id):
-        content = await self.request("GET", f"/profile/{user_id}/displayname")
-        return content.get('displayname', None)
-
-    async def get_avatar_url(self, user_id):
-        content = await self.request("GET", f"/profile/{user_id}/avatar_url")
-        return content.get('avatar_url', None)
-
-    async def get_room_id(self, room_alias):
-        content = await self.request("GET", f"/directory/room/{quote(room_alias)}")
-        return content.get("room_id", None)
-
-    def set_typing(self, room_id, is_typing=True, timeout=5000, user=None):
-        content = {
-            "typing": is_typing
-        }
-        if is_typing:
-            content["timeout"] = timeout
-        user = user or self.identity
-        return self.request("PUT", f"/rooms/{room_id}/typing/{user}", content)
-
-
-class ChildHTTPAPI(HTTPAPI):
-    def __init__(self, user, parent):
-        super().__init__(parent.base_url, parent.domain, parent.bot_mxid, parent.token, user,
-                         parent.log, parent.state_store, parent.session, child=True)
-        self.parent = parent
-
-    @property
-    def txn_id(self):
-        return self.parent.txn_id
-
-    @txn_id.setter
-    def txn_id(self, value):
-        self.parent.txn_id = value
-
-
-class IntentAPI:
-    mxid_regex = re.compile("@(.+):(.+)")
-
-    def __init__(self, mxid, client, bot=None, state_store=None, log=None):
-        self.client = client
-        self.bot = bot
-        self.mxid = mxid
-        self.log = log
-
-        results = self.mxid_regex.search(mxid)
-        if not results:
-            raise ValueError("invalid MXID")
-        self.localpart = results.group(1)
-
-        self.state_store = state_store
-
-    def user(self, user):
-        if not self.bot:
-            return self.client.intent(user)
-        else:
-            self.log.warning("Called IntentAPI#user() of child intent object.")
-            return self.bot.client.intent(user)
-
-    # region User actions
-
-    async def get_joined_rooms(self):
-        await self.ensure_registered()
-        response = await self.client.request("GET", "/joined_rooms")
-        return response["joined_rooms"]
-
-    async def set_display_name(self, name):
-        await self.ensure_registered()
-        content = {"displayname": name}
-        return await self.client.request("PUT", f"/profile/{self.mxid}/displayname", content)
-
-    async def set_presence(self, status="online"):
-        await self.ensure_registered()
-        content = {
-            "presence": status
-        }
-        return await self.client.request("PUT", f"/presence/{self.mxid}/status", content)
-
-    async def set_avatar(self, url):
-        await self.ensure_registered()
-        content = {"avatar_url": url}
-        return await self.client.request("PUT", f"/profile/{self.mxid}/avatar_url", content)
-
-    async def upload_file(self, data, mime_type=None):
-        await self.ensure_registered()
-        mime_type = mime_type or magic.from_buffer(data, mime=True)
-        return await self.client.request("POST", "", content=data,
-                                         headers={"Content-Type": mime_type},
-                                         api_path="/_matrix/media/r0/upload")
-
-    async def download_file(self, url):
-        await self.ensure_registered()
-        url = self.client.get_download_url(url)
-        async with self.client.session.get(url) as response:
-            return await response.read()
-
-    # endregion
-    # region Room actions
-
-    async def create_room(self, alias=None, is_public=False, name=None, topic=None,
-                          is_direct=False, invitees=None, initial_state=None):
-        await self.ensure_registered()
-        content = {
-            "visibility": "public" if is_public else "private",
-            "is_direct": is_direct,
-        }
-        if alias:
-            content["room_alias_name"] = alias
-        if invitees:
-            content["invite"] = invitees
-        if name:
-            content["name"] = name
-        if topic:
-            content["topic"] = topic
-        if initial_state:
-            content["initial_state"] = initial_state
-
-        return await self.client.request("POST", "/createRoom", content)
-
-    def _invite_direct(self, room_id, user_id):
-        content = {"user_id": user_id}
-        return self.client.request("POST", "/rooms/" + room_id + "/invite", content)
-
-    async def invite(self, room_id, user_id, check_cache=False):
-        await self.ensure_joined(room_id)
-        try:
-            ok_states = {"invite", "join"}
-            do_invite = (not check_cache
-                         or self.state_store.get_membership(room_id, user_id) not in ok_states)
-            if do_invite:
-                response = await self._invite_direct(room_id, user_id)
-                self.state_store.invited(room_id, user_id)
-                return response
-        except MatrixRequestError as e:
-            if e.errcode != "M_FORBIDDEN":
-                raise IntentError(f"Failed to invite {user_id} to {room_id}", e)
-            if "is already in the room" in e.message:
-                self.state_store.joined(room_id, user_id)
-
-    def set_room_avatar(self, room_id, avatar_url, info=None):
-        content = {
-            "url": avatar_url,
-        }
-        if info:
-            content["info"] = info
-        return self.send_state_event(room_id, "m.room.avatar", content)
-
-    async def add_room_alias(self, room_id, localpart):
-        await self.ensure_registered()
-        content = {"room_id": room_id}
-        alias = f"#{localpart}:{self.client.domain}"
-        return await self.client.request("PUT", f"/directory/room/{quote(alias)}", content)
-
-    async def remove_room_alias(self, localpart):
-        await self.ensure_registered()
-        alias = f"#{localpart}:{self.client.domain}"
-        return await self.client.request("DELETE", f"/directory/room/{quote(alias)}")
-
-    def set_room_name(self, room_id, name):
-        body = {"name": name}
-        return self.send_state_event(room_id, "m.room.name", body)
-
-    async def get_power_levels(self, room_id, ignore_cache=False):
-        await self.ensure_joined(room_id)
-        if not ignore_cache:
-            try:
-                return self.state_store.get_power_levels(room_id)
-            except KeyError:
-                pass
-        levels = await self.client.request("GET",
-                                           f"/rooms/{quote(room_id)}/state/m.room.power_levels")
-        self.state_store.set_power_levels(room_id, levels)
-        return levels
-
-    async def set_power_levels(self, room_id, content):
-        if "events" not in content:
-            content["events"] = {}
-        response = await self.send_state_event(room_id, "m.room.power_levels", content)
-        self.state_store.set_power_levels(room_id, content)
-        return response
-
-    async def get_pinned_messages(self, room_id):
-        await self.ensure_joined(room_id)
-        response = await self.client.request("GET", f"/rooms/{room_id}/state/m.room.pinned_events")
-        return response["content"]["pinned"]
-
-    def set_pinned_messages(self, room_id, events):
-        return self.send_state_event(room_id, "m.room.pinned_events", {
-            "pinned": events
-        })
-
-    async def pin_message(self, room_id, event_id):
-        events = await self.get_pinned_messages(room_id)
-        if event_id not in events:
-            events.append(event_id)
-            await self.set_pinned_messages(room_id, events)
-
-    async def unpin_message(self, room_id, event_id):
-        events = await self.get_pinned_messages(room_id)
-        if event_id in events:
-            events.remove(event_id)
-            await self.set_pinned_messages(room_id, events)
-
-    async def get_event(self, room_id, event_id):
-        await self.ensure_joined(room_id)
-        return await self.client.request("GET", f"/rooms/{room_id}/event/{event_id}")
-
-    async def set_typing(self, room_id, is_typing=True, timeout=5000):
-        await self.ensure_joined(room_id)
-        content = {
-            "typing": is_typing
-        }
-        if is_typing:
-            content["timeout"] = timeout
-        return await self.client.request("PUT", f"/rooms/{room_id}/typing/{self.mxid}", content)
-
-    async def mark_read(self, room_id, event_id):
-        await self.ensure_joined(room_id)
-        return await self.client.request("POST", f"/rooms/{room_id}/receipt/m.read/{event_id}",
-                                         content={})
-
-    def send_notice(self, room_id, text, html=None, relates_to=None):
-        return self.send_text(room_id, text, html, "m.notice", relates_to)
-
-    def send_emote(self, room_id, text, html=None, relates_to=None):
-        return self.send_text(room_id, text, html, "m.emote", relates_to)
-
-    def send_image(self, room_id, url, info=None, text=None, relates_to=None):
-        return self.send_file(room_id, url, info or {}, text, "m.image", relates_to)
-
-    def send_file(self, room_id, url, info=None, text=None, file_type="m.file", relates_to=None):
-        return self.send_message(room_id, {
-            "msgtype": file_type,
-            "url": url,
-            "body": text or "Uploaded file",
-            "info": info or {},
-            "m.relates_to": relates_to or None,
-        })
-
-    def send_text(self, room_id, text, html=None, msgtype="m.text", relates_to=None):
-        if html:
-            if not text:
-                text = html
-            return self.send_message(room_id, {
-                "body": text,
-                "msgtype": msgtype,
-                "format": "org.matrix.custom.html",
-                "formatted_body": html or text,
-                "m.relates_to": relates_to or None,
-            })
-        else:
-            return self.send_message(room_id, {
-                "body": text,
-                "msgtype": msgtype,
-                "m.relates_to": relates_to or None,
-            })
-
-    def send_message(self, room_id, body):
-        return self.send_event(room_id, "m.room.message", body)
-
-    async def error_and_leave(self, room_id, text, html=None):
-        await self.ensure_joined(room_id)
-        await self.send_notice(room_id, text, html=html)
-        await self.leave_room(room_id)
-
-    def kick(self, room_id, user_id, message):
-        return self.set_membership(room_id, user_id, "leave", message)
-
-    def get_membership(self, room_id, user_id):
-        return self.get_state_event(room_id, "m.room.member", state_key=user_id)
-
-    def set_membership(self, room_id, user_id, membership, reason="", profile=None):
-        body = {
-            "membership": membership,
-            "reason": reason
-        }
-        profile = profile or {}
-        if "displayname" in profile:
-            body["displayname"] = profile["displayname"]
-        if "avatar_url" in profile:
-            body["avatar_url"] = profile["avatar_url"]
-
-        return self.send_state_event(room_id, "m.room.member", body, state_key=user_id)
-
-    @staticmethod
-    def _get_event_url(room_id, event_type, txn_id):
-        return f"/rooms/{quote(room_id)}/send/{quote(event_type)}/{quote(txn_id)}"
-
-    async def send_event(self, room_id, event_type, content, txn_id=None):
-        await self.ensure_joined(room_id)
-        await self._ensure_has_power_level_for(room_id, event_type)
-
-        txn_id = txn_id or str(self.client.txn_id) + str(int(time() * 1000))
-        self.client.txn_id += 1
-
-        url = self._get_event_url(room_id, event_type, txn_id)
-
-        return await self.client.request("PUT", url, content)
-
-    @staticmethod
-    def _get_state_url(room_id, event_type, state_key=""):
-        url = f"/rooms/{quote(room_id)}/state/{quote(event_type)}"
-        if state_key:
-            url += f"/{quote(state_key)}"
-        return url
-
-    async def send_state_event(self, room_id, event_type, content, state_key=""):
-        await self.ensure_joined(room_id)
-        await self._ensure_has_power_level_for(room_id, event_type)
-        url = self._get_state_url(room_id, event_type, state_key)
-        return await self.client.request("PUT", url, content)
-
-    async def get_state_event(self, room_id, event_type, state_key=""):
-        await self.ensure_joined(room_id)
-        url = self._get_state_url(room_id, event_type, state_key)
-        return await self.client.request("GET", url)
-
-    def join_room(self, room_id):
-        return self.ensure_joined(room_id, ignore_cache=True)
-
-    def _join_room_direct(self, room):
-        return self.client.request("POST", f"/join/{quote(room)}")
-
-    def leave_room(self, room_id):
-        try:
-            self.state_store.left(room_id, self.mxid)
-            return self.client.request("POST", f"/rooms/{quote(room_id)}/leave")
-        except MatrixRequestError as e:
-            if "not in room" not in e.message:
-                raise
-
-    def get_room_memberships(self, room_id):
-        return self.client.request("GET", f"/rooms/{quote(room_id)}/members")
-
-    async def get_room_members(self, room_id, allowed_memberships=("join",)):
-        memberships = await self.get_room_memberships(room_id)
-        return [membership["state_key"] for membership in memberships["chunk"] if
-                membership["content"]["membership"] in allowed_memberships]
-
-    async def get_room_state(self, room_id):
-        await self.ensure_joined(room_id)
-        state = await self.client.request("GET", f"/rooms/{quote(room_id)}/state")
-        # TODO update values based on state?
-        return state
-
-    # endregion
-    # region Ensure functions
-
-    async def ensure_joined(self, room_id, ignore_cache=False):
-        if not ignore_cache and self.state_store.is_joined(room_id, self.mxid):
-            return
-        await self.ensure_registered()
-        try:
-            await self._join_room_direct(room_id)
-            self.state_store.joined(room_id, self.mxid)
-        except MatrixRequestError as e:
-            if e.errcode != "M_FORBIDDEN" or not self.bot:
-                raise IntentError(f"Failed to join room {room_id} as {self.mxid}", e)
-            try:
-                await self.bot.invite(room_id, self.mxid)
-                await self._join_room_direct(room_id)
-                self.state_store.joined(room_id, self.mxid)
-            except MatrixRequestError as e2:
-                raise IntentError(f"Failed to join room {room_id} as {self.mxid}", e2)
-
-    def _register(self):
-        content = {"username": self.localpart}
-        query_params = {"kind": "user"}
-        return self.client.request("POST", "/register", content, query_params)
-
-    async def ensure_registered(self):
-        if self.state_store.is_registered(self.mxid):
-            return
-        try:
-            await self._register()
-        except MatrixRequestError as e:
-            if e.errcode != "M_USER_IN_USE":
-                self.log.exception(f"Failed to register {self.mxid}!")
-                # raise IntentError(f"Failed to register {self.mxid}", e)
-                return
-        self.state_store.registered(self.mxid)
-
-    async def _ensure_has_power_level_for(self, room_id, event_type):
-        if not self.state_store.has_power_levels(room_id):
-            await self.get_power_levels(room_id)
-        if self.state_store.has_power_level(room_id, self.mxid, event_type):
-            return
-        elif not self.bot:
-            self.log.warning(
-                f"Power level of {self.mxid} is not enough for {event_type} in {room_id}")
-            # raise IntentError(f"Power level of {self.mxid} is not enough"
-            #                  + f"for {event_type} in {room_id}")
-            return
-        # TODO implement
-
-    # endregion

mautrix_appservice/state_store.py

@@ -1,123 +0,0 @@
-# -*- coding: future_fstrings -*-
-# matrix-appservice-python - A Matrix Application Service framework written in Python.
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-import json
-
-
-class StateStore:
-    def __init__(self, autosave_file=None):
-        self.registrations = set()
-        self.memberships = {}
-        self.power_levels = {}
-        self.autosave_file = autosave_file
-
-    def save(self, file):
-        if isinstance(file, str):
-            output = open(file, "w")
-        else:
-            output = file
-
-        json.dump({
-            "registrations": list(self.registrations),
-            "memberships": self.memberships,
-            "power_levels": self.power_levels,
-        }, output)
-
-        if isinstance(file, str):
-            output.close()
-
-    def load(self, file):
-        if isinstance(file, str):
-            try:
-                input_source = open(file, "r")
-            except FileNotFoundError:
-                return
-        else:
-            input_source = file
-
-        data = json.load(input_source)
-        if "registrations" in data:
-            self.registrations = set(data["registrations"])
-        if "memberships" in data:
-            self.memberships = data["memberships"]
-        if "power_levels" in data:
-            self.power_levels = data["power_levels"]
-
-        if isinstance(file, str):
-            input_source.close()
-
-    def _autosave(self):
-        if self.autosave_file:
-            self.save(self.autosave_file)
-
-    def is_registered(self, user):
-        return user in self.registrations
-
-    def registered(self, user):
-        self.registrations.add(user)
-        self._autosave()
-
-    def get_membership(self, room, user):
-        return self.memberships.get(room, {}).get(user, "left")
-
-    def is_joined(self, room, user):
-        return self.get_membership(room, user) == "join"
-
-    def set_membership(self, room, user, membership):
-        if room not in self.memberships:
-            self.memberships[room] = {}
-        self.memberships[room][user] = membership
-        self._autosave()
-
-    def joined(self, room, user):
-        return self.set_membership(room, user, "join")
-
-    def invited(self, room, user):
-        return self.set_membership(room, user, "invite")
-
-    def left(self, room, user):
-        return self.set_membership(room, user, "left")
-
-    def has_power_levels(self, room):
-        return room in self.power_levels
-
-    def get_power_levels(self, room):
-        return self.power_levels[room]
-
-    def has_power_level(self, room, user, event):
-        room_levels = self.power_levels.get(room, {})
-        required = room_levels.get("events", {}).get(event, 95)
-        has = room_levels.get("users", {}).get(user, 0)
-        return has >= required
-
-    def set_power_level(self, room, user, level):
-        if room not in self.power_levels:
-            self.power_levels[room] = {
-                "users": {},
-                "events": {},
-            }
-        elif "users" not in self.power_levels[room]:
-            self.power_levels[room]["users"] = {}
-        self.power_levels[room]["users"][user] = level
-        self._autosave()
-
-    def set_power_levels(self, room, content):
-        if "events" not in content:
-            content["events"] = {}
-        if "users" not in content:
-            content["users"] = {}
-        self.power_levels[room] = content
-        self._autosave()

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.1.1"
+__version__ = "0.6.1rc2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,57 +1,71 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, List, Any
+from time import time
 import argparse
-import sys
-import logging
 import asyncio
+import logging.config
+import sys
+import copy
+import signal
+import os
 
 import sqlalchemy as sql
-from sqlalchemy import orm
 
 from mautrix_appservice import AppService
+from alchemysession import AlchemySessionContainer
 
-from .base import Base
+from .web.provisioning import ProvisioningAPI
+from .web.public import PublicBridgeWebsite
+from .abstract_user import init as init_abstract_user
+from .bot import init as init_bot
 from .config import Config
+from .context import Context
+from .db import Base, init as init_db
+from .formatter import init as init_formatter
 from .matrix import MatrixHandler
-
-from .db import init as init_db
-from .user import init as init_user, User
 from .portal import init as init_portal
 from .puppet import init as init_puppet
+from .sqlstatestore import SQLStateStore
+from .user import User, init as init_user
+from . import __version__
 
-log = logging.getLogger("mau")
-time_formatter = logging.Formatter("[%(asctime)s] [%(levelname)s@%(name)s] %(message)s")
-handler = logging.StreamHandler()
-handler.setFormatter(time_formatter)
-log.addHandler(handler)
+try:
+    import prometheus_client as prometheus
+except ImportError:
+    prometheus = None
 
 parser = argparse.ArgumentParser(
     description="A Matrix-Telegram puppeting bridge.",
     prog="python -m mautrix-telegram")
 parser.add_argument("-c", "--config", type=str, default="config.yaml",
                     metavar="<path>", help="the path to your config file")
+parser.add_argument("-b", "--base-config", type=str, default="example-config.yaml",
+                    metavar="<path>", help="the path to the example config "
+                                           "(for automatic config updates)")
 parser.add_argument("-g", "--generate-registration", action="store_true",
                     help="generate registration and quit")
 parser.add_argument("-r", "--registration", type=str, default="registration.yaml",
                     metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()
 
-config = Config(args.config, args.registration)
+config = Config(args.config, args.registration, args.base_config, os.environ)
 config.load()
+config.update()
 
 if args.generate_registration:
     config.generate_registration()
@@ -59,35 +73,91 @@ if args.generate_registration:
     print(f"Registration generated and saved to {config.registration_path}")
     sys.exit(0)
 
-if config["appservice.debug"]:
-    telethon_log = logging.getLogger("telethon")
-    telethon_log.addHandler(handler)
-    telethon_log.setLevel(logging.DEBUG)
-    log.setLevel(logging.DEBUG)
-    log.debug("Debug messages enabled.")
+logging.config.dictConfig(copy.deepcopy(config["logging"]))
+log = logging.getLogger("mau.init")  # type: logging.Logger
+log.debug(f"Initializing mautrix-telegram {__version__}")
 
-db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
-db_factory = orm.sessionmaker(bind=db_engine)
-db_session = orm.scoping.scoped_session(db_factory)
+db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
 Base.metadata.bind = db_engine
-Base.metadata.create_all()
 
-loop = asyncio.get_event_loop()
+session_container = AlchemySessionContainer(engine=db_engine, table_base=Base, session=False,
+                                            table_prefix="telethon_", manage_tables=False)
+session_container.core_mode = True
+
+try:
+    import uvloop
+
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+    log.debug("Using uvloop for asyncio")
+except ImportError:
+    pass
+
+loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
+
+state_store = SQLStateStore()
+mebibyte = 1024 ** 2
 appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      config["appservice.as_token"], config["appservice.hs_token"],
-                     config["appservice.bot_username"], log="mau.as", loop=loop)
-context = (appserv, db_session, config, loop)
+                     config["appservice.bot_username"], log="mau.as", loop=loop,
+                     verify_ssl=config["homeserver.verify_ssl"], state_store=state_store,
+                     real_user_content_key="net.maunium.telegram.puppet",
+                     aiohttp_params={
+                         "client_max_size": config["appservice.max_body_size"] * mebibyte
+                     })
+bot = init_bot(config)
+context = Context(appserv, config, loop, session_container, bot)
+
+if config["appservice.public.enabled"]:
+    public_website = PublicBridgeWebsite(loop)
+    appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public_website.app)
+    context.public_website = public_website
+
+if config["appservice.provisioning.enabled"]:
+    provisioning_api = ProvisioningAPI(context)
+    appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning",
+                           provisioning_api.app)
+    context.provisioning_api = provisioning_api
+
+context.mx = MatrixHandler(context)
+
+if config["metrics.enabled"]:
+    if prometheus:
+        prometheus.start_http_server(config["metrics.listen_port"])
+    else:
+        log.warn("Metrics are enabled in the config, but prometheus-async is not installed.")
 
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
-    MatrixHandler(context)
-    init_db(db_session)
+    start_ts = time()
+    init_db(db_engine)
+    init_abstract_user(context)
+    init_formatter(context)
     init_portal(context)
-    init_puppet(context)
-    startup_actions = init_user(context) + [start]
+    startup_actions = (init_puppet(context) +
+                       init_user(context) +
+                       [start, context.mx.init_as_bot()])  # type: List[Awaitable[Any]]
+
+    if context.bot:
+        startup_actions.append(context.bot.start())
+
+    signal.signal(signal.SIGINT, signal.default_int_handler)
+    signal.signal(signal.SIGTERM, signal.default_int_handler)
+
+    end_ts = time()
     try:
+        log.debug(f"Initialization complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " running startup actions")
+        start_ts = time()
         loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
+        end_ts = time()
+        log.debug(f"Startup actions complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " now running forever")
         loop.run_forever()
     except KeyboardInterrupt:
-        for user in User.by_tgid.values():
-            user.stop()
+        log.debug("Interrupt received, stopping clients")
+        loop.run_until_complete(
+            asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
+        log.debug("Clients stopped, shutting down")
         sys.exit(0)
+    except Exception as e:
+        log.exception("Unexpected error")
+        sys.exit(1)

mautrix_telegram/abstract_user.py

@@ -0,0 +1,423 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Optional, List, Union, Dict, TYPE_CHECKING
+from abc import ABC, abstractmethod
+import asyncio
+import logging
+import platform
+import time
+
+from telethon.tl.patched import MessageService, Message
+from telethon.tl.types import (
+    Channel, ChannelForbidden, Chat, ChatForbidden, MessageActionChannelMigrateFrom, PeerUser,
+    TypeUpdate, UpdateChannelPinnedMessage, UpdateChatPinnedMessage, UpdateChatParticipantAdmin,
+    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateDeleteMessages,
+    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateNewMessage,
+    UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, UpdateUserName,
+    UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, UserStatusOnline)
+
+from mautrix_appservice import MatrixRequestError, AppService
+from alchemysession import AlchemySessionContainer
+
+from . import portal as po, puppet as pu, __version__
+from .db import Message as DBMessage
+from .types import TelegramID, MatrixUserID
+from .tgclient import MautrixTelegramClient
+
+if TYPE_CHECKING:
+    from .context import Context
+    from .config import Config
+    from .bot import Bot
+
+config = None  # type: Config
+# Value updated from config in init()
+MAX_DELETIONS = 10  # type: int
+
+UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                      UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
+UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
+
+try:
+    from prometheus_client import Histogram
+
+    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
+                            ["update_type"])
+except ImportError:
+    Histogram = None
+    UPDATE_TIME = None
+
+class AbstractUser(ABC):
+    session_container = None  # type: AlchemySessionContainer
+    loop = None  # type: asyncio.AbstractEventLoop
+    log = None  # type: logging.Logger
+    az = None  # type: AppService
+    bot = None  # type: Bot
+    ignore_incoming_bot_events = True  # type: bool
+
+    def __init__(self) -> None:
+        self.is_admin = False  # type: bool
+        self.matrix_puppet_whitelisted = False  # type: bool
+        self.puppet_whitelisted = False  # type: bool
+        self.whitelisted = False  # type: bool
+        self.relaybot_whitelisted = False  # type: bool
+        self.client = None  # type: MautrixTelegramClient
+        self.tgid = None  # type: TelegramID
+        self.mxid = None  # type: MatrixUserID
+        self.is_relaybot = False  # type: bool
+        self.is_bot = False  # type: bool
+        self.relaybot = None  # type: Optional[Bot]
+
+    @property
+    def connected(self) -> bool:
+        return self.client and self.client.is_connected()
+
+    @property
+    def _proxy_settings(self) -> Optional[Tuple[int, str, str, str, str, str]]:
+        proxy_type = config["telegram.proxy.type"].lower()
+        if proxy_type == "disabled":
+            return None
+        elif proxy_type == "socks4":
+            proxy_type = 1
+        elif proxy_type == "socks5":
+            proxy_type = 2
+        elif proxy_type == "http":
+            proxy_type = 3
+
+        return (proxy_type,
+                config["telegram.proxy.address"], config["telegram.proxy.port"],
+                config["telegram.proxy.rdns"],
+                config["telegram.proxy.username"], config["telegram.proxy.password"])
+
+    def _init_client(self) -> None:
+        self.log.debug(f"Initializing client for {self.name}")
+
+        self.session = self.session_container.new_session(self.name)
+        if config["telegram.server.enabled"]:
+            self.session.set_dc(config["telegram.server.dc"],
+                                config["telegram.server.ip"],
+                                config["telegram.server.port"])
+
+        if self.is_relaybot:
+            base_logger = logging.getLogger("telethon.relaybot")
+        else:
+            base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
+
+        device = config["telegram.device_info.device_model"]
+        sysversion = config["telegram.device_info.system_version"]
+        appversion = config["telegram.device_info.app_version"]
+
+        self.client = MautrixTelegramClient(
+            session=self.session,
+
+            api_id=config["telegram.api_id"],
+            api_hash=config["telegram.api_hash"],
+
+            app_version=__version__ if appversion == "auto" else appversion,
+            system_version=MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion,
+            device_model=f"{platform.system()} {platform.release()}" if device == "auto" else device,
+
+            timeout=config["telegram.connection.timeout"],
+            connection_retries=config["telegram.connection.retries"],
+            retry_delay=config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
+            request_retries=config["telegram.connection.request_retries"],
+
+            proxy=self._proxy_settings,
+
+            loop=self.loop,
+            base_logger=base_logger
+        )
+        self.client.add_event_handler(self._update_catch)
+
+    @abstractmethod
+    async def update(self, update: TypeUpdate) -> bool:
+        return False
+
+    @abstractmethod
+    async def post_login(self) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def register_portal(self, portal: po.Portal) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def unregister_portal(self, portal: po.Portal) -> None:
+        raise NotImplementedError()
+
+    async def _update_catch(self, update: TypeUpdate) -> None:
+        start_time = time.time()
+        try:
+            if not await self.update(update):
+                await self._update(update)
+        except Exception:
+            self.log.exception("Failed to handle Telegram update")
+        if UPDATE_TIME:
+            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
+
+    async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
+        if self.is_bot:
+            return []
+        dialogs = await self.client.get_dialogs(limit=limit)
+        return [dialog.entity for dialog in dialogs if (
+            not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden))
+            and not (isinstance(dialog.entity, Chat)
+                     and (dialog.entity.deactivated or dialog.entity.left)))]
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        raise NotImplementedError()
+
+    async def is_logged_in(self) -> bool:
+        return self.client and self.client.is_connected() and await self.client.is_user_authorized()
+
+    async def has_full_access(self, allow_bot: bool = False) -> bool:
+        return (self.puppet_whitelisted
+                and (not self.is_bot or allow_bot)
+                and await self.is_logged_in())
+
+    async def start(self, delete_unless_authenticated: bool = False) -> 'AbstractUser':
+        if not self.client:
+            self._init_client()
+        await self.client.connect()
+        self.log.debug("%s connected: %s", self.mxid, self.connected)
+        return self
+
+    async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
+        if not self.puppet_whitelisted or self.connected:
+            return self
+        self.log.debug("ensure_started(%s, even_if_no_session=%s)", self.mxid, even_if_no_session)
+        if even_if_no_session or self.session_container.has_session(self.mxid):
+            await self.start(delete_unless_authenticated=not even_if_no_session)
+        return self
+
+    async def stop(self) -> None:
+        await self.client.disconnect()
+        self.client = None
+
+    # region Telegram update handling
+
+    async def _update(self, update: TypeUpdate) -> None:
+        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+                              loop=self.loop)
+        if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                               UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
+            await self.update_message(update)
+        elif isinstance(update, UpdateDeleteMessages):
+            await self.delete_message(update)
+        elif isinstance(update, UpdateDeleteChannelMessages):
+            await self.delete_channel_message(update)
+        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
+            await self.update_typing(update)
+        elif isinstance(update, UpdateUserStatus):
+            await self.update_status(update)
+        elif isinstance(update, UpdateChatParticipantAdmin):
+            await self.update_admin(update)
+        elif isinstance(update, UpdateChatParticipants):
+            await self.update_participants(update)
+        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
+            await self.update_pinned_messages(update)
+        elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
+            await self.update_others_info(update)
+        elif isinstance(update, UpdateReadHistoryOutbox):
+            await self.update_read_receipt(update)
+        else:
+            self.log.debug("Unhandled update: %s", update)
+
+    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
+                                                         UpdateChatPinnedMessage]) -> None:
+        if isinstance(update, UpdateChatPinnedMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        else:
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+        if portal and portal.mxid:
+            await portal.receive_telegram_pin_id(update.id, self.tgid)
+
+    @staticmethod
+    async def update_participants(update: UpdateChatParticipants) -> None:
+        portal = po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
+        if portal and portal.mxid:
+            await portal.update_telegram_participants(update.participants.participants)
+
+    async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
+        if not isinstance(update.peer, PeerUser):
+            self.log.debug("Unexpected read receipt peer: %s", update.peer)
+            return
+
+        portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
+        if not portal or not portal.mxid:
+            return
+
+        # We check that these are user read receipts, so tg_space is always the user ID.
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
+        if not message:
+            return
+
+        puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
+        await puppet.intent.mark_read(portal.mxid, message.mxid)
+
+    async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
+        # TODO duplication not checked
+        portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
+        if not portal or not portal.mxid:
+            return
+
+        await portal.set_telegram_admin(TelegramID(update.user_id))
+
+    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
+        if isinstance(update, UpdateUserTyping):
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
+        else:
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
+
+        if not portal or not portal.mxid:
+            return
+
+        sender = pu.Puppet.get(TelegramID(update.user_id))
+        await portal.handle_telegram_typing(sender, update)
+
+    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
+                                     ) -> None:
+        try:
+            users = (entity for entity in entities.values() if isinstance(entity, User))
+            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
+            await asyncio.gather(*[puppet.update_info(self, info)
+                                   for puppet, info in puppets if puppet])
+        except Exception:
+            self.log.exception("Failed to handle entity updates")
+
+    async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
+        # TODO duplication not checked
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
+        if isinstance(update, UpdateUserName):
+            puppet.username = update.username
+            if await puppet.update_displayname(self, update):
+                puppet.save()
+        elif isinstance(update, UpdateUserPhoto):
+            if await puppet.update_avatar(self, update.photo):
+                puppet.save()
+        else:
+            self.log.warning("Unexpected other user info update: %s", update)
+
+    async def update_status(self, update: UpdateUserStatus) -> None:
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
+        if isinstance(update.status, UserStatusOnline):
+            await puppet.default_mxid_intent.set_presence("online")
+        elif isinstance(update.status, UserStatusOffline):
+            await puppet.default_mxid_intent.set_presence("offline")
+        else:
+            self.log.warning("Unexpected user status update: %s", update)
+        return
+
+    def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
+                                                                  Optional[pu.Puppet],
+                                                                  Optional[po.Portal]]:
+        if isinstance(update, UpdateShortChatMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
+            sender = pu.Puppet.get(TelegramID(update.from_id))
+        elif isinstance(update, UpdateShortMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
+            sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
+        elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
+                                 UpdateEditMessage, UpdateEditChannelMessage)):
+            update = update.message
+            if isinstance(update.to_id, PeerUser) and not update.out:
+                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
+            else:
+                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
+            sender = pu.Puppet.get(update.from_id) if update.from_id else None
+        else:
+            self.log.warning(
+                f"Unexpected message type in User#get_message_details: {type(update)}")
+            return update, None, None
+        return update, sender, portal
+
+    @staticmethod
+    async def _try_redact(message: DBMessage) -> None:
+        portal = po.Portal.get_by_mxid(message.mx_room)
+        if not portal:
+            return
+        try:
+            await portal.main_intent.redact(message.mx_room, message.mxid)
+        except MatrixRequestError:
+            pass
+
+    async def delete_message(self, update: UpdateDeleteMessages) -> None:
+        if len(update.messages) > MAX_DELETIONS:
+            return
+
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
+            for message in messages:
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    portal = po.Portal.get_by_mxid(message.mx_room)
+                    await self._try_redact(message)
+
+    async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
+        if len(update.messages) > MAX_DELETIONS:
+            return
+
+        channel_id = TelegramID(update.channel_id)
+
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
+            for message in messages:
+                message.delete()
+                await self._try_redact(message)
+
+    async def update_message(self, original_update: UpdateMessage) -> None:
+        update, sender, portal = self.get_message_details(original_update)
+
+        if self.is_bot and not portal.mxid:
+            self.log.debug(f"Ignoring message received by bot in unbridged chat %s",
+                           portal.tgid_log)
+            return
+
+        if self.ignore_incoming_bot_events and self.bot and sender.id == self.bot.tgid:
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
+            return
+
+        if isinstance(update, MessageService):
+            if isinstance(update.action, MessageActionChannelMigrateFrom):
+                self.log.debug(f"Ignoring action %s to %s by %d", update.action,
+                               portal.tgid_log,
+                               sender.id)
+                return
+            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
+                           sender.id)
+            return await portal.handle_telegram_action(self, sender, update)
+
+        user = sender.tgid if sender else "admin"
+        if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
+            return await portal.handle_telegram_edit(self, sender, update)
+
+        self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
+        return await portal.handle_telegram_message(self, sender, update)
+
+    # endregion
+
+
+def init(context: "Context") -> None:
+    global config, MAX_DELETIONS
+    AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
+    AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
+    AbstractUser.session_container = context.session_container
+    MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/base.py

@@ -1,2 +0,0 @@
-from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()

mautrix_telegram/bot.py

@@ -0,0 +1,298 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Callable, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
+import logging
+import re
+
+from telethon.tl.patched import Message, MessageService
+from telethon.tl.types import (
+    ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
+    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
+    MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
+from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
+from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
+from telethon.errors import ChannelInvalidError, ChannelPrivateError
+
+from .types import MatrixUserID
+from .abstract_user import AbstractUser
+from .db import BotChat
+from .types import TelegramID
+from . import puppet as pu, portal as po, user as u
+
+if TYPE_CHECKING:
+    from .config import Config
+    from .context import Context
+
+config = None  # type: Config
+
+ReplyFunc = Callable[[str], Awaitable[Message]]
+
+
+class Bot(AbstractUser):
+    log = logging.getLogger("mau.bot")  # type: logging.Logger
+    mxid_regex = re.compile("@.+:.+")  # type: Pattern
+
+    def __init__(self, token: str) -> None:
+        super().__init__()
+        self.token = token  # type: str
+        self.puppet_whitelisted = True  # type: bool
+        self.whitelisted = True  # type: bool
+        self.relaybot_whitelisted = True  # type: bool
+        self.username = None  # type: str
+        self.is_relaybot = True  # type: bool
+        self.is_bot = True  # type: bool
+        self.chats = {}  # type: Dict[int, str]
+        self.tg_whitelist = []  # type: List[int]
+        self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
+                                       or False)  # type: bool
+        self._me_info = None  # type: Optional[User]
+        self._me_mxid = None  # type: Optional[MatrixUserID]
+
+    async def get_me(self, use_cache: bool = True) -> Tuple[User, MatrixUserID]:
+        if not use_cache or not self._me_mxid:
+            self._me_info = await self.client.get_me()
+            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
+        return self._me_info, self._me_mxid
+
+    async def init_permissions(self) -> None:
+        whitelist = config["bridge.relaybot.whitelist"] or []
+        for user_id in whitelist:
+            if isinstance(user_id, str):
+                entity = await self.client.get_input_entity(user_id)
+                if isinstance(entity, InputUser):
+                    user_id = entity.user_id
+                else:
+                    user_id = None
+            if isinstance(user_id, int):
+                self.tg_whitelist.append(user_id)
+
+    async def start(self, delete_unless_authenticated: bool = False) -> 'Bot':
+        self.chats = {chat.id: chat.type for chat in BotChat.all()}
+        await super().start(delete_unless_authenticated)
+        if not await self.is_logged_in():
+            await self.client.sign_in(bot_token=self.token)
+        await self.post_login()
+        return self
+
+    async def post_login(self) -> None:
+        await self.init_permissions()
+        info = await self.client.get_me()
+        self.tgid = info.id
+        self.username = info.username
+        self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
+
+        chat_ids = [chat_id for chat_id, chat_type in self.chats.items() if chat_type == "chat"]
+        response = await self.client(GetChatsRequest(chat_ids))
+        for chat in response.chats:
+            if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
+                self.remove_chat(TelegramID(chat.id))
+
+        channel_ids = [InputChannel(chat_id, 0)
+                       for chat_id, chat_type in self.chats.items()
+                       if chat_type == "channel"]
+        for channel_id in channel_ids:
+            try:
+                await self.client(GetChannelsRequest([channel_id]))
+            except (ChannelPrivateError, ChannelInvalidError):
+                self.remove_chat(TelegramID(channel_id.channel_id))
+
+        if config["bridge.catch_up"]:
+            try:
+                await self.client.catch_up()
+            except Exception:
+                self.log.exception("Failed to run catch_up() for bot")
+
+    def register_portal(self, portal: po.Portal) -> None:
+        self.add_chat(portal.tgid, portal.peer_type)
+
+    def unregister_portal(self, portal: po.Portal) -> None:
+        self.remove_chat(portal.tgid)
+
+    def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
+        if chat_id not in self.chats:
+            self.chats[chat_id] = chat_type
+            BotChat(id=TelegramID(chat_id), type=chat_type).insert()
+
+    def remove_chat(self, chat_id: TelegramID) -> None:
+        try:
+            del self.chats[chat_id]
+        except KeyError:
+            pass
+        BotChat.delete(chat_id)
+
+    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
+        if tgid in self.tg_whitelist:
+            return True
+
+        user = u.User.get_by_tgid(tgid)
+        if user and user.is_admin:
+            self.tg_whitelist.append(user.tgid)
+            return True
+
+        if self.whitelist_group_admins:
+            if isinstance(chat, PeerChannel):
+                p = await self.client(GetParticipantRequest(chat, tgid))
+                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
+            elif isinstance(chat, PeerChat):
+                chat = await self.client(GetFullChatRequest(chat.chat_id))
+                participants = chat.full_chat.participants.participants
+                for p in participants:
+                    if p.user_id == tgid:
+                        return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+        return False
+
+    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
+        if not await self._can_use_commands(event.to_id, event.from_id):
+            await reply("You do not have the permission to use that command.")
+            return False
+        return True
+
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> Message:
+        if not config["bridge.relaybot.authless_portals"]:
+            return await reply("This bridge doesn't allow portal creation from Telegram.")
+
+        if not portal.allow_bridging():
+            return await reply("This bridge doesn't allow bridging this chat.")
+
+        await portal.create_matrix_room(self)
+        if portal.mxid:
+            if portal.username:
+                return await reply(
+                    f"Portal is public: [{portal.alias}](https://matrix.to/#/{portal.alias})")
+            else:
+                return await reply(
+                    "Portal is not public. Use `/invite <mxid>` to get an invite.")
+
+    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc,
+                                    mxid_input: MatrixUserID) -> Message:
+        if len(mxid_input) == 0:
+            return await reply("Usage: `/invite <mxid>`")
+        elif not portal.mxid:
+            return await reply("Portal does not have Matrix room. "
+                               "Create one with /portal first.")
+        if not self.mxid_regex.match(mxid_input):
+            return await reply("That doesn't look like a Matrix ID.")
+        user = await u.User.get_by_mxid(MatrixUserID(mxid_input)).ensure_started()
+        if not user.relaybot_whitelisted:
+            return await reply("That user is not whitelisted to use the bridge.")
+        elif await user.is_logged_in():
+            displayname = f"@{user.username}" if user.username else user.displayname
+            return await reply("That user seems to be logged in. "
+                               f"Just invite [{displayname}](tg://user?id={user.tgid})")
+        else:
+            await portal.main_intent.invite(portal.mxid, user.mxid)
+            return await reply(f"Invited `{user.mxid}` to the portal.")
+
+    @staticmethod
+    def handle_command_id(message: Message, reply: ReplyFunc) -> Awaitable[Message]:
+        # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
+        # chat is a normal group or a supergroup/channel when using the ID.
+        if isinstance(message.to_id, PeerChannel):
+            return reply(f"-100{message.to_id.channel_id}")
+        return reply(str(-message.to_id.chat_id))
+
+    def match_command(self, text: str, command: str) -> bool:
+        text = text.lower()
+        command = f"/{command.lower()}"
+        command_targeted = f"{command}@{self.username.lower()}"
+
+        is_plain_command = text == command or text == command_targeted
+        if is_plain_command:
+            return True
+
+        is_arg_command = text.startswith(command + " ") or text.startswith(command_targeted + " ")
+        if is_arg_command:
+            return True
+
+        return False
+
+    async def handle_command(self, message: Message) -> None:
+        def reply(reply_text: str) -> Awaitable[Message]:
+            return self.client.send_message(message.to_id, reply_text, reply_to=message.id)
+
+        text = message.message
+
+        if self.match_command(text, "id"):
+            await self.handle_command_id(message, reply)
+            return
+
+        portal = po.Portal.get_by_entity(message.to_id)
+
+        if self.match_command(text, "portal"):
+            if not await self.check_can_use_commands(message, reply):
+                return
+            await self.handle_command_portal(portal, reply)
+        elif self.match_command(text, "invite"):
+            if not await self.check_can_use_commands(message, reply):
+                return
+            try:
+                mxid = text[text.index(" ") + 1:]
+            except ValueError:
+                mxid = ""
+            await self.handle_command_invite(portal, reply, mxid_input=mxid)
+
+    def handle_service_message(self, message: MessageService) -> None:
+        to_id = message.to_id  # type: TelegramID
+        if isinstance(to_id, PeerChannel):
+            to_id = to_id.channel_id
+            chat_type = "channel"
+        elif isinstance(to_id, PeerChat):
+            to_id = to_id.chat_id
+            chat_type = "chat"
+        else:
+            return
+
+        action = message.action
+        if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
+            self.add_chat(to_id, chat_type)
+        elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
+            self.remove_chat(to_id)
+        elif isinstance(action, MessageActionChatMigrateTo):
+            self.remove_chat(to_id)
+            self.add_chat(TelegramID(action.channel_id), "channel")
+
+    async def update(self, update) -> bool:
+        if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
+            return False
+        if isinstance(update.message, MessageService):
+            self.handle_service_message(update.message)
+            return False
+
+        is_command = (isinstance(update.message, Message)
+                      and update.message.entities and len(update.message.entities) > 0
+                      and isinstance(update.message.entities[0], MessageEntityBotCommand))
+        if is_command:
+            await self.handle_command(update.message)
+            return True
+        return False
+
+    def is_in_chat(self, peer_id) -> bool:
+        return peer_id in self.chats
+
+    @property
+    def name(self) -> str:
+        return "bot"
+
+
+def init(cfg: 'Config') -> Optional[Bot]:
+    global config
+    config = cfg
+    token = config["telegram.bot_token"]
+    if token and not token.lower().startswith("disable"):
+        return Bot(token)
+    return None

mautrix_telegram/commands/__init__.py

@@ -1,2 +1,5 @@
-from .handler import command_handler, CommandHandler
-from . import clean_rooms, auth, meta, telegram
+from .handler import (command_handler, command_handlers as _command_handlers,
+                      CommandHandler, CommandProcessor, CommandEvent,
+                      SECTION_GENERAL, SECTION_AUTH, SECTION_CREATING_PORTALS,
+                      SECTION_PORTAL_MANAGEMENT, SECTION_MISC, SECTION_ADMIN)
+from . import portal, telegram, clean_rooms, matrix_auth, meta

mautrix_telegram/commands/auth.py

@@ -1,119 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-import asyncio
-
-from telethon.errors import *
-
-from . import command_handler
-
-
-@command_handler(needs_auth=False)
-async def ping(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
-    me = await evt.sender.client.get_me()
-    if me:
-        return await evt.reply(f"You're logged in as @{me.username}")
-    else:
-        return await evt.reply("You're not logged in.")
-
-
-@command_handler(needs_auth=False, management_only=True)
-def register(evt):
-    return evt.reply("Not yet implemented.")
-
-
-@command_handler(needs_auth=False, management_only=True)
-async def login(evt):
-    if evt.sender.logged_in:
-        return await evt.reply("You are already logged in.")
-    elif len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp login <phone number>`")
-    phone_number = evt.args[0]
-    await evt.sender.client.sign_in(phone_number)
-    evt.sender.command_status = {
-        "next": enter_code,
-        "action": "Login",
-    }
-    return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
-
-
-@command_handler(needs_auth=False)
-async def enter_code(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
-
-    try:
-        user = await evt.sender.client.sign_in(code=evt.args[0])
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PhoneNumberUnoccupiedError:
-        return await evt.reply("That phone number has not been registered."
-                               "Please register with `$cmdprefix+sp register <phone>`.")
-    except PhoneCodeExpiredError:
-        return await evt.reply(
-            "Phone code expired. Try again with `$cmdprefix+sp login <phone>`.")
-    except PhoneCodeInvalidError:
-        return await evt.reply("Invalid phone code.")
-    except PhoneNumberAppSignupForbiddenError:
-        return await evt.reply(
-            "Your phone number does not allow 3rd party apps to sign in.")
-    except PhoneNumberFloodError:
-        return await evt.reply(
-            "Your phone number has been temporarily blocked for flooding. "
-            "The block is usually applied for around a day.")
-    except PhoneNumberBannedError:
-        return await evt.reply("Your phone number has been banned from Telegram.")
-    except SessionPasswordNeededError:
-        evt.sender.command_status = {
-            "next": enter_password,
-            "action": "Login (password entry)",
-        }
-        return await evt.reply("Your account has two-factor authentication."
-                               "Please send your password here.")
-    except Exception:
-        evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code."
-                               "Check console for more details.")
-
-
-@command_handler(needs_auth=False)
-async def enter_password(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
-
-    try:
-        user = await evt.sender.client.sign_in(password=evt.args[0])
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PasswordHashInvalidError:
-        return await evt.reply("Incorrect password.")
-    except Exception:
-        evt.log.exception("Error sending password")
-        return await evt.reply("Unhandled exception while sending password. "
-                               "Check console for more details.")
-
-
-@command_handler(needs_auth=False)
-async def logout(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
-    if await evt.sender.log_out():
-        return await evt.reply("Logged out successfully.")
-    return await evt.reply("Failed to log out.")

mautrix_telegram/commands/clean_rooms.py

@@ -1,33 +1,40 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from mautrix_appservice import MatrixRequestError
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, NewType, Optional, Tuple, Union
 
-from . import command_handler
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from ..types import MatrixRoomID, MatrixUserID
+from . import command_handler, CommandEvent, SECTION_ADMIN
 from .. import puppet as pu, portal as po
 
+ManagementRoom = NewType('ManagementRoom', Tuple[MatrixRoomID, MatrixUserID])
 
-async def _find_rooms(intent):
-    management_rooms = []
-    unidentified_rooms = []
-    portals = []
-    empty_portals = []
+
+async def _find_rooms(intent: IntentAPI) -> Tuple[List[ManagementRoom], List[MatrixRoomID],
+                                                  List['po.Portal'], List['po.Portal']]:
+    management_rooms = []  # type: List[ManagementRoom]
+    unidentified_rooms = []  # type: List[MatrixRoomID]
+    portals = []  # type: List[po.Portal]
+    empty_portals = []  # type: List[po.Portal]
 
     rooms = await intent.get_joined_rooms()
-    for room in rooms:
+    for room_str in rooms:
+        room = MatrixRoomID(room_str)
         portal = po.Portal.get_by_mxid(room)
         if not portal:
             try:
@@ -35,11 +42,11 @@ async def _find_rooms(intent):
             except MatrixRequestError:
                 members = []
             if len(members) == 2:
-                other_member = members[0] if members[0] != intent.mxid else members[1]
+                other_member = MatrixUserID(members[0] if members[0] != intent.mxid else members[1])
                 if pu.Puppet.get_id_from_mxid(other_member):
                     unidentified_rooms.append(room)
                 else:
-                    management_rooms.append((room, other_member))
+                    management_rooms.append(ManagementRoom((room, other_member)))
             else:
                 unidentified_rooms.append(room)
         else:
@@ -52,12 +59,10 @@ async def _find_rooms(intent):
     return management_rooms, unidentified_rooms, portals, empty_portals
 
 
-@command_handler(needs_admin=True, name="clean-rooms")
-async def clean_rooms(evt):
-    if not evt.is_management:
-        return await evt.reply("`clean-rooms` is a particularly spammy command. Please don't "
-                               "run it in non-management rooms.")
-
+@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
+                 help_section=SECTION_ADMIN,
+                 help_text="Clean up unused portal/management rooms.")
+async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
     management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
 
     reply = ["#### Management rooms (M)"]
@@ -65,8 +70,8 @@ async def clean_rooms(evt):
                for n, (room, other_member) in enumerate(management_rooms)]
               or ["No management rooms found."])
     reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [P{n+1}](https://matrix.to/#/{portal.mxid}) "
-               + f"(to Telegram chat \"{portal.title}\")"
+    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
+               f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(portals)]
               or ["No active portal rooms found."])
     reply.append("#### Unidentified rooms (U)")
@@ -74,8 +79,8 @@ async def clean_rooms(evt):
                for n, room in enumerate(unidentified_rooms)]
               or ["No unidentified rooms found."])
     reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [E{n}](https://matrix.to/#/{portal.mxid}) "
-               + f"(to Telegram chat \"{portal.title}\")"
+    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
+               f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(empty_portals)]
               or ["No inactive portal rooms found."])
 
@@ -88,9 +93,9 @@ async def clean_rooms(evt):
               "",
               ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
                "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name."),
+               "the group name. (e.g. `I2-6`)"),
               "",
-              ("Please note that you will have to re-run `$cmdprefix+sp cleanrooms` "
+              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
                "between each use of the commands above.")]
 
     evt.sender.command_status = {
@@ -102,17 +107,20 @@ async def clean_rooms(evt):
     return await evt.reply("\n".join(reply))
 
 
-async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals, empty_portals):
+async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
+                             unidentified_rooms: List[MatrixRoomID], portals: List["po.Portal"],
+                             empty_portals: List["po.Portal"]) -> None:
     command = evt.args[0]
-    rooms_to_clean = []
+    rooms_to_clean = []  # type: List[Union[po.Portal, MatrixRoomID]]
     if command == "clean-recommended":
-        rooms_to_clean = empty_portals + unidentified_rooms
+        rooms_to_clean += empty_portals
+        rooms_to_clean += unidentified_rooms
     elif command == "clean-groups":
         if len(evt.args) < 2:
             return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1]
+        groups_to_clean = evt.args[1].upper()
         if "M" in groups_to_clean:
-            rooms_to_clean += management_rooms
+            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
         if "A" in groups_to_clean:
             rooms_to_clean += portals
         if "U" in groups_to_clean:
@@ -121,12 +129,12 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
             rooms_to_clean += empty_portals
     elif command == "clean-range":
         try:
-            range = evt.args[1]
-            group, range = range[0], range[1:]
-            start, end = range.split("-")
+            clean_range = evt.args[1]
+            group, clean_range = clean_range[0], clean_range[1:]
+            start, end = clean_range.split("-")
             start, end = int(start), int(end)
             if group == "M":
-                group = management_rooms
+                group = [room_id for (room_id, user_id) in management_rooms]
             elif group == "A":
                 group = portals
             elif group == "U":
@@ -141,28 +149,28 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
                 "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
     else:
         return await evt.reply(f"Unknown room cleaning action `{command}`. "
-                               + "Use `$cmdprefix+sp cancel` to cancel room "
-                               + "cleaning.")
+                               "Use `$cmdprefix+sp cancel` to cancel room "
+                               "cleaning.")
 
     evt.sender.command_status = {
         "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
         "action": "Room cleaning",
     }
     await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type"
-                    + "`$cmdprefix+sp confirm-clean`.")
+                    "`$cmdprefix+sp confirm-clean`.")
 
 
-async def execute_room_cleanup(evt, rooms_to_clean):
+async def execute_room_cleanup(evt, rooms_to_clean: List[Union[po.Portal, MatrixRoomID]]) -> None:
     if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
         await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
-                        + "This might take a while.")
+                        "This might take a while.")
         cleaned = 0
         for room in rooms_to_clean:
             if isinstance(room, po.Portal):
                 await room.cleanup_and_delete()
                 cleaned += 1
-            elif isinstance(room, str):
-                await po.Portal.cleanup_room(evt.az.intent, room, type="Room")
+            elif isinstance(room, str):  # str is aliased by MatrixRoomID
+                await po.Portal.cleanup_room(evt.az.intent, room, message="Room deleted")
                 cleaned += 1
         evt.sender.command_status = None
         await evt.reply(f"{cleaned} rooms cleaned up successfully.")

mautrix_telegram/commands/handler.py

@@ -1,115 +1,365 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-import markdown
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""This module contains classes handling commands issued by Matrix users."""
+from typing import Awaitable, Callable, Dict, List, NamedTuple, Optional
 import logging
+import traceback
+
+import commonmark
 
 from telethon.errors import FloodWaitError
 
-command_handlers = {}
+from ..types import MatrixRoomID, MatrixEventID
+from ..util import format_duration
+from .. import user as u, context as c
+
+command_handlers = {}  # type: Dict[str, CommandHandler]
+
+HelpSection = NamedTuple('HelpSection', [('name', str), ('order', int), ('description', str)])
+
+SECTION_GENERAL = HelpSection("General", 0, "")
+SECTION_AUTH = HelpSection("Authentication", 10, "")
+SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
+SECTION_PORTAL_MANAGEMENT = HelpSection("Portal management", 30, "")
+SECTION_MISC = HelpSection("Miscellaneous", 40, "")
+SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 
-def command_handler(needs_auth=True, management_only=False, needs_admin=False, name=None):
-    def decorator(func):
-        def wrapper(evt):
-            if management_only and not evt.is_management:
-                return evt.reply(f"`{evt.command}` is a restricted command:"
-                                 + "you may only run it in management rooms.")
-            elif needs_auth and not evt.sender.logged_in:
-                return evt.reply("This command requires you to be logged in.")
-            elif needs_admin and not evt.sender.is_admin:
-                return evt.reply("This is command requires administrator privileges.")
-            return func(evt)
+class HtmlEscapingRenderer(commonmark.HtmlRenderer):
+    def __init__(self, allow_html: bool = False):
+        super().__init__()
+        self.allow_html = allow_html
 
-        command_handlers[name or func.__name__.replace("_", "-")] = wrapper
-        return wrapper
+    def lit(self, s):
+        if self.allow_html:
+            return super().lit(s)
+        return super().lit(s.replace("<", "&lt;").replace(">", "&gt;"))
 
-    return decorator
+    def image(self, node, entering):
+        prev = self.allow_html
+        self.allow_html = True
+        super().image(node, entering)
+        self.allow_html = prev
+
+
+md_parser = commonmark.Parser()
+md_renderer = HtmlEscapingRenderer()
+
+
+def ensure_trailing_newline(s: str) -> str:
+    """Returns the passed string, but with a guaranteed trailing newline."""
+    return s + ("" if s[-1] == "\n" else "\n")
 
 
 class CommandEvent:
-    def __init__(self, handler, room, sender, command, args, is_management, is_portal):
-        self.az = handler.az
-        self.log = handler.log
-        self.loop = handler.loop
-        self.command_prefix = handler.command_prefix
+    """Holds information about a command issued in a Matrix room.
+
+    When a Matrix command was issued to the bot, CommandEvent will hold
+    information regarding the event.
+
+    Attributes:
+        room_id: The id of the Matrix room in which the command was issued.
+        event_id: The id of the matrix event which contained the command.
+        sender: The user who issued the command.
+        command: The issued command.
+        args: Arguments given with the issued command.
+        is_management: Determines whether the room in which the command wa
+            issued is a management room.
+        is_portal: Determines whether the room in which the command was issued
+            is a portal.
+    """
+
+    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, event: MatrixEventID,
+                 sender: u.User, command: str, args: List[str], is_management: bool,
+                 is_portal: bool) -> None:
+        self.az = processor.az
+        self.log = processor.log
+        self.loop = processor.loop
+        self.tgbot = processor.tgbot
+        self.config = processor.config
+        self.public_website = processor.public_website
+        self.command_prefix = processor.command_prefix
         self.room_id = room
+        self.event_id = event
         self.sender = sender
         self.command = command
         self.args = args
         self.is_management = is_management
         self.is_portal = is_portal
 
-    def reply(self, message, allow_html=False, render_markdown=True):
-        message = message.replace("$cmdprefix+sp ",
-                                  "" if self.is_management else f"{self.command_prefix} ")
-        message = message.replace("$cmdprefix", self.command_prefix)
-        html = None
+    def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True
+              ) -> Awaitable[Dict]:
+        """Write a reply to the room in which the command was issued.
+
+        Replaces occurences of "$cmdprefix" in the message with the command
+        prefix and replaces occurences of "$cmdprefix+sp " with the command
+        prefix if the command was not issued in a management room.
+        If allow_html and render_markdown are both False, the message will not
+        be rendered to html and sending of html is disabled.
+
+        Args:
+            message: The message to post in the room.
+            allow_html: Escape html in the message or don't render html at all
+                if markdown is disabled.
+            render_markdown: Use markdown formatting to render the passed
+                message to html.
+
+        Returns:
+            Handler for the message sending function.
+        """
+        message_cmd = self._replace_command_prefix(message)
+        html = self._render_message(message_cmd, allow_html=allow_html,
+                                    render_markdown=render_markdown)
+
+        return self.az.intent.send_notice(self.room_id, message_cmd, html=html)
+
+    def mark_read(self) -> Awaitable[Dict]:
+        """Marks the command as read by the bot."""
+        return self.az.intent.mark_read(self.room_id, self.event_id)
+
+    def _replace_command_prefix(self, message: str) -> str:
+        """Returns the string with the proper command prefix entered."""
+        message = message.replace(
+            "$cmdprefix+sp ", "" if self.is_management else f"{self.command_prefix} "
+        )
+        return message.replace("$cmdprefix", self.command_prefix)
+
+    @staticmethod
+    def _render_message(message: str, allow_html: bool, render_markdown: bool) -> Optional[str]:
+        """Renders the message as HTML.
+
+        Args:
+            allow_html: Flag to allow custom HTML in the message.
+            render_markdown: If true, markdown styling is applied to the message.
+
+        Returns:
+            The message rendered as HTML.
+            None is returned if no styled output is required.
+        """
+        html = ""
         if render_markdown:
-            html = markdown.markdown(message, safe_mode="escape" if allow_html else False)
+            md_renderer.allow_html = allow_html
+            html = md_renderer.render(md_parser.parse(message))
         elif allow_html:
             html = message
-        return self.az.intent.send_notice(self.room_id, message, html=html)
-
-
-def format_duration(seconds):
-    def pluralize(count, singular): return singular if count == 1 else singular + "s"
-
-    def include(count, word): return f"{count} {pluralize(count, word)}" if count > 0 else ""
-
-    minutes, seconds = divmod(seconds, 60)
-    hours, minutes = divmod(minutes, 60)
-    days, hours = divmod(hours, 24)
-    parts = [a for a in [
-        include(days, "day"),
-        include(hours, "hour"),
-        include(minutes, "minute"),
-        include(seconds, "second")] if a]
-    if len(parts) > 2:
-        return "{} and {}".format(", ".join(parts[:-1]), parts[-1])
-    return " and ".join(parts)
+        return ensure_trailing_newline(html) if html else None
 
 
 class CommandHandler:
+    """A command which can be executed from a Matrix room.
+
+    The command manages its permission and help texts.
+    When called, it will check the permission of the command event and execute
+    the command or, in case of error, report back to the user.
+
+    Attributes:
+        needs_auth: Flag indicating if the sender is required to be logged in.
+        needs_puppeting: Flag indicating if the sender is required to use
+            Telegram puppeteering for this command.
+        needs_matrix_puppeting: Flag indicating if the sender is required to use
+            Matrix pupeteering.
+        needs_admin: Flag for whether only admin users can issue this command.
+        management_only: Whether the command can exclusively be issued in a
+            management room.
+        name: The name of this command.
+        help_section: Section of the help in which this command will appear.
+    """
+
+    def __init__(self, handler: Callable[[CommandEvent], Awaitable[Dict]], needs_auth: bool,
+                 needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
+                 management_only: bool, name: str, help_text: str, help_args: str,
+                 help_section: HelpSection) -> None:
+        """
+        Args:
+            handler: The function handling the execution of this command.
+            needs_auth: Flag indicating if the sender is required to be logged in.
+            needs_puppeting: Flag indicating if the sender is required to use
+                Telegram puppeteering for this command.
+            needs_matrix_puppeting: Flag indicating if the sender is required to
+                use Matrix pupeteering.
+            needs_admin: Flag for whether only admin users can issue this command.
+            management_only: Whether the command can exclusively be issued
+                in a management room.
+            name: The name of this command.
+            help_text: The text displayed in the help for this command.
+            help_args: Help text for the arguments of this command.
+            help_section: Section of the help in which this command will appear.
+        """
+        self._handler = handler
+        self.needs_auth = needs_auth
+        self.needs_puppeting = needs_puppeting
+        self.needs_matrix_puppeting = needs_matrix_puppeting
+        self.needs_admin = needs_admin
+        self.management_only = management_only
+        self.name = name
+        self._help_text = help_text
+        self._help_args = help_args
+        self.help_section = help_section
+
+    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        """Returns the reason why the command could not be issued.
+
+        Args:
+            evt: The event for which to get the error information.
+
+        Returns:
+            A string describing the error or None if there was no error.
+        """
+        if self.management_only and not evt.is_management:
+            return (f"`{evt.command}` is a restricted command: "
+                    "you may only run it in management rooms.")
+        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
+            return "This command requires puppeting privileges."
+        elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
+            return "This command requires Matrix puppeting privileges."
+        elif self.needs_admin and not evt.sender.is_admin:
+            return "This command requires administrator privileges."
+        elif self.needs_auth and not await evt.sender.is_logged_in():
+            return "This command requires you to be logged in."
+        return None
+
+    def has_permission(self, is_management: bool, puppet_whitelisted: bool,
+                       matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
+        """Checks the permission for this command with the given status.
+
+        Args:
+            is_management: If the room in which the command will be issued is a
+                management room.
+            puppet_whitelisted: If the connected Telegram account puppet is
+                allowed to issue the command.
+            matrix_puppet_whitelisted: If the connected Matrix account puppet is
+                allowed to issue the command.
+            is_admin: If the issuing user is an admin.
+            is_logged_in: If the issuing user is logged in.
+
+        Returns:
+            True if a user with the given state is allowed to issue the
+            command.
+        """
+        return ((not self.management_only or is_management) and
+                (not self.needs_puppeting or puppet_whitelisted) and
+                (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
+                (not self.needs_admin or is_admin) and
+                (not self.needs_auth or is_logged_in))
+
+    async def __call__(self, evt: CommandEvent) -> Dict:
+        """Executes the command if evt was issued with proper rights.
+
+        Args:
+            evt: The CommandEvent for which to check permissions.
+
+        Returns:
+            The result of the command or the error message function.
+
+        Raises:
+            FloodWaitError
+        """
+        error = await self.get_permission_error(evt)
+        if error is not None:
+            return await evt.reply(error)
+        return await self._handler(evt)
+
+    @property
+    def has_help(self) -> bool:
+        """Returns true if this command has a help text."""
+        return bool(self.help_section) and bool(self._help_text)
+
+    @property
+    def help(self) -> str:
+        """Returns the help text to this command."""
+        return f"**{self.name}** {self._help_args} - {self._help_text}"
+
+
+def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
+                    needs_auth: bool = True, needs_puppeting: bool = True,
+                    needs_matrix_puppeting: bool = False, needs_admin: bool = False,
+                    management_only: bool = False, name: Optional[str] = None,
+                    help_text: str = "", help_args: str = "", help_section: HelpSection = None
+                    ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
+                                  CommandHandler]:
+    def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:
+        actual_name = name or func.__name__.replace("_", "-")
+        handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
+                                 needs_admin, management_only, actual_name, help_text, help_args,
+                                 help_section)
+        command_handlers[handler.name] = handler
+        return handler
+
+    return decorator if _func is None else decorator(_func)
+
+
+class CommandProcessor:
+    """Handles the raw commands issued by a user to the Matrix bot."""
     log = logging.getLogger("mau.commands")
 
-    def __init__(self, context):
-        self.az, self.db, self.config, self.loop = context
+    def __init__(self, context: c.Context) -> None:
+        self.az, self.config, self.loop, self.tgbot = context.core
+        self.public_website = context.public_website
         self.command_prefix = self.config["bridge.command_prefix"]
 
-    # region Utility functions for handling commands
+    async def handle(self, room: MatrixRoomID, event_id: MatrixEventID, sender: u.User,
+                     command: str, args: List[str], is_management: bool, is_portal: bool
+                     ) -> Optional[Dict]:
+        """Handles the raw commands issued by a user to the Matrix bot.
 
-    async def handle(self, room, sender, command, args, is_management, is_portal):
-        evt = CommandEvent(self, room, sender, command, args,
-                           is_management, is_portal)
+        If the command is not known, it might be a followup command and is
+        delegated to a command handler registered for that purpose in the
+        senders command_status as "next".
+
+        Args:
+            room: ID of the Matrix room in which the command was issued.
+            event_id: ID of the event by which the command was issued.
+            sender: The sender who issued the command.
+            command: The issued command, case insensitive.
+            args: Arguments given with the command.
+            is_management: Whether the room is a management room.
+            is_portal: Whether the room is a portal.
+
+        Returns:
+            The result of the error message function or None if no error
+            occured. Unknown and delegated commands do not count as errors.
+        """
+        if not command_handlers or "unknown-command" not in command_handlers:
+            raise ValueError("command_handlers are not properly initialized.")
+
+        evt = CommandEvent(self, room, event_id, sender, command, args, is_management, is_portal)
+        orig_command = command
         command = command.lower()
         try:
-            command = command_handlers[command]
+            handler = command_handlers[command]
         except KeyError:
             if sender.command_status and "next" in sender.command_status:
-                args.insert(0, command)
+                args.insert(0, orig_command)
                 evt.command = ""
-                command = sender.command_status["next"]
+                handler = sender.command_status["next"]
             else:
-                command = command_handlers["unknown-command"]
+                handler = command_handlers["unknown-command"]
         try:
-            await command(evt)
+            await handler(evt)
         except FloodWaitError as e:
-            return evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
+            return await evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
         except Exception:
-            self.log.exception(f"Fatal error handling command "
-                               + f"{evt.command} {' '.join(args)} from {sender.mxid}")
-            return evt.reply("Fatal error while handling command. Check logs for more details.")
+            self.log.exception("Unhandled error while handling command "
+                               f"{evt.command} {' '.join(args)} from {sender.mxid}")
+            if evt.sender.is_admin and evt.is_management:
+                return await evt.reply("Unhandled error while handling command:\n\n"
+                                       "```traceback\n"
+                                       f"{traceback.format_exc()}"
+                                       "```")
+            return await evt.reply("Unhandled error while handling command. "
+                                   "Check logs for more details.")
+        return None

mautrix_telegram/commands/matrix_auth.py

@@ -0,0 +1,101 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from . import command_handler, CommandEvent, SECTION_AUTH
+from .. import puppet as pu
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
+                           "account.")
+async def logout_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
+
+
+@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account.")
+async def login_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        token = evt.public_website.make_token(evt.sender.mxid, "/matrix-login")
+        url = f"{prefix}/matrix-login?token={token}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Pings the server with the stored matrix authentication.")
+async def ping_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    resp = await puppet.init_custom_mxid()
+    if resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Your access token is invalid.")
+    elif resp == pu.PuppetError.Success:
+        return await evt.reply("Your Matrix login is working.")
+    return await evt.reply(f"Unknown response while checking your Matrix login: {resp}.")
+
+
+async def enter_matrix_token(evt: CommandEvent) -> Dict:
+    evt.sender.command_status = None
+
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+
+    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    if resp == pu.PuppetError.OnlyLoginSelf:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    elif resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Failed to verify access token.")
+    assert resp == pu.PuppetError.Success, "Encountered an unhandled PuppetError."
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")

mautrix_telegram/commands/meta.py

@@ -1,76 +1,72 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from . import command_handler
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Optional, Tuple
+
+from . import command_handler, CommandEvent, _command_handlers, SECTION_GENERAL
+from .handler import HelpSection
 
 
-@command_handler()
-def cancel(evt):
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Cancel an ongoing action (such as login)")
+async def cancel(evt: CommandEvent) -> Optional[Dict]:
     if evt.sender.command_status:
         action = evt.sender.command_status["action"]
         evt.sender.command_status = None
-        return evt.reply(f"{action} cancelled.")
+        return await evt.reply(f"{action} cancelled.")
     else:
-        return evt.reply("No ongoing command.")
+        return await evt.reply("No ongoing command.")
 
 
-@command_handler()
-def unknown_command(evt):
-    return evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
+@command_handler(needs_auth=False, needs_puppeting=False)
+async def unknown_command(evt: CommandEvent) -> Optional[Dict]:
+    return await evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
 
 
-@command_handler()
-def help(evt):
+help_cache = {}  # type: Dict[Tuple[bool, bool, bool, bool, bool], str]
+
+
+async def _get_help_text(evt: CommandEvent) -> str:
+    cache_key = (evt.is_management, evt.sender.puppet_whitelisted,
+                 evt.sender.matrix_puppet_whitelisted, evt.sender.is_admin,
+                 await evt.sender.is_logged_in())
+    if cache_key not in help_cache:
+        help_sections = {}  # type: Dict[HelpSection, List[str]]
+        for handler in _command_handlers.values():
+            if handler.has_help and handler.has_permission(*cache_key):
+                help_sections.setdefault(handler.help_section, [])
+                help_sections[handler.help_section].append(handler.help + "  ")
+        help_sorted = sorted(help_sections.items(), key=lambda item: item[0].order)
+        helps = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help_sorted]
+        help_cache[cache_key] = "\n".join(helps)
+    return help_cache[cache_key]
+
+
+def _get_management_status(evt: CommandEvent) -> str:
     if evt.is_management:
-        management_status = ("This is a management room: prefixing commands "
-                             "with `$cmdprefix` is not required.\n")
+        return "This is a management room: prefixing commands with `$cmdprefix` is not required."
     elif evt.is_portal:
-        management_status = ("**This is a portal room**: you must always "
-                             "prefix commands with `$cmdprefix`.\n"
-                             "Management commands will not be sent to Telegram.")
-    else:
-        management_status = ("**This is not a management room**: you must "
-                             "prefix commands with `$cmdprefix`.\n")
-    help = """\n
-#### Generic bridge commands
-**help**   - Show this help message.  
-**cancel** - Cancel an ongoing action (such as login).
+        return ("**This is a portal room**: you must always prefix commands with `$cmdprefix`.\n"
+                "Management commands will not be sent to Telegram.")
+    return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
 
-#### Authentication
-**login** <_phone_> - Request an authentication code.  
-**logout**          - Log out from Telegram.  
-**ping**            - Check if you're logged into Telegram.
 
-#### Initiating chats
-**search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users.  
-**pm** <_identifier_>                - Open a private chat with the given Telegram user. The
-                                       identifier is either the internal user ID, the username or
-                                       the phone number.  
-**join** <_link_>                    - Join a chat with an invite link.  
-**create** [_type_]                  - Create a Telegram chat of the given type for the current
-                                       Matrix room. The type is either `group`, `supergroup` or
-                                       `channel` (defaults to `group`).
-
-#### Portal management  
-**upgrade**                 - Upgrade a normal Telegram group to a supergroup.  
-**invite-link**             - Get a Telegram invite link to the current chat.  
-**delete-portal**           - Forget the current portal room. Only works for group chats; to delete
-                             a private chat portal, simply leave the room.  
-**group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash
-                             (`-`) as the name.  
-**clean-rooms**             - Clean up unused portal/management rooms.
-"""
-    return evt.reply(management_status + help)
+@command_handler(name="help", needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Show this help message.")
+async def help_cmd(evt: CommandEvent) -> Optional[Dict]:
+    return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))

mautrix_telegram/commands/portal/__init__.py

@@ -0,0 +1 @@
+from . import admin, bridge, config, create_chat, filter, misc, unbridge

mautrix_telegram/commands/portal/admin.py

@@ -0,0 +1,101 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+import asyncio
+
+from mautrix_appservice import MatrixRequestError
+
+from ... import portal as po, puppet as pu, user as u
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
+                 help_section=SECTION_ADMIN,
+                 help_args="<_level_> [_mxid_]",
+                 help_text="Set a temporary power level without affecting Telegram.")
+async def set_power_level(evt: CommandEvent) -> Dict:
+    try:
+        level = int(evt.args[0])
+    except KeyError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp set-pl <level> [mxid]`")
+    except ValueError:
+        return await evt.reply("The level must be an integer.")
+    levels = await evt.az.intent.get_power_levels(evt.room_id)
+    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
+    levels["users"][mxid] = level
+    try:
+        await evt.az.intent.set_power_levels(evt.room_id, levels)
+    except MatrixRequestError:
+        evt.log.exception("Failed to set power level.")
+        return await evt.reply("Failed to set power level.")
+    return {}
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`portal`|`puppet`|`user`>",
+                 help_text="Clear internal bridge caches")
+async def clear_db_cache(evt: CommandEvent) -> Dict:
+    try:
+        section = evt.args[0].lower()
+    except IndexError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+    if section == "portal":
+        po.Portal.by_tgid = {}
+        po.Portal.by_mxid = {}
+        await evt.reply("Cleared portal cache")
+    elif section == "puppet":
+        pu.Puppet.cache = {}
+        for puppet in pu.Puppet.by_custom_mxid.values():
+            puppet.sync_task.cancel()
+        pu.Puppet.by_custom_mxid = {}
+        await asyncio.gather(
+            *[puppet.init_custom_mxid() for puppet in pu.Puppet.all_with_custom_mxid()],
+            loop=evt.loop)
+        await evt.reply("Cleared puppet cache and restarted custom puppet syncers")
+    elif section == "user":
+        u.User.by_mxid = {
+            user.mxid: user
+            for user in u.User.by_tgid.values()
+        }
+        await evt.reply("Cleared non-logged-in user cache")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="[_mxid_]",
+                 help_text="Reload and reconnect a user")
+async def reload_user(evt: CommandEvent) -> Dict:
+    if len(evt.args) > 0:
+        mxid = evt.args[0]
+    else:
+        mxid = evt.sender.mxid
+    user = u.User.get_by_mxid(mxid, create=False)
+    if not user:
+        return await evt.reply("User not found")
+    puppet = pu.Puppet.get_by_custom_mxid(mxid)
+    if puppet:
+        puppet.sync_task.cancel()
+    await user.stop()
+    user.delete(delete_db=False)
+    user = u.User.get_by_mxid(mxid)
+    await user.ensure_started()
+    if puppet:
+        await puppet.init_custom_mxid()
+    await evt.reply(f"Reloaded and reconnected {user.mxid} (telegram: {user.human_tg_id})")

mautrix_telegram/commands/portal/bridge.py

@@ -0,0 +1,187 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional, Tuple, Coroutine
+import asyncio
+
+from telethon.tl.types import ChatForbidden, ChannelForbidden
+
+from ...types import MatrixRoomID, TelegramID
+from ...util import ignore_coro
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_id_]",
+                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
+                           "ID. The ID must be the prefixed version that you get with the `/id` "
+                           "command of the Telegram-side bot.")
+async def bridge(evt: CommandEvent) -> Dict:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** "
+                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    force_use_bot = False
+    if evt.args[0] == "--usebot" and evt.sender.is_admin:
+        force_use_bot = True
+        evt.args = evt.args[1:]
+    room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
+    that_this = "This" if room_id == evt.room_id else "That"
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if portal:
+        return await evt.reply(f"{that_this} room is already a portal room.")
+
+    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
+
+    # The /id bot command provides the prefixed ID, so we assume
+    tgid_str = evt.args[0]
+    if tgid_str.startswith("-100"):
+        tgid = TelegramID(int(tgid_str[4:]))
+        peer_type = "channel"
+    elif tgid_str.startswith("-"):
+        tgid = TelegramID(-int(tgid_str))
+        peer_type = "chat"
+    else:
+        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
+                               "If you did not get the ID using the `/id` bot command, please "
+                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+                               "Bridging private chats to existing rooms is not allowed.")
+
+    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    if not portal.allow_bridging():
+        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
+                               "If you're the bridge admin, try "
+                               "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
+    if portal.mxid:
+        has_portal_message = (
+            "That Telegram chat already has a portal at "
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
+        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+            return await evt.reply(f"{has_portal_message}"
+                                   "Additionally, you do not have the permissions to unbridge "
+                                   "that room.")
+        evt.sender.command_status = {
+            "next": confirm_bridge,
+            "action": "Room bridging",
+            "mxid": portal.mxid,
+            "bridge_to_mxid": room_id,
+            "tgid": portal.tgid,
+            "peer_type": portal.peer_type,
+            "force_use_bot": force_use_bot,
+        }
+        return await evt.reply(f"{has_portal_message}"
+                               "However, you have the permissions to unbridge that room.\n\n"
+                               "To delete that portal completely and continue bridging, use "
+                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
+    evt.sender.command_status = {
+        "next": confirm_bridge,
+        "action": "Room bridging",
+        "bridge_to_mxid": room_id,
+        "tgid": portal.tgid,
+        "peer_type": portal.peer_type,
+        "force_use_bot": force_use_bot,
+    }
+    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
+                           "chat to this room, use `$cmdprefix+sp continue`")
+
+
+async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
+                                            ) -> Tuple[bool, Optional[Coroutine[None, None, None]]]:
+    if not portal.mxid:
+        await evt.reply("The portal seems to have lost its Matrix room between you"
+                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                        "Continuing without touching previous Matrix room...")
+        return True, None
+    elif evt.args[0] == "delete-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Portal deleted (moving to another room)")
+    elif evt.args[0] == "unbridge-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Room unbridged (portal moving to another room)",
+                                         puppets_only=True)
+    else:
+        await evt.reply(
+            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
+            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
+            "continue` to either delete or unbridge the existing room (respectively) and "
+            "continue with the bridging.\n\n"
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
+        return False, None
+
+
+async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
+    status = evt.sender.command_status
+    try:
+        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        bridge_to_mxid = status["bridge_to_mxid"]
+    except KeyError:
+        evt.sender.command_status = None
+        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
+                               "This shouldn't happen unless you're messing with the command "
+                               "handler code.")
+    if "mxid" in status:
+        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
+        if not ok:
+            return None
+        elif coro:
+            ignore_coro(asyncio.ensure_future(coro, loop=evt.loop))
+            await evt.reply("Cleaning up previous portal room...")
+    elif portal.mxid:
+        evt.sender.command_status = None
+        return await evt.reply("The portal seems to have created a Matrix room between you "
+                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                               "Please start over by calling the bridge command again.")
+    elif evt.args[0] != "continue":
+        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
+                               "`$cmdprefix+sp cancel` to cancel.")
+
+    evt.sender.command_status = None
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+    user = evt.sender if is_logged_in else evt.tgbot
+    try:
+        entity = await user.client.get_entity(portal.peer)
+    except Exception:
+        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+        if is_logged_in:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You are logged in, are you in that chat?")
+        else:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You're not logged in, is the relay bot in the chat?")
+    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
+        if is_logged_in:
+            return await evt.reply("You don't seem to be in that chat.")
+        else:
+            return await evt.reply("The bot doesn't seem to be in that chat.")
+
+    direct = False
+
+    portal.mxid = bridge_to_mxid
+    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    portal.photo_id = ""
+    portal.save()
+
+    ignore_coro(asyncio.ensure_future(portal.update_matrix_room(user, entity, direct,
+                                                                levels=levels),
+                                      loop=evt.loop))
+
+    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -0,0 +1,132 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Awaitable
+from io import StringIO
+
+from ...config import yaml
+from ... import portal as po, util
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="View or change per-portal settings.",
+                 help_args="<`help`|_subcommand_> [...]")
+async def config(evt: CommandEvent) -> None:
+    cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
+    if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
+        await config_help(evt)
+        return
+    elif cmd == "defaults":
+        await config_defaults(evt)
+        return
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        await evt.reply("This is not a portal room.")
+        return
+    elif cmd == "view":
+        await config_view(evt, portal)
+        return
+
+    key = evt.args[1] if len(evt.args) > 1 else None
+    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    if cmd == "set":
+        await config_set(evt, portal, key, value)
+    elif cmd == "unset":
+        await config_unset(evt, portal, key)
+    elif cmd == "add" or cmd == "del":
+        await config_add_del(evt, portal, key, value, cmd)
+    else:
+        return
+    portal.save()
+
+
+def config_help(evt: CommandEvent) -> Awaitable[Dict]:
+    return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
+
+* **help** - View this help text.
+* **view** - View the current config data.
+* **defaults** - View the default config values.
+* **set** <_key_> <_value_> - Set a config value.
+* **unset** <_key_> - Remove a config value.
+* **add** <_key_> <_value_> - Add a value to an array.
+* **del** <_key_> <_value_> - Remove a value from an array.
+""")
+
+
+def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump(portal.local_config, stream)
+    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
+
+
+def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump({
+        "bridge_notices": {
+            "default": evt.config["bridge.bridge_notices.default"],
+            "exceptions": evt.config["bridge.bridge_notices.exceptions"],
+        },
+        "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
+        "inline_images": evt.config["bridge.inline_images"],
+        "message_formats": evt.config["bridge.message_formats"],
+        "state_event_formats": evt.config["bridge.state_event_formats"],
+        "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
+    }, stream)
+    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
+
+
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[Dict]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
+    elif util.recursive_set(portal.local_config, key, value):
+        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
+    else:
+        return evt.reply(f"Failed to set value of `{key}`. "
+                         "Does the path contain non-map types?")
+
+
+def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Dict]:
+    if not key:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config unset <key>`")
+    elif util.recursive_del(portal.local_config, key):
+        return evt.reply(f"Successfully deleted `{key}` from config.")
+    else:
+        return evt.reply(f"`{key}` not found in config.")
+
+
+def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
+                   ) -> Awaitable[Dict]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
+
+    arr = util.recursive_get(portal.local_config, key)
+    if not arr:
+        return evt.reply(f"`{key}` not found in config. "
+                         f"Maybe do `$cmdprefix+sp config set {key} []` first?")
+    elif not isinstance(arr, list):
+        return evt.reply("`{key}` does not seem to be an array.")
+    elif cmd == "add":
+        if value in arr:
+            return evt.reply(f"The array at `{key}` already contains `{value}`.")
+        arr.append(value)
+        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
+    else:
+        if value not in arr:
+            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
+        arr.remove(value)
+        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -0,0 +1,61 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+
+from ... import portal as po
+from ...types import TelegramID
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_type_]",
+                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
+                           "The type is either `group`, `supergroup` or `channel` (defaults to "
+                           "`group`).")
+async def create(evt: CommandEvent) -> Dict:
+    type = evt.args[0] if len(evt.args) > 0 else "group"
+    if type not in {"chat", "group", "supergroup", "channel"}:
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
+
+    if po.Portal.get_by_mxid(evt.room_id):
+        return await evt.reply("This is already a portal room.")
+
+    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply("You do not have the permissions to bridge this room.")
+
+    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    if not title:
+        return await evt.reply("Please set a title before creating a Telegram chat.")
+
+    supergroup = type == "supergroup"
+    type = {
+        "supergroup": "channel",
+        "channel": "channel",
+        "chat": "chat",
+        "group": "chat",
+    }[type]
+
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+                       mxid=evt.room_id, title=title, about=about)
+    try:
+        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+    except ValueError as e:
+        portal.delete()
+        return await evt.reply(e.args[0])
+    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/filter.py

@@ -0,0 +1,95 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`>",
+                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
+                           "default.")
+async def filter_mode(evt: CommandEvent) -> Dict:
+    try:
+        mode = evt.args[0]
+        if mode not in ("whitelist", "blacklist"):
+            raise ValueError()
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
+
+    evt.config["bridge.filter.mode"] = mode
+    evt.config.save()
+    po.Portal.filter_mode = mode
+    if mode == "whitelist":
+        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
+                               "To allow bridging a specific chat, use"
+                               "`!filter whitelist <chat ID>`.")
+    else:
+        return await evt.reply("The bridge will now allow bridging chats by default.\n"
+                               "To disallow bridging a specific chat, use"
+                               "`!filter blacklist <chat ID>`.")
+
+
+@command_handler(name="filter", needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+                 help_text="Allow or disallow bridging a specific chat.")
+async def edit_filter(evt: CommandEvent) -> Optional[Dict]:
+    try:
+        action = evt.args[0]
+        if action not in ("whitelist", "blacklist", "add", "remove"):
+            raise ValueError()
+
+        id_str = evt.args[1]
+        if id_str.startswith("-100"):
+            filter_id = int(id_str[4:])
+        elif id_str.startswith("-"):
+            filter_id = int(id_str[1:])
+        else:
+            filter_id = int(id_str)
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
+
+    mode = evt.config["bridge.filter.mode"]
+    if mode not in ("blacklist", "whitelist"):
+        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
+
+    filter_id_list = evt.config["bridge.filter.list"]
+
+    if action in ("blacklist", "whitelist"):
+        action = "add" if mode == action else "remove"
+
+    def save() -> None:
+        evt.config["bridge.filter.list"] = filter_id_list
+        evt.config.save()
+        po.Portal.filter_list = filter_id_list
+
+    if action == "add":
+        if filter_id in filter_id_list:
+            return await evt.reply(f"That chat is already {mode}ed.")
+        filter_id_list.append(filter_id)
+        save()
+        return await evt.reply(f"Chat ID added to {mode}.")
+    elif action == "remove":
+        if filter_id not in filter_id_list:
+            return await evt.reply(f"That chat is not {mode}ed.")
+        filter_id_list.remove(filter_id)
+        save()
+        return await evt.reply(f"Chat ID removed from {mode}.")
+    return None

mautrix_telegram/commands/portal/misc.py

@@ -0,0 +1,127 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+
+from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
+                             UsernameNotModifiedError, UsernameOccupiedError)
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT, SECTION_MISC
+from .util import user_has_power_level
+
+
+@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.")
+async def sync_state(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to synchronize this room.")
+
+    await portal.sync_matrix_members()
+    await evt.reply("Synchronization complete")
+
+
+@command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Get the ID of the Telegram chat where this room is bridged.")
+async def get_id(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    tgid = portal.tgid
+    if portal.peer_type == "chat":
+        tgid = -tgid
+    elif portal.peer_type == "channel":
+        tgid = f"-100{tgid}"
+    await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Get a Telegram invite link to the current chat.")
+async def invite_link(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if portal.peer_type == "user":
+        return await evt.reply("You can't invite users to private chats.")
+
+    try:
+        link = await portal.get_invite_link(evt.sender)
+        return await evt.reply(f"Invite link to {portal.title}: {link}")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to create an invite link.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Upgrade a normal Telegram group to a supergroup.")
+async def upgrade(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type == "channel":
+        return await evt.reply("This is already a supergroup or a channel.")
+    elif portal.peer_type == "user":
+        return await evt.reply("You can't upgrade private chats.")
+
+    try:
+        await portal.upgrade_telegram_chat(evt.sender)
+        return await evt.reply(f"Group upgraded to supergroup. New ID: -100{portal.tgid}")
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to upgrade this group.")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="<_name_|`-`>",
+                 help_text="Change the username of a supergroup/channel. "
+                           "To disable, use a dash (`-`) as the name.")
+async def group_name(evt: CommandEvent) -> Dict:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type != "channel":
+        return await evt.reply("Only channels and supergroups have usernames.")
+
+    try:
+        await portal.set_telegram_username(evt.sender,
+                                           evt.args[0] if evt.args[0] != "-" else "")
+        if portal.username:
+            return await evt.reply(f"Username of channel changed to {portal.username}.")
+        else:
+            return await evt.reply(f"Channel is now private.")
+    except ChatAdminRequiredError:
+        return await evt.reply(
+            "You don't have the permission to set the username of this channel.")
+    except UsernameNotModifiedError:
+        if portal.username:
+            return await evt.reply("That is already the username of this channel.")
+        else:
+            return await evt.reply("This channel is already private")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username")

mautrix_telegram/commands/portal/unbridge.py

@@ -0,0 +1,97 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Callable, Optional
+
+from ...types import MatrixRoomID
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+from .util import user_has_power_level
+
+
+async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
+                                           action: Optional[str] = None
+                                           ) -> Optional[po.Portal]:
+    room_id = MatrixRoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if not portal:
+        that_this = "This" if room_id == evt.room_id else "That"
+        await evt.reply(f"{that_this} is not a portal room.")
+        return None
+
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
+        action = action or f"{permission.replace('_', ' ')}s"
+        await evt.reply(f"You do not have the permissions to {action} that portal.")
+        return None
+    return portal
+
+
+def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
+                                completed_message: str) -> Dict:
+    async def post_confirm(confirm) -> Optional[Dict]:
+        confirm.sender.command_status = None
+        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
+            await function()
+            if confirm.room_id != room_id:
+                return await confirm.reply(completed_message)
+        else:
+            return await confirm.reply(f"{action} cancelled.")
+        return None
+
+    return {
+        "next": post_confirm,
+        "action": action,
+    }
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove all users from the current portal room and forget the portal. "
+                           "Only works for group chats; to delete a private chat portal, simply "
+                           "leave the room.")
+async def delete_portal(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
+                                                            portal.cleanup_and_delete, "delete",
+                                                            "Portal successfully deleted.")
+    return await evt.reply("Please confirm deletion of portal "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           f"to Telegram chat \"{portal.title}\" "
+                           "by typing `$cmdprefix+sp confirm-delete`"
+                           "\n\n"
+                           "**WARNING:** If the bridge bot has the power level to do so, **this "
+                           "will kick ALL users** in the room. If you just want to remove the "
+                           "bridge, use `$cmdprefix+sp unbridge` instead.")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove puppets from the current portal room and forget the portal.")
+async def unbridge(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
+                                                            portal.unbridge, "unbridge",
+                                                            "Room successfully unbridged.")
+    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           "by typing `$cmdprefix+sp confirm-unbridge`")

mautrix_telegram/commands/portal/util.py

@@ -0,0 +1,56 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Tuple
+
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from ... import user as u
+
+
+async def get_initial_state(intent: IntentAPI, room_id: str) -> Tuple[str, str, Dict]:
+    state = await intent.get_room_state(room_id)
+    title = None
+    about = None
+    levels = None
+    for event in state:
+        try:
+            if event["type"] == "m.room.name":
+                title = event["content"]["name"]
+            elif event["type"] == "m.room.topic":
+                about = event["content"]["topic"]
+            elif event["type"] == "m.room.power_levels":
+                levels = event["content"]
+            elif event["type"] == "m.room.canonical_alias":
+                title = title or event["content"]["alias"]
+        except KeyError:
+            # Some state event probably has empty content
+            pass
+    return title, about, levels
+
+
+async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50
+                               ) -> bool:
+    if sender.is_admin:
+        return True
+    # Make sure the state store contains the power levels.
+    try:
+        await intent.get_power_levels(room)
+    except MatrixRequestError:
+        return False
+    return intent.state_store.has_power_level(room, sender.mxid,
+                                              event=f"net.maunium.telegram.{event}",
+                                              default=default)

mautrix_telegram/commands/telegram.py

@@ -1,261 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from telethon.errors import *
-from telethon.tl.types import User as TLUser
-from telethon.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest
-from telethon.tl.functions.channels import JoinChannelRequest
-
-from .. import puppet as pu, portal as po
-from . import command_handler
-
-
-@command_handler()
-async def search(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
-
-    force_remote = False
-    if evt.args[0] in {"-r", "--remote"}:
-        force_remote = True
-        evt.args.pop(0)
-
-    query = " ".join(evt.args)
-    if force_remote and len(query) < 5:
-        return await evt.reply("Minimum length of query for remote search is 5 characters.")
-
-    results, remote = await evt.sender.search(query, force_remote)
-
-    if not results:
-        if len(query) < 5 and remote:
-            return await evt.reply("No local results. "
-                                   "Minimum length of remote query is 5 characters.")
-        return await evt.reply("No results 3:")
-
-    reply = []
-    if remote:
-        reply += ["**Results from Telegram server:**", ""]
-    else:
-        reply += ["**Results in contacts:**", ""]
-    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
-               + f"{puppet.id} ({similarity}% match)")
-              for puppet, similarity in results]
-
-    # TODO somehow show remote channel results when joining by alias is possible?
-
-    return await evt.reply("\n".join(reply))
-
-
-@command_handler()
-async def pm(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
-
-    user = await evt.sender.client.get_entity(evt.args[0])
-    if not user:
-        return await evt.reply("User not found.")
-    elif not isinstance(user, TLUser):
-        return await evt.reply("That doesn't seem to be a user.")
-    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
-    await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply(
-        f"Created private chat room with {pu.Puppet.get_displayname(user, False)}")
-
-
-@command_handler()
-async def invite_link(evt):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
-        return await evt.reply("You can't invite users to private chats.")
-
-    try:
-        link = await portal.get_invite_link(evt.sender)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to create an invite link.")
-
-
-@command_handler(needs_admin=True)
-async def delete_portal(evt):
-    room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if not portal:
-        that_this = "This" if room_id == evt.room_id else "That"
-        return await evt.reply(f"{that_this} is not a portal room.")
-
-    async def post_confirm(confirm):
-        evt.sender.command_status = None
-        if len(confirm.args) > 0 and confirm.args[0] == "confirm-delete":
-            await portal.cleanup_and_delete()
-            if confirm.room_id != room_id:
-                return await confirm.reply("Portal successfully deleted.")
-        else:
-            return await confirm.reply("Portal deletion cancelled.")
-
-    evt.sender.command_status = {
-        "next": post_confirm,
-        "action": "Portal deletion",
-    }
-    return await evt.reply("Please confirm deletion of portal "
-                           + f"[{room_id}](https://matrix.to/#/{room_id}) "
-                           + f"to Telegram chat \"{portal.title}\" "
-                           + "by typing `$cmdprefix+sp confirm-delete`")
-
-
-@command_handler()
-async def join(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
-
-    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
-    arg = regex.match(evt.args[0])
-    if not arg:
-        return await evt.reply("That doesn't look like a Telegram invite link.")
-    arg = arg.group(1)
-    if arg.startswith("joinchat/"):
-        invite_hash = arg[len("joinchat/"):]
-        try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
-        except InviteHashInvalidError:
-            return await evt.reply("Invalid invite link.")
-        except InviteHashExpiredError:
-            return await evt.reply("Invite link expired.")
-        try:
-            updates = evt.sender.client(ImportChatInviteRequest(invite_hash))
-        except UserAlreadyParticipantError:
-            return await evt.reply("You are already in that chat.")
-    else:
-        channel = await evt.sender.client.get_entity(arg)
-        if not channel:
-            return await evt.reply("Channel/supergroup not found.")
-        updates = await evt.sender.client(JoinChannelRequest(channel))
-    for chat in updates.chats:
-        portal = po.Portal.get_by_entity(chat)
-        if portal.mxid:
-            await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
-            return await evt.reply(f"Created room for {portal.title}")
-        else:
-            await portal.invite_matrix([evt.sender.mxid])
-            return await evt.reply(f"Invited you to portal of {portal.title}")
-
-
-@command_handler()
-async def create(evt):
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
-
-    if po.Portal.get_by_mxid(evt.room_id):
-        return await evt.reply("This is already a portal room.")
-
-    state = await evt.az.intent.get_room_state(evt.room_id)
-    title = None
-    about = None
-    levels = None
-    for event in state:
-        if event["type"] == "m.room.name":
-            title = event["content"]["name"]
-        elif event["type"] == "m.room.topic":
-            about = event["content"]["topic"]
-        elif event["type"] == "m.room.power_levels":
-            levels = event["content"]
-    if not title:
-        return await evt.reply("Please set a title before creating a Telegram chat.")
-    elif (not levels or not levels["users"] or evt.az.intent.mxid not in levels["users"] or
-          levels["users"][evt.az.intent.mxid] < 100):
-        return await evt.reply(f"Please give "
-                               + f"[the bridge bot](https://matrix.to/#/{evt.az.intent.mxid})"
-                               + f" a power level of 100 before creating a Telegram chat.")
-    else:
-        for user, level in levels["users"].items():
-            if level >= 100 and user != evt.az.intent.mxid:
-                return await evt.reply(
-                    f"Please make sure only the bridge bot has power level above"
-                    + f"99 before creating a Telegram chat.\n\n"
-                    + f"Use power level 95 instead of 100 for admins.")
-
-    supergroup = type == "supergroup"
-    type = {
-        "supergroup": "channel",
-        "channel": "channel",
-        "chat": "chat",
-        "group": "chat",
-    }[type]
-
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
-    try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
-
-
-@command_handler()
-async def upgrade(evt):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type == "channel":
-        return await evt.reply("This is already a supergroup or a channel.")
-    elif portal.peer_type == "user":
-        return await evt.reply("You can't upgrade private chats.")
-
-    try:
-        await portal.upgrade_telegram_chat(evt.sender)
-        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to upgrade this group.")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-
-
-@command_handler()
-async def group_name(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type != "channel":
-        return await evt.reply("Only channels and supergroups have usernames.")
-
-    try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
-        if portal.username:
-            return await evt.reply(f"Username of channel changed to {portal.username}.")
-        else:
-            return await evt.reply(f"Channel is now private.")
-    except ChatAdminRequiredError:
-        return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
-    except UsernameNotModifiedError:
-        if portal.username:
-            return await evt.reply("That is already the username of this channel.")
-        else:
-            return await evt.reply("This channel is already private")
-    except UsernameOccupiedError:
-        return await evt.reply("That username is already in use.")
-    except UsernameInvalidError:
-        return await evt.reply("Invalid username")

mautrix_telegram/commands/telegram/__init__.py

@@ -0,0 +1 @@
+from . import account, auth, misc

mautrix_telegram/commands/telegram/account.py

@@ -0,0 +1,124 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
+from telethon.tl.types import Authorization
+from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
+                                           ResetAuthorizationRequest, UpdateProfileRequest)
+
+from .. import command_handler, CommandEvent, SECTION_AUTH
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new username_>",
+                 help_text="Change your Telegram username.")
+async def username(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp username <new username>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own username.")
+    new_name = evt.args[0]
+    if new_name == "-":
+        new_name = ""
+    try:
+        await evt.sender.client(UpdateUsernameRequest(username=new_name))
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username. Usernames must be between 5 and 30 alphanumeric "
+                               "characters.")
+    except UsernameNotModifiedError:
+        return await evt.reply("That is your current username.")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    await evt.sender.update_info()
+    if not evt.sender.username:
+        await evt.reply("Username removed")
+    else:
+        await evt.reply(f"Username changed to {evt.sender.username}")
+
+
+@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
+                 help_text="Change your Telegram displayname.")
+async def displayname(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own displayname.")
+
+    first_name, last_name = ((evt.args[0], "")
+                             if len(evt.args) == 1
+                             else (" ".join(evt.args[:-1]), evt.args[-1]))
+    try:
+        await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid first name")
+    await evt.sender.update_info()
+    await evt.reply("Displayname updated")
+
+
+def _format_session(sess: Authorization) -> str:
+    return (f"**{sess.app_name} {sess.app_version}**  \n"
+            f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
+            f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
+            f"  **From:** {sess.ip} - {sess.region}, {sess.country}")
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<`list`|`terminate`> [_hash_]",
+                 help_text="View or delete other Telegram sessions.")
+async def session(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't manage their sessions")
+    cmd = evt.args[0].lower()
+    if cmd == "list":
+        res = await evt.sender.client(GetAuthorizationsRequest())
+        session_list = res.authorizations
+        current = [s for s in session_list if s.current][0]
+        current_text = _format_session(current)
+        other_text = "\n".join(f"* {_format_session(sess)}  \n"
+                               f"  **Hash:** {sess.hash}"
+                               for sess in session_list if not sess.current)
+        return await evt.reply(f"### Current session\n"
+                               f"{current_text}\n"
+                               f"\n"
+                               f"### Other active sessions\n"
+                               f"{other_text}")
+    elif cmd == "terminate" and len(evt.args) > 1:
+        try:
+            session_hash = int(evt.args[1])
+        except ValueError:
+            return await evt.reply("Hash must be an integer")
+        try:
+            ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
+        except HashInvalidError:
+            return await evt.reply("Invalid session hash.")
+        except AuthKeyError as e:
+            if e.message == "FRESH_RESET_AUTHORISATION_FORBIDDEN":
+                return await evt.reply("New sessions can't terminate other sessions. "
+                                       "Please wait a while.")
+            raise
+        if ok:
+            return await evt.reply("Session terminated successfully.")
+        else:
+            return await evt.reply("Session not found.")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")

mautrix_telegram/commands/telegram/auth.py

@@ -0,0 +1,271 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Any, Dict, Optional
+import asyncio
+
+from telethon.errors import (
+    AccessTokenExpiredError, AccessTokenInvalidError, FirstNameInvalidError, FloodWaitError,
+    PasswordHashInvalidError, PhoneCodeExpiredError, PhoneCodeInvalidError,
+    PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
+    PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError)
+
+from ... import puppet as pu, user as u
+from ...commands import command_handler, CommandEvent, SECTION_AUTH
+from ...util import format_duration, ignore_coro
+
+
+@command_handler(needs_auth=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Check if you're logged into Telegram.")
+async def ping(evt: CommandEvent) -> Optional[Dict]:
+    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
+    if me:
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
+        return await evt.reply(f"You're logged in as {human_tg_id}")
+    else:
+        return await evt.reply("You're not logged in.")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Get the info of the message relay Telegram bot.")
+async def ping_bot(evt: CommandEvent) -> Optional[Dict]:
+    if not evt.tgbot:
+        return await evt.reply("Telegram message relay bot not configured.")
+    info, mxid = await evt.tgbot.get_me(use_cache=False)
+    return await evt.reply("Telegram message relay bot is active: "
+                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
+                           "To use the bot, simply invite it to a portal room.")
+
+
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_phone_> <_full name_>",
+                 help_text="Register to Telegram")
+async def register(evt: CommandEvent) -> Optional[Dict]:
+    if await evt.sender.is_logged_in():
+        return await evt.reply("You are already logged in.")
+    elif len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
+
+    phone_number = evt.args[0]
+    if len(evt.args) == 2:
+        full_name = evt.args[1], ""
+    else:
+        full_name = " ".join(evt.args[1:-1]), evt.args[-1]
+
+    await _request_code(evt, phone_number, {
+        "next": enter_code_register,
+        "action": "Register",
+        "full_name": full_name,
+    })
+    return None
+
+
+async def enter_code_register(evt: CommandEvent) -> Dict:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        first_name, last_name = evt.sender.command_status["full_name"]
+        user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
+        evt.sender.command_status = None
+        return await evt.reply(f"Successfully registered to Telegram.")
+    except PhoneNumberOccupiedError:
+        return await evt.reply("That phone number has already been registered. "
+                               "You can log in with `$cmdprefix+sp login`.")
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
+    except PhoneCodeExpiredError:
+        return await evt.reply(
+            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`.")
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
+    except Exception:
+        evt.log.exception("Error sending phone code")
+        return await evt.reply("Unhandled exception while sending code. "
+                               "Check console for more details.")
+
+
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Get instructions on how to log in.")
+async def login(evt: CommandEvent) -> Optional[Dict]:
+    override_sender = False
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        evt.sender = await u.User.get_by_mxid(evt.args[0]).ensure_started()
+        override_sender = True
+    if await evt.sender.is_logged_in():
+        return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
+
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login and not override_sender:
+        evt.sender.command_status = {
+            "next": enter_phone_or_token,
+            "action": "Login",
+        }
+
+    nb = "**N.B. Logging in grants the bridge full access to your Telegram account.**"
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
+        if override_sender:
+            return await evt.reply(f"[Click here to log in]({url}) as "
+                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+        elif allow_matrix_login:
+            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
+                                   f" number (or bot auth token) here to log in.\n\n{nb}")
+        return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
+    elif allow_matrix_login:
+        if override_sender:
+            return await evt.reply(
+                "This bridge instance does not allow you to log in outside of Matrix. "
+                "Logging in as another user inside Matrix is not currently possible.")
+        return await evt.reply("Please send your phone number (or bot auth token) here to start "
+                               f"the login process.\n\n{nb}")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
+                        ) -> Dict:
+    ok = False
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        await evt.sender.client.sign_in(phone_number)
+        ok = True
+        return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
+    except PhoneNumberAppSignupForbiddenError:
+        return await evt.reply(
+            "Your phone number does not allow 3rd party apps to sign in.")
+    except PhoneNumberFloodError:
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            "The ban is usually applied for around a day.")
+    except FloodWaitError as e:
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            f"Please wait for {format_duration(e.seconds)} before trying again.")
+    except PhoneNumberBannedError:
+        return await  evt.reply("Your phone number has been banned from Telegram.")
+    except PhoneNumberUnoccupiedError:
+        return await  evt.reply("That phone number has not been registered. "
+                                "Please register with `$cmdprefix+sp register <phone>`.")
+    except Exception:
+        evt.log.exception("Error requesting phone code")
+        return await evt.reply("Unhandled exception while requesting code. "
+                               "Check console for more details.")
+    finally:
+        evt.sender.command_status = next_status if ok else None
+
+
+@command_handler(needs_auth=False)
+async def enter_phone_or_token(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+
+    # phone numbers don't contain colons but telegram bot auth tokens do
+    if evt.args[0].find(":") > 0:
+        try:
+            await _sign_in(evt, bot_token=evt.args[0])
+        except Exception:
+            evt.log.exception("Error sending auth token")
+            return await evt.reply("Unhandled exception while sending auth token. "
+                                   "Check console for more details.")
+    else:
+        await _request_code(evt, evt.args[0], {
+            "next": enter_code,
+            "action": "Login",
+        })
+    return None
+
+
+@command_handler(needs_auth=False)
+async def enter_code(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+    try:
+        await _sign_in(evt, code=evt.args[0])
+    except Exception:
+        evt.log.exception("Error sending phone code")
+        return await evt.reply("Unhandled exception while sending code. "
+                               "Check console for more details.")
+    return None
+
+
+@command_handler(needs_auth=False)
+async def enter_password(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+    try:
+        await _sign_in(evt, password=" ".join(evt.args))
+    except AccessTokenInvalidError:
+        return await evt.reply("That bot token is not valid.")
+    except AccessTokenExpiredError:
+        return await evt.reply("That bot token has expired.")
+    except Exception:
+        evt.log.exception("Error sending password")
+        return await evt.reply("Unhandled exception while sending password. "
+                               "Check console for more details.")
+    return None
+
+
+async def _sign_in(evt: CommandEvent, **sign_in_info) -> Dict:
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        user = await evt.sender.client.sign_in(**sign_in_info)
+        existing_user = u.User.get_by_tgid(user.id)
+        if existing_user and existing_user != evt.sender:
+            await existing_user.log_out()
+            await evt.reply(f"[{existing_user.displayname}]"
+                            f"(https://matrix.to/#/{existing_user.mxid})"
+                            " was logged out from the account.")
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
+        evt.sender.command_status = None
+        name = f"@{user.username}" if user.username else f"+{user.phone}"
+        return await evt.reply(f"Successfully logged in as {name}")
+    except PhoneCodeExpiredError:
+        return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
+    except PasswordHashInvalidError:
+        return await evt.reply("Incorrect password.")
+    except SessionPasswordNeededError:
+        evt.sender.command_status = {
+            "next": enter_password,
+            "action": "Login (password entry)",
+        }
+        return await evt.reply("Your account has two-factor authentication. "
+                               "Please send your password here.")
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Log out from Telegram.")
+async def logout(evt: CommandEvent) -> Optional[Dict]:
+    if await evt.sender.log_out():
+        return await evt.reply("Logged out successfully.")
+    return await evt.reply("Failed to log out.")

mautrix_telegram/commands/telegram/misc.py

@@ -0,0 +1,283 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Optional, Tuple
+import codecs
+import base64
+import re
+
+from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
+                             UserAlreadyParticipantError)
+from telethon.tl.patched import Message
+from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
+                               TypePeer)
+from telethon.tl.types.messages import BotCallbackAnswer
+from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
+                                            GetBotCallbackAnswerRequest, SendVoteRequest)
+from telethon.tl.functions.channels import JoinChannelRequest
+
+from ... import puppet as pu, portal as po
+from ...abstract_user import AbstractUser
+from ...db import Message as DBMessage
+from ...types import TelegramID
+from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[_-r|--remote_] <_query_>",
+                 help_text="Search your contacts or the Telegram servers for users.")
+async def search(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
+
+    force_remote = False
+    if evt.args[0] in {"-r", "--remote"}:
+        force_remote = True
+        evt.args.pop(0)
+
+    query = " ".join(evt.args)
+    if force_remote and len(query) < 5:
+        return await evt.reply("Minimum length of query for remote search is 5 characters.")
+
+    results, remote = await evt.sender.search(query, force_remote)
+
+    if not results:
+        if len(query) < 5 and remote:
+            return await evt.reply("No local results. "
+                                   "Minimum length of remote query is 5 characters.")
+        return await evt.reply("No results 3:")
+
+    reply = []  # type: List[str]
+    if remote:
+        reply += ["**Results from Telegram server:**", ""]
+    else:
+        reply += ["**Results in contacts:**", ""]
+    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
+               f"{puppet.id} ({similarity}% match)")
+              for puppet, similarity in results]
+
+    # TODO somehow show remote channel results when joining by alias is possible?
+
+    return await evt.reply("\n".join(reply))
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_identifier_>",
+                 help_text="Open a private chat with the given Telegram user. The identifier is "
+                           "either the internal user ID, the username or the phone number. "
+                           "**N.B.** The phone numbers you start chats with must already be in "
+                           "your contacts.")
+async def pm(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
+
+    try:
+        user = await evt.sender.client.get_entity(evt.args[0])
+    except ValueError:
+        return await evt.reply("Invalid user identifier or user not found.")
+
+    if not user:
+        return await evt.reply("User not found.")
+    elif not isinstance(user, TLUser):
+        return await evt.reply("That doesn't seem to be a user.")
+    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
+    await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
+    return await evt.reply("Created private chat room with "
+                           f"{pu.Puppet.get_displayname(user, False)}")
+
+
+async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[Dict]]:
+    if arg.startswith("joinchat/"):
+        invite_hash = arg[len("joinchat/"):]
+        try:
+            await evt.sender.client(CheckChatInviteRequest(invite_hash))
+        except InviteHashInvalidError:
+            return None, await evt.reply("Invalid invite link.")
+        except InviteHashExpiredError:
+            return None, await evt.reply("Invite link expired.")
+        try:
+            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
+        except UserAlreadyParticipantError:
+            return None, await evt.reply("You are already in that chat.")
+    else:
+        channel = await evt.sender.client.get_entity(arg)
+        if not channel:
+            return None, await evt.reply("Channel/supergroup not found.")
+        return await evt.sender.client(JoinChannelRequest(channel)), None
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_link_>",
+                 help_text="Join a chat with an invite link.")
+async def join(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
+
+    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
+    arg = regex.match(evt.args[0])
+    if not arg:
+        return await evt.reply("That doesn't look like a Telegram invite link.")
+
+    updates, _ = await _join(evt, arg.group(1))
+    if not updates:
+        return None
+
+    for chat in updates.chats:
+        portal = po.Portal.get_by_entity(chat)
+        if portal.mxid:
+            await portal.invite_to_matrix([evt.sender.mxid])
+            return await evt.reply(f"Invited you to portal of {portal.title}")
+        else:
+            await evt.reply(f"Creating room for {chat.title}... This might take a while.")
+            await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
+            return await evt.reply(f"Created room for {portal.title}")
+    return None
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[`chats`|`contacts`|`me`]",
+                 help_text="Synchronize your chat portals, contacts and/or own info.")
+async def sync(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) > 0:
+        sync_only = evt.args[0]
+        if sync_only not in ("chats", "contacts", "me"):
+            return await evt.reply("**Usage:** `$cmdprefix+sp sync [chats|contacts|me]`")
+    else:
+        sync_only = None
+
+    if not sync_only or sync_only == "chats":
+        await evt.sender.sync_dialogs(synchronous_create=True)
+    if not sync_only or sync_only == "contacts":
+        await evt.sender.sync_contacts()
+    if not sync_only or sync_only == "me":
+        await evt.sender.update_info()
+    return await evt.reply("Synchronization complete.")
+
+
+PEER_TYPE_CHAT = b"g"
+
+
+class MessageIDError(ValueError):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+
+async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
+                               ) -> Tuple[TypePeer, Message]:
+    try:
+        enc_id += (4 - len(enc_id) % 4) * "="
+        enc_id = base64.b64decode(enc_id)
+        peer_type, enc_id = bytes([enc_id[0]]), enc_id[1:]
+        tgid = TelegramID(int(codecs.encode(enc_id[0:5], "hex_codec"), 16))
+        msg_id = TelegramID(int(codecs.encode(enc_id[5:10], "hex_codec"), 16))
+        space = None
+        if peer_type == PEER_TYPE_CHAT:
+            space = TelegramID(int(codecs.encode(enc_id[10:15], "hex_codec"), 16))
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (format)") from e
+
+    if peer_type == PEER_TYPE_CHAT:
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
+        if not orig_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
+        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
+        if not new_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (your copy of message not found in db)")
+        msg_id = new_msg.tgid
+    try:
+        peer = await user.client.get_input_entity(tgid)
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (chat not found)") from e
+
+    msg = await user.client.get_messages(entity=peer, ids=msg_id)
+    if not msg:
+        raise MessageIDError(f"Invalid {type_name} ID (message not found)")
+    return peer, msg
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_play ID_>",
+                 help_text="Play a Telegram game.")
+async def play(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp play <play ID>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to play games.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't play games :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="play")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaGame):
+        return await evt.reply("Invalid play ID (message doesn't look like a game)")
+
+    game = await evt.sender.client(GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
+    if not isinstance(game, BotCallbackAnswer):
+        return await evt.reply("Game request response invalid")
+
+    await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
+                    f"{msg.media.game.description}")
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_poll ID_> <_choice number_>",
+                 help_text="Vote in a Telegram poll.")
+async def vote(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to vote in polls.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't vote in polls :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="poll")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaPoll):
+        return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
+
+    options = []
+    for option in evt.args[1:]:
+        try:
+            if len(option) > 10:
+                raise ValueError("option index too long")
+            option_index = int(option) - 1
+        except ValueError:
+            option_index = None
+        if option_index is None:
+            return await evt.reply(f"Invalid option number \"{option}\"",
+                                   render_markdown=False, allow_html=False)
+        elif option_index < 0:
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"Option numbers must be positive.")
+        elif option_index >= len(msg.media.poll.answers):
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"The poll only has {len(msg.media.poll.answers)} options.")
+        options.append(msg.media.poll.answers[option_index].option)
+    options = [msg.media.poll.answers[int(option) - 1].option
+               for option in evt.args[1:]]
+    try:
+        resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
+    except OptionsTooMuchError:
+        return await evt.reply("You passed too many options.")
+    # TODO use response
+    return await evt.mark_read()

mautrix_telegram/config.py

@@ -1,77 +1,136 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Any, Dict, Optional, Tuple
 from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
 import random
 import string
 
-yaml = YAML()
+yaml = YAML()  # type: YAML
+yaml.indent(4)
 
 
 class DictWithRecursion:
-    def __init__(self, data=None):
-        self._data = data or {}
+    def __init__(self, data: Optional[CommentedMap] = None) -> None:
+        self._data = data or CommentedMap()  # type: CommentedMap
 
-    def _recursive_get(self, data, key, default_value):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
-            next_data = data.get(key, {})
+    @staticmethod
+    def _parse_key(key: str) -> Tuple[str, Optional[str]]:
+        if '.' not in key:
+            return key, None
+        key, next_key = key.split('.', 1)
+        if len(key) > 0 and key[0] == "[":
+            end_index = next_key.index("]")
+            key = key[1:] + "." + next_key[:end_index]
+            next_key = next_key[end_index + 2:] if len(next_key) > end_index + 1 else None
+        return key, next_key
+
+    def _recursive_get(self, data: CommentedMap, key: str, default_value: Any) -> Any:
+        key, next_key = self._parse_key(key)
+        if next_key is not None:
+            next_data = data.get(key, CommentedMap())
             return self._recursive_get(next_data, next_key, default_value)
         return data.get(key, default_value)
 
-    def get(self, key, default_value, allow_recursion=True):
+    def get(self, key: str, default_value: Any, allow_recursion: bool = True) -> Any:
         if allow_recursion and '.' in key:
             return self._recursive_get(self._data, key, default_value)
         return self._data.get(key, default_value)
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> Any:
         return self.get(key, None)
 
-    def _recursive_set(self, data, key, value):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
+    def __contains__(self, key: str) -> bool:
+        return self[key] is not None
+
+    def _recursive_set(self, data: CommentedMap, key: str, value: Any) -> None:
+        key, next_key = self._parse_key(key)
+        if next_key is not None:
             if key not in data:
-                data[key] = {}
-            next_data = data.get(key, {})
-            self._recursive_set(next_data, next_key, value)
-            return
+                data[key] = CommentedMap()
+            next_data = data.get(key, CommentedMap())
+            return self._recursive_set(next_data, next_key, value)
         data[key] = value
 
-    def set(self, key, value, allow_recursion=True):
+    def set(self, key: str, value: Any, allow_recursion: bool = True) -> None:
         if allow_recursion and '.' in key:
             self._recursive_set(self._data, key, value)
             return
         self._data[key] = value
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value: Any) -> None:
         self.set(key, value)
 
+    def _recursive_del(self, data: CommentedMap, key: str) -> None:
+        key, next_key = self._parse_key(key)
+        if next_key is not None:
+            if key not in data:
+                return
+            next_data = data[key]
+            return self._recursive_del(next_data, next_key)
+        try:
+            del data[key]
+            del data.ca.items[key]
+        except KeyError:
+            pass
+
+    def delete(self, key: str, allow_recursion: bool = True) -> None:
+        if allow_recursion and '.' in key:
+            self._recursive_del(self._data, key)
+            return
+        try:
+            del self._data[key]
+            del self._data.ca.items[key]
+        except KeyError:
+            pass
+
+    def __delitem__(self, key: str) -> None:
+        self.delete(key)
+
 
 class Config(DictWithRecursion):
-    def __init__(self, path, registration_path):
+    def __init__(self, path: str, registration_path: str, base_path: str,
+                 overrides: Dict[str, Any] = None) -> None:
         super().__init__()
-        self.path = path
-        self.registration_path = registration_path
-        self._registration = None
+        self.path = path  # type: str
+        self.registration_path = registration_path  # type: str
+        self.base_path = base_path  # type: str
+        self._registration = None  # type: Optional[Dict]
+        self._overrides = overrides or {}  # type: Dict[str, Any]
 
-    def load(self):
+    def __getitem__(self, key: str) -> Any:
+        try:
+            return self._overrides[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
+        except KeyError:
+            return super().__getitem__(key)
+
+    def load(self) -> None:
         with open(self.path, 'r') as stream:
             self._data = yaml.load(stream)
 
-    def save(self):
+    def load_base(self) -> Optional[DictWithRecursion]:
+        try:
+            with open(self.base_path, 'r') as stream:
+                return DictWithRecursion(yaml.load(stream))
+        except OSError:
+            pass
+        return None
+
+    def save(self) -> None:
         with open(self.path, 'w') as stream:
             yaml.dump(self._data, stream)
         if self._registration and self.registration_path:
@@ -79,10 +138,194 @@ class Config(DictWithRecursion):
                 yaml.dump(self._registration, stream)
 
     @staticmethod
-    def _new_token():
-        return "".join(random.choices(string.ascii_lowercase + string.digits, k=64))
+    def _new_token() -> str:
+        return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
 
-    def generate_registration(self):
+    def update(self) -> None:
+        base = self.load_base()
+        if not base:
+            return
+
+        def copy(from_path, to_path=None) -> None:
+            if from_path in self:
+                base[to_path or from_path] = self[from_path]
+
+        def copy_dict(from_path, to_path=None, override_existing_map=True) -> None:
+            if from_path in self:
+                to_path = to_path or from_path
+                if override_existing_map or to_path not in base:
+                    base[to_path] = CommentedMap()
+                for key, value in self[from_path].items():
+                    base[to_path][key] = value
+
+        copy("homeserver.address")
+        copy("homeserver.domain")
+        copy("homeserver.verify_ssl")
+
+        if "appservice.protocol" in self and "appservice.address" not in self:
+            protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
+                                        self["appservice.port"])
+            base["appservice.address"] = f"{protocol}://{hostname}:{port}"
+        else:
+            copy("appservice.address")
+        copy("appservice.hostname")
+        copy("appservice.port")
+        copy("appservice.max_body_size")
+
+        copy("appservice.database")
+
+        copy("appservice.public.enabled")
+        copy("appservice.public.prefix")
+        copy("appservice.public.external")
+
+        copy("appservice.provisioning.enabled")
+        copy("appservice.provisioning.prefix")
+        copy("appservice.provisioning.shared_secret")
+        if base["appservice.provisioning.shared_secret"] == "generate":
+            base["appservice.provisioning.shared_secret"] = self._new_token()
+
+        copy("appservice.id")
+        copy("appservice.bot_username")
+        copy("appservice.bot_displayname")
+        copy("appservice.bot_avatar")
+
+        copy("appservice.community_id")
+
+        copy("appservice.as_token")
+        copy("appservice.hs_token")
+
+        copy("metrics.enabled")
+        copy("metrics.listen_port")
+
+        copy("bridge.username_template")
+        copy("bridge.alias_template")
+        copy("bridge.displayname_template")
+
+        copy("bridge.displayname_preference")
+
+        copy("bridge.max_initial_member_sync")
+        copy("bridge.sync_channel_members")
+        copy("bridge.skip_deleted_members")
+        copy("bridge.startup_sync")
+        copy("bridge.sync_dialog_limit")
+        copy("bridge.max_telegram_delete")
+        copy("bridge.sync_matrix_state")
+        copy("bridge.allow_matrix_login")
+        copy("bridge.plaintext_highlights")
+        copy("bridge.public_portals")
+        copy("bridge.catch_up")
+        copy("bridge.sync_with_custom_puppets")
+        copy("bridge.telegram_link_preview")
+        copy("bridge.inline_images")
+        copy("bridge.image_as_file_size")
+        copy("bridge.max_document_size")
+
+        copy("bridge.bot_messages_as_notices")
+        if isinstance(self["bridge.bridge_notices"], bool):
+            base["bridge.bridge_notices"] = {
+                "default": self["bridge.bridge_notices"],
+                "exceptions": ["@importantbot:example.com"],
+            }
+        else:
+            copy("bridge.bridge_notices")
+
+        copy("bridge.deduplication.pre_db_check")
+        copy("bridge.deduplication.cache_queue_length")
+
+        if "bridge.message_formats.m_text" in self:
+            del self["bridge.message_formats"]
+        copy_dict("bridge.message_formats", override_existing_map=False)
+
+        copy("bridge.state_event_formats.join")
+        copy("bridge.state_event_formats.leave")
+        copy("bridge.state_event_formats.name_change")
+
+        copy("bridge.filter.mode")
+        copy("bridge.filter.list")
+
+        copy("bridge.command_prefix")
+
+        migrate_permissions = ("bridge.permissions" not in self
+                               or "bridge.whitelist" in self
+                               or "bridge.admins" in self)
+        if migrate_permissions:
+            permissions = self["bridge.permissions"] or CommentedMap()
+            for entry in self["bridge.whitelist"] or []:
+                permissions[entry] = "full"
+            for entry in self["bridge.admins"] or []:
+                permissions[entry] = "admin"
+            base["bridge.permissions"] = permissions
+        else:
+            copy_dict("bridge.permissions")
+
+        if "bridge.relaybot" not in self:
+            copy("bridge.authless_relaybot_portals", "bridge.relaybot.authless_portals")
+        else:
+            copy("bridge.relaybot.authless_portals")
+            copy("bridge.relaybot.whitelist_group_admins")
+            copy("bridge.relaybot.whitelist")
+            copy("bridge.relaybot.ignore_own_incoming_events")
+
+        copy("telegram.api_id")
+        copy("telegram.api_hash")
+        copy("telegram.bot_token")
+
+        copy("telegram.connection.timeout")
+        copy("telegram.connection.retries")
+        copy("telegram.connection.retry_delay")
+        copy("telegram.connection.flood_sleep_threshold")
+        copy("telegram.connection.request_retries")
+
+        copy("telegram.device_info.device_model")
+        copy("telegram.device_info.system_version")
+        copy("telegram.device_info.app_version")
+        copy("telegram.device_info.lang_code")
+        copy("telegram.device_info.system_lang_code")
+
+        copy("telegram.server.enabled")
+        copy("telegram.server.dc")
+        copy("telegram.server.ip")
+        copy("telegram.server.port")
+
+        copy("telegram.proxy.type")
+        copy("telegram.proxy.address")
+        copy("telegram.proxy.port")
+        copy("telegram.proxy.rdns")
+        copy("telegram.proxy.username")
+        copy("telegram.proxy.password")
+
+        if "appservice.debug" in self and "logging" not in self:
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
+            base["logging.root.level"] = level
+            base["logging.loggers.mau.level"] = level
+            base["logging.loggers.telethon.level"] = level
+        else:
+            copy("logging")
+
+        self._data = base._data
+        self.save()
+
+    def _get_permissions(self, key: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
+        level = self["bridge.permissions"].get(key, "")
+        admin = level == "admin"
+        matrix_puppeting = level == "full" or admin
+        puppeting = level == "puppeting" or matrix_puppeting
+        user = level == "user" or puppeting
+        relaybot = level == "relaybot" or user
+        return relaybot, user, puppeting, matrix_puppeting, admin, level
+
+    def get_permissions(self, mxid: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
+        permissions = self["bridge.permissions"] or {}
+        if mxid in permissions:
+            return self._get_permissions(mxid)
+
+        homeserver = mxid[mxid.index(":") + 1:]
+        if homeserver in permissions:
+            return self._get_permissions(homeserver)
+
+        return self._get_permissions("*")
+
+    def generate_registration(self) -> None:
         homeserver = self["homeserver.domain"]
 
         username_format = self.get("bridge.username_template", "telegram_{userid}") \
@@ -93,10 +336,8 @@ class Config(DictWithRecursion):
         self.set("appservice.as_token", self._new_token())
         self.set("appservice.hs_token", self._new_token())
 
-        url = (f"{self['appservice.protocol']}://"
-               + f"{self['appservice.hostname']}:{self['appservice.port']}")
         self._registration = {
-            "id": self.get("appservice.id", "telegram"),
+            "id": self["appservice.id"] or "telegram",
             "as_token": self["appservice.as_token"],
             "hs_token": self["appservice.hs_token"],
             "namespaces": {
@@ -109,7 +350,10 @@ class Config(DictWithRecursion):
                     "regex": f"#{alias_format}:{homeserver}"
                 }]
             },
-            "url": url,
+            "url": self["appservice.address"],
             "sender_localpart": self["appservice.bot_username"],
             "rate_limited": False
         }
+        if self["appservice.community_id"]:
+            self._registration["namespaces"]["users"][0]["group_id"] \
+                = self["appservice.community_id"]

mautrix_telegram/context.py

@@ -0,0 +1,45 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Tuple, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import asyncio
+
+    from alchemysession import AlchemySessionContainer
+    from mautrix_appservice import AppService
+
+    from .web import PublicBridgeWebsite, ProvisioningAPI
+    from .config import Config
+    from .bot import Bot
+    from .matrix import MatrixHandler
+
+
+class Context:
+    def __init__(self, az: 'AppService', config: 'Config', loop: 'asyncio.AbstractEventLoop',
+                 session_container: 'AlchemySessionContainer', bot: Optional['Bot']) -> None:
+        self.az = az  # type: AppService
+        self.config = config  # type: Config
+        self.loop = loop  # type: asyncio.AbstractEventLoop
+        self.bot = bot  # type: Optional[Bot]
+        self.mx = None  # type: Optional[MatrixHandler]
+        self.session_container = session_container  # type: AlchemySessionContainer
+        self.public_website = None  # type: Optional[PublicBridgeWebsite]
+        self.provisioning_api = None  # type: Optional[ProvisioningAPI]
+
+    @property
+    def core(self) -> Tuple['AppService', 'Config', 'asyncio.AbstractEventLoop', Optional['Bot']]:
+        return self.az, self.config, self.loop, self.bot

mautrix_telegram/db.py

@@ -1,103 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from sqlalchemy import Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer, String
-from sqlalchemy.orm import relationship
-
-from .base import Base
-
-
-class Portal(Base):
-    query = None
-    __tablename__ = "portal"
-
-    # Telegram chat information
-    tgid = Column(Integer, primary_key=True)
-    tg_receiver = Column(Integer, primary_key=True)
-    peer_type = Column(String)
-
-    # Matrix portal information
-    mxid = Column(String, unique=True, nullable=True)
-
-    # Telegram chat metadata
-    username = Column(String, nullable=True)
-    title = Column(String, nullable=True)
-    about = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-
-
-class Message(Base):
-    query = None
-    __tablename__ = "message"
-
-    mxid = Column(String)
-    mx_room = Column(String)
-    tgid = Column(Integer, primary_key=True)
-    tg_space = Column(Integer, primary_key=True)
-
-    __table_args__ = (UniqueConstraint('mxid', 'mx_room', 'tg_space', name='_mx_id_room'),)
-
-
-class UserPortal(Base):
-    query = None
-    __tablename__ = "user_portal"
-
-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
-    portal = Column(Integer, primary_key=True)
-    portal_receiver = Column(Integer, primary_key=True)
-
-    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
-                                           ("portal.tgid", "portal.tg_receiver")),)
-
-
-class User(Base):
-    query = None
-    __tablename__ = "user"
-
-    mxid = Column(String, primary_key=True)
-    tgid = Column(Integer, nullable=True)
-    tg_username = Column(String, nullable=True)
-    saved_contacts = Column(Integer, default=0)
-    contacts = relationship("Contact", uselist=True,
-                            cascade="save-update, merge, delete, delete-orphan")
-    portals = relationship("Portal", secondary="user_portal", single_parent=True,
-                           cascade="save-update, merge, delete, delete-orphan")
-
-
-class Contact(Base):
-    query = None
-    __tablename__ = "contact"
-
-    user = Column("user", Integer, ForeignKey("user.tgid"), primary_key=True)
-    contact = Column("contact", Integer, ForeignKey("puppet.id"), primary_key=True)
-
-
-class Puppet(Base):
-    query = None
-    __tablename__ = "puppet"
-
-    id = Column(Integer, primary_key=True)
-    displayname = Column(String, nullable=True)
-    username = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-
-
-def init(db_session):
-    Portal.query = db_session.query_property()
-    Message.query = db_session.query_property()
-    UserPortal.query = db_session.query_property()
-    User.query = db_session.query_property()
-    Puppet.query = db_session.query_property()

mautrix_telegram/db/__init__.py

@@ -0,0 +1,33 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from .base import Base
+from .bot_chat import BotChat
+from .message import Message
+from .portal import Portal
+from .puppet import Puppet
+from .room_state import RoomState
+from .telegram_file import TelegramFile
+from .user import User, UserPortal, Contact
+from .user_profile import UserProfile
+
+
+def init(db_engine) -> None:
+    for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
+                  RoomState, BotChat):
+        table.db = db_engine
+        table.t = table.__table__
+        table.c = table.t.c

mautrix_telegram/db/base.py

@@ -0,0 +1,58 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+
+class BaseBase:
+    db = None  # type: Engine
+    t = None  # type: Table
+    __table__ = None  # type: Table
+    c = None  # type: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy):
+        pass
+
+    @classmethod
+    def _select_one_or_none(cls, *args):
+        return cls._one_or_none(cls.db.execute(cls.t.select().where(*args)))
+
+    @property
+    @abstractmethod
+    def _edit_identity(self):
+        pass
+
+    def update(self, **values) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self._edit_identity)
+                         .values(**values))
+        for key, value in values.items():
+            setattr(self, key, value)
+
+    def delete(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.delete().where(self._edit_identity))
+
+Base = declarative_base(cls=BaseBase)

mautrix_telegram/db/base.pyi

@@ -0,0 +1,26 @@
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+class Base(declarative_base):
+	db: Engine
+	t: Table
+	__table__: Table
+	c: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy): ...
+
+	@classmethod
+	def _select_one_or_none(cls, *args): ...
+
+    def _edit_identity(self): ...
+
+    def update(self, **values) -> None: ...
+
+    def delete(self) -> None: ...

mautrix_telegram/db/bot_chat.py

@@ -0,0 +1,45 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Iterable
+
+from sqlalchemy import Column, Integer, String
+
+from ..types import TelegramID
+from .base import Base
+
+
+# Fucking Telegram not telling bots what chats they are in 3:<
+class BotChat(Base):
+    __tablename__ = "bot_chat"
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    type = Column(String, nullable=False)
+
+    @classmethod
+    def delete(cls, chat_id: TelegramID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.id == chat_id))
+
+    @classmethod
+    def all(cls) -> Iterable['BotChat']:
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            chat_id, chat_type = row
+            yield cls(id=chat_id, type=chat_type)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(id=self.id, type=self.type))

mautrix_telegram/db/message.py

@@ -0,0 +1,113 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional, List
+
+from ..types import MatrixRoomID, MatrixEventID, TelegramID
+from .base import Base
+
+
+class Message(Base):
+    __tablename__ = "message"
+
+    mxid = Column(String)  # type: MatrixEventID
+    mx_room = Column(String)  # type: MatrixRoomID
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+    edit_index = Column(Integer, primary_key=True)  # type: int
+
+    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
+        try:
+            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
+                       edit_index=edit_index)
+        except StopIteration:
+            return None
+
+    @staticmethod
+    def _all(rows: RowProxy) -> List['Message']:
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
+                        edit_index=row[4])
+                for row in rows]
+
+    @classmethod
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
+        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
+                                                                 cls.c.tg_space == tg_space))))
+
+    @classmethod
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
+        query = cls.t.select()
+        if edit_index < 0:
+            query = (query
+                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                     .order_by(desc(cls.c.edit_index))
+                     .limit(1)
+                     .offset(-edit_index - 1))
+        else:
+            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                     cls.c.edit_index == edit_index))
+        return cls._one_or_none(cls.db.execute(query))
+
+    @classmethod
+    def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
+        rows = cls.db.execute(select([func.count(cls.c.tg_space)])
+                              .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
+        try:
+            count, = next(rows)
+            return count
+        except StopIteration:
+            return 0
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID, tg_space: TelegramID
+                    ) -> Optional['Message']:
+        return cls._select_one_or_none(and_(cls.c.mxid == mxid,
+                                            cls.c.mx_room == mx_room,
+                                            cls.c.tg_space == tg_space))
+
+    @classmethod
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
+                         .values(**values))
+
+    @classmethod
+    def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
+                         .values(**values))
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
+                    self.c.edit_index == self.edit_index)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
+                                                tgid=self.tgid, tg_space=self.tg_space,
+                                                edit_index=self.edit_index))

mautrix_telegram/db/portal.py

@@ -0,0 +1,81 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, Integer, String, Boolean, Text, and_
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional
+
+from ..types import MatrixRoomID, TelegramID
+from .base import Base
+
+
+class Portal(Base):
+    __tablename__ = "portal"
+
+    # Telegram chat information
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_receiver = Column(Integer, primary_key=True)  # type: TelegramID
+    peer_type = Column(String, nullable=False)
+    megagroup = Column(Boolean)
+
+    # Matrix portal information
+    mxid = Column(String, unique=True, nullable=True)  # type: Optional[MatrixRoomID]
+
+    config = Column(Text, nullable=True)
+
+    # Telegram chat metadata
+    username = Column(String, nullable=True)
+    title = Column(String, nullable=True)
+    about = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+
+    @classmethod
+    def scan(cls, row) -> Optional['Portal']:
+        (tgid, tg_receiver, peer_type, megagroup, mxid, config, username, title, about,
+         photo_id) = row
+        return cls(tgid=tgid, tg_receiver=tg_receiver, peer_type=peer_type, megagroup=megagroup,
+                   mxid=mxid, config=config, username=username, title=title, about=about,
+                   photo_id=photo_id)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Portal']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
+        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver))
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixRoomID) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.username == username)
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_receiver == self.tg_receiver)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
+                megagroup=self.megagroup, mxid=self.mxid, config=self.config,
+                username=self.username, title=self.title, about=self.about, photo_id=self.photo_id))

mautrix_telegram/db/puppet.py

@@ -0,0 +1,88 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql import expression
+from typing import Optional, Iterable
+
+from ..types import MatrixUserID, MatrixRoomID, TelegramID
+from .base import Base
+
+
+class Puppet(Base):
+    __tablename__ = "puppet"
+
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    custom_mxid = Column(String, nullable=True)  # type: Optional[MatrixUserID]
+    access_token = Column(String, nullable=True)
+    displayname = Column(String, nullable=True)
+    displayname_source = Column(Integer, nullable=True)  # type: Optional[TelegramID]
+    username = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+    is_bot = Column(Boolean, nullable=True)
+    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
+    disable_updates = Column(Boolean, nullable=False, server_default=expression.false())
+
+    @classmethod
+    def scan(cls, row) -> Optional['Puppet']:
+        (id, custom_mxid, access_token, displayname, displayname_source, username, photo_id,
+         is_bot, matrix_registered, disable_updates) = row
+        return cls(id=id, custom_mxid=custom_mxid, access_token=access_token,
+                   displayname=displayname, displayname_source=displayname_source,
+                   username=username, photo_id=photo_id, is_bot=is_bot,
+                   matrix_registered=matrix_registered, disable_updates=disable_updates)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Puppet']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
+
+    @classmethod
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.custom_mxid != None))
+        for row in rows:
+            yield cls.scan(row)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.id == tgid)
+
+    @classmethod
+    def get_by_custom_mxid(cls, mxid: MatrixUserID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.custom_mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.username == username)
+
+    @classmethod
+    def get_by_displayname(cls, displayname: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.displayname == displayname)
+
+    @property
+    def _edit_identity(self):
+        return self.c.id == self.id
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
+                displayname=self.displayname, displayname_source=self.displayname_source,
+                username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
+                matrix_registered=self.matrix_registered, disable_updates=self.disable_updates))

mautrix_telegram/db/room_state.py

@@ -0,0 +1,62 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, Text
+from typing import Dict, Optional
+import json
+
+from ..types import MatrixRoomID
+from .base import Base
+
+
+class RoomState(Base):
+    __tablename__ = "mx_room_state"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    power_levels = Column("power_levels", Text, nullable=True)  # type: Optional[Dict]
+
+    @property
+    def _power_levels_text(self) -> Optional[str]:
+        return json.dumps(self.power_levels) if self.power_levels else None
+
+    @property
+    def has_power_levels(self) -> bool:
+        return bool(self.power_levels)
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID) -> Optional['RoomState']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.room_id == room_id))
+        try:
+            room_id, power_levels_text = next(rows)
+            return cls(room_id=room_id, power_levels=(json.loads(power_levels_text)
+                                                      if power_levels_text else None))
+        except StopIteration:
+            return None
+
+    def update(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self.c.room_id == self.room_id)
+                         .values(power_levels=self._power_levels_text))
+
+    @property
+    def _edit_identity(self):
+        return self.c.room_id == self.room_id
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id,
+                                                power_levels=self._power_levels_text))

mautrix_telegram/db/telegram_file.py

@@ -0,0 +1,56 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
+from typing import Optional
+
+from .base import Base
+
+
+class TelegramFile(Base):
+    __tablename__ = "telegram_file"
+
+    id = Column(String, primary_key=True)
+    mxc = Column(String)
+    mime_type = Column(String)
+    was_converted = Column(Boolean)
+    timestamp = Column(BigInteger)
+    size = Column(Integer, nullable=True)
+    width = Column(Integer, nullable=True)
+    height = Column(Integer, nullable=True)
+    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+    thumbnail = None  # type: Optional[TelegramFile]
+
+    @classmethod
+    def get(cls, loc_id: str) -> Optional['TelegramFile']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.id == loc_id))
+        try:
+            loc_id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
+            thumb = None
+            if thumb_id:
+                thumb = cls.get(thumb_id)
+            return cls(id=loc_id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
+                       size=s, width=w, height=h, thumbnail_id=thumb_id, thumbnail=thumb)
+        except StopIteration:
+            return None
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, mxc=self.mxc, mime_type=self.mime_type,
+                was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
+                width=self.width, height=self.height,
+                thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/user.py

@@ -0,0 +1,130 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional, Iterable, Tuple
+
+from ..types import MatrixUserID, TelegramID
+from .base import Base
+
+
+class User(Base):
+    __tablename__ = "user"
+
+    mxid = Column(String, primary_key=True)  # type: MatrixUserID
+    tgid = Column(Integer, nullable=True, unique=True)  # type: Optional[TelegramID]
+    tg_username = Column(String, nullable=True)
+    tg_phone = Column(String, nullable=True)
+    saved_contacts = Column(Integer, default=0, nullable=False)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['User']:
+        try:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = next(rows)
+            return cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                       saved_contacts=saved_contacts)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def all(cls) -> Iterable['User']:
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = row
+            yield cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                      saved_contacts=saved_contacts)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tgid == tgid)
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixUserID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tg_username == username)
+
+    @property
+    def _edit_identity(self):
+        return self.c.mxid == self.mxid
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                mxid=self.mxid, tgid=self.tgid, tg_username=self.tg_username,
+                tg_phone=self.tg_phone, saved_contacts=self.saved_contacts))
+
+    @property
+    def contacts(self) -> Iterable[TelegramID]:
+        rows = self.db.execute(Contact.t.select().where(Contact.c.user == self.tgid))
+        for row in rows:
+            user, contact = row
+            yield contact
+
+    @contacts.setter
+    def contacts(self, puppets: Iterable[TelegramID]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
+            insert_puppets = [{"user": self.tgid, "contact": tgid} for tgid in puppets]
+            if insert_puppets:
+                conn.execute(Contact.t.insert(), insert_puppets)
+
+    @property
+    def portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        rows = self.db.execute(UserPortal.t.select().where(UserPortal.c.user == self.tgid))
+        for row in rows:
+            user, portal, portal_receiver = row
+            yield (portal, portal_receiver)
+
+    @portals.setter
+    def portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
+            insert_portals = [{
+                "user": self.tgid,
+                "portal": tgid,
+                "portal_receiver": tg_receiver
+            } for tgid, tg_receiver in portals]
+            if insert_portals:
+                conn.execute(UserPortal.t.insert(), insert_portals)
+
+    def delete(self) -> None:
+        super().delete()
+        self.portals = []
+        self.contacts = []
+
+
+class UserPortal(Base):
+    __tablename__ = "user_portal"
+
+    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
+                  primary_key=True)  # type: TelegramID
+    portal = Column(Integer, primary_key=True)  # type: TelegramID
+    portal_receiver = Column(Integer, primary_key=True)  # type: TelegramID
+
+    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
+                                           ("portal.tgid", "portal.tg_receiver"),
+                                           onupdate="CASCADE", ondelete="CASCADE"),)
+
+
+class Contact(Base):
+    __tablename__ = "contact"
+
+    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)  # type: TelegramID
+    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)  # type: TelegramID

mautrix_telegram/db/user_profile.py

@@ -0,0 +1,69 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, and_
+from typing import Dict, Optional
+
+from ..types import MatrixUserID, MatrixRoomID
+from .base import Base
+
+
+class UserProfile(Base):
+    __tablename__ = "mx_user_profile"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    user_id = Column(String, primary_key=True)  # type: MatrixUserID
+    membership = Column(String, nullable=False, default="leave")
+    displayname = Column(String, nullable=True)
+    avatar_url = Column(String, nullable=True)
+
+    def dict(self) -> Dict[str, str]:
+        return {
+            "membership": self.membership,
+            "displayname": self.displayname,
+            "avatar_url": self.avatar_url,
+        }
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID, user_id: MatrixUserID) -> Optional['UserProfile']:
+        rows = cls.db.execute(
+            cls.t.select().where(and_(cls.c.room_id == room_id, cls.c.user_id == user_id)))
+        try:
+            room_id, user_id, membership, displayname, avatar_url = next(rows)
+            return cls(room_id=room_id, user_id=user_id, membership=membership,
+                       displayname=displayname, avatar_url=avatar_url)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def delete_all(cls, room_id: MatrixRoomID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.room_id == room_id))
+
+    def update(self) -> None:
+        super().update(membership=self.membership, displayname=self.displayname,
+                       avatar_url=self.avatar_url)
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.room_id == self.room_id, self.c.user_id == self.user_id)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id, user_id=self.user_id,
+                                                membership=self.membership,
+                                                displayname=self.displayname,
+                                                avatar_url=self.avatar_url))

mautrix_telegram/formatter.py

@@ -1,380 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from html import escape, unescape
-from html.parser import HTMLParser
-from collections import deque
-import re
-import logging
-
-from mautrix_appservice import MatrixRequestError
-
-from telethon.tl.types import *
-
-from . import user as u, puppet as p
-from .db import Message as DBMessage
-
-log = logging.getLogger("mau.formatter")
-
-# TEXT LEN EXPLANATION:
-# Telegram formatting counts two bytes in an UTF-16 string as one character.
-#
-# For Telegram -> Matrix formatting, we get the same counting mechanism by encoding the input
-# text as UTF-16 Little Endian and doubling all the offsets and lengths given by Telegram. With
-# those doubled values, we process the input entities and text. The text is converted back to
-# native str format before it's inserted into the output HTML.
-#
-# For Matrix -> Telegram formatting, do the same input encoding, but divide the length by two
-# instead of multiplying when generating the lengths and offsets of Telegram entities.
-#
-# The endianness doesn't matter, but it has to be specified to avoid the two BOM bits messing
-# everything up.
-TEMP_ENC = "utf-16-le"
-
-
-# region Matrix to Telegram
-
-
-class MatrixParser(HTMLParser):
-    mention_regex = re.compile("https://matrix.to/#/(@.+)")
-
-    def __init__(self):
-        super().__init__()
-        self.text = ""
-        self.entities = []
-        self._building_entities = {}
-        self._list_counter = 0
-        self._open_tags = deque()
-        self._open_tags_meta = deque()
-        self._previous_ended_line = True
-
-    def handle_starttag(self, tag, attrs):
-        self._open_tags.appendleft(tag)
-        self._open_tags_meta.appendleft(0)
-        attrs = dict(attrs)
-        entity_type = None
-        args = {}
-        if tag == "strong" or tag == "b":
-            entity_type = MessageEntityBold
-        elif tag == "em" or tag == "i":
-            entity_type = MessageEntityItalic
-        elif tag == "code":
-            try:
-                pre = self._building_entities["pre"]
-                try:
-                    pre.language = attrs["class"][len("language-"):]
-                except KeyError:
-                    pass
-            except KeyError:
-                entity_type = MessageEntityCode
-        elif tag == "pre":
-            entity_type = MessageEntityPre
-            args["language"] = ""
-        elif tag == "a":
-            try:
-                url = attrs["href"]
-            except KeyError:
-                return
-            mention = self.mention_regex.search(url)
-            if mention:
-                mxid = mention.group(1)
-                user = p.Puppet.get_by_mxid(mxid, create=False)
-                if not user:
-                    user = u.User.get_by_mxid(mxid, create=False)
-                    if not user:
-                        return
-                if user.username:
-                    entity_type = MessageEntityMention
-                    url = f"@{user.username}"
-                else:
-                    entity_type = MessageEntityMentionName
-                    args["user_id"] = user.tgid
-            elif url.startswith("mailto:"):
-                url = url[len("mailto:"):]
-                entity_type = MessageEntityEmail
-            else:
-                if self.get_starttag_text() == url:
-                    entity_type = MessageEntityUrl
-                else:
-                    entity_type = MessageEntityTextUrl
-                    args["url"] = url
-                    url = None
-            self._open_tags_meta.popleft()
-            self._open_tags_meta.appendleft(url)
-
-        if entity_type and tag not in self._building_entities:
-            # See "TEXT LEN EXPLANATION" near start of file
-            offset = int(len(self.text.encode(TEMP_ENC)) / 2)
-            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
-
-    def _list_depth(self):
-        depth = 0
-        for tag in self._open_tags:
-            if tag == "ol" or tag == "ul":
-                depth += 1
-        return depth
-
-    def handle_data(self, text):
-        text = unescape(text)
-        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
-        list_format_offset = 0
-        if previous_tag == "a":
-            url = self._open_tags_meta[0]
-            if url:
-                text = url
-        elif len(self._open_tags) > 1 and self._previous_ended_line and previous_tag == "li":
-            list_type = self._open_tags[1]
-            indent = (self._list_depth() - 1) * 4 * " "
-            text = text.strip("\n")
-            if len(text) == 0:
-                return
-            elif list_type == "ul":
-                text = f"{indent}* {text}"
-                list_format_offset = len(indent) + 2
-            elif list_type == "ol":
-                n = self._open_tags_meta[1]
-                n += 1
-                self._open_tags_meta[1] = n
-                text = f"{indent}{n}. {text}"
-                list_format_offset = len(indent) + 3
-        for tag, entity in self._building_entities.items():
-            # See "TEXT LEN EXPLANATION" near start of file
-            entity.length += int(len(text.strip("\n").encode(TEMP_ENC)) / 2)
-            entity.offset += list_format_offset
-
-        if text.endswith("\n"):
-            self._previous_ended_line = True
-        else:
-            self._previous_ended_line = False
-
-        self.text += text
-
-    def handle_endtag(self, tag):
-        try:
-            self._open_tags.popleft()
-            self._open_tags_meta.popleft()
-        except IndexError:
-            pass
-        if (tag == "ul" or tag == "ol") and self.text.endswith("\n"):
-            self.text = self.text[:-1]
-        entity = self._building_entities.pop(tag, None)
-        if entity:
-            self.entities.append(entity)
-
-
-def matrix_to_telegram(html):
-    try:
-        parser = MatrixParser()
-        parser.feed(html)
-        return parser.text, parser.entities
-    except Exception:
-        log.exception("Failed to convert Matrix format:\nhtml=%s", html)
-
-
-def matrix_reply_to_telegram(content, tg_space, room_id=None):
-    try:
-        reply = content["m.relates_to"]["m.in_reply_to"]
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-        message = DBMessage.query.filter(DBMessage.mxid == event_id and
-                                         DBMessage.tg_space == tg_space and
-                                         DBMessage.mx_room == room_id).one_or_none()
-        if message:
-            return message.tgid
-    except KeyError:
-        pass
-    return None
-
-
-# endregion
-# region Telegram to Matrix
-
-def telegram_reply_to_matrix(evt, source):
-    if evt.reply_to_msg_id:
-        space = (evt.to_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
-                 else source.tgid)
-        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
-        if msg:
-            return {
-                "m.in_reply_to": {
-                    "event_id": msg.mxid,
-                    "room_id": msg.mx_room,
-                }
-            }
-    return {}
-
-
-async def telegram_event_to_matrix(evt, source, native_replies=False, message_link_in_reply=False,
-                                   main_intent=None, reply_text="Reply"):
-    text = evt.message
-    html = telegram_to_matrix(evt.message, evt.entities) if evt.entities else None
-    relates_to = {}
-
-    if evt.fwd_from:
-        if not html:
-            html = escape(text)
-        from_id = evt.fwd_from.from_id
-        user = u.User.get_by_tgid(from_id)
-        if user:
-            fwd_from = f"<a href='https://matrix.to/#/{user.mxid}'>{user.mxid}</a>"
-        else:
-            puppet = p.Puppet.get(from_id, create=False)
-            if puppet and puppet.displayname:
-                fwd_from = f"<a href='https://matrix.to/#/{puppet.mxid}'>{puppet.displayname}</a>"
-            else:
-                user = await source.client.get_entity(from_id)
-                if user:
-                    fwd_from = p.Puppet.get_displayname(user, format=False)
-                else:
-                    fwd_from = None
-        if not fwd_from:
-            fwd_from = "Unknown user"
-        html = (f"Forwarded message from <b>{fwd_from}</b><br/>"
-                + f"<blockquote>{html}</blockquote>")
-
-    if evt.reply_to_msg_id:
-        space = (evt.to_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
-                 else source.tgid)
-        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
-        if msg:
-            if native_replies:
-                relates_to["m.in_reply_to"] = {
-                    "event_id": msg.mxid,
-                    "room_id": msg.mx_room,
-                }
-                if reply_text == "Edit":
-                    html = "<u>Edit:</u> " + (html or escape(text))
-            else:
-                try:
-                    event = await main_intent.get_event(msg.mx_room, msg.mxid)
-                    content = event["content"]
-                    body = (content["formatted_body"]
-                            if "formatted_body" in content
-                            else content["body"])
-                    sender = event['sender']
-                    puppet = p.Puppet.get_by_mxid(sender, create=False)
-                    displayname = puppet.displayname if puppet else sender
-                    reply_to_user = f"<a href='https://matrix.to/#/{sender}'>{displayname}</a>"
-                    reply_to_msg = (("<a href='https://matrix.to/#/"
-                                     + f"{msg.mx_room}/{msg.mxid}'>{reply_text}</a>")
-                                    if message_link_in_reply else "Reply")
-                    quote = f"{reply_to_msg} to {reply_to_user}<blockquote>{body}</blockquote>"
-                except (ValueError, KeyError, MatrixRequestError):
-                    quote = "{reply_text} to unknown user <em>(Failed to fetch message)</em>:<br/>"
-                if html:
-                    html = quote + html
-                else:
-                    html = quote + escape(text)
-
-    if isinstance(evt, Message) and evt.post and evt.post_author:
-        if not html:
-            html = escape(text)
-        text += f"\n- {evt.post_author}"
-        html += f"<br/><i>- <u>{evt.post_author}</u></i>"
-
-    if html:
-        html = html.replace("\n", "<br/>")
-
-    return text, html, relates_to
-
-
-def telegram_to_matrix(text, entities):
-    try:
-        return _telegram_to_matrix(text, entities)
-    except Exception:
-        log.exception("Failed to convert Telegram format:\n"
-                      "message=%s\n"
-                      "entities=%s",
-                      text, entities)
-
-
-def _telegram_to_matrix(text, entities):
-    if not entities:
-        return text
-    # See "TEXT LEN EXPLANATION" near start of file
-    text = text.encode(TEMP_ENC)
-    html = []
-    last_offset = 0
-    for entity in entities:
-        entity.offset *= 2
-        entity.length *= 2
-        if entity.offset > last_offset:
-            html.append(escape(text[last_offset:entity.offset].decode(TEMP_ENC)))
-        elif entity.offset < last_offset:
-            continue
-
-        skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length].decode(TEMP_ENC))
-        entity_type = type(entity)
-
-        if entity_type == MessageEntityBold:
-            html.append(f"<strong>{entity_text}</strong>")
-        elif entity_type == MessageEntityItalic:
-            html.append(f"<em>{entity_text}</em>")
-        elif entity_type == MessageEntityCode:
-            html.append(f"<code>{entity_text}</code>")
-        elif entity_type == MessageEntityPre:
-            if entity.language:
-                html.append("<pre>"
-                            + f"<code class='language-{entity.language}'>{entity_text}</code>"
-                            + "</pre>")
-            else:
-                html.append(f"<pre><code>{entity_text}</code></pre>")
-        elif entity_type == MessageEntityMention:
-            username = entity_text[1:]
-
-            user = u.User.find_by_username(username)
-            if user:
-                mxid = user.mxid
-            else:
-                puppet = p.Puppet.find_by_username(username)
-                mxid = puppet.mxid if puppet else None
-            if mxid:
-                html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
-            else:
-                skip_entity = True
-        elif entity_type == MessageEntityMentionName:
-            user = u.User.get_by_tgid(entity.user_id)
-            if user:
-                mxid = user.mxid
-            else:
-                puppet = p.Puppet.get(entity.user_id, create=False)
-                mxid = puppet.mxid if puppet else None
-            if mxid:
-                html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
-            else:
-                skip_entity = True
-        elif entity_type == MessageEntityEmail:
-            html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
-        elif entity_type in {MessageEntityTextUrl, MessageEntityUrl}:
-            url = escape(entity.url) if entity_type == MessageEntityTextUrl else entity_text
-            if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
-                url = "http://" + url
-            html.append(f"<a href='{url}'>{entity_text}</a>")
-        elif entity_type == MessageEntityBotCommand:
-            html.append(f"<font color='blue'>!{entity_text[1:]}")
-        elif entity_type == MessageEntityHashtag:
-            html.append(f"<font color='blue'>{entity_text}</font>")
-        else:
-            skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:].decode(TEMP_ENC))
-
-    return "".join(html)
-
-# endregion

mautrix_telegram/formatter/__init__.py

@@ -0,0 +1,8 @@
+from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
+                          init_mx)
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
+from .. import context as c
+
+
+def init(context: c.Context) -> None:
+    init_mx(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -0,0 +1,154 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple, Callable, Pattern, Match, TYPE_CHECKING, Dict, Any
+import re
+import logging
+
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
+                               TypeMessageEntity)
+
+from ... import puppet as pu
+from ...types import TelegramID, MatrixRoomID
+from ...db import Message as DBMessage
+from ..util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                    trim_reply_fallback_text)
+from .parser import ParsedMessage, parse_html
+
+if TYPE_CHECKING:
+    from ...context import Context
+
+log = logging.getLogger("mau.fmt.mx")  # type: logging.Logger
+should_bridge_plaintext_highlights = False  # type: bool
+
+command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
+not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
+plain_mention_regex = None  # type: Optional[Pattern]
+
+
+def plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
+def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+    if len(message) > 4096:
+        message = message[0:4082] + " [message cut]"
+        new_entities = []
+        for entity in entities:
+            if entity.offset > 4082:
+                continue
+            if entity.offset + entity.length > 4082:
+                entity.length = 4082 - entity.offset
+            new_entities.append(entity)
+        new_entities.append(MessageEntityItalic(4082, len(" [message cut]")))
+        entities = new_entities
+    return message, entities
+
+
+class FormatError(Exception):
+    pass
+
+
+def matrix_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(plain_mention_to_html, html)
+
+        text, entities = parse_html(add_surrogates(html))
+        text = remove_surrogates(text.strip())
+        text, entities = cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
+                             room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
+    relates_to = content.get("m.relates_to", None) or {}
+    if not relates_to:
+        return None
+    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
+             else relates_to.get("m.in_reply_to", None) or {})
+    if not reply:
+        return None
+    room_id = room_id or reply.get("room_id", None)
+    event_id = reply.get("event_id", None)
+    if not event_id:
+        return
+
+    try:
+        if content["format"] == "org.matrix.custom.html":
+            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
+    except KeyError:
+        pass
+    content["body"] = trim_reply_fallback_text(content["body"])
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
+    return None
+
+
+def matrix_text_to_telegram(text: str) -> ParsedMessage:
+    text = command_regex.sub(r"/\1", text)
+    text = text.replace("\t", " " * 4)
+    text = not_command_regex.sub(r"\1", text)
+    if should_bridge_plaintext_highlights:
+        entities, pmr_replacer = plain_mention_to_text()
+        text = plain_mention_regex.sub(pmr_replacer, text)
+    else:
+        entities = []
+    return text, entities
+
+
+def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
+    entities = []
+
+    def replacer(match) -> str:
+        puppet = pu.Puppet.find_by_displayname(match.group(2))
+        if puppet:
+            offset = match.start()
+            length = match.end() - offset
+            if puppet.username:
+                entity = MessageEntityMention(offset, length)
+                text = f"@{puppet.username}"
+            else:
+                entity = MessageEntityMentionName(offset, length, user_id=puppet.tgid)
+                text = puppet.displayname
+            entities.append(entity)
+            return text
+        return "".join(match.groups())
+
+    return entities, replacer
+
+
+def init_mx(context: "Context") -> None:
+    global plain_mention_regex, should_bridge_plaintext_highlights
+    config = context.config
+    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
+    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
+    plain_mention_regex = re.compile(f"^({dn_template})")
+    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -0,0 +1,66 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Tuple
+
+from html.parser import HTMLParser
+
+
+class HTMLNode(list):
+    def __init__(self, tag: str, attrs: List[Tuple[str, str]]):
+        super().__init__()
+        self.tag = tag  # type: str
+        self.text = ""  # type: str
+        self.tail = ""  # type: str
+        self.attrib = dict(attrs)  # type: Dict[str, str]
+
+
+class NodeifyingParser(HTMLParser):
+    # From https://www.w3.org/TR/html5/syntax.html#writing-html-documents-elements
+    void_tags = ("area", "base", "br", "col", "command", "embed", "hr", "img", "input", "link",
+                 "meta", "param", "source", "track", "wbr")
+
+    def __init__(self):
+        super().__init__()
+        self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
+
+    def handle_starttag(self, tag, attrs):
+        node = HTMLNode(tag, attrs)
+        self.stack[-1].append(node)
+        if tag not in self.void_tags:
+            self.stack.append(node)
+
+    def handle_startendtag(self, tag, attrs):
+        self.stack[-1].append(HTMLNode(tag, attrs))
+
+    def handle_endtag(self, tag):
+        if tag == self.stack[-1].tag:
+            self.stack.pop()
+
+    def handle_data(self, data):
+        if len(self.stack[-1]) > 0:
+            self.stack[-1][-1].tail += data
+        else:
+            self.stack[-1].text += data
+
+    def error(self, message):
+        pass
+
+
+def read_html(data: str) -> HTMLNode:
+    parser = NodeifyingParser()
+    parser.feed(data)
+    return parser.stack[0]

mautrix_telegram/formatter/from_matrix/html_reader.pyi

@@ -0,0 +1,11 @@
+from typing import Dict, List
+
+
+class HTMLNode(List['HTMLNode']):
+    tag: str
+    text: str
+    tail: str
+    attrib: Dict[str, str]
+
+
+def read_html(data: str) -> HTMLNode: ...

mautrix_telegram/formatter/from_matrix/parser.py

@@ -0,0 +1,253 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Tuple, Pattern
+import re
+
+from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
+                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
+                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
+                               MessageEntityBold as Bold, MessageEntityItalic as Italic,
+                               MessageEntityCode as Code, MessageEntityPre as Pre,
+                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
+                               MessageEntityBlockquote as Blockquote, TypeMessageEntity)
+
+from ... import user as u, puppet as pu, portal as po
+from ...types import MatrixUserID
+from .telegram_message import TelegramMessage, Entity, offset_length_multiply
+
+from .html_reader import HTMLNode, read_html
+
+ParsedMessage = Tuple[str, List[TypeMessageEntity]]
+
+
+def parse_html(input_html: str) -> ParsedMessage:
+    return MatrixParser.parse(input_html)
+
+
+class RecursionContext:
+    def __init__(self, strip_linebreaks: bool = True, ul_depth: int = 0):
+        self.strip_linebreaks = strip_linebreaks  # type: bool
+        self.ul_depth = ul_depth  # type: int
+        self._inited = True  # type: bool
+
+    def __setattr__(self, key, value):
+        if getattr(self, "_inited", False) is True:
+            raise TypeError("'RecursionContext' object is immutable")
+        super(RecursionContext, self).__setattr__(key, value)
+
+    def enter_list(self) -> 'RecursionContext':
+        return RecursionContext(strip_linebreaks=self.strip_linebreaks, ul_depth=self.ul_depth + 1)
+
+    def enter_code_block(self) -> 'RecursionContext':
+        return RecursionContext(strip_linebreaks=False, ul_depth=self.ul_depth)
+
+
+class MatrixParser:
+    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")  # type: Pattern
+    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")  # type: Pattern
+    block_tags = ("p", "pre", "blockquote",
+                  "ol", "ul", "li",
+                  "h1", "h2", "h3", "h4", "h5", "h6",
+                  "div", "hr", "table")  # type: Tuple[str, ...]
+    list_bullets = ("●", "○", "■", "‣")  # type: Tuple[str, ...]
+
+    @classmethod
+    def list_bullet(cls, depth: int) -> str:
+        return cls.list_bullets[(depth - 1) % len(cls.list_bullets)] + " "
+
+    @classmethod
+    def list_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        ordered = node.tag == "ol"
+        tagged_children = cls.node_to_tagged_tmessages(node, ctx)
+        counter = 1
+        indent_length = 0
+        if ordered:
+            try:
+                counter = int(node.attrib.get("start", "1"))
+            except ValueError:
+                counter = 1
+
+            longest_index = counter - 1 + len(tagged_children)
+            indent_length = len(str(longest_index))
+        indent = (indent_length + 4) * " "
+        children = []  # type: List[TelegramMessage]
+        for child, tag in tagged_children:
+            if tag != "li":
+                continue
+
+            if ordered:
+                prefix = f"{counter}. "
+                counter += 1
+            else:
+                prefix = cls.list_bullet(ctx.ul_depth)
+            child = child.prepend(prefix)
+            parts = child.split("\n")
+            parts = parts[:1] + [part.prepend(indent) for part in parts[1:]]
+            child = TelegramMessage.join(parts, "\n")
+            children.append(child)
+        return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def header_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        children = cls.node_to_tmessages(node, ctx)
+        length = int(node.tag[1])
+        prefix = "#" * length + " "
+        return TelegramMessage.join(children, "").prepend(prefix).format(Bold)
+
+    @classmethod
+    def basic_format_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        if node.tag in ("b", "strong"):
+            msg.format(Bold)
+        elif node.tag in ("i", "em"):
+            msg.format(Italic)
+        elif node.tag in ("s", "strike", "del"):
+            msg.format(Strike)
+        elif node.tag in ("u", "ins"):
+            msg.format(Underline)
+        elif node == "blockquote":
+            msg.format(Blockquote)
+        elif node.tag == "command":
+            msg.format(Command)
+
+        return msg
+
+    @classmethod
+    def link_to_tstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        href = node.attrib.get("href", "")
+        if not href:
+            return msg
+
+        if href.startswith("mailto:"):
+            return TelegramMessage(href[len("mailto:"):]).format(Email)
+
+        mention = cls.mention_regex.match(href)
+        if mention:
+            mxid = MatrixUserID(mention.group(1))
+            user = (pu.Puppet.get_by_mxid(mxid)
+                    or u.User.get_by_mxid(mxid, create=False))
+            if not user:
+                return msg
+            if user.username:
+                return TelegramMessage(f"@{user.username}").format(Mention)
+            elif user.tgid:
+                displayname = user.plain_displayname or msg.text
+                return TelegramMessage(displayname).format(MentionName, user_id=user.tgid)
+            return msg
+
+        room = cls.room_regex.match(href)
+        if room:
+            username = po.Portal.get_username_from_mx_alias(room.group(1))
+            portal = po.Portal.find_by_username(username)
+            if portal and portal.username:
+                return TelegramMessage(f"@{portal.username}").format(Mention)
+
+        return (msg.format(URL)
+                if msg.text == href
+                else msg.format(TextURL, url=href))
+
+    @classmethod
+    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def node_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        if node.tag == "mx-reply":
+            return TelegramMessage("")
+        elif node.tag == "ol":
+            return cls.list_to_tmessage(node, ctx)
+        elif node.tag == "ul":
+            return cls.list_to_tmessage(node, ctx.enter_list())
+        elif node.tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            return cls.header_to_tmessage(node, ctx)
+        elif node.tag == "br":
+            return TelegramMessage("\n")
+        elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
+            return cls.basic_format_to_tmessage(node, ctx)
+        elif node.tag == "blockquote":
+            # Telegram already has blockquote entities in the protocol schema, but it strips them
+            # server-side and none of the official clients support them.
+            # TODO once Telegram changes that, use the above if block for blockquotes too.
+            return cls.blockquote_to_tmessage(node, ctx)
+        elif node.tag == "a":
+            return cls.link_to_tstring(node, ctx)
+        elif node.tag == "p":
+            return cls.tag_aware_parse_node(node, ctx).append("\n")
+        elif node.tag == "pre":
+            lang = ""
+            try:
+                if node[0].tag == "code":
+                    node = node[0]
+                    lang = node.attrib["class"][len("language-"):]
+            except (IndexError, KeyError):
+                pass
+            return cls.parse_node(node, ctx.enter_code_block()).format(Pre, language=lang)
+        elif node.tag == "code":
+            return cls.parse_node(node, ctx.enter_code_block()).format(Code)
+        return cls.tag_aware_parse_node(node, ctx)
+
+    @staticmethod
+    def text_to_tmessage(text: str, ctx: RecursionContext) -> TelegramMessage:
+        if ctx.strip_linebreaks:
+            text = text.replace("\n", "")
+        return TelegramMessage(text)
+
+    @classmethod
+    def node_to_tagged_tmessages(cls, node: HTMLNode, ctx: RecursionContext
+                                 ) -> List[Tuple[TelegramMessage, str]]:
+        output = []
+
+        if node.text:
+            output.append((cls.text_to_tmessage(node.text, ctx), "text"))
+        for child in node:
+            output.append((cls.node_to_tmessage(child, ctx), child.tag))
+            if child.tail:
+                output.append((cls.text_to_tmessage(child.tail, ctx), "text"))
+        return output
+
+    @classmethod
+    def node_to_tmessages(cls, node: HTMLNode, ctx: RecursionContext
+                          ) -> List[TelegramMessage]:
+        return [msg for (msg, tag) in cls.node_to_tagged_tmessages(node, ctx)]
+
+    @classmethod
+    def tag_aware_parse_node(cls, node: HTMLNode, ctx: RecursionContext
+                             ) -> TelegramMessage:
+        msgs = cls.node_to_tagged_tmessages(node, ctx)
+        output = TelegramMessage()
+        prev_was_block = False
+        for msg, tag in msgs:
+            if tag in cls.block_tags:
+                msg = msg.append("\n")
+                if not prev_was_block:
+                    msg = msg.prepend("\n")
+                prev_was_block = True
+            output = output.append(msg)
+        return output.trim()
+
+    @classmethod
+    def parse_node(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        return TelegramMessage.join(cls.node_to_tmessages(node, ctx))
+
+    @classmethod
+    def parse(cls, data: str) -> ParsedMessage:
+        msg = cls.node_to_tmessage(read_html(f"<body>{data}</body>"), RecursionContext())
+        return msg.text, msg.entities

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -0,0 +1,158 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Callable, List, Optional, Sequence, Type, Union
+
+from telethon.tl.types import (MessageEntityMentionName as MentionName,
+                               MessageEntityTextUrl as TextURL, MessageEntityPre as Pre,
+                               TypeMessageEntity, InputMessageEntityMentionName as InputMentionName)
+
+
+class Entity:
+    @staticmethod
+    def copy(entity: TypeMessageEntity) -> Optional[TypeMessageEntity]:
+        if not entity:
+            return None
+        kwargs = {
+            "offset": entity.offset,
+            "length": entity.length,
+        }
+        if isinstance(entity, Pre):
+            kwargs["language"] = entity.language
+        elif isinstance(entity, TextURL):
+            kwargs["url"] = entity.url
+        elif isinstance(entity, (MentionName, InputMentionName)):
+            kwargs["user_id"] = entity.user_id
+        return entity.__class__(**kwargs)
+
+    @classmethod
+    def adjust(cls, entity: Union[TypeMessageEntity, List[TypeMessageEntity]],
+               func: Callable[[TypeMessageEntity], None]
+               ) -> Union[Optional[TypeMessageEntity], List[TypeMessageEntity]]:
+        if isinstance(entity, list):
+            return [Entity.adjust(element, func) for element in entity if entity]
+        elif not entity:
+            return None
+        entity = cls.copy(entity)
+        func(entity)
+        if entity.offset < 0:
+            entity.length += entity.offset
+            entity.offset = 0
+        return entity
+
+
+def offset_diff(amount: int) -> Callable[[TypeMessageEntity], None]:
+    def func(entity: TypeMessageEntity) -> None:
+        entity.offset += amount
+
+    return func
+
+
+def offset_length_multiply(amount: int) -> Callable[[TypeMessageEntity], None]:
+    def func(entity: TypeMessageEntity) -> None:
+        entity.offset *= amount
+        entity.length *= amount
+
+    return func
+
+
+class TelegramMessage:
+    def __init__(self, text: str = "", entities: Optional[List[TypeMessageEntity]] = None) -> None:
+        self.text = text  # type: str
+        self.entities = entities or []  # type: List[TypeMessageEntity]
+
+    def offset_entities(self, offset: int) -> 'TelegramMessage':
+        def apply_offset(entity: TypeMessageEntity, inner_offset: int
+                         ) -> Optional[TypeMessageEntity]:
+            entity = Entity.copy(entity)
+            entity.offset += inner_offset
+            if entity.offset < 0:
+                entity.offset = 0
+            elif entity.offset > len(self.text):
+                return None
+            elif entity.offset + entity.length > len(self.text):
+                entity.length = len(self.text) - entity.offset
+            return entity
+
+        self.entities = [apply_offset(entity, offset) for entity in self.entities if entity]
+        self.entities = [x for x in self.entities if x is not None]
+        return self
+
+    def append(self, *args: Union[str, 'TelegramMessage']) -> 'TelegramMessage':
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities += Entity.adjust(msg.entities, offset_diff(len(self.text)))
+            self.text += msg.text
+        return self
+
+    def prepend(self, *args: Union[str, 'TelegramMessage']) -> 'TelegramMessage':
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities = msg.entities + Entity.adjust(self.entities, offset_diff(len(msg.text)))
+            self.text = msg.text + self.text
+        return self
+
+    def format(self, entity_type: Type[TypeMessageEntity], offset: int = None, length: int = None,
+               **kwargs) -> 'TelegramMessage':
+        self.entities.append(entity_type(offset=offset or 0,
+                                         length=length if length is not None else len(self.text),
+                                         **kwargs))
+        return self
+
+    def concat(self, *args: Union[str, 'TelegramMessage']) -> 'TelegramMessage':
+        return TelegramMessage().append(self, *args)
+
+    def trim(self) -> 'TelegramMessage':
+        orig_len = len(self.text)
+        self.text = self.text.lstrip()
+        diff = orig_len - len(self.text)
+        self.text = self.text.rstrip()
+        self.offset_entities(-diff)
+        return self
+
+    def split(self, separator, max_items: int = 0) -> List['TelegramMessage']:
+        text_parts = self.text.split(separator, max_items - 1)
+        output = []  # type: List[TelegramMessage]
+
+        offset = 0
+        for part in text_parts:
+            msg = TelegramMessage(part)
+            for entity in self.entities:
+                start_in_range = len(part) > entity.offset - offset >= 0
+                end_in_range = len(part) >= entity.offset - offset + entity.length > 0
+                if start_in_range and end_in_range:
+                    msg.entities.append(Entity.adjust(entity, offset_diff(-offset)))
+            output.append(msg)
+
+            offset += len(part)
+            offset += len(separator)
+
+        return output
+
+    @staticmethod
+    def join(items: Sequence[Union[str, 'TelegramMessage']],
+             separator: str = " ") -> 'TelegramMessage':
+        main = TelegramMessage()
+        for msg in items:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            main.entities += Entity.adjust(msg.entities, offset_diff(len(main.text)))
+            main.text += msg.text + separator
+        if len(separator) > 0:
+            main.text = main.text[:-len(separator)]
+        return main

mautrix_telegram/formatter/from_telegram.py

@@ -0,0 +1,337 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
+from html import escape
+import logging
+import re
+
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityUrl,
+                               MessageEntityEmail, MessageEntityTextUrl, MessageEntityBold,
+                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
+                               MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
+                               MessageEntityPhone, TypeMessageEntity, Message, PeerChannel,
+                               MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
+                               MessageEntityUnderline, PeerUser)
+
+from mautrix_appservice import MatrixRequestError
+from mautrix_appservice.intent_api import IntentAPI
+
+from .. import user as u, puppet as pu, portal as po
+from ..types import TelegramID
+from ..db import Message as DBMessage
+from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                   trim_reply_fallback_text)
+
+if TYPE_CHECKING:
+    from ..abstract_user import AbstractUser
+
+log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
+
+
+def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
+    if evt.reply_to_msg_id:
+        space = (evt.to_id.channel_id
+                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+                 else source.tgid)
+        msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
+        if msg:
+            return {
+                "m.in_reply_to": {
+                    "event_id": msg.mxid,
+                    "room_id": msg.mx_room,
+                },
+                "rel_type": "m.reference",
+                "event_id": msg.mxid,
+                "room_id": msg.mx_room,
+            }
+    return {}
+
+
+async def _add_forward_header(source, text: str, html: Optional[str],
+                              fwd_from: MessageFwdHeader) -> Tuple[str, str]:
+    if not html:
+        html = escape(text)
+    fwd_from_html, fwd_from_text = None, None
+    if fwd_from.from_id:
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
+        if user:
+            fwd_from_text = user.displayname or user.mxid
+            fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
+                             f"{escape(fwd_from_text)}</a>")
+
+        if not fwd_from_text:
+            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
+            if puppet and puppet.displayname:
+                fwd_from_text = puppet.displayname or puppet.mxid
+                fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                                 f"{escape(fwd_from_text)}</a>")
+
+        if not fwd_from_text:
+            user = await source.client.get_entity(PeerUser(fwd_from.from_id))
+            if user:
+                fwd_from_text = pu.Puppet.get_displayname(user, False)
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+    else:
+        portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
+        if portal:
+            fwd_from_text = portal.title
+            if portal.alias:
+                fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
+                                 f"{escape(fwd_from_text)}</a>")
+            else:
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+        else:
+            channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
+            if channel:
+                fwd_from_text = channel.title
+                fwd_from_html = f"<b>{fwd_from_text}</b>"
+
+    if not fwd_from_text:
+        if fwd_from.from_id:
+            fwd_from_text = "Unknown user"
+        else:
+            fwd_from_text = "Unknown source"
+        fwd_from_html = f"<b>{fwd_from_text}</b>"
+
+    text = "\n".join([f"> {line}" for line in text.split("\n")])
+    text = f"Forwarded from {fwd_from_text}:\n{text}"
+    html = (f"Forwarded message from {fwd_from_html}<br/>"
+            f"<tg-forward><blockquote>{html}</blockquote></tg-forward>")
+    return text, html
+
+
+async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
+                            relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]:
+    space = (evt.to_id.channel_id
+             if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+             else source.tgid)
+
+    msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
+    if not msg:
+        return text, html
+
+    relates_to["rel_type"] = "m.reference"
+    relates_to["event_id"] = msg.mxid
+    relates_to["room_id"] = msg.mx_room
+    relates_to["m.in_reply_to"] = {
+        "event_id": msg.mxid,
+        "room_id": msg.mx_room,
+    }
+
+    try:
+        event = await main_intent.get_event(msg.mx_room, msg.mxid)
+
+        content = event["content"]
+        r_sender = event["sender"]
+
+        r_text_body = trim_reply_fallback_text(content["body"])
+        r_html_body = trim_reply_fallback_html(content["formatted_body"]
+                                               if "formatted_body" in content
+                                               else escape(content["body"]))
+
+        puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
+        r_displayname = puppet.displayname if puppet else r_sender
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{escape(r_displayname)}</a>"
+    except (ValueError, KeyError, MatrixRequestError):
+        r_sender_link = "unknown user"
+        r_displayname = "unknown user"
+        r_text_body = "Failed to fetch message"
+        r_html_body = "<em>Failed to fetch message</em>"
+
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>In reply to</a>"
+    html = (
+        f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
+        + (html or escape(text)))
+
+    lines = r_text_body.strip().split("\n")
+    text_with_quote = f"> <{r_displayname}> {lines.pop(0)}"
+    for line in lines:
+        if line:
+            text_with_quote += f"\n> {line}"
+    text_with_quote += "\n\n"
+    text_with_quote += text
+    return text_with_quote, html
+
+
+async def telegram_to_matrix(evt: Message, source: "AbstractUser",
+                             main_intent: Optional[IntentAPI] = None,
+                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
+                             override_text: str = None,
+                             override_entities: List[TypeMessageEntity] = None,
+                             no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
+    text = add_surrogates(override_text or evt.message)
+    entities = override_entities or evt.entities
+    html = _telegram_entities_to_matrix_catch(text, entities) if entities else None
+    relates_to = {}  # type: Dict
+
+    if prefix_html:
+        html = prefix_html + (html or escape(text))
+    if prefix_text:
+        text = prefix_text + text
+
+    if evt.fwd_from:
+        text, html = await _add_forward_header(source, text, html, evt.fwd_from)
+
+    if evt.reply_to_msg_id and not no_reply_fallback:
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent)
+
+    if isinstance(evt, Message) and evt.post and evt.post_author:
+        if not html:
+            html = escape(text)
+        text += f"\n- {evt.post_author}"
+        html += f"<br/><i>- <u>{evt.post_author}</u></i>"
+
+    if html:
+        html = html.replace("\n", "<br/>")
+
+    return remove_surrogates(text), remove_surrogates(html), relates_to
+
+
+def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEntity]) -> str:
+    try:
+        return _telegram_entities_to_matrix(text, entities)
+    except Exception:
+        log.exception("Failed to convert Telegram format:\n"
+                      "message=%s\n"
+                      "entities=%s",
+                      text, entities)
+    return "[failed conversion in _telegram_entities_to_matrix]"
+
+
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity],
+                                 offset: int = 0, length: int = None) -> str:
+    if not entities:
+        return escape(text)
+    if length is None:
+        length = len(text)
+    html = []
+    last_offset = 0
+    for i, entity in enumerate(entities):
+        if entity.offset > offset + length:
+            break
+        relative_offset = entity.offset - offset
+        if relative_offset > last_offset:
+            html.append(escape(text[last_offset:relative_offset]))
+        elif relative_offset < last_offset:
+            continue
+
+        skip_entity = False
+        entity_text = _telegram_entities_to_matrix(
+            text=text[relative_offset:relative_offset + entity.length],
+            entities=entities[i + 1:], offset=entity.offset, length=entity.length)
+        entity_type = type(entity)
+
+        if entity_type == MessageEntityBold:
+            html.append(f"<strong>{entity_text}</strong>")
+        elif entity_type == MessageEntityItalic:
+            html.append(f"<em>{entity_text}</em>")
+        elif entity_type == MessageEntityUnderline:
+            html.append(f"<u>{entity_text}</u>")
+        elif entity_type == MessageEntityStrike:
+            html.append(f"<del>{entity_text}</del>")
+        elif entity_type == MessageEntityBlockquote:
+            html.append(f"<blockquote>{entity_text}</blockquote>")
+        elif entity_type == MessageEntityCode:
+            html.append(f"<pre><code>{entity_text}</code></pre>"
+                        if "\n" in entity_text
+                        else f"<code>{entity_text}</code>")
+        elif entity_type == MessageEntityPre:
+            skip_entity = _parse_pre(html, entity_text, entity.language)
+        elif entity_type == MessageEntityMention:
+            skip_entity = _parse_mention(html, entity_text)
+        elif entity_type == MessageEntityMentionName:
+            skip_entity = _parse_name_mention(html, entity_text, TelegramID(entity.user_id))
+        elif entity_type == MessageEntityEmail:
+            html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
+        elif entity_type in (MessageEntityTextUrl, MessageEntityUrl):
+            skip_entity = _parse_url(html, entity_text,
+                                     entity.url if entity_type == MessageEntityTextUrl else None)
+        elif entity_type == MessageEntityBotCommand:
+            html.append(f"<font color='blue'>!{entity_text[1:]}</font>")
+        elif entity_type in (MessageEntityHashtag, MessageEntityCashtag, MessageEntityPhone):
+            html.append(f"<font color='blue'>{entity_text}</font>")
+        else:
+            skip_entity = True
+        last_offset = relative_offset + (0 if skip_entity else entity.length)
+    html.append(escape(text[last_offset:]))
+
+    return "".join(html)
+
+
+def _parse_pre(html: List[str], entity_text: str, language: str) -> bool:
+    if language:
+        html.append("<pre>"
+                    f"<code class='language-{language}'>{entity_text}</code>"
+                    "</pre>")
+    else:
+        html.append(f"<pre><code>{entity_text}</code></pre>")
+    return False
+
+
+def _parse_mention(html: List[str], entity_text: str) -> bool:
+    username = entity_text[1:]
+
+    user = u.User.find_by_username(username) or pu.Puppet.find_by_username(username)
+    if user:
+        mxid = user.mxid
+    else:
+        portal = po.Portal.find_by_username(username)
+        mxid = portal.alias or portal.mxid if portal else None
+
+    if mxid:
+        html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
+    else:
+        return True
+    return False
+
+
+def _parse_name_mention(html: List[str], entity_text: str, user_id: TelegramID) -> bool:
+    user = u.User.get_by_tgid(user_id)
+    if user:
+        mxid = user.mxid
+    else:
+        puppet = pu.Puppet.get(user_id, create=False)
+        mxid = puppet.mxid if puppet else None
+    if mxid:
+        html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
+    else:
+        return True
+    return False
+
+
+message_link_regex = re.compile(
+    r"https?://t(?:elegram)?\.(?:me|dog)/([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})")
+
+
+def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
+    url = escape(url) if url else entity_text
+    if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
+        url = "http://" + url
+
+    message_link_match = message_link_regex.match(url)
+    if message_link_match:
+        group, msgid_str = message_link_match.groups()
+        msgid = int(msgid_str)
+
+        portal = po.Portal.find_by_username(group)
+        if portal:
+            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
+            if message:
+                url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
+
+    html.append(f"<a href='{url}'>{entity_text}</a>")
+    return False

mautrix_telegram/formatter/util.py

@@ -0,0 +1,57 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Pattern
+from html import escape
+import struct
+import re
+
+
+# add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
+# Licensed under the MIT license.
+# https://github.com/LonamiWebs/Telethon/blob/7cce7aa3e4c6c7019a55530391b1761d33e5a04e/telethon/helpers.py
+def add_surrogates(text: Optional[str]) -> Optional[str]:
+    if text is None:
+        return None
+    return "".join("".join(chr(y) for y in struct.unpack("<HH", x.encode("utf-16-le")))
+                   if (0x10000 <= ord(x) <= 0x10FFFF) else x for x in text)
+
+
+def remove_surrogates(text: Optional[str]) -> Optional[str]:
+    if text is None:
+        return None
+    return text.encode("utf-16", "surrogatepass").decode("utf-16")
+
+
+# trim_reply_fallback_text, html_reply_fallback_regex and trim_reply_fallback_html are Matrix
+# reply fallback utility functions.
+# You may copy and use them under any OSI-approved license.
+def trim_reply_fallback_text(text: str) -> str:
+    if not text.startswith("> ") or "\n" not in text:
+        return text
+    lines = text.split("\n")
+    while len(lines) > 0 and lines[0].startswith("> "):
+        lines.pop(0)
+    return "\n".join(lines)
+
+
+html_reply_fallback_regex = re.compile("^<mx-reply>"
+                                       r"[\s\S]+?"
+                                       "</mx-reply>")  # type: Pattern
+
+
+def trim_reply_fallback_html(html: str) -> str:
+    return html_reply_fallback_regex.sub("", html)

mautrix_telegram/matrix.py

@@ -1,171 +1,266 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Match, Optional, Set, Tuple, TYPE_CHECKING
 import logging
+import asyncio
+import time
+import re
 
-from mautrix_appservice import MatrixRequestError
+from mautrix_appservice import MatrixRequestError, IntentError
 
-from .user import User
-from .portal import Portal
-from .puppet import Puppet
-from .commands import CommandHandler
+from .types import MatrixEvent, MatrixEventID, MatrixRoomID, MatrixUserID
+from . import user as u, portal as po, puppet as pu, commands as com
+
+if TYPE_CHECKING:
+    from .context import Context
+
+try:
+    from prometheus_client import Histogram
+
+    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events",
+                           ["event_type"])
+except ImportError:
+    Histogram = None
+    EVENT_TIME = None
 
 
 class MatrixHandler:
-    log = logging.getLogger("mau.mx")
+    log = logging.getLogger("mau.mx")  # type: logging.Logger
 
-    def __init__(self, context):
-        self.az, self.db, self.config, _ = context
-        self.commands = CommandHandler(context)
+    def __init__(self, context: 'Context') -> None:
+        self.az, self.config, _, self.tgbot = context.core
+        self.commands = com.CommandProcessor(context)  # type: com.CommandProcessor
+        self.previously_typing = []  # type: List[MatrixUserID]
 
         self.az.matrix_event_handler(self.handle_event)
 
-    async def init_as_bot(self):
-        self.az.intent.set_display_name(
-            self.config.get("appservice.bot_displayname", "Telegram bridge bot"))
+    async def init_as_bot(self) -> None:
+        displayname = self.config["appservice.bot_displayname"]
+        if displayname:
+            try:
+                await self.az.intent.set_display_name(
+                    displayname if displayname != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set displayname")
 
-    async def handle_puppet_invite(self, room, puppet, inviter):
-        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}")
-        if not inviter.logged_in:
-            await puppet.intent.error_and_leave(
-                room, text="Please log in before inviting Telegram puppets.")
+        avatar = self.config["appservice.bot_avatar"]
+        if avatar:
+            try:
+                await self.az.intent.set_avatar(avatar if avatar != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set avatar")
+
+    async def handle_puppet_invite(self, room_id: MatrixRoomID, puppet: pu.Puppet, inviter: u.User
+                                   ) -> None:
+        intent = puppet.default_mxid_intent
+        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}")
+        if not await inviter.is_logged_in():
+            await intent.error_and_leave(
+                room_id, text="Please log in before inviting Telegram puppets.")
             return
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if portal:
             if portal.peer_type == "user":
-                await puppet.intent.error_and_leave(
-                    room, text="You can not invite additional users to private chats.")
+                await intent.error_and_leave(
+                    room_id, text="You can not invite additional users to private chats.")
                 return
             await portal.invite_telegram(inviter, puppet)
-            await puppet.intent.join_room(room)
+            await intent.join_room(room_id)
             return
         try:
-            members = await self.az.intent.get_room_members(room)
+            members = await self.az.intent.get_room_members(room_id)
         except MatrixRequestError:
             members = []
-        if self.az.intent.mxid not in members:
+        if self.az.bot_mxid not in members:
             if len(members) > 1:
-                await puppet.intent.error_and_leave(room, text=None, html=(
+                await intent.error_and_leave(room_id, text=None, html=(
                     f"Please invite "
-                    + f"<a href='https://matrix.to/#/{self.az.intent.mxid}'>the bridge bot</a> "
-                    + f"first if you want to create a Telegram chat."))
+                    f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
+                    f"first if you want to create a Telegram chat."))
                 return
 
-            await puppet.intent.join_room(room)
-            portal = Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
+            await intent.join_room(room_id)
+            portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
+            # TODO: if portal is None:
             if portal.mxid:
                 try:
-                    await puppet.intent.invite(portal.mxid, inviter.mxid)
-                    await puppet.intent.send_notice(room, text=None, html=(
+                    await intent.invite(portal.mxid, inviter.mxid)
+                    await intent.send_notice(room_id, text=None, html=(
                         "You already have a private chat with me: "
-                        + f"<a href='https://matrix.to/#/{portal.mxid}'>"
-                        + "Link to room"
-                        + "</a>"))
-                    await puppet.intent.leave_room(room)
+                        f"<a href='https://matrix.to/#/{portal.mxid}'>"
+                        "Link to room"
+                        "</a>"))
+                    await intent.leave_room(room_id)
                     return
                 except MatrixRequestError:
                     pass
-            portal.mxid = room
+            portal.mxid = room_id
             portal.save()
             inviter.register_portal(portal)
-            await puppet.intent.send_notice(room, "Portal to private chat created.")
+            await intent.send_notice(room_id, "Portal to private chat created.")
         else:
-            await puppet.intent.join_room(room)
-            await puppet.intent.send_notice(room, "This puppet will remain inactive until a "
-                                                  "Telegram chat is created for this room.")
+            await intent.join_room(room_id)
+            await intent.send_notice(room_id, "This puppet will remain inactive until a "
+                                              "Telegram chat is created for this room.")
+
+    async def accept_bot_invite(self, room_id: MatrixRoomID, inviter: u.User) -> None:
+        tries = 0
+        while tries < 5:
+            try:
+                await self.az.intent.join_room(room_id)
+                break
+            except (IntentError, MatrixRequestError):
+                tries += 1
+                wait_for_seconds = (tries + 1) * 10
+                if tries < 5:
+                    self.log.exception(f"Failed to join room {room_id} with bridge bot, "
+                                       f"retrying in {wait_for_seconds} seconds...")
+                    await asyncio.sleep(wait_for_seconds)
+                else:
+                    self.log.exception("Failed to join room {room}, giving up.")
+                    return
 
-    async def handle_invite(self, room, user, inviter):
-        inviter = User.get_by_mxid(inviter)
         if not inviter.whitelisted:
-            return
-        elif user == self.az.bot_mxid:
-            await self.az.intent.join_room(room)
+            await self.az.intent.send_notice(
+                room_id,
+                text="You are not whitelisted to use this bridge.\n\n"
+                     "If you are the owner of this bridge, see the "
+                     "`bridge.permissions` section in your config file.",
+                html="<p>You are not whitelisted to use this bridge.</p>"
+                     "<p>If you are the owner of this bridge, see the "
+                     "<code>bridge.permissions</code> section in your config file.</p>")
+            await self.az.intent.leave_room(room_id)
+
+        try:
+            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
+        except MatrixRequestError:
+            is_management = False
+        cmd_prefix = self.commands.command_prefix
+        text = html = "Hello, I'm a Telegram bridge bot. "
+        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
+            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
+            html += (f"Use <code>{cmd_prefix} help</code> for help"
+                     f" or <code>{cmd_prefix} login</code> to log in.")
+            pass
+        else:
+            text += f"Use `{cmd_prefix} help` for help."
+            html += f"Use <code>{cmd_prefix} help</code> for help."
+        await self.az.intent.send_notice(room_id, text=text, html=html)
+
+    async def handle_invite(self, room_id: MatrixRoomID, user_id: MatrixUserID,
+                            inviter_mxid: MatrixUserID) -> None:
+        self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
+        inviter = u.User.get_by_mxid(inviter_mxid)
+        if inviter is None:
+            self.log.exception("Failed to find user with Matrix ID {inviter_mxid}")
+        await inviter.ensure_started()
+        if user_id == self.az.bot_mxid:
+            return await self.accept_bot_invite(room_id, inviter)
+        elif not inviter.whitelisted:
             return
 
-        puppet = Puppet.get_by_mxid(user)
+        puppet = pu.Puppet.get_by_mxid(user_id)
         if puppet:
-            await self.handle_puppet_invite(room, puppet, inviter)
+            await self.handle_puppet_invite(room_id, puppet, inviter)
             return
 
-        user = User.get_by_mxid(user, create=False)
-        portal = Portal.get_by_mxid(room)
-        if user and user.has_full_access and portal:
+        user = u.User.get_by_mxid(user_id, create=False)
+        if not user:
+            return
+        await user.ensure_started()
+        portal = po.Portal.get_by_mxid(room_id)
+        if user and await user.has_full_access(allow_bot=True) and portal:
             await portal.invite_telegram(inviter, user)
             return
 
         # The rest can probably be ignored
-        self.log.debug(f"{inviter} invited {user} to {room}")
 
-    async def handle_join(self, room, user):
-        user = User.get_by_mxid(user)
+    async def handle_join(self, room_id: MatrixRoomID, user_id: MatrixUserID,
+                          event_id: MatrixEventID) -> None:
+        user = await u.User.get_by_mxid(user_id).ensure_started()
 
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
-        if not user.whitelisted:
-            await portal.main_intent.kick(room, user.mxid,
+        if not user.relaybot_whitelisted:
+            await portal.main_intent.kick(room_id, user.mxid,
                                           "You are not whitelisted on this Telegram bridge.")
             return
-        elif not user.logged_in:
-            # TODO[waiting-for-bots] once we have bot support, this won't be needed.
-            await portal.main_intent.kick(room, user.mxid,
-                                          "You are not logged into this Telegram bridge.")
+        elif not await user.is_logged_in() and not portal.has_bot:
+            await portal.main_intent.kick(room_id, user.mxid,
+                                          "This chat does not have a bot relaying "
+                                          "messages for unauthenticated users.")
             return
 
-        self.log.debug(f"{user} joined {room}")
-        # TODO join Telegram chat if applicable
+        self.log.debug(f"{user} joined {room_id}")
+        if await user.is_logged_in() or portal.has_bot:
+            await portal.join_matrix(user, event_id)
 
-    async def handle_part(self, room, user, sender):
-        self.log.debug(f"{user} left {room}")
+    async def handle_part(self, room_id: MatrixRoomID, user_id: MatrixUserID,
+                          sender_mxid: MatrixUserID, event_id: MatrixEventID) -> None:
+        self.log.debug(f"{user_id} left {room_id}")
 
-        sender = User.get_by_mxid(sender, create=False)
+        sender = u.User.get_by_mxid(sender_mxid, create=False)
+        if not sender:
+            return
+        await sender.ensure_started()
 
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
-        puppet = Puppet.get_by_mxid(user)
-        if sender and puppet:
-            await portal.leave_matrix(puppet, sender)
+        puppet = pu.Puppet.get_by_mxid(user_id)
+        if puppet:
+            if sender:
+                await portal.kick_matrix(puppet, sender)
+            return
 
-        user = User.get_by_mxid(user, create=False)
-        if user and user.logged_in:
-            await portal.leave_matrix(user, sender)
+        user = u.User.get_by_mxid(user_id, create=False)
+        if not user:
+            return
+        await user.ensure_started()
+        if await user.is_logged_in() or portal.has_bot:
+            await portal.leave_matrix(user, sender, event_id)
 
-    def is_command(self, message):
+    def is_command(self, message: Dict) -> Tuple[bool, str]:
         text = message.get("body", "")
         prefix = self.config["bridge.command_prefix"]
         is_command = text.startswith(prefix)
         if is_command:
-            text = text[len(prefix) + 1:]
+            text = text[len(prefix) + 1:].lstrip()
         return is_command, text
 
-    async def handle_message(self, room, sender, message, event_id):
-        self.log.debug(f"{sender} sent {message} to ${room}")
-
+    async def handle_message(self, room: MatrixRoomID, sender_id: MatrixUserID, message: Dict,
+                             event_id: MatrixEventID) -> None:
         is_command, text = self.is_command(message)
-        sender = User.get_by_mxid(sender)
+        sender = await u.User.get_by_mxid(sender_id).ensure_started()
+        if not sender.relaybot_whitelisted:
+            self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:"
+                           " User is not whitelisted.")
+            return
+        self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}")
 
-        portal = Portal.get_by_mxid(room)
-        if sender.has_full_access and portal and not is_command:
+        portal = po.Portal.get_by_mxid(room)
+        if not is_command and portal and (await sender.is_logged_in() or portal.has_bot):
             await portal.handle_matrix_message(sender, message, event_id)
             return
 
-        if message["msgtype"] != "m.text":
+        if not sender.whitelisted or message.get("msgtype", "m.unknown") != "m.text":
             return
 
         try:
@@ -182,59 +277,201 @@ class MatrixHandler:
                 # Not enough values to unpack, i.e. no arguments
                 command = text
                 args = []
-            await self.commands.handle(room, sender, command, args, is_management,
+            await self.commands.handle(room, event_id, sender, command, args, is_management,
                                        is_portal=portal is not None)
 
-    async def handle_redaction(self, room, sender, event_id):
-        portal = Portal.get_by_mxid(room)
-        sender = User.get_by_mxid(sender)
-        if sender.has_full_access and portal:
-            await portal.handle_matrix_deletion(sender, event_id)
+    @staticmethod
+    async def handle_redaction(room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                               event_id: MatrixEventID) -> None:
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if not sender.relaybot_whitelisted:
+            return
 
-    async def handle_power_levels(self, room, sender, new, old):
-        portal = Portal.get_by_mxid(room)
-        sender = User.get_by_mxid(sender)
-        if sender.has_full_access and portal:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        await portal.handle_matrix_deletion(sender, event_id)
+
+    @staticmethod
+    async def handle_power_levels(room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                                  new: Dict, old: Dict) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             await portal.handle_matrix_power_levels(sender, new["users"], old["users"])
 
-    async def handle_room_meta(self, type, room, sender, content):
-        portal = Portal.get_by_mxid(room)
-        sender = User.get_by_mxid(sender)
-        if sender.has_full_access and portal:
+    @staticmethod
+    async def handle_room_meta(evt_type: str, room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                               content: dict) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             handler, content_key = {
                 "m.room.name": (portal.handle_matrix_title, "name"),
                 "m.room.topic": (portal.handle_matrix_about, "topic"),
                 "m.room.avatar": (portal.handle_matrix_avatar, "url"),
-            }[type]
+            }[evt_type]
             if content_key not in content:
-                # FIXME handle
-                pass
-                await handler(sender, content[content_key])
+                return
+            await handler(sender, content[content_key])
 
-    def filter_matrix_event(self, event):
-        return (event["sender"] == self.az.bot_mxid
-                or Puppet.get_id_from_mxid(event["sender"]) is not None)
+    @staticmethod
+    async def handle_room_pin(room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                              new_events: Set[str], old_events: Set[str]) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
+            events = new_events - old_events
+            if len(events) > 0:
+                # New event pinned, set that as pinned in Telegram.
+                await portal.handle_matrix_pin(sender, MatrixEventID(events.pop()))
+            elif len(new_events) == 0:
+                # All pinned events removed, remove pinned event in Telegram.
+                await portal.handle_matrix_pin(sender, None)
 
-    async def handle_event(self, evt):
+    @staticmethod
+    async def handle_room_upgrade(room_id: MatrixRoomID, new_room_id: MatrixRoomID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if portal:
+            await portal.handle_matrix_upgrade(new_room_id)
+
+    @staticmethod
+    async def handle_name_change(room_id: MatrixRoomID, user_id: MatrixUserID, displayname: str,
+                                 prev_displayname: str, event_id: MatrixEventID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.has_bot:
+            return
+
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if await user.needs_relaybot(portal):
+            await portal.name_change_matrix(user, displayname, prev_displayname, event_id)
+
+    @staticmethod
+    def parse_read_receipts(content: Dict) -> Dict[MatrixUserID, MatrixEventID]:
+        return {user_id: event_id
+                for event_id, receipts in content.items()
+                for user_id in receipts.get("m.read", {})}
+
+    @staticmethod
+    async def handle_read_receipts(room_id: MatrixRoomID,
+                                   receipts: Dict[MatrixUserID, MatrixEventID]) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id, event_id in receipts.items():
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+            await portal.mark_read(user, event_id)
+
+    @staticmethod
+    async def handle_presence(user_id: MatrixUserID, presence: str) -> None:
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if not await user.is_logged_in():
+            return
+        await user.set_presence(presence == "online")
+
+    async def handle_typing(self, room_id: MatrixRoomID, now_typing: List[MatrixUserID]) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id in set(self.previously_typing + now_typing):
+            is_typing = user_id in now_typing
+            was_typing = user_id in self.previously_typing
+            if is_typing and was_typing:
+                continue
+
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+
+            await portal.set_typing(user, is_typing)
+
+        self.previously_typing = now_typing
+
+    def filter_matrix_event(self, event: MatrixEvent) -> bool:
+        sender = event.get("sender", None)
+        if not sender:
+            return False
+        return (sender == self.az.bot_mxid
+                or pu.Puppet.get_id_from_mxid(sender) is not None)
+
+    async def try_handle_ephemeral_event(self, evt: MatrixEvent) -> None:
+        try:
+            await self.handle_ephemeral_event(evt)
+        except Exception:
+            self.log.exception("Error handling manually received Matrix event")
+
+    async def handle_ephemeral_event(self, evt: MatrixEvent) -> None:
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
+        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        content = evt.get("content", {})  # type: Dict
+        if evt_type == "m.receipt":
+            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
+        elif evt_type == "m.presence":
+            await self.handle_presence(sender, content.get("presence", "offline"))
+        elif evt_type == "m.typing":
+            await self.handle_typing(room_id, content.get("user_ids", []))
+
+    async def handle_event(self, evt: MatrixEvent) -> None:
         if self.filter_matrix_event(evt):
             return
+        start_time = time.time()
         self.log.debug("Received event: %s", evt)
-        type = evt["type"]
-        content = evt.get("content", {})
-        if type == "m.room.member":
-            membership = content.get("membership", "")
-            if membership == "invite":
-                await self.handle_invite(evt["room_id"], evt["state_key"], evt["sender"])
-            elif membership == "leave":
-                await self.handle_part(evt["room_id"], evt["state_key"], evt["sender"])
-            elif membership == "join":
-                await self.handle_join(evt["room_id"], evt["state_key"])
-        elif type == "m.room.message":
-            await self.handle_message(evt["room_id"], evt["sender"], content, evt["event_id"])
-        elif type == "m.room.redaction":
-            await self.handle_redaction(evt["room_id"], evt["sender"], evt["redacts"])
-        elif type == "m.room.power_levels":
-            await self.handle_power_levels(evt["room_id"], evt["sender"], evt["content"],
-                                           evt["prev_content"])
-        elif type == "m.room.name" or type == "m.room.avatar" or type == "m.room.topic":
-            await self.handle_room_meta(type, evt["room_id"], evt["sender"], evt["content"])
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
+        event_id = evt.get("event_id", None)  # type: Optional[MatrixEventID]
+        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        state_key = evt.get("state_key", None)
+        content = evt.get("content", {})  # type: Dict
+        if state_key is not None:
+            if evt_type == "m.room.member":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
+                membership = content.get("membership", "")  # type: str
+                prev_membership = prev_content.get("membership", "leave")  # type: str
+                if membership == prev_membership:
+                    match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
+                    mxid = match.group(0)  # type: str
+                    displayname = content.get("displayname", None) or mxid  # type: str
+                    prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
+                    if displayname != prev_displayname:
+                        await self.handle_name_change(room_id, state_key, displayname,
+                                                      prev_displayname, event_id)
+                elif membership == "invite":
+                    await self.handle_invite(room_id, state_key, sender)
+                elif prev_membership == "join" and membership == "leave":
+                    await self.handle_part(room_id, state_key, sender, event_id)
+                elif membership == "join":
+                    await self.handle_join(room_id, state_key, event_id)
+            elif evt_type == "m.room.power_levels":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})
+                await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
+            elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
+                await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
+            elif evt_type == "m.room.pinned_events":
+                new_events = set(evt["content"]["pinned"])
+                try:
+                    old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+                except KeyError:
+                    old_events = set()
+                await self.handle_room_pin(room_id, sender, new_events, old_events)
+            elif evt_type == "m.room.tombstone":
+                await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
+            else:
+                return
+        else:
+            if evt_type in ("m.room.message", "m.sticker"):
+                if evt_type != "m.room.message":
+                    content["msgtype"] = evt_type
+                await self.handle_message(room_id, sender, content, event_id)
+            elif evt_type == "m.room.redaction":
+                await self.handle_redaction(room_id, sender, evt["redacts"])
+            else:
+                return
+
+        if EVENT_TIME:
+            EVENT_TIME.labels(event_type=evt_type).observe(time.time() - start_time)

mautrix_telegram/puppet.py

@@ -1,79 +1,322 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Any, Dict, List, Iterable, Optional, Pattern, Union, TYPE_CHECKING
 from difflib import SequenceMatcher
-import re
+from enum import Enum
+from aiohttp import ServerDisconnectedError
+import asyncio
 import logging
+import re
 
-from telethon.tl.types import UserProfilePhoto, PeerUser
-from telethon.errors.rpc_error_list import LocationInvalidError
+from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
+                               InputPeerPhotoFileLocation, UserProfilePhotoEmpty)
+from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
+from .types import MatrixUserID, TelegramID
 from .db import Puppet as DBPuppet
+from . import util
 
-config = None
+if TYPE_CHECKING:
+    from .matrix import MatrixHandler
+    from .config import Config
+    from .context import Context
+    from .abstract_user import AbstractUser
+
+PuppetError = Enum('PuppetError', 'Success OnlyLoginSelf InvalidAccessToken')
+
+config = None  # type: Config
 
 
 class Puppet:
-    log = logging.getLogger("mau.puppet")
-    db = None
-    az = None
-    mxid_regex = None
-    cache = {}
+    log = logging.getLogger("mau.puppet")  # type: logging.Logger
+    az = None  # type: AppService
+    mx = None  # type: MatrixHandler
+    loop = None  # type: asyncio.AbstractEventLoop
+    mxid_regex = None  # type: Pattern
+    username_template = None  # type: str
+    hs_domain = None  # type: str
+    cache = {}  # type: Dict[TelegramID, Puppet]
+    by_custom_mxid = {}  # type: Dict[str, Puppet]
 
-    def __init__(self, id=None, username=None, displayname=None, photo_id=None):
-        self.id = id
+    def __init__(self,
+                 id: TelegramID,
+                 access_token: Optional[str] = None,
+                 custom_mxid: Optional[MatrixUserID] = None,
+                 username: Optional[str] = None,
+                 displayname: Optional[str] = None,
+                 displayname_source: Optional[TelegramID] = None,
+                 photo_id: Optional[str] = None,
+                 is_bot: bool = False,
+                 is_registered: bool = False,
+                 disable_updates: bool = False,
+                 db_instance: Optional[DBPuppet] = None) -> None:
+        self.id = id  # type: TelegramID
+        self.access_token = access_token  # type: Optional[str]
+        self.custom_mxid = custom_mxid  # type: Optional[MatrixUserID]
+        self.default_mxid = self.get_mxid_from_id(self.id)  # type: MatrixUserID
 
-        self.localpart = config.get("bridge.username_template", "telegram_{userid}").format(
-            userid=self.id)
-        hs = config["homeserver"]["domain"]
-        self.mxid = f"@{self.localpart}:{hs}"
-        self.username = username
-        self.displayname = displayname
-        self.photo_id = photo_id
-        self.intent = self.az.intent.user(self.mxid)
+        self.username = username  # type: Optional[str]
+        self.displayname = displayname  # type: Optional[str]
+        self.displayname_source = displayname_source  # type: Optional[TelegramID]
+        self.photo_id = photo_id  # type: Optional[str]
+        self.is_bot = is_bot  # type: bool
+        self.is_registered = is_registered  # type: bool
+        self.disable_updates = disable_updates  # type: bool
+        self._db_instance = db_instance  # type: Optional[DBPuppet]
+
+        self.default_mxid_intent = self.az.intent.user(self.default_mxid)
+        self.intent = self._fresh_intent()  # type: IntentAPI
+        self.sync_task = None  # type: Optional[asyncio.Future]
 
         self.cache[id] = self
+        if self.custom_mxid:
+            self.by_custom_mxid[self.custom_mxid] = self
 
     @property
-    def tgid(self):
+    def mxid(self) -> MatrixUserID:
+        return self.custom_mxid or self.default_mxid
+
+    @property
+    def tgid(self) -> TelegramID:
         return self.id
 
-    def to_db(self):
-        return self.db.merge(
-            DBPuppet(id=self.id, username=self.username, displayname=self.displayname,
-                     photo_id=self.photo_id))
+    @property
+    def is_real_user(self) -> bool:
+        """ Is True when the puppet is a real Matrix user. """
+        return bool(self.custom_mxid and self.access_token)
+
+    @staticmethod
+    async def is_logged_in() -> bool:
+        """ Is True if the puppet is logged in. """
+        return True
+
+    @property
+    def plain_displayname(self) -> str:
+        tpl = config["bridge.displayname_template"]
+        if tpl == "{displayname}":
+            # Template has no extra stuff, no need to parse.
+            return self.displayname
+        regex = re.compile("^" + re.escape(tpl).replace(re.escape("{displayname}"), "(.+?)") + "$")
+        match = regex.match(self.displayname)
+        return match.group(1) or self.displayname
+
+    def get_input_entity(self, user: 'AbstractUser') -> Awaitable[TypeInputPeer]:
+        return user.client.get_input_entity(PeerUser(user_id=self.tgid))
+
+    # region Custom puppet management
+    def _fresh_intent(self) -> IntentAPI:
+        return (self.az.intent.user(self.custom_mxid, self.access_token)
+                if self.is_real_user else self.default_mxid_intent)
+
+    async def switch_mxid(self, access_token: Optional[str],
+                          mxid: Optional[MatrixUserID]) -> PuppetError:
+        prev_mxid = self.custom_mxid
+        self.custom_mxid = mxid
+        self.access_token = access_token
+        self.intent = self._fresh_intent()
+
+        err = await self.init_custom_mxid()
+        if err != PuppetError.Success:
+            return err
+
+        try:
+            del self.by_custom_mxid[prev_mxid]  # type: ignore
+        except KeyError:
+            pass
+        if self.mxid != self.default_mxid:
+            self.by_custom_mxid[self.mxid] = self
+            await self.leave_rooms_with_default_user()
+        self.save()
+        return PuppetError.Success
+
+    async def init_custom_mxid(self) -> PuppetError:
+        if not self.is_real_user:
+            return PuppetError.Success
+
+        mxid = await self.intent.whoami()
+        if not mxid or mxid != self.custom_mxid:
+            self.custom_mxid = None
+            self.access_token = None
+            self.intent = self._fresh_intent()
+            if mxid != self.custom_mxid:
+                return PuppetError.OnlyLoginSelf
+            return PuppetError.InvalidAccessToken
+        if config["bridge.sync_with_custom_puppets"]:
+            self.sync_task = asyncio.ensure_future(self.sync(), loop=self.loop)
+        return PuppetError.Success
+
+    async def leave_rooms_with_default_user(self) -> None:
+        for room_id in await self.default_mxid_intent.get_joined_rooms():
+            try:
+                await self.default_mxid_intent.leave_room(room_id)
+                await self.intent.ensure_joined(room_id)
+            except (IntentError, MatrixRequestError):
+                pass
+
+    def create_sync_filter(self) -> Awaitable[str]:
+        return self.intent.client.create_filter(self.custom_mxid, {
+            "room": {
+                "include_leave": False,
+                "state": {
+                    "types": []
+                },
+                "timeline": {
+                    "types": [],
+                },
+                "ephemeral": {
+                    "types": ["m.typing", "m.receipt"],
+                },
+                "account_data": {
+                    "types": []
+                }
+            },
+            "account_data": {
+                "types": [],
+            },
+            "presence": {
+                "types": ["m.presence"],
+                "senders": [self.custom_mxid],
+            },
+        })
+
+    def filter_events(self, events: List[Dict]) -> List:
+        new_events = []
+        for event in events:
+            evt_type = event.get("type", None)
+            event.setdefault("content", {})
+            if evt_type == "m.typing":
+                is_typing = self.custom_mxid in event["content"].get("user_ids", [])
+                event["content"]["user_ids"] = [self.custom_mxid] if is_typing else []
+            elif evt_type == "m.receipt":
+                val = None
+                evt = None
+                for event_id in event["content"]:
+                    try:
+                        val = event["content"][event_id]["m.read"][self.custom_mxid]
+                        evt = event_id
+                        break
+                    except KeyError:
+                        pass
+                if val and evt:
+                    event["content"] = {evt: {"m.read": {
+                        self.custom_mxid: val
+                    }}}
+                else:
+                    continue
+            new_events.append(event)
+        return new_events
+
+    def handle_sync(self, presence: List, ephemeral: Dict) -> None:
+        presence_events = [self.mx.try_handle_ephemeral_event(event) for event in presence]
+
+        for room_id, events in ephemeral.items():
+            for event in events:
+                event["room_id"] = room_id
+
+        ephemeral_events = [self.mx.try_handle_ephemeral_event(event)
+                            for events in ephemeral.values()
+                            for event in self.filter_events(events)]
+
+        events = ephemeral_events + presence_events  # List[Callable[[int], Awaitable[None]]]
+        coro = asyncio.gather(*events, loop=self.loop)
+        asyncio.ensure_future(coro, loop=self.loop)
+
+    async def sync(self) -> None:
+        try:
+            await self._sync()
+        except asyncio.CancelledError:
+            self.log.info("Syncing cancelled")
+        except Exception:
+            self.log.exception("Fatal error syncing")
+
+    async def _sync(self) -> None:
+        if not self.is_real_user:
+            self.log.warning("Called sync() for non-custom puppet.")
+            return
+        custom_mxid = self.custom_mxid
+        access_token_at_start = self.access_token
+        errors = 0
+        next_batch = None
+        filter_id = await self.create_sync_filter()
+        self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.")
+        while access_token_at_start == self.access_token:
+            try:
+                sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch,
+                                                          set_presence="offline")  # type: Dict
+                errors = 0
+                if next_batch is not None:
+                    presence = sync_resp.get("presence", {}).get("events", [])  # type: List
+                    ephemeral = {room: data.get("ephemeral", {}).get("events", [])
+                                 for room, data
+                                 in sync_resp.get("rooms", {}).get("join", {}).items()
+                                 }  # type: Dict
+                    self.handle_sync(presence, ephemeral)
+                next_batch = sync_resp.get("next_batch", None)
+            except (MatrixRequestError, ServerDisconnectedError) as e:
+                wait = min(errors, 11) ** 2
+                self.log.warning(f"Syncer for {custom_mxid} errored: {e}. "
+                                 f"Waiting for {wait} seconds...")
+                errors += 1
+                await asyncio.sleep(wait)
+        self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.")
+
+    # endregion
+    # region DB conversion
+
+    @property
+    def db_instance(self) -> DBPuppet:
+        if not self._db_instance:
+            self._db_instance = self.new_db_instance()
+        return self._db_instance
+
+    def new_db_instance(self) -> DBPuppet:
+        return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
+                        username=self.username, displayname=self.displayname,
+                        displayname_source=self.displayname_source, photo_id=self.photo_id,
+                        is_bot=self.is_bot, matrix_registered=self.is_registered,
+                        disable_updates=self.disable_updates)
 
     @classmethod
-    def from_db(cls, db_puppet):
-        return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname, db_puppet.photo_id)
+    def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
+        return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
+                      db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
-    def save(self):
-        self.to_db()
-        self.db.commit()
+    def save(self) -> None:
+        self.db_instance.update(access_token=self.access_token, custom_mxid=self.custom_mxid,
+                                username=self.username, displayname=self.displayname,
+                                displayname_source=self.displayname_source, photo_id=self.photo_id,
+                                is_bot=self.is_bot, matrix_registered=self.is_registered,
+                                disable_updates=self.disable_updates)
 
-    def similarity(self, query):
+    # endregion
+    # region Info updating
+
+    def similarity(self, query: str) -> int:
         username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                                if self.username else 0)
         displayname_similarity = (SequenceMatcher(None, self.displayname, query).ratio()
                                   if self.displayname else 0)
         similarity = max(username_similarity, displayname_similarity)
-        return round(similarity * 1000) / 10
+        return int(round(similarity * 100))
 
     @staticmethod
-    def get_displayname(info, format=True):
+    def get_displayname(info: User, enable_format: bool = True) -> str:
         data = {
             "phone number": info.phone if hasattr(info, "phone") else None,
             "username": info.username,
@@ -82,22 +325,26 @@ class Puppet:
             "first name": info.first_name,
             "last name": info.last_name,
         }
-        preferences = config.get("bridge.displayname_preference",
-                                 ["full name", "username", "phone"])
+        preferences = config["bridge.displayname_preference"]
         name = None
         for preference in preferences:
             name = data[preference]
             if name:
                 break
-        if not name:
+
+        if isinstance(info, User) and info.deleted:
+            name = f"Deleted account {info.id}"
+        elif not name:
             name = info.id
 
-        if not format:
+        if not enable_format:
             return name
-        return config.get("bridge.displayname_template", "{displayname} (Telegram)").format(
+        return config["bridge.displayname_template"].format(
             displayname=name)
 
-    async def update_info(self, source, info):
+    async def update_info(self, source: 'AbstractUser', info: User) -> None:
+        if self.disable_updates:
+            return
         changed = False
         if self.username != info.username:
             self.username = info.username
@@ -105,78 +352,182 @@ class Puppet:
 
         changed = await self.update_displayname(source, info) or changed
         if isinstance(info.photo, UserProfilePhoto):
-            changed = await self.update_avatar(source, info.photo.photo_big) or changed
+            changed = await self.update_avatar(source, info.photo) or changed
+
+        self.is_bot = info.bot
 
         if changed:
             self.save()
 
-    async def update_displayname(self, source, info):
+    async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
+                                 ) -> bool:
+        if self.disable_updates:
+            return False
+        allow_source = (source.is_relaybot
+                        or self.displayname_source == source.tgid
+                        # No displayname source, so just trust anything
+                        or self.displayname_source is None
+                        # No phone -> not in contact list -> can't set custom name
+                        or (isinstance(info, User) and info.phone is None))
+        if not allow_source:
+            return False
+        elif isinstance(info, UpdateUserName):
+            info = await source.client.get_entity(PeerUser(self.tgid))
+
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
-            await self.intent.set_display_name(displayname)
             self.displayname = displayname
-            return True
-
-    async def update_avatar(self, source, photo):
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
-        if self.photo_id != photo_id:
+            self.displayname_source = source.tgid
             try:
-                file = await source.client.download_file_bytes(photo)
-            except LocationInvalidError:
-                return False
-            uploaded = await self.intent.upload_file(file)
-            await self.intent.set_avatar(uploaded["content_uri"])
-            self.photo_id = photo_id
+                await self.default_mxid_intent.set_display_name(displayname[:100])
+            except MatrixRequestError:
+                self.log.exception("Failed to set displayname")
+                self.displayname = ""
+                self.displayname_source = None
+            return True
+        elif source.is_relaybot or self.displayname_source is None:
+            self.displayname_source = source.tgid
             return True
         return False
 
+    async def update_avatar(self, source: 'AbstractUser',
+                            photo: Union[UserProfilePhoto, UserProfilePhotoEmpty]) -> bool:
+        if self.disable_updates:
+            return False
+
+        if isinstance(photo, UserProfilePhotoEmpty):
+            photo_id = ""
+        else:
+            photo_id = str(photo.photo_id)
+        if self.photo_id != photo_id:
+            if not photo_id:
+                self.photo_id = ""
+                try:
+                    await self.default_mxid_intent.set_avatar("")
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
+                return True
+
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(source),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
+            if file:
+                self.photo_id = photo_id
+                try:
+                    await self.default_mxid_intent.set_avatar(file.mxc)
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
+                return True
+        return False
+
+    # endregion
+    # region Getters
+
     @classmethod
-    def get(cls, id, create=True):
+    def get(cls, tgid: TelegramID, create: bool = True) -> Optional['Puppet']:
         try:
-            return cls.cache[id]
+            return cls.cache[tgid]
         except KeyError:
             pass
 
-        puppet = DBPuppet.query.get(id)
+        puppet = DBPuppet.get_by_tgid(tgid)
         if puppet:
             return cls.from_db(puppet)
 
         if create:
-            puppet = cls(id)
-            cls.db.add(puppet.to_db())
-            cls.db.commit()
+            puppet = cls(tgid)
+            puppet.db_instance.insert()
             return puppet
 
         return None
 
     @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid: MatrixUserID, create: bool = True) -> Optional['Puppet']:
         tgid = cls.get_id_from_mxid(mxid)
-        return cls.get(tgid, create) if tgid else None
+        if tgid:
+            return cls.get(tgid, create)
+
+        return None
 
     @classmethod
-    def get_id_from_mxid(cls, mxid):
+    def get_by_custom_mxid(cls, mxid: MatrixUserID) -> Optional['Puppet']:
+        if not mxid:
+            raise ValueError("Matrix ID can't be empty")
+
+        try:
+            return cls.by_custom_mxid[mxid]
+        except KeyError:
+            pass
+
+        puppet = DBPuppet.get_by_custom_mxid(mxid)
+        if puppet:
+            puppet = cls.from_db(puppet)
+            return puppet
+
+        return None
+
+    @classmethod
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        return (cls.by_custom_mxid[puppet.mxid]
+                if puppet.custom_mxid in cls.by_custom_mxid
+                else cls.from_db(puppet)
+                for puppet in DBPuppet.all_with_custom_mxid())
+
+    @classmethod
+    def get_id_from_mxid(cls, mxid: MatrixUserID) -> Optional[TelegramID]:
         match = cls.mxid_regex.match(mxid)
         if match:
-            return int(match.group(1))
+            return TelegramID(int(match.group(1)))
         return None
 
     @classmethod
-    def find_by_username(cls, username):
+    def get_mxid_from_id(cls, tgid: TelegramID) -> MatrixUserID:
+        return MatrixUserID(f"@{cls.username_template.format(userid=tgid)}:{cls.hs_domain}")
+
+    @classmethod
+    def find_by_username(cls, username: str) -> Optional['Puppet']:
+        if not username:
+            return None
+
         for _, puppet in cls.cache.items():
-            if puppet.username == username:
+            if puppet.username and puppet.username.lower() == username.lower():
                 return puppet
 
-        puppet = DBPuppet.query.filter(DBPuppet.username == username).one_or_none()
-        if puppet:
-            return cls.from_db(puppet)
+        dbpuppet = DBPuppet.get_by_username(username)
+        if dbpuppet:
+            return cls.from_db(dbpuppet)
 
         return None
 
+    @classmethod
+    def find_by_displayname(cls, displayname: str) -> Optional['Puppet']:
+        if not displayname:
+            return None
 
-def init(context):
+        for _, puppet in cls.cache.items():
+            if puppet.displayname and puppet.displayname == displayname:
+                return puppet
+
+        dbpuppet = DBPuppet.get_by_displayname(displayname)
+        if dbpuppet:
+            return cls.from_db(dbpuppet)
+
+        return None
+    # endregion
+
+
+def init(context: 'Context') -> List[Awaitable[Any]]:  # [None, None, PuppetError]
     global config
-    Puppet.az, Puppet.db, config, _ = context
-    localpart = config.get("bridge.username_template", "telegram_{userid}").format(userid="(.+)")
-    hs = config["homeserver"]["domain"]
-    Puppet.mxid_regex = re.compile(f"@{localpart}:{hs}")
+    Puppet.az, config, Puppet.loop, _ = context.core
+    Puppet.mx = context.mx
+    Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
+    Puppet.hs_domain = config["homeserver"]["domain"]
+    Puppet.mxid_regex = re.compile(
+        f"@{Puppet.username_template.format(userid='([0-9]+)')}:{Puppet.hs_domain}")
+    return [puppet.init_custom_mxid() for puppet in Puppet.all_with_custom_mxid()]

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -0,0 +1,81 @@
+import argparse
+import sqlalchemy as sql
+from sqlalchemy import orm
+from sqlalchemy.ext.declarative import declarative_base
+
+from alchemysession import AlchemySessionContainer
+
+parser = argparse.ArgumentParser(description="mautrix-telegram dbms migration script",
+                                 prog="python -m mautrix_telegram.scripts.dbms_migrate")
+parser.add_argument("-f", "--from-url", type=str, required=True, metavar="<url>",
+                    help="the old database path")
+parser.add_argument("-t", "--to-url", type=str, required=True, metavar="<url>",
+                    help="the new database path")
+parser.add_argument("-v", "--verbose", action="store_true", help="Verbose logs while migrating")
+args = parser.parse_args()
+verbose = args.verbose or False
+
+
+def log(message, end="\n"):
+    if verbose:
+        print(message, end=end, flush=True)
+
+
+def connect(to):
+    import mautrix_telegram.db.base as base
+    base.Base = declarative_base(cls=base.BaseBase)
+    from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile,
+                                     Contact, Puppet, BotChat, TelegramFile)
+    db_engine = sql.create_engine(to)
+    db_factory = orm.sessionmaker(bind=db_engine)
+    db_session = orm.scoped_session(db_factory)  # type: orm.Session
+    base.Base.metadata.bind = db_engine
+    session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
+                                                table_base=base.Base, table_prefix="telethon_",
+                                                manage_tables=False)
+
+    return db_session, {
+        "Version": session_container.Version,
+        "Session": session_container.Session,
+        "Entity": session_container.Entity,
+        "SentFile": session_container.SentFile,
+        "UpdateState": session_container.UpdateState,
+        "Portal": Portal,
+        "Message": Message,
+        "Puppet": Puppet,
+        "User": User,
+        "UserPortal": UserPortal,
+        "RoomState": RoomState,
+        "UserProfile": UserProfile,
+        "Contact": Contact,
+        "BotChat": BotChat,
+        "TelegramFile": TelegramFile,
+    }
+
+log("Connecting to old database")
+session, tables = connect(args.from_url)
+
+data = {}
+for name, table in tables.items():
+    log("Reading table {name}...".format(name=name), end=" ")
+    data[name] = session.query(table).all()
+    log("Done!")
+
+log("Connecting to new database")
+session, tables = connect(args.to_url)
+
+for name, table in tables.items():
+    log("Writing table {name}".format(name=name), end="")
+    length = len(data[name])
+    n = 0
+    for row in data[name]:
+        session.merge(row)
+        n += 5
+        if n >= length:
+            log(".", end="")
+            n = 0
+    log(" Done!")
+
+log("Committing changes to database...", end=" ")
+session.commit()
+log("Done!")

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -0,0 +1,124 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+from sqlalchemy import orm
+import sqlalchemy as sql
+import argparse
+
+from mautrix_telegram.db import Base, Portal, Message, Puppet, BotChat
+from mautrix_telegram.config import Config
+
+from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as TelematrixBase
+
+parser = argparse.ArgumentParser(
+    description="mautrix-telegram telematrix import script",
+    prog="python -m mautrix_telegram.scripts.telematrix_import")
+parser.add_argument("-c", "--config", type=str, default="config.yaml",
+                    metavar="<path>", help="the path to your mautrix-telegram config file")
+parser.add_argument("-b", "--bot-id", type=int, required=True,
+                    metavar="<id>", help="the telegram user ID of your relay bot")
+parser.add_argument("-t", "--telematrix-database", type=str, default="sqlite:///database.db",
+                    metavar="<url>", help="your telematrix database URL")
+args = parser.parse_args()
+
+config = Config(args.config, None, None)
+config.load()
+
+mxtg_db_engine = sql.create_engine(
+    config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+mxtg = orm.sessionmaker(bind=mxtg_db_engine)()
+Base.metadata.bind = mxtg_db_engine
+
+telematrix_db_engine = sql.create_engine(args.telematrix_database)
+telematrix = orm.sessionmaker(bind=telematrix_db_engine)()
+TelematrixBase.metadata.bind = telematrix_db_engine
+
+chat_links = telematrix.query(ChatLink).all()
+tg_users = telematrix.query(TgUser).all()
+mx_users = telematrix.query(MatrixUser).all()
+tm_messages = telematrix.query(TMMessage).all()
+
+telematrix.close()
+telematrix_db_engine.dispose()
+
+portals_by_tgid = {}  # type: Dict[int, Portal]
+portals_by_mxid = {}  # type: Dict[str, Portal]
+chats = {}  # type: Dict[int, BotChat]
+messages = {}  # type: Dict[str, Message]
+puppets = {}  # type: Dict[int, Puppet]
+
+for chat_link in chat_links:
+    if type(chat_link.tg_room) is str:
+        print("Expected tg_room to be a number, got a string. Ignoring %s" % chat_link.tg_room)
+        continue
+    if chat_link.tg_room >= 0:
+        print("Unexpected unprefixed telegram chat ID: %s, ignoring..." % chat_link.tg_room)
+        continue
+    tgid = str(chat_link.tg_room)
+    if tgid.startswith("-100"):
+        tgid = int(tgid[4:])
+        peer_type = "channel"
+        megagroup = True
+    else:
+        tgid = -chat_link.tg_room
+        peer_type = "chat"
+        megagroup = False
+
+    portal = Portal(tgid=tgid, tg_receiver=tgid, peer_type=peer_type, megagroup=megagroup,
+                    mxid=chat_link.matrix_room)
+    chats[tgid] = BotChat(id=tgid, type=peer_type)
+    if chat_link.tg_room in portals_by_tgid:
+        print(f"Warning: Ignoring bridge from {portal.tgid} to {portal.mxid} "
+              f"in favor of {portals_by_tgid[portal.tgid].mxid}")
+        continue
+    elif chat_link.matrix_room in portals_by_mxid:
+        print(f"Warning: Ignoring bridge from {portal.mxid} to {portal.tgid} "
+              f"in favor of {portals_by_mxid[portal.mxid].tgid}")
+        continue
+    portals_by_tgid[portal.tgid] = portal
+    portals_by_mxid[portal.mxid] = portal
+
+for tm_msg in tm_messages:
+    try:
+        portal = portals_by_tgid[tm_msg.tg_group_id]
+    except KeyError:
+        print(f"Found message entry {tm_msg.tg_message_id} in unlinked chat {tm_msg.tg_group_id},"
+              " ignoring...")
+        continue
+    if tm_msg.matrix_room_id != portal.mxid:
+        print(f"Found message entry {tm_msg.tg_message_id} with "
+              f"mismatching matrix room ID {tm_msg.matrix_room_id} (expected {portal.mxid})")
+        continue
+    tg_space = portal.tgid if portal.peer_type == "channel" else args.bot_id
+    message = Message(mxid=tm_msg.matrix_event_id, mx_room=tm_msg.matrix_room_id,
+                      tgid=tm_msg.tg_message_id, tg_space=tg_space)
+    messages[tm_msg.matrix_event_id] = message
+
+for user in tg_users:
+    puppets[user.tg_id] = Puppet(id=user.tg_id, displayname=user.name,
+                                 displayname_source=args.bot_id)
+
+for k, v in portals_by_tgid.items():
+    mxtg.add(v)
+for k, v in chats.items():
+    mxtg.add(v)
+for k, v in messages.items():
+    mxtg.add(v)
+for k, v in puppets.items():
+    mxtg.add(v)
+
+mxtg.commit()

mautrix_telegram/scripts/telematrix_import/models.py

@@ -0,0 +1,44 @@
+import sqlalchemy as sa
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+
+class ChatLink(Base):
+    __tablename__ = "chat_link"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    matrix_room = sa.Column(sa.String)
+    tg_room = sa.Column(sa.BigInteger)
+    active = sa.Column(sa.Boolean)
+
+
+class TgUser(Base):
+    __tablename__ = "tg_user"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    tg_id = sa.Column(sa.BigInteger)
+    name = sa.Column(sa.String)
+    profile_pic_id = sa.Column(sa.String, nullable=True)
+
+
+class MatrixUser(Base):
+    __tablename__ = "matrix_user"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    matrix_id = sa.Column(sa.String)
+    name = sa.Column(sa.String)
+
+
+class Message(Base):
+    """Describes a message in a room bridged between Telegram and Matrix"""
+    __tablename__ = "message"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    tg_group_id = sa.Column(sa.BigInteger)
+    tg_message_id = sa.Column(sa.BigInteger)
+
+    matrix_room_id = sa.Column(sa.String)
+    matrix_event_id = sa.Column(sa.String)
+
+    displayname = sa.Column(sa.String)

mautrix_telegram/sqlstatestore.py

@@ -0,0 +1,119 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Tuple
+
+from mautrix_appservice import StateStore
+
+from .types import MatrixUserID, MatrixRoomID
+from . import puppet as pu
+from .db import RoomState, UserProfile
+
+
+class SQLStateStore(StateStore):
+    def __init__(self) -> None:
+        super().__init__()
+        self.profile_cache = {}  # type: Dict[Tuple[str, str], UserProfile]
+        self.room_state_cache = {}  # type: Dict[str, RoomState]
+
+    @staticmethod
+    def is_registered(user: MatrixUserID) -> bool:
+        puppet = pu.Puppet.get_by_mxid(user)
+        return puppet.is_registered if puppet else False
+
+    @staticmethod
+    def registered(user: MatrixUserID) -> None:
+        puppet = pu.Puppet.get_by_mxid(user)
+        if puppet:
+            puppet.is_registered = True
+            puppet.save()
+
+    def update_state(self, event: Dict) -> None:
+        event_type = event["type"]
+        if event_type == "m.room.power_levels":
+            self.set_power_levels(event["room_id"], event["content"])
+        elif event_type == "m.room.member":
+            self.set_member(event["room_id"], event["state_key"], event["content"])
+
+    def _get_user_profile(self, room_id: MatrixRoomID, user_id: MatrixUserID, create: bool = True
+                          ) -> UserProfile:
+        key = (room_id, user_id)
+        try:
+            return self.profile_cache[key]
+        except KeyError:
+            pass
+
+        profile = UserProfile.get(*key)
+        if profile:
+            self.profile_cache[key] = profile
+        elif create:
+            profile = UserProfile(room_id=room_id, user_id=user_id, membership="leave")
+            profile.insert()
+            self.profile_cache[key] = profile
+        return profile
+
+    def get_member(self, room: MatrixRoomID, user: MatrixUserID) -> Dict:
+        return self._get_user_profile(room, user).dict()
+
+    def set_member(self, room: MatrixRoomID, user: MatrixUserID, member: Dict) -> None:
+        profile = self._get_user_profile(room, user)
+        profile.membership = member.get("membership", profile.membership or "leave")
+        profile.displayname = member.get("displayname", profile.displayname)
+        profile.avatar_url = member.get("avatar_url", profile.avatar_url)
+        profile.update()
+
+    def set_membership(self, room: MatrixRoomID, user: MatrixUserID, membership: str) -> None:
+        self.set_member(room, user, {
+            "membership": membership,
+        })
+
+    def _get_room_state(self, room_id: MatrixRoomID, create: bool = True) -> RoomState:
+        try:
+            return self.room_state_cache[room_id]
+        except KeyError:
+            pass
+
+        room = RoomState.get(room_id)
+        if room:
+            self.room_state_cache[room_id] = room
+        elif create:
+            room = RoomState(room_id=room_id)
+            room.insert()
+            self.room_state_cache[room_id] = room
+        return room
+
+    def has_power_levels(self, room: MatrixRoomID) -> bool:
+        return bool(self._get_room_state(room).power_levels)
+
+    def get_power_levels(self, room: MatrixRoomID) -> Dict:
+        return self._get_room_state(room).power_levels
+
+    def set_power_level(self, room: MatrixRoomID, user: MatrixUserID, level: int) -> None:
+        room_state = self._get_room_state(room)
+        power_levels = room_state.power_levels
+        if not power_levels:
+            power_levels = {
+                "users": {},
+                "events": {},
+            }
+        power_levels[room]["users"][user] = level
+        room_state.power_levels = power_levels
+        room_state.update()
+
+    def set_power_levels(self, room: MatrixRoomID, content: Dict) -> None:
+        state = self._get_room_state(room)
+        state.power_levels = content
+        state.update()

mautrix_telegram/tgclient.py

@@ -1,84 +1,53 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from io import BytesIO
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Union, Optional
 
-from telethon import TelegramClient
-from telethon.tl.functions.messages import SendMessageRequest, SendMediaRequest
-from telethon.tl.types import *
+from telethon import TelegramClient, utils
+from telethon.tl.functions.messages import SendMediaRequest
+from telethon.tl.types import (
+    InputMediaUploadedDocument, InputMediaUploadedPhoto, TypeDocumentAttribute, TypeInputMedia,
+    TypeInputPeer, TypeMessageEntity, TypeMessageMedia, TypePeer)
+from telethon.tl.patched import Message
 
 
 class MautrixTelegramClient(TelegramClient):
-    async def send_message(self, entity, message, reply_to=None, entities=None, link_preview=True):
-        entity = await self.get_input_entity(entity)
+    async def upload_file_direct(self, file: bytes, mime_type: str = None,
+                                 attributes: List[TypeDocumentAttribute] = None,
+                                 file_name: str = None, max_image_size: float = 10 * 1000 ** 2,
+                                 ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
+        file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)
 
-        request = SendMessageRequest(
-            peer=entity,
-            message=message,
-            entities=entities,
-            no_webpage=not link_preview,
-            reply_to_msg_id=self._get_message_id(reply_to)
-        )
-        result = await self(request)
-        if isinstance(result, UpdateShortSentMessage):
-            return Message(
-                id=result.id,
-                to_id=entity,
-                message=message,
-                date=result.date,
-                out=result.out,
-                media=result.media,
-                entities=result.entities
-            )
-
-        return self._get_response_message(request, result)
-
-    async def send_file(self, entity, file, mime_type=None, caption=None, attributes=None,
-                        file_name=None, reply_to=None, **kwargs):
-        entity = await self.get_input_entity(entity)
-        reply_to = self._get_message_id(reply_to)
-
-        file_handle = await self.upload_file(file, file_name=file_name, use_cache=False)
-
-        if mime_type == "image/png":
-            media = InputMediaUploadedPhoto(file_handle, caption or "")
+        if (mime_type == "image/png" or mime_type == "image/jpeg") and len(file) < max_image_size:
+            return InputMediaUploadedPhoto(file_handle)
         else:
             attributes = attributes or []
             attr_dict = {type(attr): attr for attr in attributes}
 
-            media = InputMediaUploadedDocument(
+            return InputMediaUploadedDocument(
                 file=file_handle,
                 mime_type=mime_type or "application/octet-stream",
-                attributes=list(attr_dict.values()),
-                caption=caption or "")
+                attributes=list(attr_dict.values()))
 
-        request = SendMediaRequest(entity, media, reply_to_msg_id=reply_to)
-        return self._get_response_message(request, await self(request))
-
-    async def download_file_bytes(self, location):
-        if isinstance(location, Document):
-            location = InputDocumentFileLocation(location.id, location.access_hash,
-                                                 location.version)
-        elif not isinstance(location, (InputFileLocation, InputDocumentFileLocation)):
-            location = InputFileLocation(location.volume_id, location.local_id, location.secret)
-
-        file = BytesIO()
-
-        await self.download_file(location, file)
-
-        data = file.getvalue()
-        file.close()
-        return data
+    async def send_media(self, entity: Union[TypeInputPeer, TypePeer],
+                         media: Union[TypeInputMedia, TypeMessageMedia],
+                         caption: str = None, entities: List[TypeMessageEntity] = None,
+                         reply_to: int = None) -> Optional[Message]:
+        entity = await self.get_input_entity(entity)
+        reply_to = utils.get_message_id(reply_to)
+        request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],
+                                   reply_to_msg_id=reply_to)
+        return self._get_response_message(request, await self(request), entity)

mautrix_telegram/types.py

@@ -0,0 +1,9 @@
+from typing import Dict, NewType
+
+MatrixUserID = NewType('MatrixUserID', str)
+MatrixRoomID = NewType('MatrixRoomID', str)
+MatrixEventID = NewType('MatrixEventID', str)
+
+MatrixEvent = NewType('MatrixEvent', Dict)
+
+TelegramID = NewType('TelegramID', int)

mautrix_telegram/user.py

@@ -1,194 +1,259 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Dict, List, Iterable, Match, NewType, Optional, Tuple, TYPE_CHECKING
 import logging
 import asyncio
-import platform
+import re
 
-from telethon.tl.types import *
+from telethon.tl.types import (
+    TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
+    UpdateShortChatMessage, UpdateShortMessage, User as TLUser)
 from telethon.tl.types.contacts import ContactsNotModified
-from telethon.tl.types import User as TLUser
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
+from telethon.tl.functions.account import UpdateStatusRequest
 from mautrix_appservice import MatrixRequestError
 
-from .db import User as DBUser, Message as DBMessage, Contact as DBContact
-from .tgclient import MautrixTelegramClient
-from . import portal as po, puppet as pu, __version__
+from .types import MatrixUserID, TelegramID
+from .db import User as DBUser
+from .abstract_user import AbstractUser
+from . import portal as po, puppet as pu
 
-config = None
+if TYPE_CHECKING:
+    from .config import Config
+    from .context import Context
+
+config = None  # type: Config
+
+SearchResult = NewType('SearchResult', Tuple['pu.Puppet', int])
 
 
-class User:
-    loop = None
-    log = logging.getLogger("mau.user")
-    db = None
-    az = None
-    by_mxid = {}
-    by_tgid = {}
+class User(AbstractUser):
+    log = logging.getLogger("mau.user")  # type: logging.Logger
+    by_mxid = {}  # type: Dict[str, User]
+    by_tgid = {}  # type: Dict[int, User]
 
-    def __init__(self, mxid, tgid=None, username=None, db_contacts=None, saved_contacts=0,
-                 db_portals=None):
-        self.mxid = mxid
-        self.tgid = tgid
-        self.username = username
-        self.contacts = []
-        self.saved_contacts = saved_contacts
+    def __init__(self, mxid: MatrixUserID, tgid: Optional[TelegramID] = None,
+                 username: Optional[str] = None, phone: Optional[str] = None,
+                 db_contacts: Optional[Iterable[TelegramID]] = None,
+                 saved_contacts: int = 0, is_bot: bool = False,
+                 db_portals: Optional[Iterable[Tuple[TelegramID, TelegramID]]] = None,
+                 db_instance: Optional[DBUser] = None) -> None:
+        super().__init__()
+        self.mxid = mxid  # type: MatrixUserID
+        self.tgid = tgid  # type: TelegramID
+        self.is_bot = is_bot  # type: bool
+        self.username = username  # type: str
+        self.phone = phone  # type: str
+        self.contacts = []  # type: List[pu.Puppet]
+        self.saved_contacts = saved_contacts  # type: int
         self.db_contacts = db_contacts
-        self.portals = {}
-        self.db_portals = db_portals
+        self.portals = {}  # type: Dict[Tuple[TelegramID, TelegramID], po.Portal]
+        self.db_portals = db_portals or []
+        self._db_instance = db_instance  # type: Optional[DBUser]
 
-        self.command_status = None
-        self.connected = False
-        self.client = None
-        self._init_client()
+        self.command_status = None  # type: Optional[Dict]
 
-        self.is_admin = self.mxid in config.get("bridge.admins", [])
-
-        whitelist = config.get("bridge.whitelist", None) or [self.mxid]
-        self.whitelisted = not whitelist or self.mxid in whitelist
-        if not self.whitelisted:
-            homeserver = self.mxid[self.mxid.index(":") + 1:]
-            self.whitelisted = homeserver in whitelist
+        (self.relaybot_whitelisted,
+         self.whitelisted,
+         self.puppet_whitelisted,
+         self.matrix_puppet_whitelisted,
+         self.is_admin,
+         self.permissions) = config.get_permissions(self.mxid)
 
         self.by_mxid[mxid] = self
         if tgid:
             self.by_tgid[tgid] = self
 
     @property
-    def logged_in(self):
-        return self.client.is_user_authorized()
+    def name(self) -> str:
+        return self.mxid
 
     @property
-    def has_full_access(self):
-        return self.logged_in and self.whitelisted
+    def mxid_localpart(self) -> str:
+        match = re.compile("@(.+):(.+)").match(self.mxid)  # type: Match
+        return match.group(1)
 
     @property
-    def db_contacts(self):
-        return [self.db.merge(DBContact(user=self.tgid, contact=puppet.id))
-                for puppet in self.contacts]
+    def human_tg_id(self) -> str:
+        return f"@{self.username}" if self.username else f"+{self.phone}" or None
+
+    # TODO replace with proper displayname getting everywhere
+    @property
+    def displayname(self) -> str:
+        return self.mxid_localpart
+
+    @property
+    def plain_displayname(self) -> str:
+        return self.displayname
+
+    @property
+    def db_contacts(self) -> Iterable[TelegramID]:
+        return (puppet.id
+                for puppet in self.contacts
+                if puppet)
 
     @db_contacts.setter
-    def db_contacts(self, contacts):
-        if contacts:
-            self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts]
-        else:
-            self.contacts = []
+    def db_contacts(self, contacts: Iterable[TelegramID]) -> None:
+        self.contacts = [pu.Puppet.get(entry) for entry in contacts] if contacts else []
 
     @property
-    def db_portals(self):
-        return [portal.to_db(merge=False) for _, portal in self.portals.items()]
+    def db_portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        return (portal.tgid_full
+                for portal in self.portals.values()
+                if portal and not portal.deleted)
 
     @db_portals.setter
-    def db_portals(self, portals):
-        if portals:
-            self.portals = {(portal.tgid, portal.tg_receiver):
-                                po.Portal.get_by_tgid(portal.tgid, portal.tg_receiver)
-                            for portal in portals}
-        else:
-            self.portals = {}
+    def db_portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
+        self.portals = {
+            tgid_full: po.Portal.get_by_tgid(*tgid_full)
+            for tgid_full in portals
+        } if portals else {}
 
     # region Database conversion
 
-    def to_db(self):
-        return self.db.merge(
-            DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                   contacts=self.db_contacts, saved_contacts=self.saved_contacts,
-                   portals=self.db_portals))
+    @property
+    def db_instance(self) -> DBUser:
+        if not self._db_instance:
+            self._db_instance = self.new_db_instance()
+        return self._db_instance
 
-    def save(self):
-        self.to_db()
-        self.db.commit()
+    def new_db_instance(self) -> DBUser:
+        return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
+                      saved_contacts=self.saved_contacts, portals=self.db_portals)
 
-    def delete(self):
+    def save(self, contacts: bool = False, portals: bool = False) -> None:
+        self.db_instance.update(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
+                                saved_contacts=self.saved_contacts)
+        if contacts:
+            self.db_instance.contacts = self.db_contacts
+        if portals:
+            self.db_instance.portals = self.db_portals
+
+    def delete(self, delete_db: bool = True) -> None:
         try:
             del self.by_mxid[self.mxid]
             del self.by_tgid[self.tgid]
         except KeyError:
             pass
-        self.db.delete(self.to_db())
-        self.db.commit()
+        if delete_db and self._db_instance:
+            self._db_instance.delete()
 
     @classmethod
-    def from_db(cls, db_user):
-        return User(db_user.mxid, db_user.tgid, db_user.tg_username, db_user.contacts,
-                    db_user.saved_contacts, db_user.portals)
+    def from_db(cls, db_user: DBUser) -> 'User':
+        return User(db_user.mxid, db_user.tgid, db_user.tg_username, db_user.tg_phone,
+                    db_user.contacts, db_user.saved_contacts, False, db_user.portals,
+                    db_instance=db_user)
 
     # endregion
     # region Telegram connection management
 
-    def _init_client(self):
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
-        self.client = MautrixTelegramClient(self.mxid,
-                                            config["telegram.api_id"],
-                                            config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device)
-        self.client.add_update_handler(self.update_catch)
+    async def try_ensure_started(self) -> None:
+        try:
+            await self.ensure_started()
+        except Exception:
+            self.log.exception("Exception in ensure_started")
 
-    async def start(self, delete_unless_authenticated=False):
-        self.connected = await self.client.connect()
-        if self.logged_in:
+    def ensure_started(self, even_if_no_session=False) -> Awaitable['User']:
+        return super().ensure_started(even_if_no_session)
+
+    async def start(self, delete_unless_authenticated: bool = False) -> 'User':
+        await super().start()
+        if await self.is_logged_in():
+            self.log.debug(f"Ensuring post_login() for {self.name}")
             asyncio.ensure_future(self.post_login(), loop=self.loop)
         elif delete_unless_authenticated:
-            # User not logged in -> forget user
-            self.client.disconnect()
+            self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
+            await self.client.disconnect()
             self.client.session.delete()
-            self.delete()
         return self
 
-    async def post_login(self, info=None):
+    async def post_login(self, info: TLUser = None) -> None:
         try:
             await self.update_info(info)
-            await self.sync_dialogs()
-            await self.sync_contacts()
+            if not self.is_bot and config["bridge.startup_sync"]:
+                await self.sync_dialogs()
+                await self.sync_contacts()
+            if config["bridge.catch_up"]:
+                await self.client.catch_up()
         except Exception:
-            self.log.exception("Failed to run post-login functions")
+            self.log.exception("Failed to run post-login functions for %s", self.mxid)
 
-    def stop(self):
-        self.client.disconnect()
-        self.client = None
-        self.connected = False
+    async def update(self, update: TypeUpdate) -> bool:
+        if not self.is_bot:
+            return False
+
+        if isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
+            message = update.message
+            if isinstance(message.to_id, PeerUser) and not message.out:
+                portal = po.Portal.get_by_tgid(message.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
+            else:
+                portal = po.Portal.get_by_entity(message.to_id, receiver_id=self.tgid)
+        elif isinstance(update, UpdateShortChatMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
+        elif isinstance(update, UpdateShortMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
+        else:
+            return False
+
+        if portal:
+            self.register_portal(portal)
+
+        return True
 
     # endregion
     # region Telegram actions that need custom methods
 
-    async def update_info(self, info=None):
+    async def set_presence(self, online: bool = True) -> None:
+        if not self.is_bot:
+            await self.client(UpdateStatusRequest(offline=not online))
+
+    async def update_info(self, info: TLUser = None) -> None:
         info = info or await self.client.get_me()
         changed = False
+        if self.is_bot != info.bot:
+            self.is_bot = info.bot
+            changed = True
         if self.username != info.username:
             self.username = info.username
             changed = True
+        if self.phone != info.phone:
+            self.phone = info.phone
+            changed = True
         if self.tgid != info.id:
             self.tgid = info.id
             self.by_tgid[self.tgid] = self
         if changed:
             self.save()
 
-    async def log_out(self):
+    async def log_out(self) -> bool:
+        puppet = pu.Puppet.get(self.tgid)
+        if puppet.is_real_user:
+            await puppet.switch_mxid(None, None)
         for _, portal in self.portals.items():
+            if not portal or portal.deleted or not portal.mxid or portal.has_bot:
+                continue
             try:
                 await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
             except MatrixRequestError:
                 pass
         self.portals = {}
         self.contacts = []
-        self.save()
+        self.save(portals=True, contacts=True)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
@@ -202,28 +267,30 @@ class User:
         self.delete()
         return True
 
-    def _search_local(self, query, max_results=5, min_similarity=45):
-        results = []
+    def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
+                      ) -> List[SearchResult]:
+        results = []  # type: List[SearchResult]
         for contact in self.contacts:
             similarity = contact.similarity(query)
             if similarity >= min_similarity:
-                results.append((contact, similarity))
+                results.append(SearchResult((contact, similarity)))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
-    async def _search_remote(self, query, max_results=5):
+    async def _search_remote(self, query: str, max_results: int = 5) -> List[SearchResult]:
         if len(query) < 5:
             return []
         server_results = await self.client(SearchRequest(q=query, limit=max_results))
-        results = []
+        results = []  # type: List[SearchResult]
         for user in server_results.users:
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
-            results.append((puppet, puppet.similarity(query)))
+            results.append(SearchResult((puppet, puppet.similarity(query))))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
-    async def search(self, query, force_remote=False):
+    async def search(self, query: str, force_remote: bool = False
+                     ) -> Tuple[List[SearchResult], bool]:
         if force_remote:
             return await self._search_remote(query), True
 
@@ -233,232 +300,105 @@ class User:
 
         return await self._search_remote(query), True
 
-    async def sync_dialogs(self):
-        dialogs = await self.client.get_dialogs(limit=30)
+    async def sync_dialogs(self, synchronous_create: bool = False) -> None:
         creators = []
-        for dialog in dialogs:
-            entity = dialog.entity
-            invalid = (isinstance(entity, (TLUser, ChatForbidden, ChannelForbidden))
-                       or (isinstance(entity, Chat) and (entity.deactivated or entity.left)))
-            if invalid:
-                continue
+        for entity in await self.get_dialogs(limit=config["bridge.sync_dialog_limit"] or None):
             portal = po.Portal.get_by_entity(entity)
             self.portals[portal.tgid_full] = portal
-            creators.append(portal.create_matrix_room(self, entity, invites=[self.mxid]))
-        self.save()
+            creators.append(
+                portal.create_matrix_room(self, entity, invites=[self.mxid],
+                                          synchronous=synchronous_create))
+        self.save(portals=True)
         await asyncio.gather(*creators, loop=self.loop)
 
-    def register_portal(self, portal):
+    def register_portal(self, portal: po.Portal) -> None:
         try:
             if self.portals[portal.tgid_full] == portal:
                 return
         except KeyError:
             pass
         self.portals[portal.tgid_full] = portal
-        self.save()
+        self.save(portals=True)
 
-    def unregister_portal(self, portal):
+    def unregister_portal(self, portal: po.Portal) -> None:
         try:
             del self.portals[portal.tgid_full]
-            self.save()
+            self.save(portals=True)
         except KeyError:
             pass
 
-    def _hash_contacts(self):
+    async def needs_relaybot(self, portal: po.Portal) -> bool:
+        return not await self.is_logged_in() or (
+            (portal.has_bot or self.bot) and portal.tgid_full not in self.portals)
+
+    def _hash_contacts(self) -> int:
         acc = 0
-        for id in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
-            acc = (acc * 20261 + id) & 0xffffffff
+        for contact in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
+            acc = (acc * 20261 + contact) & 0xffffffff
         return acc & 0x7fffffff
 
-    async def sync_contacts(self):
+    async def sync_contacts(self) -> None:
         response = await self.client(GetContactsRequest(hash=self._hash_contacts()))
         if isinstance(response, ContactsNotModified):
             return
-        self.log.debug("Updating contacts...")
+        self.log.debug(f"Updating contacts of {self.name}...")
         self.contacts = []
         self.saved_contacts = response.saved_count
         for user in response.users:
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
             self.contacts.append(puppet)
-        self.save()
-
-    # endregion
-    # region Telegram update handling
-
-    async def update_catch(self, update):
-        try:
-            await self.update(update)
-        except Exception:
-            self.log.exception("Failed to handle Telegram update")
-
-    async def update(self, update):
-        if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
-                               UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
-            await self.update_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
-            await self.update_typing(update)
-        elif isinstance(update, UpdateUserStatus):
-            await self.update_status(update)
-        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
-            await self.update_admin(update)
-        elif isinstance(update, UpdateChatParticipants):
-            portal = po.Portal.get_by_tgid(update.participants.chat_id)
-            if portal and portal.mxid:
-                await portal.update_telegram_participants(update.participants.participants)
-        elif isinstance(update, UpdateChannelPinnedMessage):
-            portal = po.Portal.get_by_tgid(update.channel_id)
-            if portal and portal.mxid:
-                await portal.update_telegram_pin(self, update.id)
-        elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
-            await self.update_others_info(update)
-        elif isinstance(update, UpdateReadHistoryOutbox):
-            await self.update_read_receipt(update)
-        else:
-            self.log.debug("Unhandled update: %s", update)
-
-    async def update_read_receipt(self, update):
-        if not isinstance(update.peer, PeerUser):
-            self.log.debug("Unexpected read receipt peer: %s", update.peer)
-            return
-
-        portal = po.Portal.get_by_tgid(update.peer.user_id, self.tgid)
-        if not portal or not portal.mxid:
-            return
-
-        # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.query.get((update.max_id, self.tgid))
-        if not message:
-            return
-
-        puppet = pu.Puppet.get(update.peer.user_id)
-        await puppet.intent.mark_read(portal.mxid, message.mxid)
-
-    async def update_admin(self, update):
-        portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        if isinstance(update, UpdateChatAdmins):
-            await portal.set_telegram_admins_enabled(update.enabled)
-        elif isinstance(update, UpdateChatParticipantAdmin):
-            puppet = pu.Puppet.get(update.user_id)
-            user = User.get_by_tgid(update.user_id)
-            await portal.set_telegram_admin(puppet, user)
-
-    async def update_typing(self, update):
-        if isinstance(update, UpdateUserTyping):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
-        else:
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        sender = pu.Puppet.get(update.user_id)
-        await portal.handle_telegram_typing(sender, update)
-
-    async def update_others_info(self, update):
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update, UpdateUserName):
-            if await puppet.update_displayname(self, update):
-                puppet.save()
-        elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
-                puppet.save()
-
-    async def update_status(self, update):
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update.status, UserStatusOnline):
-            await puppet.intent.set_presence("online")
-        elif isinstance(update.status, UserStatusOffline):
-            await puppet.intent.set_presence("offline")
-        else:
-            self.log.warning("Unexpected user status update: %s", update)
-        return
-
-    def get_message_details(self, update):
-        if isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-            sender = pu.Puppet.get(update.from_id)
-        elif isinstance(update, UpdateShortMessage):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
-            sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
-        elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
-                                 UpdateEditMessage, UpdateEditChannelMessage)):
-            update = update.message
-            if isinstance(update.to_id, PeerUser) and not update.out:
-                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
-            else:
-                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
-            sender = pu.Puppet.get(update.from_id) if update.from_id else None
-        else:
-            self.log.warning(
-                f"Unexpected message type in User#get_message_details: {type(update)}")
-            return update, None, None
-        return update, sender, portal
-
-    def update_message(self, original_update):
-        update, sender, portal = self.get_message_details(original_update)
-
-        if isinstance(update, MessageService):
-            if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.debug(f"Ignoring action %s to %s by %d", update.action, portal.tgid_log,
-                               sender.id)
-                return
-            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
-                           sender.id)
-            return portal.handle_telegram_action(self, sender, update.action)
-
-        user = sender.tgid if sender else "admin"
-        if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-            return portal.handle_telegram_edit(self, sender, update)
-
-        self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
-        return portal.handle_telegram_message(self, sender, update)
+        self.save(contacts=True)
 
     # endregion
     # region Class instance lookup
 
     @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid: MatrixUserID, create: bool = True) -> Optional['User']:
+        if not mxid:
+            raise ValueError("Matrix ID can't be empty")
+
         try:
             return cls.by_mxid[mxid]
         except KeyError:
             pass
 
-        user = DBUser.query.get(mxid)
+        user = DBUser.get_by_mxid(mxid)
         if user:
             user = cls.from_db(user)
-            asyncio.ensure_future(user.start(), loop=cls.loop)
             return user
 
         if create:
             user = cls(mxid)
-            cls.db.add(user.to_db())
-            cls.db.commit()
-            asyncio.ensure_future(user.start(), loop=cls.loop)
+            user.db_instance.insert()
             return user
 
         return None
 
     @classmethod
-    def get_by_tgid(cls, tgid):
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
         try:
             return cls.by_tgid[tgid]
         except KeyError:
             pass
 
-        user = DBUser.query.filter(DBUser.tgid == tgid).one_or_none()
+        user = DBUser.get_by_tgid(tgid)
         if user:
             user = cls.from_db(user)
-            asyncio.ensure_future(user.start(), loop=cls.loop)
             return user
 
         return None
 
     @classmethod
-    def find_by_username(cls, username):
+    def find_by_username(cls, username: str) -> Optional['User']:
+        if not username:
+            return None
+
         for _, user in cls.by_tgid.items():
-            if user.username == username:
+            if user.username and user.username.lower() == username.lower():
                 return user
 
-        puppet = DBUser.query.filter(DBUser.tg_username == username).one_or_none()
+        puppet = DBUser.get_by_username(username)
         if puppet:
             return cls.from_db(puppet)
 
@@ -466,9 +406,9 @@ class User:
     # endregion
 
 
-def init(context):
+def init(context: 'Context') -> List[Awaitable[None]]:
     global config
-    User.az, User.db, config, User.loop = context
+    config = context.config
 
-    users = [User.from_db(user) for user in DBUser.query.all()]
-    return [user.start(delete_unless_authenticated=True) for user in users]
+    users = [User.from_db(user) for user in DBUser.all()]
+    return [user.try_ensure_started() for user in users if user.tgid]

mautrix_telegram/util/__init__.py

@@ -0,0 +1,7 @@
+from .file_transfer import transfer_file_to_matrix, convert_image
+from .format_duration import format_duration
+from .signed_token import sign_token, verify_token
+from .recursive_dict import recursive_del, recursive_set, recursive_get
+
+def ignore_coro(coro):
+    pass

mautrix_telegram/util/file_transfer.py

@@ -0,0 +1,223 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Tuple, Union, Dict
+from io import BytesIO
+import time
+import logging
+import asyncio
+
+import magic
+from sqlalchemy.exc import IntegrityError, InvalidRequestError
+
+from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLocation,
+                               TypePhotoSize, PhotoSize, PhotoCachedSize, InputPhotoFileLocation,
+                               InputPeerPhotoFileLocation)
+from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
+                             SecurityError, FileIdInvalidError)
+from mautrix_appservice import IntentAPI
+
+from ..tgclient import MautrixTelegramClient
+from ..db import TelegramFile as DBTelegramFile
+from ..util import sane_mimetypes
+
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+try:
+    from moviepy.editor import VideoFileClip
+    import random
+    import string
+    import os
+    import mimetypes
+except ImportError:
+    VideoFileClip = random = string = os = mimetypes = None
+
+log = logging.getLogger("mau.util")  # type: logging.Logger
+
+TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
+                     InputFileLocation, InputPhotoFileLocation]
+
+
+def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
+                  thumbnail_to: Optional[Tuple[int, int]] = None
+                  ) -> Tuple[str, bytes, Optional[int], Optional[int]]:
+    if not Image:
+        return source_mime, file, None, None
+    try:
+        image = Image.open(BytesIO(file)).convert("RGBA")  # type: Image.Image
+        if thumbnail_to:
+            image.thumbnail(thumbnail_to, Image.ANTIALIAS)
+        new_file = BytesIO()
+        image.save(new_file, target_type)
+        w, h = image.size
+        return f"image/{target_type}", new_file.getvalue(), w, h
+    except Exception:
+        log.exception(f"Failed to convert {source_mime} to {target_type}")
+        return source_mime, file, None, None
+
+
+def _temp_file_name(ext: str) -> str:
+    return ("/tmp/mxtg-video-"
+            + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
+            + ext)
+
+
+def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str = "png",
+                          max_size: Tuple[int, int] = (1024, 720)) -> Tuple[bytes, int, int]:
+    # We don't have any way to read the video from memory, so save it to disk.
+    temp_file = _temp_file_name(video_ext)
+    with open(temp_file, "wb") as file:
+        file.write(data)
+
+    # Read temp file and get frame
+    clip = VideoFileClip(temp_file)
+    frame = clip.get_frame(0)
+
+    # Convert to png and save to BytesIO
+    image = Image.fromarray(frame).convert("RGBA")
+    thumbnail_file = BytesIO()
+    if max_size:
+        image.thumbnail(max_size, Image.ANTIALIAS)
+    image.save(thumbnail_file, frame_ext)
+
+    os.remove(temp_file)
+
+    w, h = image.size
+    return thumbnail_file.getvalue(), w, h
+
+
+def _location_to_id(location: TypeLocation) -> str:
+    if isinstance(location, (Document, InputDocumentFileLocation, InputPhotoFileLocation)):
+        return f"{location.id}-{location.access_hash}"
+    elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
+        return f"{location.volume_id}-{location.local_id}"
+
+
+async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                       thumbnail_loc: TypeLocation, video: bytes,
+                                       mime: str) -> Optional[DBTelegramFile]:
+    if not Image or not VideoFileClip:
+        return None
+
+    loc_id = _location_to_id(thumbnail_loc)
+    if not loc_id:
+        return None
+
+    db_file = DBTelegramFile.get(loc_id)
+    if db_file:
+        return db_file
+
+    video_ext = sane_mimetypes.guess_extension(mime)
+    if VideoFileClip and video_ext:
+        try:
+            file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
+        except OSError:
+            return None
+        mime_type = "image/png"
+    else:
+        file = await client.download_file(thumbnail_loc)
+        width, height = None, None
+        mime_type = magic.from_buffer(file, mime=True)
+
+    content_uri = await intent.upload_file(file, mime_type)
+
+    db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
+                             was_converted=False, timestamp=int(time.time()), size=len(file),
+                             width=width, height=height)
+    try:
+        db_file.insert()
+    except (IntegrityError, InvalidRequestError) as e:
+        log.exception(f"{e.__class__.__name__} while saving transferred file thumbnail data. "
+                      "This was probably caused by two simultaneous transfers of the same file, "
+                      "and might (but probably won't) cause problems with thumbnails or something.")
+    return db_file
+
+
+transfer_locks = {}  # type: Dict[str, asyncio.Lock]
+
+TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
+
+
+async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
+                                  is_sticker: bool = False) -> Optional[DBTelegramFile]:
+    location_id = _location_to_id(location)
+    if not location_id:
+        return None
+
+    db_file = DBTelegramFile.get(location_id)
+    if db_file:
+        return db_file
+
+    try:
+        lock = transfer_locks[location_id]
+    except KeyError:
+        lock = asyncio.Lock()
+        transfer_locks[location_id] = lock
+    async with lock:
+        return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
+                                                       thumbnail, is_sticker)
+
+
+async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                            loc_id: str, location: TypeLocation,
+                                            thumbnail: TypeThumbnail, is_sticker: bool
+                                            ) -> Optional[DBTelegramFile]:
+    db_file = DBTelegramFile.get(loc_id)
+    if db_file:
+        return db_file
+
+    try:
+        file = await client.download_file(location)
+    except (LocationInvalidError, FileIdInvalidError):
+        return None
+    except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
+        log.exception(f"{e.__class__.__name__} while downloading a file.")
+        return None
+
+    width, height = None, None
+    mime_type = magic.from_buffer(file, mime=True)
+
+    image_converted = False
+    if mime_type == "image/webp":
+        new_mime_type, file, width, height = convert_image(
+            file, source_mime="image/webp", target_type="png",
+            thumbnail_to=(256, 256) if is_sticker else None)
+        image_converted = new_mime_type != mime_type
+        mime_type = new_mime_type
+        thumbnail = None
+
+    content_uri = await intent.upload_file(file, mime_type)
+
+    db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
+                             mime_type=mime_type, was_converted=image_converted,
+                             timestamp=int(time.time()), size=len(file),
+                             width=width, height=height)
+    if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
+        if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
+            thumbnail = thumbnail.location
+        db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
+                                                               mime_type)
+
+    try:
+        db_file.insert()
+    except (IntegrityError, InvalidRequestError) as e:
+        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
+                      "This was probably caused by two simultaneous transfers of the same file, "
+                      "and should not cause any problems.")
+    return db_file

mautrix_telegram/util/format_duration.py

@@ -0,0 +1,36 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+def format_duration(seconds: int) -> str:
+    def pluralize(count: int, singular: str) -> str:
+        return singular if count == 1 else singular + "s"
+
+    def include(count: int, word: str) -> str:
+        return f"{count} {pluralize(count, word)}" if count > 0 else ""
+
+    minutes, seconds = divmod(seconds, 60)
+    hours, minutes = divmod(minutes, 60)
+    days, hours = divmod(hours, 24)
+    parts = [a for a in [
+        include(days, "day"),
+        include(hours, "hour"),
+        include(minutes, "minute"),
+        include(seconds, "second")] if a]
+    if len(parts) > 2:
+        return "{} and {}".format(", ".join(parts[:-1]), parts[-1])
+    return " and ".join(parts)

mautrix_telegram/util/recursive_dict.py

@@ -0,0 +1,54 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Any
+from ..config import DictWithRecursion
+
+
+def recursive_set(data: Dict[str, Any], key: str, value: Any) -> bool:
+    key, next_key = DictWithRecursion._parse_key(key)
+    if next_key is not None:
+        if key not in data:
+            data[key] = {}
+        next_data = data.get(key, {})
+        if not isinstance(next_data, dict):
+            return False
+        return recursive_set(next_data, next_key, value)
+    data[key] = value
+    return True
+
+
+def recursive_get(data: Dict[str, Any], key: str) -> Any:
+    key, next_key = DictWithRecursion._parse_key(key)
+    if next_key is not None:
+        next_data = data.get(key, None)
+        if not next_data:
+            return None
+        return recursive_get(next_data, next_key)
+    return data.get(key, None)
+
+
+def recursive_del(data: Dict[str, any], key: str) -> bool:
+    key, next_key = DictWithRecursion._parse_key(key)
+    if next_key is not None:
+        if key not in data:
+            return False
+        next_data = data.get(key, {})
+        return recursive_del(next_data, next_key)
+    if key in data:
+        del data[key]
+        return True
+    return False