Bump version to 0.2.0rc3

Rework Dockerfile to remove virtualenv

.editorconfig

@@ -8,5 +8,8 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 
+[*.py]
+max_line_length = 99
+
 [*.{yaml,yml,py}]
 indent_style = space

Dockerfile

@@ -0,0 +1,30 @@
+FROM docker.io/alpine:3.7
+
+ENV UID=1337 \
+    GID=1337
+
+COPY . /opt/mautrixtelegram
+RUN apk add --no-cache \
+      python3-dev \
+      py3-virtualenv \
+      py3-pillow \
+      py3-aiohttp \
+      py3-lxml \
+      py3-magic \
+      py3-numpy \
+      py3-asn1crypto \
+      py3-sqlalchemy \
+      build-base \
+      ffmpeg \
+      bash \
+      ca-certificates \
+      su-exec \
+      s6 \
+ && cd /opt/mautrixtelegram \
+ && cp -r docker/root/* / \
+ && rm docker -rf \
+ && pip3 install -r requirements.txt -r optional-requirements.txt
+
+VOLUME /data
+
+CMD ["/bin/s6-svscan", "/etc/s6.d"]

LICENSE

@@ -1,23 +1,21 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
                             Preamble
 
-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
 
   The licenses for most software and other practical works are designed
 to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
+our General Public Licenses are intended to guarantee your freedom to
 share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
+software for all its users.
 
   When we speak of free software, we are referring to freedom, not
 price.  Our General Public Licenses are designed to make sure that you
@@ -26,44 +24,34 @@ them if you wish), that you receive source code or can get it if you
 want it, that you can change the software or use pieces of it in new
 free programs, and that you know you can do these things.
 
-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
 
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
 
-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
 
-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
@@ -72,7 +60,7 @@ modification follow.
 
   0. Definitions.
 
-  "This License" refers to version 3 of the GNU General Public License.
+  "This License" refers to version 3 of the GNU Affero General Public License.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
@@ -549,35 +537,45 @@ to collect a royalty for further conveying from those to whom you convey
 the Program, the only way you could satisfy both those terms and this
 License would be to refrain entirely from conveying the Program.
 
-  13. Use with the GNU Affero General Public License.
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
 
   Notwithstanding any other provision of this License, you have
 permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
+under version 3 of the GNU General Public License into a single
 combined work, and to convey the resulting work.  The terms of this
 License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
 
   14. Revised Versions of this License.
 
   The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
 address new problems or concerns.
 
   Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
+Program specifies that a certain numbered version of the GNU Affero General
 Public License "or any later version" applies to it, you have the
 option of following the terms and conditions either of that numbered
 version or of any later version published by the Free Software
 Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
+GNU Affero General Public License, you may choose any version ever published
 by the Free Software Foundation.
 
   If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
+versions of the GNU Affero General Public License can be used, that proxy's
 public statement of acceptance of a version permanently authorizes you
 to choose that version for the Program.
 
@@ -635,40 +633,29 @@ the "copyright" line and a pointer to where the full notice is found.
     Copyright (C) <year>  <name of author>
 
     This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
+    it under the terms of the GNU Affero General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
     (at your option) any later version.
 
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
+    GNU Affero General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
 
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

README.md

@@ -1,12 +1,14 @@
 # mautrix-telegram
-A Matrix-Telegram puppeting bridge.
+A Matrix-Telegram hybrid puppeting/relaybot bridge.
 
 ### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)
 
-### [Features & Roadmap](ROADMAP.md)
+### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)
 
 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)
 
-A Telegram chat bridged to the Matrix room will be created once the bridge supports using a bot
-for unauthenticated users.
+Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
+
+## Preview
+![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)

ROADMAP.md

@@ -1,95 +1,53 @@
 # Features & roadmap
 
 * Matrix → Telegram
-  * [x] Plaintext messages
-  * [x] Formatted messages
-    * [ ] Bot commands (!command -> /command)
-    * [x] Mentions
-  * [x] Rich quotes
-  * [ ] Locations (not implemented in Riot)
-  * [x] Images
-  * [x] Files
+  * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [ ] ‡ Message history
   * [ ] † Presence
   * [ ] † Typing notifications
   * [ ] † Read receipts
-  * [ ] Pinning messages
+  * [x] Pinning messages
   * [x] Power level
     * [x] Normal chats
       * [ ] Non-hardcoded PL requirements
 	* [x] Supergroups/channels
 	  * [ ] Precise bridging (non-hardcoded PL requirements, bridge specific permissions, etc..)
-  * [ ] Membership actions
-    * [x] Inviting
-      * [x] Puppets
-      * [x] Matrix users who have logged into Telegram
-    * [x] Kicking
-    * [ ] Joining
-      * [ ] Chat name as alias
-      * [ ] ‡ Chat invite link as alias
-    * [x] Leaving
+  * [x] Membership actions (invite/kick/join/leave)
   * [x] Room metadata changes (name, topic, avatar)
   * [x] Initial room metadata
   * [ ] User metadata
     * [ ] Initial displayname/username/avatar at register
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
-  * [x] Plaintext messages
-  * [x] Formatted messages
-    * [x] Bot commands (/command -> !command)
-    * [x] Mentions
-  * [x] Replies
-  * [x] Forwards
-  * [x] Images
-  * [x] Locations
-  * [x] Stickers
-  * [x] Audio messages
-  * [x] Video messages
-  * [x] Documents
-  * [ ] Message deletions (no way to tell difference between user-specific deletion and global deletion)
-  * [ ] Message edits (not supported in Matrix)
+  * [x] Message content (text, formatting, files, etc..)
+  * [x] Message deletions
+  * [x] Message edits
+  * [ ] Message history
   * [x] Avatars
   * [x] Presence
   * [x] Typing notifications
   * [x] Read receipts (private chat only)
   * [x] Pinning messages
   * [x] Admin/chat creator status
-  * [ ] Supergroup/channel permissions (precise per-user not supported in Matrix)
-  * [x] Membership actions
-    * [x] Inviting
-    * [x] Kicking
-    * [x] Joining/leaving
+  * [ ] Supergroup/channel permissions (precise per-user permissions not supported in Matrix)
+  * [x] Membership actions (invite/kick/join/leave)
   * [ ] Chat metadata changes
     * [x] Title
     * [x] Avatar
     * [ ] † About text
     * [ ] † Public channel username
   * [x] Initial chat metadata (about text missing)
-  * [x] User metadata
-    * [x] Initial displayname/avatar
-    * [x] Changes to displayname/avatar
+  * [x] User metadata (displayname/avatar)
   * [x] Supergroup upgrade
 * Misc
   * [x] Automatic portal creation
     * [x] At startup
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [ ] Option to use bot to relay messages for unauthenticated Matrix users
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users
   * [ ] Option to use own Matrix account for messages sent from other Telegram clients
-* [Commands](https://github.com/tulir/mautrix-telegram/wiki/Management-commands)
-  * [x] Logging in and out (`login` + code entering)
-  * [x] Logging out
-  * [ ] Registering (`register`)
-  * [x] Searching for users (`search`)
-  * [x] Starting private chats (`pm`)
-  * [x] Joining chats with invite links (`join`)
-  * [x] Creating a Telegram chat for an existing Matrix room (`create`)
-  * [x] Upgrading the chat of a portal room into a supergroup (`upgrade`)
-  * [x] Change username of supergroup/channel (`groupname`)
-  * [x] Getting the Telegram invite link to a Matrix room (`invitelink`)
-  * Bridge administration
-    * [x] Clean up and forget a portal room (`deleteportal`)
-    * [ ] Setting Matrix-only power levels (`powerlevel`)
+  * [ ] ‡ Calls (hard, not yet supported by Telethon)
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/versions/1b241f7e8530_add_telegramfile_table.py

@@ -0,0 +1,28 @@
+"""Add TelegramFile table
+
+Revision ID: 1b241f7e8530
+Revises: 97d2a942bcf8
+Create Date: 2018-02-19 23:52:06.605741
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '1b241f7e8530'
+down_revision = '97d2a942bcf8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('telegram_file',
+                    sa.Column('id', sa.String(), nullable=False),
+                    sa.Column('mxc', sa.String(), nullable=True),
+                    sa.Column('mime_type', sa.String(), nullable=True),
+                    sa.Column('was_converted', sa.Boolean(), nullable=True),
+                    sa.PrimaryKeyConstraint('id'))
+
+
+def downgrade():
+    op.drop_table('telegram_file')

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -0,0 +1,24 @@
+"""Add is_bot field to puppets
+
+Revision ID: 1fa46383a9d3
+Revises: 30eca60587f1
+Create Date: 2018-04-29 23:44:40.102333
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1fa46383a9d3'
+down_revision = '30eca60587f1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('puppet', 'is_bot')

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -0,0 +1,23 @@
+"""Add megagroup field to portals
+
+Revision ID: 30eca60587f1
+Revises: cfc972368e50
+Create Date: 2018-04-29 15:51:04.656605
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '30eca60587f1'
+down_revision = 'cfc972368e50'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
+
+
+def downgrade():
+    op.drop_column('portal', 'megagroup')

alembic/versions/501dad2868bc_move_sessions_to_main_database.py

@@ -0,0 +1,111 @@
+"""Move sessions to main database
+
+Revision ID: 501dad2868bc
+Revises: 7d47d84380b6
+Create Date: 2018-03-02 19:15:53.826985
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlite3
+import os
+
+# revision identifiers, used by Alembic.
+revision = '501dad2868bc'
+down_revision = '7d47d84380b6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    Session = op.create_table('telethon_sessions',
+                              sa.Column('session_id', sa.VARCHAR(), nullable=False),
+                              sa.Column('dc_id', sa.INTEGER(), nullable=False),
+                              sa.Column('server_address', sa.VARCHAR(), nullable=True),
+                              sa.Column('port', sa.INTEGER(), nullable=True),
+                              sa.Column('auth_key', sa.BLOB(), nullable=True),
+                              sa.PrimaryKeyConstraint('session_id', 'dc_id'))
+    SentFile = op.create_table('telethon_sent_files',
+                               sa.Column('session_id', sa.VARCHAR(), nullable=False),
+                               sa.Column('md5_digest', sa.BLOB(), nullable=False),
+                               sa.Column('file_size', sa.INTEGER(), nullable=False),
+                               sa.Column('type', sa.INTEGER(), nullable=False),
+                               sa.Column('id', sa.INTEGER(), nullable=True),
+                               sa.Column('hash', sa.INTEGER(), nullable=True),
+                               sa.PrimaryKeyConstraint('session_id', 'md5_digest', 'file_size',
+                                                       'type'))
+    Entity = op.create_table('telethon_entities',
+                             sa.Column('session_id', sa.VARCHAR(), nullable=False),
+                             sa.Column('id', sa.INTEGER(), nullable=False),
+                             sa.Column('hash', sa.INTEGER(), nullable=False),
+                             sa.Column('username', sa.VARCHAR(), nullable=True),
+                             sa.Column('phone', sa.INTEGER(), nullable=True),
+                             sa.Column('name', sa.VARCHAR(), nullable=True),
+                             sa.PrimaryKeyConstraint('session_id', 'id'))
+    Version = op.create_table('telethon_version',
+                              sa.Column('version', sa.INTEGER(), nullable=False),
+                              sa.PrimaryKeyConstraint('version'))
+    conn = op.get_bind()
+    sessions = [os.path.basename(f) for f in os.listdir(".") if f.endswith(".session")]
+    for session in sessions:
+        session_to_sqlalchemy(conn, session, Session, SentFile, Entity)
+
+
+def session_to_sqlalchemy(conn, path, Session, SentFile, Entity):
+    session_conn = sqlite3.connect(path)
+    session_id = os.path.splitext(path)[0]
+    c = session_conn.cursor()
+
+    auth_data_tuples = c.execute("SELECT * FROM sessions").fetchall()
+    auth_data_dicts = []
+    for row in auth_data_tuples:
+        dc_id, server_address, port, auth_key = row
+        auth_data_dicts.append({
+            "session_id": session_id,
+            "dc_id": dc_id,
+            "server_address": server_address,
+            "port": port,
+            "auth_key": auth_key,
+        })
+    if auth_data_dicts:
+        conn.execute(Session.insert().values(auth_data_dicts))
+
+    sent_file_tuples = c.execute("SELECT * FROM sent_files").fetchall()
+    sent_file_dicts = []
+    for row in sent_file_tuples:
+        md5_digest, file_size, type, id, hash = row
+        sent_file_dicts.append({
+            "session_id": session_id,
+            "md5_digest": md5_digest,
+            "file_size": file_size,
+            "type": type,
+            "id": id,
+            "hash": hash,
+        })
+    if sent_file_dicts:
+        conn.execute(SentFile.insert().values(sent_file_dicts))
+
+    entity_tuples = c.execute("SELECT * FROM entities").fetchall()
+    entity_dicts = []
+    for row in entity_tuples:
+        id, hash, username, phone, name = row
+        entity_dicts.append({
+            "session_id": session_id,
+            "id": id,
+            "hash": hash,
+            "username": username,
+            "phone": phone,
+            "name": name,
+        })
+    if entity_dicts:
+        conn.execute(Entity.insert().values(entity_dicts))
+
+    c.close()
+    session_conn.close()
+
+
+def downgrade():
+    op.drop_table('telethon_version')
+    op.drop_table('telethon_entities')
+    op.drop_table('telethon_sent_files')
+    op.drop_table('telethon_sessions')

alembic/versions/7d47d84380b6_add_timestamp_to_telegramfile.py

@@ -0,0 +1,25 @@
+"""Add timestamp to TelegramFile
+
+Revision ID: 7d47d84380b6
+Revises: 1b241f7e8530
+Create Date: 2018-02-19 23:53:18.050871
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '7d47d84380b6'
+down_revision = '1b241f7e8530'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('telegram_file',
+                  sa.Column('timestamp', sa.BigInteger(), nullable=False, default=0,
+                            server_default="true"))
+
+
+def downgrade():
+    op.drop_column('telegram_file', 'timestamp')

alembic/versions/97d2a942bcf8_initial_revision.py

@@ -1,14 +1,13 @@
 """initial revision
 
 Revision ID: 97d2a942bcf8
-Revises: 
+Revises:
 Create Date: 2018-02-11 18:40:55.483842
 
 """
 from alembic import op
 import sqlalchemy as sa
 
-
 # revision identifiers, used by Alembic.
 revision = '97d2a942bcf8'
 down_revision = None
@@ -17,12 +16,61 @@ depends_on = None
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    pass
-    # ### end Alembic commands ###
+    op.create_table('portal',
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_receiver', sa.Integer),
+                    sa.Column('peer_type', sa.String, nullable=False, default=""),
+                    sa.Column('mxid', sa.String, nullable=True),
+                    sa.Column('username', sa.String, nullable=True),
+                    sa.Column('title', sa.String, nullable=True),
+                    sa.Column('about', sa.String, nullable=True),
+                    sa.Column('photo_id', sa.String, nullable=True),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_receiver'),
+                    sa.UniqueConstraint('mxid'))
+    op.create_table('user',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('tgid', sa.Integer, nullable=True),
+                    sa.Column('tg_username', sa.String, nullable=True),
+                    sa.Column('saved_contacts', sa.Integer, nullable=False, default=0),
+                    sa.PrimaryKeyConstraint('mxid'))
+    op.create_table('puppet',
+                    sa.Column('id', sa.Integer),
+                    sa.Column('displayname', sa.String, nullable=True),
+                    sa.Column('username', sa.String, nullable=True),
+                    sa.Column('photo_id', sa.String, nullable=True),
+                    sa.PrimaryKeyConstraint('id'))
+    op.create_table('contact',
+                    sa.Column('user', sa.Integer),
+                    sa.Column('contact', sa.Integer),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
+                    sa.ForeignKeyConstraint(("contact",), ("puppet.id",)),
+                    sa.PrimaryKeyConstraint('user', 'contact'))
+    op.create_table('user_portal',
+                    sa.Column('user', sa.Integer),
+                    sa.Column('portal', sa.Integer),
+                    sa.Column('portal_receiver', sa.Integer),
+                    sa.PrimaryKeyConstraint('user', 'portal', 'portal_receiver'),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
+                    sa.ForeignKeyConstraint(("portal", "portal_receiver"),
+                                            ("portal.tgid", "portal.tg_receiver")))
+    op.create_table('message',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    op.create_table('bot_chat',
+                    sa.Column('id', sa.Integer),
+                    sa.Column('type', sa.String, nullable=False, default=""),
+                    sa.PrimaryKeyConstraint('id'))
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    pass
-    # ### end Alembic commands ###
+    op.drop_table('bot_chat')
+    op.drop_table('message')
+    op.drop_table('user_portal')
+    op.drop_table('contact')
+    op.drop_table('puppet')
+    op.drop_table('user')
+    op.drop_table('portal')

alembic/versions/bcfefa1f1299_add_displayname_source_fields_for_.py

@@ -0,0 +1,23 @@
+"""Add displayname source fields for puppets
+
+Revision ID: bcfefa1f1299
+Revises: bdadd173ee02
+Create Date: 2018-05-19 17:00:21.078098
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'bcfefa1f1299'
+down_revision = 'bdadd173ee02'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('puppet', sa.Column('displayname_source', sa.Integer(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('puppet', 'displayname_source')

alembic/versions/bdadd173ee02_update_telethon_update_state_table.py

@@ -0,0 +1,43 @@
+"""Update telethon update state table
+
+Revision ID: bdadd173ee02
+Revises: eeaf0dae87ce
+Create Date: 2018-05-13 10:42:59.395597
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'bdadd173ee02'
+down_revision = 'eeaf0dae87ce'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column("id", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("hash", existing_type=sa.Integer, type_=sa.BigInteger)
+
+    with op.batch_alter_table("telethon_update_state") as batch_op:
+        batch_op.alter_column("entity_id", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("pts", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("qts", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("date", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("seq", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.add_column(sa.Column("unread_count", sa.Integer))
+
+
+def downgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column("id", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("hash", existing_type=sa.BigInteger, type_=sa.Integer)
+
+    with op.batch_alter_table("telethon_update_state") as batch_op:
+        batch_op.alter_column("entity_id", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("pts", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("qts", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("date", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("seq", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.drop_column("unread_count")

alembic/versions/cfc972368e50_add_metadata_to_telegramfile.py

@@ -0,0 +1,35 @@
+"""Add metadata to TelegramFile
+
+Revision ID: cfc972368e50
+Revises: 501dad2868bc
+Create Date: 2018-03-09 16:07:01.236712
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'cfc972368e50'
+down_revision = '501dad2868bc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.add_column(sa.Column('size', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('width', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('height', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('thumbnail', sa.String(), nullable=True))
+        batch_op.create_foreign_key(constraint_name="fk_file_thumbnail",
+                                    referent_table="telegram_file",
+                                    local_cols=['thumbnail'],
+                                    remote_cols=['id'])
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column('size')
+        batch_op.drop_column('width')
+        batch_op.drop_column('height')
+        batch_op.drop_column('thumbnail')

alembic/versions/eeaf0dae87ce_add_telethon_update_state_table.py

@@ -0,0 +1,34 @@
+"""Add telethon update state table
+
+Revision ID: eeaf0dae87ce
+Revises: 1fa46383a9d3
+Create Date: 2018-04-30 17:30:59.610885
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'eeaf0dae87ce'
+down_revision = '1fa46383a9d3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column('phone', existing_type=sa.Integer, type_=sa.BigInteger)
+    op.create_table('telethon_update_state',
+                    sa.Column('session_id', sa.String, nullable=False),
+                    sa.Column('entity_id', sa.Integer, nullable=False),
+                    sa.Column('pts', sa.Integer, nullable=True),
+                    sa.Column('qts', sa.Integer, nullable=True),
+                    sa.Column('date', sa.Integer, nullable=True),
+                    sa.Column('seq', sa.Integer, nullable=True),
+                    sa.PrimaryKeyConstraint('session_id', 'entity_id'))
+
+
+def downgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column('phone', existing_type=sa.BigInteger, type_=sa.Integer)
+    op.drop_table('telethon_update_state')

docker/root/etc/s6.d/.s6-svscan/finish

@@ -0,0 +1 @@
+#!/bin/sh

docker/root/etc/s6.d/mautrix-telegram/finish

@@ -0,0 +1,2 @@
+#!/bin/bash
+s6-svscanctl -t /etc/s6.d

docker/root/etc/s6.d/mautrix-telegram/run

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Define functions
+function fixperms {
+	chown -R ${UID}:${GID} /data /opt/mautrixtelegram
+}
+
+
+# Go into env
+cd /opt/mautrixtelegram
+export FFMPEG_BINARY=/usr/bin/ffmpeg
+
+# Replace database path in alembic.ini
+sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" alembic.ini
+sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
+
+# Check that database is in the right state
+alembic upgrade head
+
+if [[ ! -f /data/config.yaml ]]; then
+	cp example-config.yaml /data/config.yaml
+	echo "Didn't find a config file."
+	echo "Copied default config file to /data/config.yaml"
+	echo "Modify that config file to your liking."
+	echo "Start the container again after that to generate the registration file."
+	fixperms
+	exit
+fi
+
+if [[ ! -f /data/registration.yaml ]]; then
+	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
+	echo "Didn't find a registration file."
+	echo "Generated ode for you."
+	echo "Copy that over to synapses app service directory."
+	fixperms
+	exit
+fi
+
+fixperms
+exec su-exec ${UID}:${GID} python3 -m mautrix_telegram -c /data/config.yaml

example-config.yaml

@@ -1,7 +1,12 @@
 # Homeserver details
 homeserver:
+    # The address that this appservice can use to connect to the homeserver.
     address: https://matrix.org
+    # The domain of the homeserver (for MXIDs, etc).
     domain: matrix.org
+    # Whether or not to verify the SSL certificate of the homeserver.
+    # Only applies if address starts with https://
+    verify_ssl: true
 
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
@@ -14,8 +19,23 @@ appservice:
     hostname: localhost
     port: 8080
 
+    # The full URI to the database.
+    database: sqlite:///mautrix-telegram.db
+
+    # Public part of web server for out-of-Matrix interaction with the bridge.
+    # Used for things like login if the user wants to make sure the 2FA password isn't stored in
+    # the HS database.
+    public:
+        # Whether or not the public-facing endpoints should be enabled.
+        enabled: true
+        # The prefix to use in the public-facing endpoints.
+        prefix: /public
+        # The base URL where the public-facing endpoints are available. The prefix is not added
+        # implicitly.
+        external: https://example.com/public
+
     # Whether or not to enable debug messages in the console.
-    debug: false
+    debug: true
 
     # The unique ID of this appservice.
     id: telegram
@@ -58,32 +78,85 @@ bridge:
     - username
     - phone number
 
-    # Whether or not to use native Matrix replies. At the time of writing, only riot-web supports
-    # replies and the format of them is subject to change.
-    native_replies: True
-    # If native replies are disabled, should the custom replies contain a link to the message being
-    # replied to?
-    link_in_reply: False
     # Show message editing as a reply to the original message.
     # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: False
+    edits_as_replies: false
+    # Highlight changed/added parts in edits. Requires lxml.
+    highlight_edits: false
+    # Whether or not Matrix bot messages (type m.notice) should be bridged.
+    bridge_notices: true
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    # Maximum number of members to sync per portal when starting up. Other members will be
+    # synced when they send messages. The maximum is 10000, after which the Telegram server
+    # will not send any more members.
+    # Defaults to no local limit (-> limited to 10000 by server)
+    max_initial_member_sync: -1
+    # The maximum number of simultaneous Telegram deletions to handle.
+    # A large number of simultaneous redactions could put strain on your homeserver.
+    max_telegram_delete: 10
+    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
+    # login website (see appservice.public config section)
+    allow_matrix_login: true
+    # Use inline images instead of m.image to make rich captions possible.
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    inline_images: false
+    # Whether or not to bridge plaintext highlights.
+    # Only enable this if your displayname_template has some static part that the bridge can use to
+    # reliably identify what is a plaintext highlight.
+    plaintext_highlights: false
+    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
+    public_portals: true
+    # Whether to send stickers as the new native m.sticker type or normal m.images.
+    # Old versions of Riot don't support the new type at all.
+    # Remember that proper sticker support always requires Pillow to convert webp into png.
+    native_stickers: true
+    # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
+    # WARNING: Probably buggy, might get stuck in infinite loop.
+    catch_up: false
+
+    filter:
+        # Filter mode to use. Either "blacklist" or "whitelist".
+        # If the mode is "blacklist", the listed chats will never be bridged. An empty blacklist disables the filter.
+        # If the mode is "whitelist", only the listed chats can be bridged.
+        # Direct chats are not affected.
+        mode: blacklist
+        # The list of group/channel IDs to filter.
+        list: []
 
     # The prefix for commands. Only required in non-management rooms.
     command_prefix: "!tg"
 
-    # Whitelist of user IDs that are allowed to use this bridge. Leave empty to disable.
-    # You can enter a domain without the localpart to allow all users from that homeserver to use the bridge.
-    whitelist:
-    - "internal.example.com"
-    - "@user:public.example.com"
+    # Permissions for using the bridge.
+    # Permitted values:
+    #   relaybot - Only use the bridge via the relaybot, no access to commands.
+    #       full - Full access to use the bridge via relaybot or logging in with Telegram account.
+    #      admin - Full access to use the bridge and some extra administration commands.
+    # Permitted keys:
+    #        * - All Matrix users
+    #   domain - All users on that homeserver
+    #     mxid - Specific user
+    permissions:
+        "*": "relaybot"
+        "example.com": "full"
+        "public.example.com": "full"
+        "@admin:example.com": "admin"
 
-    # Admins can do things like delete portal rooms. Here you must specify the exact MXID, domains
-    # are not accepted.
-    admins:
-    - "@admin:internal.example.com"
+    # Options related to the message relay Telegram bot.
+    relaybot:
+        # Whether or not to allow creating portals from Telegram.
+        authless_portals: true
+        # Whether or not to allow Telegram group admins to use the bot commands.
+        whitelist_group_admins: true
+        # List of usernames/user IDs who are also allowed to use the bot commands.
+        whitelist:
+        - myusername
+        - 12345678
 
 # Telegram config
 telegram:
     # Get your own API keys at https://my.telegram.org/apps
     api_id: 12345
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
+    # (Optional) Create your own bot at https://t.me/BotFather
+    bot_token: disabled

mautrix_appservice/__init__.py

@@ -1,5 +0,0 @@
-from .appservice import AppService
-from .errors import MatrixError, MatrixRequestError, IntentError
-
-__version__ = "0.1.0"
-__author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_appservice/appservice.py

@@ -1,180 +0,0 @@
-# -*- coding: future_fstrings -*-
-# matrix-appservice-python - A Matrix Application Service framework written in Python.
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Partly based on github.com/Cadair/python-appservice-framework (MIT license)
-from functools import partial
-from contextlib import contextmanager
-from aiohttp import web
-import aiohttp
-import asyncio
-import logging
-
-from .intent_api import HTTPAPI
-from .state_store import StateStore
-
-
-class AppService:
-    def __init__(self, server, domain, as_token, hs_token, bot_localpart, loop=None, log=None,
-                 query_user=None, query_alias=None):
-        self.server = server
-        self.domain = domain
-        self.as_token = as_token
-        self.hs_token = hs_token
-        self.bot_mxid = f"@{bot_localpart}:{domain}"
-        self.state_store = StateStore(autosave_file="mx-state.json")
-        self.state_store.load("mx-state.json")
-
-        self.transactions = []
-
-        self._http_session = None
-        self._intent = None
-
-        self.loop = loop or asyncio.get_event_loop()
-        self.log = (logging.getLogger(log) if isinstance(log, str)
-                    else log or logging.getLogger("mautrix_appservice"))
-
-        def default_query_handler(_):
-            return None
-
-        self.query_user = query_user or default_query_handler
-        self.query_alias = query_alias or default_query_handler
-
-        self.event_handlers = []
-
-        self.app = web.Application(loop=self.loop)
-        self.app.router.add_route("PUT", "/transactions/{transaction_id}",
-                                  self._http_handle_transaction)
-        self.app.router.add_route("GET", "/rooms/{alias}", self._http_query_alias)
-        self.app.router.add_route("GET", "/users/{user_id}", self._http_query_user)
-
-        self.matrix_event_handler(self.update_state_store)
-
-    @property
-    def http_session(self):
-        if self._http_session is None:
-            raise AttributeError("the http_session attribute can only be used "
-                                 "from within the `AppService.run` context manager")
-        else:
-            return self._http_session
-
-    @property
-    def intent(self):
-        if self._intent is None:
-            raise AttributeError("the intent attribute can only be used from "
-                                 "within the `AppService.run` context manager")
-        else:
-            return self._intent
-
-    @contextmanager
-    def run(self, host="127.0.0.1", port=8080):
-        self._http_session = aiohttp.ClientSession(loop=self.loop)
-        self._intent = HTTPAPI(base_url=self.server, domain=self.domain, bot_mxid=self.bot_mxid,
-                               token=self.as_token, log=self.log, state_store=self.state_store,
-                               client_session=self._http_session).bot_intent()
-
-        yield self.loop.create_server(self.app.make_handler(), host, port)
-
-        self._intent = None
-        self._http_session.close()
-        self._http_session = None
-
-    def _check_token(self, request):
-        try:
-            token = request.rel_url.query["access_token"]
-        except KeyError:
-            return False
-
-        if token != self.hs_token:
-            return False
-
-        return True
-
-    async def _http_query_user(self, request):
-        if not self._check_token(request):
-            return web.Response(status=401)
-
-        user_id = request.match_info["userId"]
-
-        try:
-            response = await self.query_user(user_id)
-        except Exception:
-            self.log.exception("Exception in user query handler")
-            return web.Response(status=500)
-
-        if not response:
-            return web.Response(status=404)
-        return web.json_response(response)
-
-    async def _http_query_alias(self, request):
-        if not self._check_token(request):
-            return web.Response(status=401)
-
-        alias = request.match_info["alias"]
-
-        try:
-            response = await self.query_alias(alias)
-        except Exception:
-            self.log.exception("Exception in alias query handler")
-            return web.Response(status=500)
-
-        if not response:
-            return web.Response(status=404)
-        return web.json_response(response)
-
-    async def _http_handle_transaction(self, request):
-        if not self._check_token(request):
-            return web.Response(status=401)
-
-        transaction_id = request.match_info["transaction_id"]
-        if transaction_id in self.transactions:
-            return web.Response(status=200)
-
-        json = await request.json()
-
-        try:
-            events = json["events"]
-        except KeyError:
-            return web.Response(status=400)
-
-        for event in events:
-            self.handle_matrix_event(event)
-
-        self.transactions.append(transaction_id)
-
-        return web.json_response({})
-
-    async def update_state_store(self, event):
-        event_type = event["type"]
-        if event_type == "m.room.power_levels":
-            self.state_store.set_power_levels(event["room_id"], event["content"])
-        elif event_type == "m.room.member":
-            self.state_store.set_membership(event["room_id"], event["state_key"],
-                                            event["content"]["membership"])
-
-    def handle_matrix_event(self, event):
-        async def try_handle(handler):
-            try:
-                await handler(event)
-            except Exception:
-                self.log.exception("Exception in Matrix event handler")
-
-        for handler in self.event_handlers:
-            asyncio.ensure_future(try_handle(handler), loop=self.loop)
-
-    def matrix_event_handler(self, func):
-        self.event_handlers.append(func)
-        return func

mautrix_appservice/errors.py

@@ -1,38 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-
-class MatrixError(Exception):
-    """A generic Matrix error. Specific errors will subclass this."""
-    pass
-
-
-class IntentError(MatrixError):
-    def __init__(self, message, source):
-        super().__init__(message)
-        self.source = source
-
-
-class MatrixRequestError(MatrixError):
-    """ The home server returned an error response. """
-
-    def __init__(self, code=0, text="", errcode=None, message=None):
-        super().__init__("%d: %s" % (code, text))
-        self.code = code
-        self.text = text
-        self.errcode = errcode
-        self.message = message

mautrix_appservice/intent_api.py

@@ -1,521 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from urllib.parse import quote
-from time import time
-from json.decoder import JSONDecodeError
-from aiohttp.client_exceptions import ContentTypeError
-import re
-import json
-import magic
-import asyncio
-
-from .errors import MatrixError, MatrixRequestError, IntentError
-
-
-class HTTPAPI:
-    def __init__(self, base_url, domain=None, bot_mxid=None, token=None, identity=None, log=None,
-                 state_store=None, client_session=None, child=False):
-        self.base_url = base_url
-        self.token = token
-        self.identity = identity
-        self.validate_cert = True
-        self.session = client_session
-
-        self.domain = domain
-        self.bot_mxid = bot_mxid
-        self._bot_intent = None
-        self.state_store = state_store
-
-        if child:
-            self.log = log
-        else:
-            self.intent_log = log.getChild("intent")
-            self.log = log.getChild("api")
-            self.txn_id = 0
-            self.children = {}
-
-    def user(self, user):
-        try:
-            return self.children[user]
-        except KeyError:
-            child = ChildHTTPAPI(user, self)
-            self.children[user] = child
-            return child
-
-    def bot_intent(self):
-        if self._bot_intent:
-            return self._bot_intent
-        return IntentAPI(self.bot_mxid, self, state_store=self.state_store, log=self.intent_log)
-
-    def intent(self, user):
-        return IntentAPI(user, self.user(user), self.bot_intent(), self.state_store,
-                         self.intent_log)
-
-    async def _send(self, method, endpoint, content, query_params, headers):
-        while True:
-            query_params["access_token"] = self.token
-            request = self.session.request(method, endpoint, params=query_params,
-                                           data=content, headers=headers)
-            async with request as response:
-                if response.status < 200 or response.status >= 300:
-                    errcode = message = None
-                    try:
-                        response_data = await response.json()
-                        errcode = response_data["errcode"]
-                        message = response_data["error"]
-                    except (JSONDecodeError, ContentTypeError, KeyError):
-                        pass
-                    raise MatrixRequestError(code=response.status, text=await response.text(),
-                                             errcode=errcode, message=message)
-
-                if response.status == 429:
-                    await asyncio.sleep(response.json()["retry_after_ms"] / 1000)
-                else:
-                    return await response.json()
-
-    def _log_request(self, method, path, content, query_params):
-        log_content = content if not isinstance(content, bytes) else f"<{len(content)} bytes>"
-        log_content = log_content or "(No content)"
-        query_identity = query_params["user_id"] if "user_id" in query_params else "No identity"
-        self.log.debug("%s %s %s as user %s", method, path, log_content, query_identity)
-
-    def request(self, method, path, content=None, query_params=None, headers=None,
-                api_path="/_matrix/client/r0"):
-        content = content or {}
-        query_params = query_params or {}
-        headers = headers or {}
-
-        method = method.upper()
-        if method not in ["GET", "PUT", "DELETE", "POST"]:
-            raise MatrixError("Unsupported HTTP method: %s" % method)
-
-        if "Content-Type" not in headers:
-            headers["Content-Type"] = "application/json"
-        if headers["Content-Type"] == "application/json":
-            content = json.dumps(content)
-
-        if self.identity:
-            query_params["user_id"] = self.identity
-
-        self._log_request(method, path, content, query_params)
-
-        endpoint = self.base_url + api_path + path
-        return self._send(method, endpoint, content, query_params, headers or {})
-
-    def get_download_url(self, mxcurl):
-        if mxcurl.startswith('mxc://'):
-            return f"{self.base_url}/_matrix/media/r0/download/{mxcurl[6:]}"
-        else:
-            raise ValueError("MXC URL did not begin with 'mxc://'")
-
-    async def get_display_name(self, user_id):
-        content = await self.request("GET", f"/profile/{user_id}/displayname")
-        return content.get('displayname', None)
-
-    async def get_avatar_url(self, user_id):
-        content = await self.request("GET", f"/profile/{user_id}/avatar_url")
-        return content.get('avatar_url', None)
-
-    async def get_room_id(self, room_alias):
-        content = await self.request("GET", f"/directory/room/{quote(room_alias)}")
-        return content.get("room_id", None)
-
-    def set_typing(self, room_id, is_typing=True, timeout=5000, user=None):
-        content = {
-            "typing": is_typing
-        }
-        if is_typing:
-            content["timeout"] = timeout
-        user = user or self.identity
-        return self.request("PUT", f"/rooms/{room_id}/typing/{user}", content)
-
-
-class ChildHTTPAPI(HTTPAPI):
-    def __init__(self, user, parent):
-        super().__init__(parent.base_url, parent.domain, parent.bot_mxid, parent.token, user,
-                         parent.log, parent.state_store, parent.session, child=True)
-        self.parent = parent
-
-    @property
-    def txn_id(self):
-        return self.parent.txn_id
-
-    @txn_id.setter
-    def txn_id(self, value):
-        self.parent.txn_id = value
-
-
-class IntentAPI:
-    mxid_regex = re.compile("@(.+):(.+)")
-
-    def __init__(self, mxid, client, bot=None, state_store=None, log=None):
-        self.client = client
-        self.bot = bot
-        self.mxid = mxid
-        self.log = log
-
-        results = self.mxid_regex.search(mxid)
-        if not results:
-            raise ValueError("invalid MXID")
-        self.localpart = results.group(1)
-
-        self.state_store = state_store
-
-    def user(self, user):
-        if not self.bot:
-            return self.client.intent(user)
-        else:
-            self.log.warning("Called IntentAPI#user() of child intent object.")
-            return self.bot.client.intent(user)
-
-    # region User actions
-
-    async def get_joined_rooms(self):
-        await self.ensure_registered()
-        response = await self.client.request("GET", "/joined_rooms")
-        return response["joined_rooms"]
-
-    async def set_display_name(self, name):
-        await self.ensure_registered()
-        content = {"displayname": name}
-        return await self.client.request("PUT", f"/profile/{self.mxid}/displayname", content)
-
-    async def set_presence(self, status="online"):
-        await self.ensure_registered()
-        content = {
-            "presence": status
-        }
-        return await self.client.request("PUT", f"/presence/{self.mxid}/status", content)
-
-    async def set_avatar(self, url):
-        await self.ensure_registered()
-        content = {"avatar_url": url}
-        return await self.client.request("PUT", f"/profile/{self.mxid}/avatar_url", content)
-
-    async def upload_file(self, data, mime_type=None):
-        await self.ensure_registered()
-        mime_type = mime_type or magic.from_buffer(data, mime=True)
-        return await self.client.request("POST", "", content=data,
-                                         headers={"Content-Type": mime_type},
-                                         api_path="/_matrix/media/r0/upload")
-
-    async def download_file(self, url):
-        await self.ensure_registered()
-        url = self.client.get_download_url(url)
-        async with self.client.session.get(url) as response:
-            return await response.read()
-
-    # endregion
-    # region Room actions
-
-    async def create_room(self, alias=None, is_public=False, name=None, topic=None,
-                          is_direct=False, invitees=None, initial_state=None):
-        await self.ensure_registered()
-        content = {
-            "visibility": "public" if is_public else "private",
-            "is_direct": is_direct,
-        }
-        if alias:
-            content["room_alias_name"] = alias
-        if invitees:
-            content["invite"] = invitees
-        if name:
-            content["name"] = name
-        if topic:
-            content["topic"] = topic
-        if initial_state:
-            content["initial_state"] = initial_state
-
-        return await self.client.request("POST", "/createRoom", content)
-
-    def _invite_direct(self, room_id, user_id):
-        content = {"user_id": user_id}
-        return self.client.request("POST", "/rooms/" + room_id + "/invite", content)
-
-    async def invite(self, room_id, user_id, check_cache=False):
-        await self.ensure_joined(room_id)
-        try:
-            ok_states = {"invite", "join"}
-            do_invite = (not check_cache
-                         or self.state_store.get_membership(room_id, user_id) not in ok_states)
-            if do_invite:
-                response = await self._invite_direct(room_id, user_id)
-                self.state_store.invited(room_id, user_id)
-                return response
-        except MatrixRequestError as e:
-            if e.errcode != "M_FORBIDDEN":
-                raise IntentError(f"Failed to invite {user_id} to {room_id}", e)
-            if "is already in the room" in e.message:
-                self.state_store.joined(room_id, user_id)
-
-    def set_room_avatar(self, room_id, avatar_url, info=None):
-        content = {
-            "url": avatar_url,
-        }
-        if info:
-            content["info"] = info
-        return self.send_state_event(room_id, "m.room.avatar", content)
-
-    async def add_room_alias(self, room_id, localpart):
-        await self.ensure_registered()
-        content = {"room_id": room_id}
-        alias = f"#{localpart}:{self.client.domain}"
-        return await self.client.request("PUT", f"/directory/room/{quote(alias)}", content)
-
-    async def remove_room_alias(self, localpart):
-        await self.ensure_registered()
-        alias = f"#{localpart}:{self.client.domain}"
-        return await self.client.request("DELETE", f"/directory/room/{quote(alias)}")
-
-    def set_room_name(self, room_id, name):
-        body = {"name": name}
-        return self.send_state_event(room_id, "m.room.name", body)
-
-    async def get_power_levels(self, room_id, ignore_cache=False):
-        await self.ensure_joined(room_id)
-        if not ignore_cache:
-            try:
-                return self.state_store.get_power_levels(room_id)
-            except KeyError:
-                pass
-        levels = await self.client.request("GET",
-                                           f"/rooms/{quote(room_id)}/state/m.room.power_levels")
-        self.state_store.set_power_levels(room_id, levels)
-        return levels
-
-    async def set_power_levels(self, room_id, content):
-        if "events" not in content:
-            content["events"] = {}
-        response = await self.send_state_event(room_id, "m.room.power_levels", content)
-        self.state_store.set_power_levels(room_id, content)
-        return response
-
-    async def get_pinned_messages(self, room_id):
-        await self.ensure_joined(room_id)
-        response = await self.client.request("GET", f"/rooms/{room_id}/state/m.room.pinned_events")
-        return response["content"]["pinned"]
-
-    def set_pinned_messages(self, room_id, events):
-        return self.send_state_event(room_id, "m.room.pinned_events", {
-            "pinned": events
-        })
-
-    async def pin_message(self, room_id, event_id):
-        events = await self.get_pinned_messages(room_id)
-        if event_id not in events:
-            events.append(event_id)
-            await self.set_pinned_messages(room_id, events)
-
-    async def unpin_message(self, room_id, event_id):
-        events = await self.get_pinned_messages(room_id)
-        if event_id in events:
-            events.remove(event_id)
-            await self.set_pinned_messages(room_id, events)
-
-    async def get_event(self, room_id, event_id):
-        await self.ensure_joined(room_id)
-        return await self.client.request("GET", f"/rooms/{room_id}/event/{event_id}")
-
-    async def set_typing(self, room_id, is_typing=True, timeout=5000):
-        await self.ensure_joined(room_id)
-        content = {
-            "typing": is_typing
-        }
-        if is_typing:
-            content["timeout"] = timeout
-        return await self.client.request("PUT", f"/rooms/{room_id}/typing/{self.mxid}", content)
-
-    async def mark_read(self, room_id, event_id):
-        await self.ensure_joined(room_id)
-        return await self.client.request("POST", f"/rooms/{room_id}/receipt/m.read/{event_id}",
-                                         content={})
-
-    def send_notice(self, room_id, text, html=None, relates_to=None):
-        return self.send_text(room_id, text, html, "m.notice", relates_to)
-
-    def send_emote(self, room_id, text, html=None, relates_to=None):
-        return self.send_text(room_id, text, html, "m.emote", relates_to)
-
-    def send_image(self, room_id, url, info=None, text=None, relates_to=None):
-        return self.send_file(room_id, url, info or {}, text, "m.image", relates_to)
-
-    def send_file(self, room_id, url, info=None, text=None, file_type="m.file", relates_to=None):
-        return self.send_message(room_id, {
-            "msgtype": file_type,
-            "url": url,
-            "body": text or "Uploaded file",
-            "info": info or {},
-            "m.relates_to": relates_to or None,
-        })
-
-    def send_text(self, room_id, text, html=None, msgtype="m.text", relates_to=None):
-        if html:
-            if not text:
-                text = html
-            return self.send_message(room_id, {
-                "body": text,
-                "msgtype": msgtype,
-                "format": "org.matrix.custom.html",
-                "formatted_body": html or text,
-                "m.relates_to": relates_to or None,
-            })
-        else:
-            return self.send_message(room_id, {
-                "body": text,
-                "msgtype": msgtype,
-                "m.relates_to": relates_to or None,
-            })
-
-    def send_message(self, room_id, body):
-        return self.send_event(room_id, "m.room.message", body)
-
-    async def error_and_leave(self, room_id, text, html=None):
-        await self.ensure_joined(room_id)
-        await self.send_notice(room_id, text, html=html)
-        await self.leave_room(room_id)
-
-    def kick(self, room_id, user_id, message):
-        return self.set_membership(room_id, user_id, "leave", message)
-
-    def get_membership(self, room_id, user_id):
-        return self.get_state_event(room_id, "m.room.member", state_key=user_id)
-
-    def set_membership(self, room_id, user_id, membership, reason="", profile=None):
-        body = {
-            "membership": membership,
-            "reason": reason
-        }
-        profile = profile or {}
-        if "displayname" in profile:
-            body["displayname"] = profile["displayname"]
-        if "avatar_url" in profile:
-            body["avatar_url"] = profile["avatar_url"]
-
-        return self.send_state_event(room_id, "m.room.member", body, state_key=user_id)
-
-    @staticmethod
-    def _get_event_url(room_id, event_type, txn_id):
-        return f"/rooms/{quote(room_id)}/send/{quote(event_type)}/{quote(txn_id)}"
-
-    async def send_event(self, room_id, event_type, content, txn_id=None):
-        await self.ensure_joined(room_id)
-        await self._ensure_has_power_level_for(room_id, event_type)
-
-        txn_id = txn_id or str(self.client.txn_id) + str(int(time() * 1000))
-        self.client.txn_id += 1
-
-        url = self._get_event_url(room_id, event_type, txn_id)
-
-        return await self.client.request("PUT", url, content)
-
-    @staticmethod
-    def _get_state_url(room_id, event_type, state_key=""):
-        url = f"/rooms/{quote(room_id)}/state/{quote(event_type)}"
-        if state_key:
-            url += f"/{quote(state_key)}"
-        return url
-
-    async def send_state_event(self, room_id, event_type, content, state_key=""):
-        await self.ensure_joined(room_id)
-        await self._ensure_has_power_level_for(room_id, event_type)
-        url = self._get_state_url(room_id, event_type, state_key)
-        return await self.client.request("PUT", url, content)
-
-    async def get_state_event(self, room_id, event_type, state_key=""):
-        await self.ensure_joined(room_id)
-        url = self._get_state_url(room_id, event_type, state_key)
-        return await self.client.request("GET", url)
-
-    def join_room(self, room_id):
-        return self.ensure_joined(room_id, ignore_cache=True)
-
-    def _join_room_direct(self, room):
-        return self.client.request("POST", f"/join/{quote(room)}")
-
-    def leave_room(self, room_id):
-        try:
-            self.state_store.left(room_id, self.mxid)
-            return self.client.request("POST", f"/rooms/{quote(room_id)}/leave")
-        except MatrixRequestError as e:
-            if "not in room" not in e.message:
-                raise
-
-    def get_room_memberships(self, room_id):
-        return self.client.request("GET", f"/rooms/{quote(room_id)}/members")
-
-    async def get_room_members(self, room_id, allowed_memberships=("join",)):
-        memberships = await self.get_room_memberships(room_id)
-        return [membership["state_key"] for membership in memberships["chunk"] if
-                membership["content"]["membership"] in allowed_memberships]
-
-    async def get_room_state(self, room_id):
-        await self.ensure_joined(room_id)
-        state = await self.client.request("GET", f"/rooms/{quote(room_id)}/state")
-        # TODO update values based on state?
-        return state
-
-    # endregion
-    # region Ensure functions
-
-    async def ensure_joined(self, room_id, ignore_cache=False):
-        if not ignore_cache and self.state_store.is_joined(room_id, self.mxid):
-            return
-        await self.ensure_registered()
-        try:
-            await self._join_room_direct(room_id)
-            self.state_store.joined(room_id, self.mxid)
-        except MatrixRequestError as e:
-            if e.errcode != "M_FORBIDDEN" or not self.bot:
-                raise IntentError(f"Failed to join room {room_id} as {self.mxid}", e)
-            try:
-                await self.bot.invite(room_id, self.mxid)
-                await self._join_room_direct(room_id)
-                self.state_store.joined(room_id, self.mxid)
-            except MatrixRequestError as e2:
-                raise IntentError(f"Failed to join room {room_id} as {self.mxid}", e2)
-
-    def _register(self):
-        content = {"username": self.localpart}
-        query_params = {"kind": "user"}
-        return self.client.request("POST", "/register", content, query_params)
-
-    async def ensure_registered(self):
-        if self.state_store.is_registered(self.mxid):
-            return
-        try:
-            await self._register()
-        except MatrixRequestError as e:
-            if e.errcode != "M_USER_IN_USE":
-                self.log.exception(f"Failed to register {self.mxid}!")
-                # raise IntentError(f"Failed to register {self.mxid}", e)
-                return
-        self.state_store.registered(self.mxid)
-
-    async def _ensure_has_power_level_for(self, room_id, event_type):
-        if not self.state_store.has_power_levels(room_id):
-            await self.get_power_levels(room_id)
-        if self.state_store.has_power_level(room_id, self.mxid, event_type):
-            return
-        elif not self.bot:
-            self.log.warning(
-                f"Power level of {self.mxid} is not enough for {event_type} in {room_id}")
-            # raise IntentError(f"Power level of {self.mxid} is not enough"
-            #                  + f"for {event_type} in {room_id}")
-            return
-        # TODO implement
-
-    # endregion

mautrix_appservice/state_store.py

@@ -1,123 +0,0 @@
-# -*- coding: future_fstrings -*-
-# matrix-appservice-python - A Matrix Application Service framework written in Python.
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-import json
-
-
-class StateStore:
-    def __init__(self, autosave_file=None):
-        self.registrations = set()
-        self.memberships = {}
-        self.power_levels = {}
-        self.autosave_file = autosave_file
-
-    def save(self, file):
-        if isinstance(file, str):
-            output = open(file, "w")
-        else:
-            output = file
-
-        json.dump({
-            "registrations": list(self.registrations),
-            "memberships": self.memberships,
-            "power_levels": self.power_levels,
-        }, output)
-
-        if isinstance(file, str):
-            output.close()
-
-    def load(self, file):
-        if isinstance(file, str):
-            try:
-                input_source = open(file, "r")
-            except FileNotFoundError:
-                return
-        else:
-            input_source = file
-
-        data = json.load(input_source)
-        if "registrations" in data:
-            self.registrations = set(data["registrations"])
-        if "memberships" in data:
-            self.memberships = data["memberships"]
-        if "power_levels" in data:
-            self.power_levels = data["power_levels"]
-
-        if isinstance(file, str):
-            input_source.close()
-
-    def _autosave(self):
-        if self.autosave_file:
-            self.save(self.autosave_file)
-
-    def is_registered(self, user):
-        return user in self.registrations
-
-    def registered(self, user):
-        self.registrations.add(user)
-        self._autosave()
-
-    def get_membership(self, room, user):
-        return self.memberships.get(room, {}).get(user, "left")
-
-    def is_joined(self, room, user):
-        return self.get_membership(room, user) == "join"
-
-    def set_membership(self, room, user, membership):
-        if room not in self.memberships:
-            self.memberships[room] = {}
-        self.memberships[room][user] = membership
-        self._autosave()
-
-    def joined(self, room, user):
-        return self.set_membership(room, user, "join")
-
-    def invited(self, room, user):
-        return self.set_membership(room, user, "invite")
-
-    def left(self, room, user):
-        return self.set_membership(room, user, "left")
-
-    def has_power_levels(self, room):
-        return room in self.power_levels
-
-    def get_power_levels(self, room):
-        return self.power_levels[room]
-
-    def has_power_level(self, room, user, event):
-        room_levels = self.power_levels.get(room, {})
-        required = room_levels.get("events", {}).get(event, 95)
-        has = room_levels.get("users", {}).get(user, 0)
-        return has >= required
-
-    def set_power_level(self, room, user, level):
-        if room not in self.power_levels:
-            self.power_levels[room] = {
-                "users": {},
-                "events": {},
-            }
-        elif "users" not in self.power_levels[room]:
-            self.power_levels[room]["users"] = {}
-        self.power_levels[room]["users"][user] = level
-        self._autosave()
-
-    def set_power_levels(self, room, content):
-        if "events" not in content:
-            content["events"] = {}
-        if "users" not in content:
-            content["users"] = {}
-        self.power_levels[room] = content
-        self._autosave()

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.1.1"
+__version__ = "0.2.0rc3"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -3,17 +3,17 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import argparse
 import sys
 import logging
@@ -22,16 +22,22 @@ import asyncio
 import sqlalchemy as sql
 from sqlalchemy import orm
 
+from alchemysession import AlchemySessionContainer
 from mautrix_appservice import AppService
 
 from .base import Base
-from .config import Config
+from .config import Config, DictWithRecursion
 from .matrix import MatrixHandler
 
 from .db import init as init_db
+from .abstract_user import init as init_abstract_user
 from .user import init as init_user, User
+from .bot import init as init_bot
 from .portal import init as init_portal
 from .puppet import init as init_puppet
+from .formatter import init as init_formatter
+from .public import PublicBridgeWebsite
+from .context import Context
 
 log = logging.getLogger("mau")
 time_formatter = logging.Formatter("[%(asctime)s] [%(levelname)s@%(name)s] %(message)s")
@@ -44,14 +50,18 @@ parser = argparse.ArgumentParser(
     prog="python -m mautrix-telegram")
 parser.add_argument("-c", "--config", type=str, default="config.yaml",
                     metavar="<path>", help="the path to your config file")
+parser.add_argument("-b", "--base-config", type=str, default="example-config.yaml",
+                    metavar="<path>", help="the path to the example config "
+                                           "(for automatic config updates)")
 parser.add_argument("-g", "--generate-registration", action="store_true",
                     help="generate registration and quit")
 parser.add_argument("-r", "--registration", type=str, default="registration.yaml",
                     metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()
 
-config = Config(args.config, args.registration)
+config = Config(args.config, args.registration, args.base_config)
 config.load()
+config.update()
 
 if args.generate_registration:
     config.generate_registration()
@@ -70,20 +80,37 @@ db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautr
 db_factory = orm.sessionmaker(bind=db_engine)
 db_session = orm.scoping.scoped_session(db_factory)
 Base.metadata.bind = db_engine
-Base.metadata.create_all()
+
+telethon_session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
+                                                     table_base=Base, table_prefix="telethon_",
+                                                     manage_tables=False)
 
 loop = asyncio.get_event_loop()
+
 appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      config["appservice.as_token"], config["appservice.hs_token"],
-                     config["appservice.bot_username"], log="mau.as", loop=loop)
-context = (appserv, db_session, config, loop)
+                     config["appservice.bot_username"], log="mau.as", loop=loop,
+                     verify_ssl=config["homeserver.verify_ssl"])
+
+context = Context(appserv, db_session, config, loop, None, None, telethon_session_container)
+
+if config["appservice.public.enabled"]:
+    public = PublicBridgeWebsite(loop)
+    appserv.app.add_subapp(config.get("appservice.public.prefix", "/public"), public.app)
 
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
-    MatrixHandler(context)
     init_db(db_session)
+    init_abstract_user(context)
+    context.bot = init_bot(context)
+    context.mx = MatrixHandler(context)
+    init_formatter(context)
     init_portal(context)
     init_puppet(context)
-    startup_actions = init_user(context) + [start]
+    startup_actions = init_user(context) + [start, context.mx.init_as_bot()]
+
+    if context.bot:
+        startup_actions.append(context.bot.start())
+
     try:
         loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
         loop.run_forever()

mautrix_telegram/abstract_user.py

@@ -0,0 +1,298 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import platform
+import os
+
+from telethon.tl.types import *
+from mautrix_appservice import MatrixRequestError
+
+from .tgclient import MautrixTelegramClient
+from .db import Message as DBMessage
+from . import portal as po, puppet as pu, __version__
+
+config = None
+# Value updated from config in init()
+MAX_DELETIONS = 10
+
+
+class AbstractUser:
+    session_container = None
+    loop = None
+    log = None
+    db = None
+    az = None
+
+    def __init__(self):
+        self.connected = False
+        self.whitelisted = False
+        self.client = None
+        self.tgid = None
+        self.mxid = None
+        self.is_relaybot = False
+
+    async def _init_client(self):
+        self.log.debug(f"Initializing client for {self.name}")
+        device = f"{platform.system()} {platform.release()}"
+        sysversion = MautrixTelegramClient.__version__
+        self.session = self.session_container.new_session(self.name)
+        self.client = MautrixTelegramClient(session=self.session,
+                                            api_id=config["telegram.api_id"],
+                                            api_hash=config["telegram.api_hash"],
+                                            loop=self.loop,
+                                            app_version=__version__,
+                                            system_version=sysversion,
+                                            device_model=device,
+                                            report_errors=False)
+        await self.client.add_event_handler(self._update_catch)
+
+    async def update(self, update):
+        return False
+
+    async def post_login(self):
+        raise NotImplementedError()
+
+    async def _update_catch(self, update):
+        try:
+            if not await self.update(update):
+                await self._update(update)
+        except Exception:
+            self.log.exception("Failed to handle Telegram update")
+
+    async def _get_dialogs(self, limit=None):
+        dialogs = await self.client.get_dialogs(limit=limit)
+        return [dialog.entity for dialog in dialogs if (
+            not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden))
+            and not (isinstance(dialog.entity, Chat)
+                     and (dialog.entity.deactivated or dialog.entity.left)))]
+
+    @property
+    def name(self):
+        raise NotImplementedError()
+
+    @property
+    def logged_in(self):
+        return self.client and self.client.is_user_authorized()
+
+    @property
+    def has_full_access(self):
+        return self.logged_in and self.whitelisted
+
+    async def start(self):
+        if not self.client:
+            await self._init_client()
+        self.connected = await self.client.connect()
+
+    async def ensure_started(self, even_if_no_session=False):
+        if not self.whitelisted:
+            return self
+        elif not self.connected and (even_if_no_session or os.path.exists(f"{self.name}.session")):
+            return await self.start()
+        return self
+
+    def stop(self):
+        self.client.disconnect()
+        self.client = None
+        self.connected = False
+
+    # region Telegram update handling
+
+    async def _update(self, update):
+        if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                               UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
+            await self.update_message(update)
+        elif isinstance(update, UpdateDeleteMessages):
+            await self.delete_message(update)
+        elif isinstance(update, UpdateDeleteChannelMessages):
+            await self.delete_channel_message(update)
+        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
+            await self.update_typing(update)
+        elif isinstance(update, UpdateUserStatus):
+            await self.update_status(update)
+        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
+            await self.update_admin(update)
+        elif isinstance(update, UpdateChatParticipants):
+            await self.update_participants(update)
+        elif isinstance(update, UpdateChannelPinnedMessage):
+            await self.update_pinned_messages(update)
+        elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
+            await self.update_others_info(update)
+        elif isinstance(update, UpdateReadHistoryOutbox):
+            await self.update_read_receipt(update)
+        else:
+            self.log.debug("Unhandled update: %s", update)
+
+    async def update_pinned_messages(self, update):
+        portal = po.Portal.get_by_tgid(update.channel_id)
+        if portal and portal.mxid:
+            await portal.receive_telegram_pin_id(update.id)
+
+    async def update_participants(self, update):
+        portal = po.Portal.get_by_tgid(update.participants.chat_id)
+        if portal and portal.mxid:
+            await portal.update_telegram_participants(update.participants.participants)
+
+    async def update_read_receipt(self, update):
+        if not isinstance(update.peer, PeerUser):
+            self.log.debug("Unexpected read receipt peer: %s", update.peer)
+            return
+
+        portal = po.Portal.get_by_tgid(update.peer.user_id, self.tgid)
+        if not portal or not portal.mxid:
+            return
+
+        # We check that these are user read receipts, so tg_space is always the user ID.
+        message = DBMessage.query.get((update.max_id, self.tgid))
+        if not message:
+            return
+
+        puppet = pu.Puppet.get(update.peer.user_id)
+        await puppet.intent.mark_read(portal.mxid, message.mxid)
+
+    async def update_admin(self, update):
+        # TODO duplication not checked
+        portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+        if isinstance(update, UpdateChatAdmins):
+            await portal.set_telegram_admins_enabled(update.enabled)
+        elif isinstance(update, UpdateChatParticipantAdmin):
+            await portal.set_telegram_admin(update.user_id)
+        else:
+            self.log.warning("Unexpected admin status update: %s", update)
+
+    async def update_typing(self, update):
+        if isinstance(update, UpdateUserTyping):
+            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+        else:
+            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+        sender = pu.Puppet.get(update.user_id)
+        await portal.handle_telegram_typing(sender, update)
+
+    async def update_others_info(self, update):
+        # TODO duplication not checked
+        puppet = pu.Puppet.get(update.user_id)
+        if isinstance(update, UpdateUserName):
+            if await puppet.update_displayname(self, update):
+                puppet.save()
+        elif isinstance(update, UpdateUserPhoto):
+            if await puppet.update_avatar(self, update.photo.photo_big):
+                puppet.save()
+        else:
+            self.log.warning("Unexpected other user info update: %s", update)
+
+    async def update_status(self, update):
+        puppet = pu.Puppet.get(update.user_id)
+        if isinstance(update.status, UserStatusOnline):
+            await puppet.intent.set_presence("online")
+        elif isinstance(update.status, UserStatusOffline):
+            await puppet.intent.set_presence("offline")
+        else:
+            self.log.warning("Unexpected user status update: %s", update)
+        return
+
+    def get_message_details(self, update):
+        if isinstance(update, UpdateShortChatMessage):
+            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+            sender = pu.Puppet.get(update.from_id)
+        elif isinstance(update, UpdateShortMessage):
+            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+            sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
+        elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
+                                 UpdateEditMessage, UpdateEditChannelMessage)):
+            update = update.message
+            if isinstance(update.to_id, PeerUser) and not update.out:
+                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
+            else:
+                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
+            sender = pu.Puppet.get(update.from_id) if update.from_id else None
+        else:
+            self.log.warning(
+                f"Unexpected message type in User#get_message_details: {type(update)}")
+            return update, None, None
+        return update, sender, portal
+
+    @staticmethod
+    async def _try_redact(portal, message):
+        if not portal:
+            return
+        try:
+            await portal.main_intent.redact(message.mx_room, message.mxid)
+        except MatrixRequestError:
+            pass
+
+    async def delete_message(self, update):
+        if len(update.messages) > MAX_DELETIONS:
+            return
+
+        for message in update.messages:
+            message = DBMessage.query.get((message, self.tgid))
+            if not message:
+                continue
+            self.db.delete(message)
+            number_left = DBMessage.query.filter(DBMessage.mxid == message.mxid,
+                                                 DBMessage.mx_room == message.mx_room).count()
+            if number_left == 0:
+                portal = po.Portal.get_by_mxid(message.mx_room)
+                await self._try_redact(portal, message)
+        self.db.commit()
+
+    async def delete_channel_message(self, update):
+        if len(update.messages) > MAX_DELETIONS:
+            return
+
+        portal = po.Portal.get_by_tgid(update.channel_id)
+        if not portal:
+            return
+
+        for message in update.messages:
+            message = DBMessage.query.get((message, portal.tgid))
+            if not message:
+                continue
+            self.db.delete(message)
+            await self._try_redact(portal, message)
+        self.db.commit()
+
+    async def update_message(self, original_update):
+        update, sender, portal = self.get_message_details(original_update)
+
+        if isinstance(update, MessageService):
+            if isinstance(update.action, MessageActionChannelMigrateFrom):
+                self.log.debug(f"Ignoring action %s to %s by %d", update.action,
+                               portal.tgid_log,
+                               sender.id)
+                return
+            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
+                           sender.id)
+            return await portal.handle_telegram_action(self, sender, update)
+
+        user = sender.tgid if sender else "admin"
+        if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
+            if config["bridge.edits_as_replies"]:
+                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
+                return await portal.handle_telegram_edit(self, sender, update)
+            return
+
+        self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
+        return await portal.handle_telegram_message(self, sender, update)
+
+    # endregion
+
+
+def init(context):
+    global config, MAX_DELETIONS
+    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, _ = context
+    AbstractUser.session_container = context.telethon_session_container
+    MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/bot.py

@@ -0,0 +1,269 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Callable
+import logging
+import re
+
+from telethon.tl.types import *
+from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
+from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
+from telethon.errors import ChannelInvalidError, ChannelPrivateError
+
+from .abstract_user import AbstractUser
+from .db import BotChat
+from . import puppet as pu, portal as po, user as u
+
+config = None
+
+ReplyFunc = Callable[[str], Awaitable[Message]]
+
+
+class Bot(AbstractUser):
+    log = logging.getLogger("mau.bot")
+    mxid_regex = re.compile("@.+:.+")
+
+    def __init__(self, token: str):
+        super().__init__()
+        self.token = token
+        self.whitelisted = True
+        self.username = None
+        self.is_relaybot = True
+        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}
+        self.tg_whitelist = []
+        self.whitelist_group_admins = config["bridge.relaybot.whitelist_group_admins"] or False
+
+    async def init_permissions(self):
+        whitelist = config["bridge.relaybot.whitelist"] or []
+        for id in whitelist:
+            if isinstance(id, str):
+                entity = await self.client.get_input_entity(id)
+                if isinstance(entity, InputUser):
+                    id = entity.user_id
+                else:
+                    id = None
+            if isinstance(id, int):
+                self.tg_whitelist.append(id)
+
+    async def start(self):
+        await super().start()
+        if not self.logged_in:
+            await self.client.sign_in(bot_token=self.token)
+        await self.post_login()
+        return self
+
+    async def post_login(self):
+        await self.init_permissions()
+        info = await self.client.get_me()
+        self.tgid = info.id
+        self.username = info.username
+        self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
+
+        chat_ids = [id for id, type in self.chats.items() if type == "chat"]
+        response = await self.client(GetChatsRequest(chat_ids))
+        for chat in response.chats:
+            if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
+                self.remove_chat(chat.id)
+
+        channel_ids = [InputChannel(id, 0)
+                       for id, type in self.chats.items()
+                       if type == "channel"]
+        for id in channel_ids:
+            try:
+                await self.client(GetChannelsRequest([id]))
+            except (ChannelPrivateError, ChannelInvalidError):
+                self.remove_chat(id.channel_id)
+
+        if config["bridge.catch_up"]:
+            try:
+                await self.client.catch_up()
+            except Exception:
+                self.log.exception("Failed to run catch_up() for bot")
+
+    def register_portal(self, portal: po.Portal):
+        self.add_chat(portal.tgid, portal.peer_type)
+
+    def unregister_portal(self, portal: po.Portal):
+        self.remove_chat(portal.tgid)
+
+    def add_chat(self, id: int, type: str):
+        if id not in self.chats:
+            self.chats[id] = type
+            self.db.add(BotChat(id=id, type=type))
+            self.db.commit()
+
+    def remove_chat(self, id: int):
+        try:
+            del self.chats[id]
+        except KeyError:
+            pass
+        existing_chat = BotChat.query.get(id)
+        if existing_chat:
+            self.db.delete(existing_chat)
+            self.db.commit()
+
+    async def _can_use_commands(self, chat, tgid):
+        if tgid in self.tg_whitelist:
+            return True
+
+        user = u.User.get_by_tgid(tgid)
+        if user and user.is_admin:
+            self.tg_whitelist.append(user.tgid)
+            return True
+
+        if self.whitelist_group_admins:
+            if isinstance(chat, PeerChannel):
+                p = await self.client(GetParticipantRequest(chat, tgid))
+                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
+            elif isinstance(chat, PeerChat):
+                chat = await self.client(GetFullChatRequest(chat.chat_id))
+                participants = chat.full_chat.participants.participants
+                for p in participants:
+                    if p.user_id == tgid:
+                        return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+
+    async def check_can_use_commands(self, event: Message, reply: ReplyFunc):
+        if not await self._can_use_commands(event.to_id, event.from_id):
+            await reply("You do not have the permission to use that command.")
+            return False
+        return True
+
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc):
+        if not config["bridge.relaybot.authless_portals"]:
+            return await reply("This bridge doesn't allow portal creation from Telegram.")
+
+        if not portal.allow_bridging():
+            return await reply("This bridge doesn't allow bridging this chat.")
+
+        await portal.create_matrix_room(self)
+        if portal.mxid:
+            if portal.username:
+                return await reply(
+                    f"Portal is public: [{portal.alias}](https://matrix.to/#/{portal.alias})")
+            else:
+                return await reply(
+                    "Portal is not public. Use `/invite <mxid>` to get an invite.")
+
+    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc, mxid: str):
+        if len(mxid) == 0:
+            return await reply("Usage: `/invite <mxid>`")
+        elif not portal.mxid:
+            return await reply("Portal does not have Matrix room. "
+                               "Create one with /portal first.")
+        if not self.mxid_regex.match(mxid):
+            return await reply("That doesn't look like a Matrix ID.")
+        user = await u.User.get_by_mxid(mxid).ensure_started()
+        if not user.whitelisted:
+            return await reply("That user is not whitelisted to use the bridge.")
+        elif user.logged_in:
+            displayname = f"@{user.username}" if user.username else user.displayname
+            return await reply("That user seems to be logged in. "
+                               f"Just invite [{displayname}](tg://user?id={user.tgid})")
+        else:
+            await portal.main_intent.invite(portal.mxid, user.mxid)
+            return await reply(f"Invited `{user.mxid}` to the portal.")
+
+    def handle_command_id(self, message: Message, reply: ReplyFunc):
+        # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
+        # chat is a normal group or a supergroup/channel when using the ID.
+        if isinstance(message.to_id, PeerChannel):
+            return reply(f"-100{message.to_id.channel_id}")
+        return reply(str(-message.to_id.chat_id))
+
+    def match_command(self, text: str, command: str) -> bool:
+        text = text.lower()
+        command = f"/{command.lower()}"
+        command_targeted = f"{command}@{self.username.lower()}"
+
+        is_plain_command = text == command or text == command_targeted
+        if is_plain_command:
+            return True
+
+        is_arg_command = text.startswith(command + " ") or text.startswith(command_targeted + " ")
+        if is_arg_command:
+            return True
+
+        return False
+
+    async def handle_command(self, message: Message):
+        def reply(reply_text):
+            return self.client.send_message(message.to_id, reply_text, markdown=True,
+                                            reply_to=message.id)
+
+        text = message.message
+
+        if self.match_command(text, "id"):
+            return await self.handle_command_id(message, reply)
+
+        portal = po.Portal.get_by_entity(message.to_id)
+
+        if self.match_command(text, "portal"):
+            if not await self.check_can_use_commands(message, reply):
+                return
+            await self.handle_command_portal(portal, reply)
+        elif self.match_command(text, "invite"):
+            if not await self.check_can_use_commands(message, reply):
+                return
+            try:
+                mxid = text[text.index(" ") + 1:]
+            except ValueError:
+                mxid = ""
+            await self.handle_command_invite(portal, reply, mxid=mxid)
+
+    def handle_service_message(self, message: MessageService):
+        to_id = message.to_id
+        if isinstance(to_id, PeerChannel):
+            to_id = to_id.channel_id
+            type = "channel"
+        elif isinstance(to_id, PeerChat):
+            to_id = to_id.chat_id
+            type = "chat"
+        else:
+            return
+
+        action = message.action
+        if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
+            self.add_chat(to_id, type)
+        elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
+            self.remove_chat(to_id)
+
+    async def update(self, update):
+        if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
+            return
+        if isinstance(update.message, MessageService):
+            return self.handle_service_message(update.message)
+
+        is_command = (isinstance(update.message, Message)
+                      and update.message.entities and len(update.message.entities) > 0
+                      and isinstance(update.message.entities[0], MessageEntityBotCommand))
+        if is_command:
+            return await self.handle_command(update.message)
+
+    def is_in_chat(self, peer_id) -> bool:
+        return peer_id in self.chats
+
+    @property
+    def name(self) -> str:
+        return "bot"
+
+
+def init(context):
+    global config
+    config = context.config
+    token = config["telegram.bot_token"]
+    if token and not token.lower().startswith("disable"):
+        return Bot(token)
+    return None

mautrix_telegram/commands/__init__.py

@@ -1,2 +1,2 @@
-from .handler import command_handler, CommandHandler
-from . import clean_rooms, auth, meta, telegram
+from .handler import command_handler, CommandHandler, CommandEvent
+from . import clean_rooms, auth, meta, telegram, portal

mautrix_telegram/commands/auth.py

@@ -3,22 +3,24 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import asyncio
 
 from telethon.errors import *
 
 from . import command_handler
+from .. import puppet as pu
+from ..util import format_duration
 
 
 @command_handler(needs_auth=False)
@@ -32,63 +34,171 @@ async def ping(evt):
         return await evt.reply("You're not logged in.")
 
 
+@command_handler()
+async def ping_bot(evt):
+    if not evt.tgbot:
+        return await evt.reply("Telegram message relay bot not configured.")
+    bot_info = await evt.tgbot.client.get_me()
+    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
+    displayname = bot_info.first_name
+    return await evt.reply("Telegram message relay bot is active: "
+                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
+                           "To use the bot, simply invite it to a portal room.")
+
+
 @command_handler(needs_auth=False, management_only=True)
 def register(evt):
     return evt.reply("Not yet implemented.")
 
 
+@command_handler(needs_auth=False, management_only=True)
+async def register(evt):
+    if evt.sender.logged_in:
+        return await evt.reply("You are already logged in.")
+    elif len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
+
+    phone_number = evt.args[0]
+    if len(evt.args) == 2:
+        full_name = evt.args[1], ""
+    else:
+        full_name = " ".join(evt.args[1:-1]), evt.args[-1]
+
+    await request_code(evt, phone_number, {
+        "next": enter_code_register,
+        "action": "Register",
+        "full_name": full_name,
+    })
+
+
+async def enter_code_register(evt):
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        first_name, last_name = evt.sender.command_status["full_name"]
+        user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
+        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        evt.sender.command_status = None
+        return await evt.reply(f"Successfully registered to Telegram.")
+    except PhoneNumberOccupiedError:
+        return await evt.reply("That phone number has already been registered. "
+                               "You can log in with `$cmdprefix+sp login`.")
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
+    except PhoneCodeExpiredError:
+        return await evt.reply(
+            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`.")
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
+    except Exception:
+        evt.log.exception("Error sending phone code")
+        return await evt.reply("Unhandled exception while sending code. "
+                               "Check console for more details.")
+
+
 @command_handler(needs_auth=False, management_only=True)
 async def login(evt):
     if evt.sender.logged_in:
         return await evt.reply("You are already logged in.")
-    elif len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp login <phone number>`")
-    phone_number = evt.args[0]
-    await evt.sender.client.sign_in(phone_number)
-    evt.sender.command_status = {
-        "next": enter_code,
-        "action": "Login",
-    }
-    return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
 
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_phone,
+            "action": "Login",
+        }
 
-@command_handler(needs_auth=False)
-async def enter_code(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
-
-    try:
-        user = await evt.sender.client.sign_in(code=evt.args[0])
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PhoneNumberUnoccupiedError:
-        return await evt.reply("That phone number has not been registered."
-                               "Please register with `$cmdprefix+sp register <phone>`.")
-    except PhoneCodeExpiredError:
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        url = f"{prefix}/login?mxid={evt.sender.mxid}"
+        if evt.config.get("bridge.allow_matrix_login", True):
+            return await evt.reply("\n\n".join((
+                "This bridge instance allows you to log in inside or outside Matrix.",
+                "If you would like to log in within Matrix, please send your phone number here.",
+                f"If you would like to log in outside of Matrix, [click here]({url}).")))
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
         return await evt.reply(
-            "Phone code expired. Try again with `$cmdprefix+sp login <phone>`.")
-    except PhoneCodeInvalidError:
-        return await evt.reply("Invalid phone code.")
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your phone number here to start the login process.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+async def request_code(evt, phone_number, next_status):
+    ok = False
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        await evt.sender.client.sign_in(phone_number)
+        ok = True
+        return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
     except PhoneNumberAppSignupForbiddenError:
         return await evt.reply(
             "Your phone number does not allow 3rd party apps to sign in.")
     except PhoneNumberFloodError:
         return await evt.reply(
             "Your phone number has been temporarily blocked for flooding. "
-            "The block is usually applied for around a day.")
+            "The ban is usually applied for around a day.")
+    except FloodWaitError as e:
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            f"Please wait for {format_duration(e.seconds)} before trying again.")
     except PhoneNumberBannedError:
-        return await evt.reply("Your phone number has been banned from Telegram.")
+        return await  evt.reply("Your phone number has been banned from Telegram.")
+    except PhoneNumberUnoccupiedError:
+        return await  evt.reply("That phone number has not been registered. "
+                                "Please register with `$cmdprefix+sp register <phone>`.")
+    except Exception:
+        evt.log.exception("Error requesting phone code")
+        return await evt.reply("Unhandled exception while requesting code. "
+                               "Check console for more details.")
+    finally:
+        evt.sender.command_status = next_status if ok else None
+
+
+@command_handler(needs_auth=False)
+async def enter_phone(evt):
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone <phone>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+
+    phone_number = evt.args[0]
+    await request_code(evt, phone_number, {
+        "next": enter_code,
+        "action": "Login",
+    })
+
+
+@command_handler(needs_auth=False)
+async def enter_code(evt):
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        user = await evt.sender.client.sign_in(code=evt.args[0])
+        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        evt.sender.command_status = None
+        return await evt.reply(f"Successfully logged in as @{user.username}")
+    except PhoneCodeExpiredError:
+        return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
     except SessionPasswordNeededError:
         evt.sender.command_status = {
             "next": enter_password,
             "action": "Login (password entry)",
         }
-        return await evt.reply("Your account has two-factor authentication."
+        return await evt.reply("Your account has two-factor authentication. "
                                "Please send your password here.")
     except Exception:
         evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code."
+        return await evt.reply("Unhandled exception while sending code. "
                                "Check console for more details.")
 
 
@@ -96,9 +206,12 @@ async def enter_code(evt):
 async def enter_password(evt):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
-
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        user = await evt.sender.client.sign_in(password=evt.args[0])
+        await evt.sender.ensure_started(even_if_no_session=True)
+        user = await evt.sender.client.sign_in(password=" ".join(evt.args))
         asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
         evt.sender.command_status = None
         return await evt.reply(f"Successfully logged in as @{user.username}")

mautrix_telegram/commands/clean_rooms.py

@@ -3,17 +3,17 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from mautrix_appservice import MatrixRequestError
 
 from . import command_handler
@@ -52,7 +52,7 @@ async def _find_rooms(intent):
     return management_rooms, unidentified_rooms, portals, empty_portals
 
 
-@command_handler(needs_admin=True, name="clean-rooms")
+@command_handler(needs_admin=True, needs_auth=False, name="clean-rooms")
 async def clean_rooms(evt):
     if not evt.is_management:
         return await evt.reply("`clean-rooms` is a particularly spammy command. Please don't "
@@ -66,7 +66,7 @@ async def clean_rooms(evt):
               or ["No management rooms found."])
     reply.append("#### Active portal rooms (A)")
     reply += ([f"{n+1}. [P{n+1}](https://matrix.to/#/{portal.mxid}) "
-               + f"(to Telegram chat \"{portal.title}\")"
+               f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(portals)]
               or ["No active portal rooms found."])
     reply.append("#### Unidentified rooms (U)")
@@ -75,7 +75,7 @@ async def clean_rooms(evt):
               or ["No unidentified rooms found."])
     reply.append("#### Inactive portal rooms (I)")
     reply += ([f"{n}. [E{n}](https://matrix.to/#/{portal.mxid}) "
-               + f"(to Telegram chat \"{portal.title}\")"
+               f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(empty_portals)]
               or ["No inactive portal rooms found."])
 
@@ -141,28 +141,28 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
                 "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
     else:
         return await evt.reply(f"Unknown room cleaning action `{command}`. "
-                               + "Use `$cmdprefix+sp cancel` to cancel room "
-                               + "cleaning.")
+                               "Use `$cmdprefix+sp cancel` to cancel room "
+                               "cleaning.")
 
     evt.sender.command_status = {
         "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
         "action": "Room cleaning",
     }
     await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type"
-                    + "`$cmdprefix+sp confirm-clean`.")
+                    "`$cmdprefix+sp confirm-clean`.")
 
 
 async def execute_room_cleanup(evt, rooms_to_clean):
     if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
         await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
-                        + "This might take a while.")
+                        "This might take a while.")
         cleaned = 0
         for room in rooms_to_clean:
             if isinstance(room, po.Portal):
                 await room.cleanup_and_delete()
                 cleaned += 1
             elif isinstance(room, str):
-                await po.Portal.cleanup_room(evt.az.intent, room, type="Room")
+                await po.Portal.cleanup_room(evt.az.intent, room, message="Room deleted")
                 cleaned += 1
         evt.sender.command_status = None
         await evt.reply(f"{cleaned} rooms cleaned up successfully.")

mautrix_telegram/commands/handler.py

@@ -3,22 +3,24 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import markdown
 import logging
 
 from telethon.errors import FloodWaitError
 
+from ..util import format_duration
+
 command_handlers = {}
 
 
@@ -27,7 +29,7 @@ def command_handler(needs_auth=True, management_only=False, needs_admin=False, n
         def wrapper(evt):
             if management_only and not evt.is_management:
                 return evt.reply(f"`{evt.command}` is a restricted command:"
-                                 + "you may only run it in management rooms.")
+                                 "you may only run it in management rooms.")
             elif needs_auth and not evt.sender.logged_in:
                 return evt.reply("This command requires you to be logged in.")
             elif needs_admin and not evt.sender.is_admin:
@@ -45,6 +47,8 @@ class CommandEvent:
         self.az = handler.az
         self.log = handler.log
         self.loop = handler.loop
+        self.tgbot = handler.tgbot
+        self.config = handler.config
         self.command_prefix = handler.command_prefix
         self.room_id = room
         self.sender = sender
@@ -65,29 +69,11 @@ class CommandEvent:
         return self.az.intent.send_notice(self.room_id, message, html=html)
 
 
-def format_duration(seconds):
-    def pluralize(count, singular): return singular if count == 1 else singular + "s"
-
-    def include(count, word): return f"{count} {pluralize(count, word)}" if count > 0 else ""
-
-    minutes, seconds = divmod(seconds, 60)
-    hours, minutes = divmod(minutes, 60)
-    days, hours = divmod(hours, 24)
-    parts = [a for a in [
-        include(days, "day"),
-        include(hours, "hour"),
-        include(minutes, "minute"),
-        include(seconds, "second")] if a]
-    if len(parts) > 2:
-        return "{} and {}".format(", ".join(parts[:-1]), parts[-1])
-    return " and ".join(parts)
-
-
 class CommandHandler:
     log = logging.getLogger("mau.commands")
 
     def __init__(self, context):
-        self.az, self.db, self.config, self.loop = context
+        self.az, self.db, self.config, self.loop, self.tgbot = context
         self.command_prefix = self.config["bridge.command_prefix"]
 
     # region Utility functions for handling commands
@@ -95,12 +81,13 @@ class CommandHandler:
     async def handle(self, room, sender, command, args, is_management, is_portal):
         evt = CommandEvent(self, room, sender, command, args,
                            is_management, is_portal)
+        orig_command = command
         command = command.lower()
         try:
             command = command_handlers[command]
         except KeyError:
             if sender.command_status and "next" in sender.command_status:
-                args.insert(0, command)
+                args.insert(0, orig_command)
                 evt.command = ""
                 command = sender.command_status["next"]
             else:
@@ -108,8 +95,9 @@ class CommandHandler:
         try:
             await command(evt)
         except FloodWaitError as e:
-            return evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
+            return await evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
         except Exception:
-            self.log.exception(f"Fatal error handling command "
-                               + f"{evt.command} {' '.join(args)} from {sender.mxid}")
-            return evt.reply("Fatal error while handling command. Check logs for more details.")
+            self.log.exception("Fatal error handling command "
+                               f"{evt.command} {' '.join(args)} from {sender.mxid}")
+            return await evt.reply("Fatal error while handling command. "
+                                   "Check logs for more details.")

mautrix_telegram/commands/meta.py

@@ -3,21 +3,21 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from . import command_handler
 
 
-@command_handler()
+@command_handler(needs_auth=False)
 def cancel(evt):
     if evt.sender.command_status:
         action = evt.sender.command_status["action"]
@@ -27,12 +27,12 @@ def cancel(evt):
         return evt.reply("No ongoing command.")
 
 
-@command_handler()
+@command_handler(needs_auth=False)
 def unknown_command(evt):
     return evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
 
 
-@command_handler()
+@command_handler(needs_auth=False)
 def help(evt):
     if evt.is_management:
         management_status = ("This is a management room: prefixing commands "
@@ -50,27 +50,39 @@ def help(evt):
 **cancel** - Cancel an ongoing action (such as login).
 
 #### Authentication
-**login** <_phone_> - Request an authentication code.  
-**logout**          - Log out from Telegram.  
-**ping**            - Check if you're logged into Telegram.
+**login**  - Request an authentication code.  
+**logout** - Log out from Telegram.  
+**ping**   - Check if you're logged into Telegram.
+
+#### Miscellaneous things
+**search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users.  
+**sync** [`chats`|`contacts`|`me`]   - Synchronize your chat portals, contacts and/or own info.  
+**ping-bot**                         - Get info of the message relay Telegram bot.  
+**set-pl** <_level_> [_mxid_]        - Set a temporary power level without affecting Telegram.
 
 #### Initiating chats
-**search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users.  
-**pm** <_identifier_>                - Open a private chat with the given Telegram user. The
-                                       identifier is either the internal user ID, the username or
-                                       the phone number.  
-**join** <_link_>                    - Join a chat with an invite link.  
-**create** [_type_]                  - Create a Telegram chat of the given type for the current
-                                       Matrix room. The type is either `group`, `supergroup` or
-                                       `channel` (defaults to `group`).
+**pm** <_identifier_> - Open a private chat with the given Telegram user. The identifier is either
+                        the internal user ID, the username or the phone number.  
+**join** <_link_>     - Join a chat with an invite link.  
+**create** [_type_]   - Create a Telegram chat of the given type for the current Matrix room. The
+                        type is either `group`, `supergroup` or `channel` (defaults to `group`).
 
 #### Portal management  
 **upgrade**                 - Upgrade a normal Telegram group to a supergroup.  
 **invite-link**             - Get a Telegram invite link to the current chat.  
-**delete-portal**           - Forget the current portal room. Only works for group chats; to delete
-                             a private chat portal, simply leave the room.  
+**delete-portal**           - Remove all users from the current portal room and forget the portal.
+                              Only works for group chats; to delete a private chat portal, simply
+                              leave the room.  
+**unbridge**                - Remove puppets from the current portal room and forget the portal.  
+**bridge** [_id_]           - Bridge the current Matrix room to the Telegram chat with the given
+                              ID. The ID must be the prefixed version that you get with the `/id`
+                              command of the Telegram-side bot.  
 **group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash
                              (`-`) as the name.  
 **clean-rooms**             - Clean up unused portal/management rooms.
+
+**filter** <`whitelist`|`blacklist`> <_chat ID_> - Allow or disallow bridging a specific chat.  
+**filter-mode** <`whitelist`|`blacklist`>      - Change whether the bridge will allow or disallow
+                                                 bridging rooms by default.
 """
     return evt.reply(management_status + help)

mautrix_telegram/commands/portal.py

@@ -0,0 +1,426 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import asyncio
+
+from telethon.errors import *
+from mautrix_appservice import MatrixRequestError
+
+from .. import portal as po
+from . import command_handler, CommandEvent
+
+
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl")
+async def set_power_level(evt: CommandEvent):
+    try:
+        level = int(evt.args[0])
+    except KeyError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp set-power <level> [mxid]`")
+    except ValueError:
+        return await evt.reply("The level must be an integer.")
+    levels = await evt.az.intent.get_power_levels(evt.room_id)
+    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
+    levels["users"][mxid] = level
+    try:
+        await evt.az.intent.set_power_levels(evt.room_id, levels)
+    except MatrixRequestError:
+        evt.log.exception("Failed to set power level.")
+        return await evt.reply("Failed to set power level.")
+
+
+@command_handler()
+async def invite_link(evt: CommandEvent):
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if portal.peer_type == "user":
+        return await evt.reply("You can't invite users to private chats.")
+
+    try:
+        link = await portal.get_invite_link(evt.sender)
+        return await evt.reply(f"Invite link to {portal.title}: {link}")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to create an invite link.")
+
+
+async def _has_access_to(room, intent, sender, event, default=50):
+    if sender.is_admin:
+        return True
+    # Make sure the state store contains the power levels.
+    try:
+        await intent.get_power_levels(room)
+    except MatrixRequestError:
+        return False
+    return intent.state_store.has_power_level(room, sender.mxid,
+                                              event=f"net.maunium.telegram.{event}",
+                                              default=default)
+
+
+async def _get_portal_and_check_permission(evt, permission, action=None):
+    room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if not portal:
+        that_this = "This" if room_id == evt.room_id else "That"
+        return await evt.reply(f"{that_this} is not a portal room."), False
+
+    if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, permission):
+        action = action or f"{permission.replace('_', ' ')}s"
+        return await evt.reply(f"You do not have the permissions to {action} that portal."), False
+    return portal, True
+
+
+def _get_portal_murder_function(action, room_id, function, command, completed_message):
+    async def post_confirm(confirm):
+        confirm.sender.command_status = None
+        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
+            await function()
+            if confirm.room_id != room_id:
+                return await confirm.reply(completed_message)
+        else:
+            return await confirm.reply(f"{action} cancelled.")
+
+    return {
+        "next": post_confirm,
+        "action": action,
+    }
+
+
+@command_handler()
+async def delete_portal(evt: CommandEvent):
+    portal, ok = await _get_portal_and_check_permission(evt, "delete_portal")
+    if not ok:
+        return
+
+    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
+                                                            portal.cleanup_and_delete, "delete",
+                                                            "Portal successfully deleted.")
+    return await evt.reply("Please confirm deletion of portal "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           f"to Telegram chat \"{portal.title}\" "
+                           "by typing `$cmdprefix+sp confirm-delete`")
+
+
+@command_handler()
+async def unbridge(evt: CommandEvent):
+    portal, ok = await _get_portal_and_check_permission(evt, "unbridge_room")
+    if not ok:
+        return
+
+    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
+                                                            portal.unbridge, "unbridge",
+                                                            "Room successfully unbridged.")
+    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           "by typing `$cmdprefix+sp confirm-unbridge`")
+
+
+@command_handler()
+async def bridge(evt: CommandEvent):
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** "
+                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    room_id = evt.args[1] if len(evt.args) > 1 else evt.room_id
+    that_this = "This" if room_id == evt.room_id else "That"
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if portal:
+        return await evt.reply(f"{that_this} room is already a portal room.")
+
+    if not await _has_access_to(room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply("You do not have the permissions to bridge that room.")
+
+    # The /id bot command provides the prefixed ID, so we assume
+    tgid = evt.args[0]
+    if tgid.startswith("-100"):
+        tgid = int(tgid[4:])
+        peer_type = "channel"
+    elif tgid.startswith("-"):
+        tgid = -int(tgid)
+        peer_type = "chat"
+    else:
+        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
+                               "If you did not get the ID using the `/id` bot command, please "
+                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+                               "Bridging private chats to existing rooms is not allowed.")
+
+    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    if not portal.allow_bridging():
+        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
+                               "If you're the bridge admin, try"
+                               "`$cmdprefix+sp whitelist <Telegram chat ID>` first.")
+    if portal.mxid:
+        has_portal_message = (
+            "That Telegram chat already has a portal at "
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
+        if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+            return await evt.reply(f"{has_portal_message}"
+                                   "Additionally, you do not have the permissions to unbridge "
+                                   "that room.")
+        evt.sender.command_status = {
+            "next": confirm_bridge,
+            "action": "Room bridging",
+            "mxid": portal.mxid,
+            "bridge_to_mxid": room_id,
+            "tgid": portal.tgid,
+            "peer_type": portal.peer_type,
+        }
+        return await evt.reply(f"{has_portal_message}"
+                               "However, you have the permissions to unbridge that room.\n\n"
+                               "To delete that portal completely and continue bridging, use "
+                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
+    evt.sender.command_status = {
+        "next": confirm_bridge,
+        "action": "Room bridging",
+        "bridge_to_mxid": room_id,
+        "tgid": portal.tgid,
+        "peer_type": portal.peer_type,
+    }
+    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
+                           "chat to this room, use `$cmdprefix+sp continue`")
+
+
+async def cleanup_old_portal_while_bridging(evt, portal):
+    if not portal.mxid:
+        await evt.reply("The portal seems to have lost its Matrix room between you"
+                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                        "Continuing without touching previous Matrix room...")
+        return True, None
+    elif evt.args[0] == "delete-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Portal deleted (moving to another room)")
+    elif evt.args[0] == "unbridge-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Room unbridged (portal moving to another room)",
+                                         puppets_only=True)
+    else:
+        await evt.reply(
+            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
+            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
+            "continue` to either delete or unbridge the existing room (respectively) and "
+            "continue with the bridging.\n\n"
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
+        return False, None
+
+
+async def confirm_bridge(evt: CommandEvent):
+    status = evt.sender.command_status
+    try:
+        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        bridge_to_mxid = status["bridge_to_mxid"]
+    except KeyError:
+        evt.sender.command_status = None
+        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
+                               "This shouldn't happen unless you're messing with the command "
+                               "handler code.")
+    if "mxid" in status:
+        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
+        if not ok:
+            return
+        elif coro:
+            asyncio.ensure_future(coro, loop=evt.loop)
+            await evt.reply("Cleaning up previous portal room...")
+    elif portal.mxid:
+        evt.sender.command_status = None
+        return await evt.reply("The portal seems to have created a Matrix room between you "
+                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                               "Please start over by calling the bridge command again.")
+    elif evt.args[0] != "continue":
+        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
+                               "`$cmdprefix+sp cancel` to cancel.")
+
+    portal.mxid = bridge_to_mxid
+    portal.title, portal.about, levels = await _get_initial_state(evt)
+    portal.photo_id = ""
+    portal.save()
+
+    entity = await evt.sender.client.get_entity(portal.peer)
+    direct = False
+    asyncio.ensure_future(portal.update_matrix_room(evt.sender, entity, direct, levels=levels),
+                          loop=evt.loop)
+
+    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")
+
+
+async def _get_initial_state(evt: CommandEvent):
+    state = await evt.az.intent.get_room_state(evt.room_id)
+    title = None
+    about = None
+    levels = None
+    for event in state:
+        if event["type"] == "m.room.name":
+            title = event["content"]["name"]
+        elif event["type"] == "m.room.topic":
+            about = event["content"]["topic"]
+        elif event["type"] == "m.room.power_levels":
+            levels = event["content"]
+        elif event["type"] == "m.room.canonical_alias":
+            title = title or event["content"]["alias"]
+    return title, about, levels
+
+
+@command_handler()
+async def create(evt: CommandEvent):
+    type = evt.args[0] if len(evt.args) > 0 else "group"
+    if type not in {"chat", "group", "supergroup", "channel"}:
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
+
+    if po.Portal.get_by_mxid(evt.room_id):
+        return await evt.reply("This is already a portal room.")
+
+    title, about, levels = await _get_initial_state(evt)
+    if not title:
+        return await evt.reply("Please set a title before creating a Telegram chat.")
+
+    supergroup = type == "supergroup"
+    type = {
+        "supergroup": "channel",
+        "channel": "channel",
+        "chat": "chat",
+        "group": "chat",
+    }[type]
+
+    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
+    try:
+        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+    except ValueError as e:
+        portal.delete()
+        return await evt.reply(e.args[0])
+    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
+
+
+@command_handler()
+async def upgrade(evt: CommandEvent):
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type == "channel":
+        return await evt.reply("This is already a supergroup or a channel.")
+    elif portal.peer_type == "user":
+        return await evt.reply("You can't upgrade private chats.")
+
+    try:
+        await portal.upgrade_telegram_chat(evt.sender)
+        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to upgrade this group.")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+
+
+@command_handler()
+async def group_name(evt: CommandEvent):
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type != "channel":
+        return await evt.reply("Only channels and supergroups have usernames.")
+
+    try:
+        await portal.set_telegram_username(evt.sender,
+                                           evt.args[0] if evt.args[0] != "-" else "")
+        if portal.username:
+            return await evt.reply(f"Username of channel changed to {portal.username}.")
+        else:
+            return await evt.reply(f"Channel is now private.")
+    except ChatAdminRequiredError:
+        return await evt.reply(
+            "You don't have the permission to set the username of this channel.")
+    except UsernameNotModifiedError:
+        if portal.username:
+            return await evt.reply("That is already the username of this channel.")
+        else:
+            return await evt.reply("This channel is already private")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username")
+
+
+@command_handler(needs_admin=True)
+async def filter_mode(evt: CommandEvent):
+    try:
+        mode = evt.args[0]
+        if mode not in ("whitelist", "blacklist"):
+            raise ValueError()
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
+
+    evt.config["bridge.filter.mode"] = mode
+    evt.config.save()
+    po.Portal.filter_mode = mode
+    if mode == "whitelist":
+        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
+                               "To allow bridging a specific chat, use"
+                               "`!filter whitelist <chat ID>`.")
+    else:
+        return await evt.reply("The bridge will now allow bridging chats by default.\n"
+                               "To disallow bridging a specific chat, use"
+                               "`!filter blacklist <chat ID>`.")
+
+
+@command_handler(needs_admin=True)
+async def filter(evt: CommandEvent):
+    try:
+        action = evt.args[0]
+        if action not in ("whitelist", "blacklist", "add", "remove"):
+            raise ValueError()
+
+        id = evt.args[1]
+        if id.startswith("-100"):
+            id = int(id[4:])
+        elif id.startswith("-"):
+            id = int(id[1:])
+        else:
+            id = int(id)
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
+
+    mode = evt.config["bridge.filter.mode"]
+    if mode not in ("blacklist", "whitelist"):
+        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
+
+    list = evt.config["bridge.filter.list"]
+
+    if action in ("blacklist", "whitelist"):
+        action = "add" if mode == action else "remove"
+
+    def save():
+        evt.config["bridge.filter.list"] = list
+        evt.config.save()
+        po.Portal.filter_list = list
+
+    if action == "add":
+        if id in list:
+            return await evt.reply(f"That chat is already {mode}ed.")
+        list.append(id)
+        save()
+        return await evt.reply(f"Chat ID added to {mode}.")
+    elif action == "remove":
+        if id not in list:
+            return await evt.reply(f"That chat is not {mode}ed.")
+        list.remove(id)
+        save()
+        return await evt.reply(f"Chat ID removed from {mode}.")

mautrix_telegram/commands/telegram.py

@@ -3,17 +3,17 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from telethon.errors import *
 from telethon.tl.types import User as TLUser
 from telethon.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest
@@ -51,7 +51,7 @@ async def search(evt):
     else:
         reply += ["**Results in contacts:**", ""]
     reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
-               + f"{puppet.id} ({similarity}% match)")
+               f"{puppet.id} ({similarity}% match)")
               for puppet, similarity in results]
 
     # TODO somehow show remote channel results when joining by alias is possible?
@@ -59,66 +59,44 @@ async def search(evt):
     return await evt.reply("\n".join(reply))
 
 
-@command_handler()
-async def pm(evt):
+@command_handler(name="pm")
+async def private_message(evt):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
 
-    user = await evt.sender.client.get_entity(evt.args[0])
+    try:
+        user = await evt.sender.client.get_entity(evt.args[0])
+    except ValueError:
+        return await evt.reply("Invalid user identifier or user not found.")
+
     if not user:
         return await evt.reply("User not found.")
     elif not isinstance(user, TLUser):
         return await evt.reply("That doesn't seem to be a user.")
     portal = po.Portal.get_by_entity(user, evt.sender.tgid)
     await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply(
-        f"Created private chat room with {pu.Puppet.get_displayname(user, False)}")
+    return await evt.reply("Created private chat room with "
+                           f"{pu.Puppet.get_displayname(user, False)}")
 
 
-@command_handler()
-async def invite_link(evt):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
-        return await evt.reply("You can't invite users to private chats.")
-
-    try:
-        link = await portal.get_invite_link(evt.sender)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to create an invite link.")
-
-
-@command_handler(needs_admin=True)
-async def delete_portal(evt):
-    room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if not portal:
-        that_this = "This" if room_id == evt.room_id else "That"
-        return await evt.reply(f"{that_this} is not a portal room.")
-
-    async def post_confirm(confirm):
-        evt.sender.command_status = None
-        if len(confirm.args) > 0 and confirm.args[0] == "confirm-delete":
-            await portal.cleanup_and_delete()
-            if confirm.room_id != room_id:
-                return await confirm.reply("Portal successfully deleted.")
-        else:
-            return await confirm.reply("Portal deletion cancelled.")
-
-    evt.sender.command_status = {
-        "next": post_confirm,
-        "action": "Portal deletion",
-    }
-    return await evt.reply("Please confirm deletion of portal "
-                           + f"[{room_id}](https://matrix.to/#/{room_id}) "
-                           + f"to Telegram chat \"{portal.title}\" "
-                           + "by typing `$cmdprefix+sp confirm-delete`")
+async def _join(evt, arg):
+    if arg.startswith("joinchat/"):
+        invite_hash = arg[len("joinchat/"):]
+        try:
+            await evt.sender.client(CheckChatInviteRequest(invite_hash))
+        except InviteHashInvalidError:
+            return None, await evt.reply("Invalid invite link.")
+        except InviteHashExpiredError:
+            return None, await evt.reply("Invite link expired.")
+        try:
+            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
+        except UserAlreadyParticipantError:
+            return None, await evt.reply("You are already in that chat.")
+    else:
+        channel = await evt.sender.client.get_entity(arg)
+        if not channel:
+            return None, await evt.reply("Channel/supergroup not found.")
+        return await evt.sender.client(JoinChannelRequest(channel)), None
 
 
 @command_handler()
@@ -130,132 +108,35 @@ async def join(evt):
     arg = regex.match(evt.args[0])
     if not arg:
         return await evt.reply("That doesn't look like a Telegram invite link.")
-    arg = arg.group(1)
-    if arg.startswith("joinchat/"):
-        invite_hash = arg[len("joinchat/"):]
-        try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
-        except InviteHashInvalidError:
-            return await evt.reply("Invalid invite link.")
-        except InviteHashExpiredError:
-            return await evt.reply("Invite link expired.")
-        try:
-            updates = evt.sender.client(ImportChatInviteRequest(invite_hash))
-        except UserAlreadyParticipantError:
-            return await evt.reply("You are already in that chat.")
-    else:
-        channel = await evt.sender.client.get_entity(arg)
-        if not channel:
-            return await evt.reply("Channel/supergroup not found.")
-        updates = await evt.sender.client(JoinChannelRequest(channel))
+
+    updates, _ = await _join(evt, arg.group(1))
+    if not updates:
+        return
+
     for chat in updates.chats:
         portal = po.Portal.get_by_entity(chat)
         if portal.mxid:
+            await portal.invite_to_matrix([evt.sender.mxid])
+            return await evt.reply(f"Invited you to portal of {portal.title}")
+        else:
+            await evt.reply(f"Creating room for {chat.title}... This might take a while.")
             await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
             return await evt.reply(f"Created room for {portal.title}")
-        else:
-            await portal.invite_matrix([evt.sender.mxid])
-            return await evt.reply(f"Invited you to portal of {portal.title}")
 
 
 @command_handler()
-async def create(evt):
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
-
-    if po.Portal.get_by_mxid(evt.room_id):
-        return await evt.reply("This is already a portal room.")
-
-    state = await evt.az.intent.get_room_state(evt.room_id)
-    title = None
-    about = None
-    levels = None
-    for event in state:
-        if event["type"] == "m.room.name":
-            title = event["content"]["name"]
-        elif event["type"] == "m.room.topic":
-            about = event["content"]["topic"]
-        elif event["type"] == "m.room.power_levels":
-            levels = event["content"]
-    if not title:
-        return await evt.reply("Please set a title before creating a Telegram chat.")
-    elif (not levels or not levels["users"] or evt.az.intent.mxid not in levels["users"] or
-          levels["users"][evt.az.intent.mxid] < 100):
-        return await evt.reply(f"Please give "
-                               + f"[the bridge bot](https://matrix.to/#/{evt.az.intent.mxid})"
-                               + f" a power level of 100 before creating a Telegram chat.")
+async def sync(evt):
+    if len(evt.args) > 0:
+        sync_only = evt.args[0]
+        if sync_only not in ("chats", "contacts", "me"):
+            return await evt.reply("**Usage:** `$cmdprefix+sp sync [chats|contacts|me]`")
     else:
-        for user, level in levels["users"].items():
-            if level >= 100 and user != evt.az.intent.mxid:
-                return await evt.reply(
-                    f"Please make sure only the bridge bot has power level above"
-                    + f"99 before creating a Telegram chat.\n\n"
-                    + f"Use power level 95 instead of 100 for admins.")
+        sync_only = None
 
-    supergroup = type == "supergroup"
-    type = {
-        "supergroup": "channel",
-        "channel": "channel",
-        "chat": "chat",
-        "group": "chat",
-    }[type]
-
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
-    try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
-
-
-@command_handler()
-async def upgrade(evt):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type == "channel":
-        return await evt.reply("This is already a supergroup or a channel.")
-    elif portal.peer_type == "user":
-        return await evt.reply("You can't upgrade private chats.")
-
-    try:
-        await portal.upgrade_telegram_chat(evt.sender)
-        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to upgrade this group.")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-
-
-@command_handler()
-async def group_name(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type != "channel":
-        return await evt.reply("Only channels and supergroups have usernames.")
-
-    try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
-        if portal.username:
-            return await evt.reply(f"Username of channel changed to {portal.username}.")
-        else:
-            return await evt.reply(f"Channel is now private.")
-    except ChatAdminRequiredError:
-        return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
-    except UsernameNotModifiedError:
-        if portal.username:
-            return await evt.reply("That is already the username of this channel.")
-        else:
-            return await evt.reply("This channel is already private")
-    except UsernameOccupiedError:
-        return await evt.reply("That username is already in use.")
-    except UsernameInvalidError:
-        return await evt.reply("Invalid username")
+    if not sync_only or sync_only == "chats":
+        await evt.sender.sync_dialogs(synchronous_create=True)
+    if not sync_only or sync_only == "contacts":
+        await evt.sender.sync_contacts()
+    if not sync_only or sync_only == "me":
+        await evt.sender.update_info()
+    return await evt.reply("Synchronization complete.")

mautrix_telegram/config.py

@@ -3,32 +3,34 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
 import random
 import string
 
 yaml = YAML()
+yaml.indent(4)
 
 
 class DictWithRecursion:
     def __init__(self, data=None):
-        self._data = data or {}
+        self._data = data or CommentedMap()
 
     def _recursive_get(self, data, key, default_value):
         if '.' in key:
             key, next_key = key.split('.', 1)
-            next_data = data.get(key, {})
+            next_data = data.get(key, CommentedMap())
             return self._recursive_get(next_data, next_key, default_value)
         return data.get(key, default_value)
 
@@ -40,12 +42,15 @@ class DictWithRecursion:
     def __getitem__(self, key):
         return self.get(key, None)
 
+    def __contains__(self, key):
+        return self[key] is not None
+
     def _recursive_set(self, data, key, value):
         if '.' in key:
             key, next_key = key.split('.', 1)
             if key not in data:
-                data[key] = {}
-            next_data = data.get(key, {})
+                data[key] = CommentedMap()
+            next_data = data.get(key, CommentedMap())
             self._recursive_set(next_data, next_key, value)
             return
         data[key] = value
@@ -59,18 +64,54 @@ class DictWithRecursion:
     def __setitem__(self, key, value):
         self.set(key, value)
 
+    def _recursive_del(self, data, key):
+        if '.' in key:
+            key, next_key = key.split('.', 1)
+            if key not in data:
+                return
+            next_data = data[key]
+            self._recursive_del(next_data, next_key)
+            return
+        try:
+            del data[key]
+            del data.ca.items[key]
+        except KeyError:
+            pass
+
+    def delete(self, key, allow_recursion=True):
+        if allow_recursion and '.' in key:
+            self._recursive_del(self._data, key)
+            return
+        try:
+            del self._data[key]
+            del self._data.ca.items[key]
+        except KeyError:
+            pass
+
+    def __delitem__(self, key):
+        self.delete(key)
+
 
 class Config(DictWithRecursion):
-    def __init__(self, path, registration_path):
+    def __init__(self, path, registration_path, base_path):
         super().__init__()
         self.path = path
         self.registration_path = registration_path
+        self.base_path = base_path
         self._registration = None
 
     def load(self):
         with open(self.path, 'r') as stream:
             self._data = yaml.load(stream)
 
+    def load_base(self):
+        try:
+            with open(self.base_path, 'r') as stream:
+                return DictWithRecursion(yaml.load(stream))
+        except OSError:
+            pass
+        return None
+
     def save(self):
         with open(self.path, 'w') as stream:
             yaml.dump(self._data, stream)
@@ -80,7 +121,115 @@ class Config(DictWithRecursion):
 
     @staticmethod
     def _new_token():
-        return "".join(random.choices(string.ascii_lowercase + string.digits, k=64))
+        return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
+
+    def update(self):
+        base = self.load_base()
+        if not base:
+            return
+
+        def copy(from_path, to_path=None):
+            if from_path in self:
+                base[to_path or from_path] = self[from_path]
+
+        def copy_dict(from_path, to_path=None):
+            if from_path in self:
+                to_path = to_path or from_path
+                base[to_path] = CommentedMap()
+                for key, value in self[from_path].items():
+                    base[to_path][key] = value
+
+        copy("homeserver.address")
+        copy("homeserver.verify_ssl")
+        copy("homeserver.domain")
+
+        copy("appservice.protocol")
+        copy("appservice.hostname")
+        copy("appservice.port")
+
+        copy("appservice.database")
+
+        copy("appservice.public.enabled")
+        copy("appservice.public.prefix")
+        copy("appservice.public.external")
+
+        copy("appservice.debug")
+
+        copy("appservice.id")
+        copy("appservice.bot_username")
+        copy("appservice.bot_displayname")
+
+        copy("appservice.as_token")
+        copy("appservice.hs_token")
+
+        copy("bridge.username_template")
+        copy("bridge.alias_template")
+        copy("bridge.displayname_template")
+
+        copy("bridge.displayname_preference")
+
+        copy("bridge.edits_as_replies")
+        copy("bridge.highlight_edits")
+        copy("bridge.bridge_notices")
+        copy("bridge.bot_messages_as_notices")
+        copy("bridge.max_initial_member_sync")
+        copy("bridge.max_telegram_delete")
+        copy("bridge.allow_matrix_login")
+        copy("bridge.inline_images")
+        copy("bridge.plaintext_highlights")
+        copy("bridge.public_portals")
+        copy("bridge.native_stickers")
+        copy("bridge.catch_up")
+
+        copy("bridge.filter.mode")
+        copy("bridge.filter.list")
+
+        copy("bridge.command_prefix")
+
+        migrate_permissions = ("bridge.permissions" not in self
+                               or "bridge.whitelist" in self
+                               or "bridge.admins" in self)
+        if migrate_permissions:
+            permissions = self["bridge.permissions"] or CommentedMap()
+            for entry in self["bridge.whitelist"] or []:
+                permissions[entry] = "full"
+            for entry in self["bridge.admins"] or []:
+                permissions[entry] = "admin"
+            base["bridge.permissions"] = permissions
+        else:
+            copy_dict("bridge.permissions")
+
+        if "bridge.relaybot" not in self:
+            copy("bridge.authless_relaybot_portals", "bridge.relaybot.authless_portals")
+        else:
+            copy("bridge.relaybot.authless_portals")
+            copy("bridge.relaybot.whitelist_group_admins")
+            copy("bridge.relaybot.whitelist")
+
+        copy("telegram.api_id")
+        copy("telegram.api_hash")
+        copy("telegram.bot_token")
+
+        self._data = base._data
+        self.save()
+
+    def _get_permissions(self, key):
+        level = self["bridge.permissions"].get(key, "")
+        admin = level == "admin"
+        whitelisted = level == "full" or admin
+        relaybot = level == "relaybot" or whitelisted
+        return relaybot, whitelisted, admin
+
+    def get_permissions(self, mxid):
+        permissions = self["bridge.permissions"] or {}
+        if mxid in permissions:
+            return self._get_permissions(mxid)
+
+        homeserver = mxid[mxid.index(":") + 1:]
+        if homeserver in permissions:
+            return self._get_permissions(homeserver)
+
+        return self._get_permissions("*")
 
     def generate_registration(self):
         homeserver = self["homeserver.domain"]
@@ -94,7 +243,7 @@ class Config(DictWithRecursion):
         self.set("appservice.hs_token", self._new_token())
 
         url = (f"{self['appservice.protocol']}://"
-               + f"{self['appservice.hostname']}:{self['appservice.port']}")
+               f"{self['appservice.hostname']}:{self['appservice.port']}")
         self._registration = {
             "id": self.get("appservice.id", "telegram"),
             "as_token": self["appservice.as_token"],

mautrix_telegram/context.py

@@ -0,0 +1,34 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+class Context:
+    def __init__(self, az, db, config, loop, bot, mx, telethon_session_container):
+        self.az = az
+        self.db = db
+        self.config = config
+        self.loop = loop
+        self.bot = bot
+        self.mx = mx
+        self.telethon_session_container = telethon_session_container
+
+    def __iter__(self):
+        yield self.az
+        yield self.db
+        yield self.config
+        yield self.loop
+        yield self.bot

mautrix_telegram/db.py

@@ -3,18 +3,19 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from sqlalchemy import Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer, String
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
+                        BigInteger, String, Boolean)
 from sqlalchemy.orm import relationship
 
 from .base import Base
@@ -28,6 +29,7 @@ class Portal(Base):
     tgid = Column(Integer, primary_key=True)
     tg_receiver = Column(Integer, primary_key=True)
     peer_type = Column(String)
+    megagroup = Column(Boolean)
 
     # Matrix portal information
     mxid = Column(String, unique=True, nullable=True)
@@ -48,7 +50,7 @@ class Message(Base):
     tgid = Column(Integer, primary_key=True)
     tg_space = Column(Integer, primary_key=True)
 
-    __table_args__ = (UniqueConstraint('mxid', 'mx_room', 'tg_space', name='_mx_id_room'),)
+    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
 
 
 class UserPortal(Base):
@@ -73,16 +75,15 @@ class User(Base):
     saved_contacts = Column(Integer, default=0)
     contacts = relationship("Contact", uselist=True,
                             cascade="save-update, merge, delete, delete-orphan")
-    portals = relationship("Portal", secondary="user_portal", single_parent=True,
-                           cascade="save-update, merge, delete, delete-orphan")
+    portals = relationship("Portal", secondary="user_portal")
 
 
 class Contact(Base):
     query = None
     __tablename__ = "contact"
 
-    user = Column("user", Integer, ForeignKey("user.tgid"), primary_key=True)
-    contact = Column("contact", Integer, ForeignKey("puppet.id"), primary_key=True)
+    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
+    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
 
 
 class Puppet(Base):
@@ -91,8 +92,34 @@ class Puppet(Base):
 
     id = Column(Integer, primary_key=True)
     displayname = Column(String, nullable=True)
+    displayname_source = Column(Integer, nullable=True)
     username = Column(String, nullable=True)
     photo_id = Column(String, nullable=True)
+    is_bot = Column(Boolean, nullable=True)
+
+
+# Fucking Telegram not telling bots what chats they are in 3:<
+class BotChat(Base):
+    query = None
+    __tablename__ = "bot_chat"
+    id = Column(Integer, primary_key=True)
+    type = Column(String, nullable=False)
+
+
+class TelegramFile(Base):
+    query = None
+    __tablename__ = "telegram_file"
+
+    id = Column(String, primary_key=True)
+    mxc = Column(String)
+    mime_type = Column(String)
+    was_converted = Column(Boolean)
+    timestamp = Column(BigInteger)
+    size = Column(Integer, nullable=True)
+    width = Column(Integer, nullable=True)
+    height = Column(Integer, nullable=True)
+    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+    thumbnail = relationship("TelegramFile", uselist=False)
 
 
 def init(db_session):
@@ -101,3 +128,5 @@ def init(db_session):
     UserPortal.query = db_session.query_property()
     User.query = db_session.query_property()
     Puppet.query = db_session.query_property()
+    BotChat.query = db_session.query_property()
+    TelegramFile.query = db_session.query_property()

mautrix_telegram/formatter.py

@@ -1,380 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from html import escape, unescape
-from html.parser import HTMLParser
-from collections import deque
-import re
-import logging
-
-from mautrix_appservice import MatrixRequestError
-
-from telethon.tl.types import *
-
-from . import user as u, puppet as p
-from .db import Message as DBMessage
-
-log = logging.getLogger("mau.formatter")
-
-# TEXT LEN EXPLANATION:
-# Telegram formatting counts two bytes in an UTF-16 string as one character.
-#
-# For Telegram -> Matrix formatting, we get the same counting mechanism by encoding the input
-# text as UTF-16 Little Endian and doubling all the offsets and lengths given by Telegram. With
-# those doubled values, we process the input entities and text. The text is converted back to
-# native str format before it's inserted into the output HTML.
-#
-# For Matrix -> Telegram formatting, do the same input encoding, but divide the length by two
-# instead of multiplying when generating the lengths and offsets of Telegram entities.
-#
-# The endianness doesn't matter, but it has to be specified to avoid the two BOM bits messing
-# everything up.
-TEMP_ENC = "utf-16-le"
-
-
-# region Matrix to Telegram
-
-
-class MatrixParser(HTMLParser):
-    mention_regex = re.compile("https://matrix.to/#/(@.+)")
-
-    def __init__(self):
-        super().__init__()
-        self.text = ""
-        self.entities = []
-        self._building_entities = {}
-        self._list_counter = 0
-        self._open_tags = deque()
-        self._open_tags_meta = deque()
-        self._previous_ended_line = True
-
-    def handle_starttag(self, tag, attrs):
-        self._open_tags.appendleft(tag)
-        self._open_tags_meta.appendleft(0)
-        attrs = dict(attrs)
-        entity_type = None
-        args = {}
-        if tag == "strong" or tag == "b":
-            entity_type = MessageEntityBold
-        elif tag == "em" or tag == "i":
-            entity_type = MessageEntityItalic
-        elif tag == "code":
-            try:
-                pre = self._building_entities["pre"]
-                try:
-                    pre.language = attrs["class"][len("language-"):]
-                except KeyError:
-                    pass
-            except KeyError:
-                entity_type = MessageEntityCode
-        elif tag == "pre":
-            entity_type = MessageEntityPre
-            args["language"] = ""
-        elif tag == "a":
-            try:
-                url = attrs["href"]
-            except KeyError:
-                return
-            mention = self.mention_regex.search(url)
-            if mention:
-                mxid = mention.group(1)
-                user = p.Puppet.get_by_mxid(mxid, create=False)
-                if not user:
-                    user = u.User.get_by_mxid(mxid, create=False)
-                    if not user:
-                        return
-                if user.username:
-                    entity_type = MessageEntityMention
-                    url = f"@{user.username}"
-                else:
-                    entity_type = MessageEntityMentionName
-                    args["user_id"] = user.tgid
-            elif url.startswith("mailto:"):
-                url = url[len("mailto:"):]
-                entity_type = MessageEntityEmail
-            else:
-                if self.get_starttag_text() == url:
-                    entity_type = MessageEntityUrl
-                else:
-                    entity_type = MessageEntityTextUrl
-                    args["url"] = url
-                    url = None
-            self._open_tags_meta.popleft()
-            self._open_tags_meta.appendleft(url)
-
-        if entity_type and tag not in self._building_entities:
-            # See "TEXT LEN EXPLANATION" near start of file
-            offset = int(len(self.text.encode(TEMP_ENC)) / 2)
-            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
-
-    def _list_depth(self):
-        depth = 0
-        for tag in self._open_tags:
-            if tag == "ol" or tag == "ul":
-                depth += 1
-        return depth
-
-    def handle_data(self, text):
-        text = unescape(text)
-        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
-        list_format_offset = 0
-        if previous_tag == "a":
-            url = self._open_tags_meta[0]
-            if url:
-                text = url
-        elif len(self._open_tags) > 1 and self._previous_ended_line and previous_tag == "li":
-            list_type = self._open_tags[1]
-            indent = (self._list_depth() - 1) * 4 * " "
-            text = text.strip("\n")
-            if len(text) == 0:
-                return
-            elif list_type == "ul":
-                text = f"{indent}* {text}"
-                list_format_offset = len(indent) + 2
-            elif list_type == "ol":
-                n = self._open_tags_meta[1]
-                n += 1
-                self._open_tags_meta[1] = n
-                text = f"{indent}{n}. {text}"
-                list_format_offset = len(indent) + 3
-        for tag, entity in self._building_entities.items():
-            # See "TEXT LEN EXPLANATION" near start of file
-            entity.length += int(len(text.strip("\n").encode(TEMP_ENC)) / 2)
-            entity.offset += list_format_offset
-
-        if text.endswith("\n"):
-            self._previous_ended_line = True
-        else:
-            self._previous_ended_line = False
-
-        self.text += text
-
-    def handle_endtag(self, tag):
-        try:
-            self._open_tags.popleft()
-            self._open_tags_meta.popleft()
-        except IndexError:
-            pass
-        if (tag == "ul" or tag == "ol") and self.text.endswith("\n"):
-            self.text = self.text[:-1]
-        entity = self._building_entities.pop(tag, None)
-        if entity:
-            self.entities.append(entity)
-
-
-def matrix_to_telegram(html):
-    try:
-        parser = MatrixParser()
-        parser.feed(html)
-        return parser.text, parser.entities
-    except Exception:
-        log.exception("Failed to convert Matrix format:\nhtml=%s", html)
-
-
-def matrix_reply_to_telegram(content, tg_space, room_id=None):
-    try:
-        reply = content["m.relates_to"]["m.in_reply_to"]
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-        message = DBMessage.query.filter(DBMessage.mxid == event_id and
-                                         DBMessage.tg_space == tg_space and
-                                         DBMessage.mx_room == room_id).one_or_none()
-        if message:
-            return message.tgid
-    except KeyError:
-        pass
-    return None
-
-
-# endregion
-# region Telegram to Matrix
-
-def telegram_reply_to_matrix(evt, source):
-    if evt.reply_to_msg_id:
-        space = (evt.to_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
-                 else source.tgid)
-        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
-        if msg:
-            return {
-                "m.in_reply_to": {
-                    "event_id": msg.mxid,
-                    "room_id": msg.mx_room,
-                }
-            }
-    return {}
-
-
-async def telegram_event_to_matrix(evt, source, native_replies=False, message_link_in_reply=False,
-                                   main_intent=None, reply_text="Reply"):
-    text = evt.message
-    html = telegram_to_matrix(evt.message, evt.entities) if evt.entities else None
-    relates_to = {}
-
-    if evt.fwd_from:
-        if not html:
-            html = escape(text)
-        from_id = evt.fwd_from.from_id
-        user = u.User.get_by_tgid(from_id)
-        if user:
-            fwd_from = f"<a href='https://matrix.to/#/{user.mxid}'>{user.mxid}</a>"
-        else:
-            puppet = p.Puppet.get(from_id, create=False)
-            if puppet and puppet.displayname:
-                fwd_from = f"<a href='https://matrix.to/#/{puppet.mxid}'>{puppet.displayname}</a>"
-            else:
-                user = await source.client.get_entity(from_id)
-                if user:
-                    fwd_from = p.Puppet.get_displayname(user, format=False)
-                else:
-                    fwd_from = None
-        if not fwd_from:
-            fwd_from = "Unknown user"
-        html = (f"Forwarded message from <b>{fwd_from}</b><br/>"
-                + f"<blockquote>{html}</blockquote>")
-
-    if evt.reply_to_msg_id:
-        space = (evt.to_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
-                 else source.tgid)
-        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
-        if msg:
-            if native_replies:
-                relates_to["m.in_reply_to"] = {
-                    "event_id": msg.mxid,
-                    "room_id": msg.mx_room,
-                }
-                if reply_text == "Edit":
-                    html = "<u>Edit:</u> " + (html or escape(text))
-            else:
-                try:
-                    event = await main_intent.get_event(msg.mx_room, msg.mxid)
-                    content = event["content"]
-                    body = (content["formatted_body"]
-                            if "formatted_body" in content
-                            else content["body"])
-                    sender = event['sender']
-                    puppet = p.Puppet.get_by_mxid(sender, create=False)
-                    displayname = puppet.displayname if puppet else sender
-                    reply_to_user = f"<a href='https://matrix.to/#/{sender}'>{displayname}</a>"
-                    reply_to_msg = (("<a href='https://matrix.to/#/"
-                                     + f"{msg.mx_room}/{msg.mxid}'>{reply_text}</a>")
-                                    if message_link_in_reply else "Reply")
-                    quote = f"{reply_to_msg} to {reply_to_user}<blockquote>{body}</blockquote>"
-                except (ValueError, KeyError, MatrixRequestError):
-                    quote = "{reply_text} to unknown user <em>(Failed to fetch message)</em>:<br/>"
-                if html:
-                    html = quote + html
-                else:
-                    html = quote + escape(text)
-
-    if isinstance(evt, Message) and evt.post and evt.post_author:
-        if not html:
-            html = escape(text)
-        text += f"\n- {evt.post_author}"
-        html += f"<br/><i>- <u>{evt.post_author}</u></i>"
-
-    if html:
-        html = html.replace("\n", "<br/>")
-
-    return text, html, relates_to
-
-
-def telegram_to_matrix(text, entities):
-    try:
-        return _telegram_to_matrix(text, entities)
-    except Exception:
-        log.exception("Failed to convert Telegram format:\n"
-                      "message=%s\n"
-                      "entities=%s",
-                      text, entities)
-
-
-def _telegram_to_matrix(text, entities):
-    if not entities:
-        return text
-    # See "TEXT LEN EXPLANATION" near start of file
-    text = text.encode(TEMP_ENC)
-    html = []
-    last_offset = 0
-    for entity in entities:
-        entity.offset *= 2
-        entity.length *= 2
-        if entity.offset > last_offset:
-            html.append(escape(text[last_offset:entity.offset].decode(TEMP_ENC)))
-        elif entity.offset < last_offset:
-            continue
-
-        skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length].decode(TEMP_ENC))
-        entity_type = type(entity)
-
-        if entity_type == MessageEntityBold:
-            html.append(f"<strong>{entity_text}</strong>")
-        elif entity_type == MessageEntityItalic:
-            html.append(f"<em>{entity_text}</em>")
-        elif entity_type == MessageEntityCode:
-            html.append(f"<code>{entity_text}</code>")
-        elif entity_type == MessageEntityPre:
-            if entity.language:
-                html.append("<pre>"
-                            + f"<code class='language-{entity.language}'>{entity_text}</code>"
-                            + "</pre>")
-            else:
-                html.append(f"<pre><code>{entity_text}</code></pre>")
-        elif entity_type == MessageEntityMention:
-            username = entity_text[1:]
-
-            user = u.User.find_by_username(username)
-            if user:
-                mxid = user.mxid
-            else:
-                puppet = p.Puppet.find_by_username(username)
-                mxid = puppet.mxid if puppet else None
-            if mxid:
-                html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
-            else:
-                skip_entity = True
-        elif entity_type == MessageEntityMentionName:
-            user = u.User.get_by_tgid(entity.user_id)
-            if user:
-                mxid = user.mxid
-            else:
-                puppet = p.Puppet.get(entity.user_id, create=False)
-                mxid = puppet.mxid if puppet else None
-            if mxid:
-                html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
-            else:
-                skip_entity = True
-        elif entity_type == MessageEntityEmail:
-            html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
-        elif entity_type in {MessageEntityTextUrl, MessageEntityUrl}:
-            url = escape(entity.url) if entity_type == MessageEntityTextUrl else entity_text
-            if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
-                url = "http://" + url
-            html.append(f"<a href='{url}'>{entity_text}</a>")
-        elif entity_type == MessageEntityBotCommand:
-            html.append(f"<font color='blue'>!{entity_text[1:]}")
-        elif entity_type == MessageEntityHashtag:
-            html.append(f"<font color='blue'>{entity_text}</font>")
-        else:
-            skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:].decode(TEMP_ENC))
-
-    return "".join(html)
-
-# endregion

mautrix_telegram/formatter/__init__.py

@@ -0,0 +1,9 @@
+from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
+                          init_mx)
+from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
+from ..context import Context
+
+
+def init(context: Context):
+    init_mx(context)
+    init_tg(context)

mautrix_telegram/formatter/from_matrix.py

@@ -0,0 +1,339 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from html import unescape
+from html.parser import HTMLParser
+from collections import deque
+from typing import Optional, List, Tuple, Type, Callable, Dict, Any
+import math
+import re
+import logging
+
+from telethon.tl.types import (MessageEntityMention,
+                               InputMessageEntityMentionName, MessageEntityEmail,
+                               MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
+                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
+                               MessageEntityBotCommand, InputUser, TypeMessageEntity)
+
+from ..context import Context
+from .. import user as u, puppet as pu, portal as po
+from ..db import Message as DBMessage
+from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                   trim_reply_fallback_text, html_to_unicode)
+
+log = logging.getLogger("mau.fmt.mx")
+should_bridge_plaintext_highlights = False
+
+
+class MatrixParser(HTMLParser):
+    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")
+    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")
+    block_tags = ("br", "p", "pre", "blockquote",
+                  "ol", "ul", "li",
+                  "h1", "h2", "h3", "h4", "h5", "h6",
+                  "div", "hr", "table")
+
+    def __init__(self):
+        super().__init__()
+        self.text = ""
+        self.entities = []
+        self._building_entities = {}
+        self._list_counter = 0
+        self._open_tags = deque()
+        self._open_tags_meta = deque()
+        self._line_is_new = True
+        self._list_entry_is_new = False
+
+    def _parse_url(self, url: str, args: Dict[str, Any]
+                   ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]:
+        mention = self.mention_regex.match(url)
+        if mention:
+            mxid = mention.group(1)
+            user = (pu.Puppet.get_by_mxid(mxid)
+                    or u.User.get_by_mxid(mxid, create=False))
+            if not user:
+                return None, None
+            if user.username:
+                return MessageEntityMention, f"@{user.username}"
+            elif user.tgid:
+                args["user_id"] = InputUser(user.tgid, 0)
+                return InputMessageEntityMentionName, user.displayname or None
+            else:
+                return None, None
+
+        room = self.room_regex.match(url)
+        if room:
+            username = po.Portal.get_username_from_mx_alias(room.group(1))
+            portal = po.Portal.find_by_username(username)
+            if portal and portal.username:
+                return MessageEntityMention, f"@{portal.username}"
+
+        if url.startswith("mailto:"):
+            return MessageEntityEmail, url[len("mailto:"):]
+        elif self.get_starttag_text() == url:
+            return MessageEntityUrl, url
+        else:
+            args["url"] = url
+            return MessageEntityTextUrl, None
+
+    def handle_starttag(self, tag: str, attrs: List[Tuple[str, str]]):
+        self._open_tags.appendleft(tag)
+        self._open_tags_meta.appendleft(0)
+
+        attrs = dict(attrs)
+        entity_type = None
+        args = {}
+        if tag in ("strong", "b"):
+            entity_type = MessageEntityBold
+        elif tag in ("em", "i"):
+            entity_type = MessageEntityItalic
+        elif tag == "code":
+            try:
+                pre = self._building_entities["pre"]
+                try:
+                    # Pre tag and language found, add language to MessageEntityPre
+                    pre.language = attrs["class"][len("language-"):]
+                except KeyError:
+                    # Pre tag found, but language not found, keep pre as-is
+                    pass
+            except KeyError:
+                # No pre tag found, this is inline code
+                entity_type = MessageEntityCode
+        elif tag == "pre":
+            entity_type = MessageEntityPre
+            args["language"] = ""
+        elif tag == "command":
+            entity_type = MessageEntityBotCommand
+        elif tag == "li":
+            self._list_entry_is_new = True
+        elif tag == "a":
+            try:
+                url = attrs["href"]
+            except KeyError:
+                return
+            entity_type, url = self._parse_url(url, args)
+            self._open_tags_meta.popleft()
+            self._open_tags_meta.appendleft(url)
+
+        if tag in self.block_tags and ("blockquote" not in self._open_tags or tag == "br"):
+            self._newline()
+
+        if entity_type and tag not in self._building_entities:
+            offset = len(self.text)
+            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
+
+    @property
+    def _list_indent(self) -> int:
+        indent = 0
+        first_skipped = False
+        for index, tag in enumerate(self._open_tags):
+            if not first_skipped and tag in ("ol", "ul"):
+                # The first list level isn't indented, so skip it.
+                first_skipped = True
+                continue
+            if tag == "ol":
+                n = self._open_tags_meta[index]
+                extra_length_for_long_index = (int(math.log(n, 10)) - 1) * 3
+                indent += 4 + extra_length_for_long_index
+            elif tag == "ul":
+                indent += 3
+        return indent
+
+    def _newline(self, allow_multi: bool = False):
+        if self._line_is_new and not allow_multi:
+            return
+        self.text += "\n"
+        self._line_is_new = True
+        for entity in self._building_entities.values():
+            entity.length += 1
+
+    def _handle_special_previous_tags(self, text: str) -> str:
+        if "pre" not in self._open_tags and "code" not in self._open_tags:
+            text = text.replace("\n", "")
+        else:
+            text = text.strip()
+
+        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
+        if previous_tag == "a":
+            url = self._open_tags_meta[0]
+            if url:
+                text = url
+        elif previous_tag == "command":
+            text = f"/{text}"
+        return text
+
+    def _html_to_unicode(self, text: str) -> str:
+        strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags
+        if strikethrough and underline:
+            text = html_to_unicode(text, "\u0336\u0332")
+        elif strikethrough:
+            text = html_to_unicode(text, "\u0336")
+        elif underline:
+            text = html_to_unicode(text, "\u0332")
+        return text
+
+    def _handle_tags_for_data(self, text: str) -> Tuple[str, int]:
+        extra_offset = 0
+        list_entry_handled_once = False
+        # In order to maintain order of things like blockquotes in lists or lists in blockquotes,
+        # we can't just have ifs/elses and we need to actually loop through the open tags in order.
+        for index, tag in enumerate(self._open_tags):
+            if tag == "blockquote" and self._line_is_new:
+                text = f"> {text}"
+                extra_offset += 2
+            elif tag == "li" and not list_entry_handled_once:
+                list_type_index = index + 1
+                list_type = self._open_tags[list_type_index]
+                indent = self._list_indent * " " if self._line_is_new else ""
+                if list_type == "ol":
+                    n = self._open_tags_meta[list_type_index]
+                    if self._list_entry_is_new:
+                        n += 1
+                        self._open_tags_meta[list_type_index] = n
+                        prefix = f"{n}. "
+                    else:
+                        prefix = int(math.log(n, 10)) * 3 * " " + 4 * " "
+                else:
+                    prefix = "* " if self._list_entry_is_new else 3 * " "
+                if not self._list_entry_is_new and not self._line_is_new:
+                    prefix = ""
+                extra_offset += len(indent) + len(prefix)
+                text = indent + prefix + text
+                self._list_entry_is_new = False
+                list_entry_handled_once = True
+        return text, extra_offset
+
+    def _extend_entities_in_construction(self, text: str, extra_offset: int):
+        for tag, entity in self._building_entities.items():
+            entity.length += len(text) - extra_offset
+            entity.offset += extra_offset
+
+    def handle_data(self, text: str):
+        text = unescape(text)
+        text = self._handle_special_previous_tags(text)
+        text = self._html_to_unicode(text)
+        text, extra_offset = self._handle_tags_for_data(text)
+        self._extend_entities_in_construction(text, extra_offset)
+        self._line_is_new = False
+        self.text += text
+
+    def handle_endtag(self, tag: str):
+        try:
+            self._open_tags.popleft()
+            self._open_tags_meta.popleft()
+        except IndexError:
+            pass
+
+        entity = self._building_entities.pop(tag, None)
+        if entity:
+            self.entities.append(entity)
+
+        if tag in self.block_tags and tag != "br" and "blockquote" not in self._open_tags:
+            self._newline(allow_multi=tag == "br")
+
+
+command_regex = re.compile(r"^!([A-Za-z0-9@]+)")
+not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")
+plain_mention_regex = None
+
+
+def plain_mention_to_html(match):
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
+def matrix_to_telegram(html: str) -> Tuple[str, List[TypeMessageEntity]]:
+    try:
+        parser = MatrixParser()
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(plain_mention_to_html, html)
+        parser.feed(add_surrogates(html))
+        return remove_surrogates(parser.text.strip()), parser.entities
+    except Exception:
+        log.exception("Failed to convert Matrix format:\nhtml=%s", html)
+
+
+def matrix_reply_to_telegram(content: dict, tg_space: int, room_id: Optional[str] = None
+                             ) -> Optional[int]:
+    try:
+        reply = content["m.relates_to"]["m.in_reply_to"]
+        room_id = room_id or reply["room_id"]
+        event_id = reply["event_id"]
+
+        try:
+            if content["format"] == "org.matrix.custom.html":
+                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
+        except KeyError:
+            pass
+        content["body"] = trim_reply_fallback_text(content["body"])
+
+        message = DBMessage.query.filter(DBMessage.mxid == event_id,
+                                         DBMessage.tg_space == tg_space,
+                                         DBMessage.mx_room == room_id).one_or_none()
+        if message:
+            return message.tgid
+    except KeyError:
+        pass
+    return None
+
+
+def matrix_text_to_telegram(text: str) -> Tuple[str, List[TypeMessageEntity]]:
+    text = command_regex.sub(r"/\1", text)
+    text = not_command_regex.sub(r"\1", text)
+    if should_bridge_plaintext_highlights:
+        entities, pmr_replacer = plain_mention_to_text()
+        text = plain_mention_regex.sub(pmr_replacer, text)
+    else:
+        entities = []
+    return text, entities
+
+
+def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
+    entities = []
+
+    def replacer(match):
+        puppet = pu.Puppet.find_by_displayname(match.group(2))
+        if puppet:
+            offset = match.start()
+            length = match.end() - offset
+            if puppet.username:
+                entity = MessageEntityMention(offset, length)
+                text = f"@{puppet.username}"
+            else:
+                entity = InputMessageEntityMentionName(offset, length,
+                                                       user_id=InputUser(puppet.tgid, 0))
+                text = puppet.displayname
+            entities.append(entity)
+            return text
+        return "".join(match.groups())
+
+    return entities, replacer
+
+
+def init_mx(context: Context):
+    global plain_mention_regex, should_bridge_plaintext_highlights
+    config = context.config
+    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
+    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
+    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
+    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_telegram.py

@@ -0,0 +1,326 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from html import escape
+from typing import Optional, List, Tuple
+
+try:
+    from lxml.html.diff import htmldiff
+except ImportError:
+    htmldiff = None  # type: function
+import logging
+import re
+
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName,
+                               MessageEntityEmail, MessageEntityUrl, MessageEntityTextUrl,
+                               MessageEntityBold, MessageEntityItalic, MessageEntityCode,
+                               MessageEntityPre, MessageEntityBotCommand, Message, PeerChannel,
+                               MessageEntityHashtag, TypeMessageEntity, MessageFwdHeader, PeerUser)
+
+from mautrix_appservice import MatrixRequestError
+from mautrix_appservice.intent_api import IntentAPI
+
+from .. import user as u, puppet as pu, portal as po
+from ..context import Context
+from ..db import Message as DBMessage
+from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                   trim_reply_fallback_text, unicode_to_html)
+
+log = logging.getLogger("mau.fmt.tg")
+should_highlight_edits = False
+
+
+def telegram_reply_to_matrix(evt: Message, source: u.User) -> dict:
+    if evt.reply_to_msg_id:
+        space = (evt.to_id.channel_id
+                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+                 else source.tgid)
+        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+        if msg:
+            return {
+                "m.in_reply_to": {
+                    "event_id": msg.mxid,
+                    "room_id": msg.mx_room,
+                }
+            }
+    return {}
+
+
+async def _add_forward_header(source, text: str, html: Optional[str],
+                              fwd_from: MessageFwdHeader) -> Tuple[str, str]:
+    if not html:
+        html = escape(text)
+    fwd_from_html, fwd_from_text = None, None
+    if fwd_from.from_id:
+        user = u.User.get_by_tgid(fwd_from.from_id)
+        if user:
+            fwd_from_text = user.displayname or user.mxid
+            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
+
+        if not fwd_from_text:
+            puppet = pu.Puppet.get(fwd_from.from_id, create=False)
+            if puppet and puppet.displayname:
+                fwd_from_text = puppet.displayname or puppet.mxid
+                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
+
+        if not fwd_from_text:
+            user = await source.client.get_entity(PeerUser(fwd_from.from_id))
+            if user:
+                fwd_from_text = pu.Puppet.get_displayname(user, format=False)
+                fwd_from_html = f"<b>{fwd_from_text}</b>"
+
+    if not fwd_from_text:
+        if fwd_from.from_id:
+            fwd_from_text = "Unknown user"
+        else:
+            fwd_from_text = "Unknown source"
+        fwd_from_html = f"<b>{fwd_from_text}</b>"
+
+    text = "\n".join([f"> {line}" for line in text.split("\n")])
+    text = f"Forwarded from {fwd_from_text}:\n{text}"
+    html = (f"Forwarded message from {fwd_from_html}<br/>"
+            f"<tg-forward><blockquote>{html}</blockquote></tg-forward>")
+    return text, html
+
+
+def highlight_edits(new_html: str, old_html: str) -> str:
+    # Don't include `Edit:` text in diff.
+    if old_html.startswith("<u>Edit:</u> "):
+        old_html = old_html[len("<u>Edit:</u> "):]
+
+    # Generate diff with lxml
+    new_html = htmldiff(old_html, new_html)
+
+    # Replace <ins> with <u> since Riot doesn't allow <ins>
+    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
+    # Remove <del>s since we just want to hide deletions.
+    new_html = re.sub("<del>.+?</del>", "", new_html)
+    return new_html
+
+
+async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, relates_to: dict,
+                            main_intent: IntentAPI, is_edit: bool) -> Tuple[str, str]:
+    space = (evt.to_id.channel_id
+             if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+             else source.tgid)
+
+    msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+    if not msg:
+        return text, html
+
+    relates_to["m.in_reply_to"] = {
+        "event_id": msg.mxid,
+        "room_id": msg.mx_room,
+    }
+
+    try:
+        event = await main_intent.get_event(msg.mx_room, msg.mxid)
+
+        content = event["content"]
+        r_sender = event["sender"]
+
+        r_text_body = trim_reply_fallback_text(content["body"])
+        r_html_body = trim_reply_fallback_html(content["formatted_body"]
+                                               if "formatted_body" in content
+                                               else escape(content["body"]))
+
+        puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
+        r_displayname = puppet.displayname if puppet else r_sender
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
+
+        if is_edit and should_highlight_edits:
+            html = highlight_edits(html or escape(text), r_html_body)
+    except (ValueError, KeyError, MatrixRequestError) as e:
+        r_sender_link = "unknown user"
+        r_displayname = "unknown user"
+        r_text_body = "Failed to fetch message"
+        r_html_body = "<em>Failed to fetch message</em>"
+
+    if is_edit:
+        html = f"<u>Edit:</u> {html or escape(text)}"
+        text = f"Edit: {text}"
+
+    r_keyword = "In reply to" if not is_edit else "Edit to"
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
+    html = (f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
+            + (html or escape(text)))
+
+    lines = r_text_body.strip().split("\n")
+    text_with_quote = f"> <{r_displayname}> {lines.pop(0)}"
+    for line in lines:
+        if line:
+            text_with_quote += f"\n> {line}"
+    text_with_quote += "\n\n"
+    text_with_quote += text
+    return text_with_quote, html
+
+
+async def telegram_to_matrix(evt: Message, source: u.User, main_intent: Optional[IntentAPI] = None,
+                             is_edit: bool = False, prefix_text: Optional[str] = None,
+                             prefix_html: Optional[str] = None) -> Tuple[str, str, dict]:
+    text = add_surrogates(evt.message)
+    html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None
+    relates_to = {}
+
+    if prefix_html:
+        html = prefix_html + (html or escape(text))
+    if prefix_text:
+        text = prefix_text + text
+
+    if evt.fwd_from:
+        text, html = await _add_forward_header(source, text, html, evt.fwd_from)
+
+    if evt.reply_to_msg_id:
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
+                                             is_edit)
+
+    if isinstance(evt, Message) and evt.post and evt.post_author:
+        if not html:
+            html = escape(text)
+        text += f"\n- {evt.post_author}"
+        html += f"<br/><i>- <u>{evt.post_author}</u></i>"
+
+    html = unicode_to_html(text, html, "\u0336", "del")
+    html = unicode_to_html(text, html, "\u0332", "u")
+
+    if html:
+        html = html.replace("\n", "<br/>")
+
+    return remove_surrogates(text), remove_surrogates(html), relates_to
+
+
+def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEntity]) -> str:
+    try:
+        return _telegram_entities_to_matrix(text, entities)
+    except Exception:
+        log.exception("Failed to convert Telegram format:\n"
+                      "message=%s\n"
+                      "entities=%s",
+                      text, entities)
+
+
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
+    if not entities:
+        return text
+    html = []
+    last_offset = 0
+    for entity in entities:
+        if entity.offset > last_offset:
+            html.append(escape(text[last_offset:entity.offset]))
+        elif entity.offset < last_offset:
+            continue
+
+        skip_entity = False
+        entity_text = escape(text[entity.offset:entity.offset + entity.length])
+        entity_type = type(entity)
+
+        if entity_type == MessageEntityBold:
+            html.append(f"<strong>{entity_text}</strong>")
+        elif entity_type == MessageEntityItalic:
+            html.append(f"<em>{entity_text}</em>")
+        elif entity_type == MessageEntityCode:
+            html.append(f"<code>{entity_text}</code>")
+        elif entity_type == MessageEntityPre:
+            skip_entity = _parse_pre(html, entity_text, entity.language)
+        elif entity_type == MessageEntityMention:
+            skip_entity = _parse_mention(html, entity_text)
+        elif entity_type == MessageEntityMentionName:
+            skip_entity = _parse_name_mention(html, entity_text, entity.user_id)
+        elif entity_type == MessageEntityEmail:
+            html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
+        elif entity_type in {MessageEntityTextUrl, MessageEntityUrl}:
+            skip_entity = _parse_url(html, entity_text,
+                                     entity.url if entity_type == MessageEntityTextUrl else None)
+        elif entity_type == MessageEntityBotCommand:
+            html.append(f"<font color='blue'>!{entity_text[1:]}</font>")
+        elif entity_type == MessageEntityHashtag:
+            html.append(f"<font color='blue'>{entity_text}</font>")
+        else:
+            skip_entity = True
+        last_offset = entity.offset + (0 if skip_entity else entity.length)
+    html.append(text[last_offset:])
+
+    return "".join(html)
+
+
+def _parse_pre(html: List[str], entity_text: str, language: str) -> bool:
+    if language:
+        html.append("<pre>"
+                    f"<code class='language-{language}'>{entity_text}</code>"
+                    "</pre>")
+    else:
+        html.append(f"<pre><code>{entity_text}</code></pre>")
+    return False
+
+
+def _parse_mention(html: List[str], entity_text: str) -> bool:
+    username = entity_text[1:]
+
+    user = u.User.find_by_username(username) or pu.Puppet.find_by_username(username)
+    if user:
+        mxid = user.mxid
+    else:
+        portal = po.Portal.find_by_username(username)
+        mxid = portal.alias or portal.mxid if portal else None
+
+    if mxid:
+        html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
+    else:
+        return True
+    return False
+
+
+def _parse_name_mention(html: List[str], entity_text: str, user_id: int) -> bool:
+    user = u.User.get_by_tgid(user_id)
+    if user:
+        mxid = user.mxid
+    else:
+        puppet = pu.Puppet.get(user_id, create=False)
+        mxid = puppet.mxid if puppet else None
+    if mxid:
+        html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
+    else:
+        return True
+    return False
+
+
+message_link_regex = re.compile(
+    r"https?://t(?:elegram)?\.(?:me|dog)/([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})")
+
+
+def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
+    url = escape(url) if url else entity_text
+    if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
+        url = "http://" + url
+
+    message_link_match = message_link_regex.match(url)
+    if message_link_match:
+        group, msgid = message_link_match.groups()
+        msgid = int(msgid)
+
+        portal = po.Portal.find_by_username(group)
+        if portal:
+            message = DBMessage.query.get((msgid, portal.tgid))
+            if message:
+                url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
+
+    html.append(f"<a href='{url}'>{entity_text}</a>")
+    return False
+
+
+def init_tg(context: Context):
+    global should_highlight_edits
+    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/formatter/util.py

@@ -0,0 +1,86 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from html import escape
+from typing import Optional
+import struct
+import re
+
+
+# add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
+# Licensed under the MIT license.
+# https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py
+def add_surrogates(text: Optional[str]) -> Optional[str]:
+    if text is None:
+        return None
+    return "".join("".join(chr(y) for y in struct.unpack("<HH", x.encode("utf-16-le")))
+                   if (0x10000 <= ord(x) <= 0x10FFFF) else x for x in text)
+
+
+def remove_surrogates(text: Optional[str]) -> Optional[str]:
+    if text is None:
+        return None
+    return text.encode("utf-16", "surrogatepass").decode("utf-16")
+
+
+def trim_reply_fallback_text(text: str) -> str:
+    if not text.startswith("> ") or "\n" not in text:
+        return text
+    lines = text.split("\n")
+    while len(lines) > 0 and lines[0].startswith("> "):
+        lines.pop(0)
+    return "\n".join(lines)
+
+
+html_reply_fallback_regex = re.compile("^<mx-reply>"
+                                       r"[\s\S]+?"
+                                       "</mx-reply>")
+
+
+def trim_reply_fallback_html(html: str) -> str:
+    return html_reply_fallback_regex.sub("", html)
+
+
+def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
+    if ctrl not in text:
+        return html
+    if not html:
+        html = escape(text)
+    tag_start = f"<{tag}>"
+    tag_end = f"</{tag}>"
+    characters = html.split(ctrl)
+    html = ""
+    in_tag = False
+    for char in characters:
+        if not in_tag:
+            if len(char) > 1:
+                html += char[0:-1]
+                char = char[-1]
+            html += tag_start
+            in_tag = True
+            html += char
+        else:
+            if len(char) > 1:
+                html += tag_end
+                in_tag = False
+            html += char
+    if in_tag:
+        html += tag_end
+    return html
+
+
+def html_to_unicode(text: str, ctrl: str) -> str:
+    return ctrl.join(text) + ctrl

mautrix_telegram/matrix.py

@@ -3,17 +3,17 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import logging
 
 from mautrix_appservice import MatrixRequestError
@@ -28,13 +28,13 @@ class MatrixHandler:
     log = logging.getLogger("mau.mx")
 
     def __init__(self, context):
-        self.az, self.db, self.config, _ = context
+        self.az, self.db, self.config, _, self.tgbot = context
         self.commands = CommandHandler(context)
 
         self.az.matrix_event_handler(self.handle_event)
 
     async def init_as_bot(self):
-        self.az.intent.set_display_name(
+        await self.az.intent.set_display_name(
             self.config.get("appservice.bot_displayname", "Telegram bridge bot"))
 
     async def handle_puppet_invite(self, room, puppet, inviter):
@@ -56,12 +56,12 @@ class MatrixHandler:
             members = await self.az.intent.get_room_members(room)
         except MatrixRequestError:
             members = []
-        if self.az.intent.mxid not in members:
+        if self.az.bot_mxid not in members:
             if len(members) > 1:
                 await puppet.intent.error_and_leave(room, text=None, html=(
                     f"Please invite "
-                    + f"<a href='https://matrix.to/#/{self.az.intent.mxid}'>the bridge bot</a> "
-                    + f"first if you want to create a Telegram chat."))
+                    f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
+                    f"first if you want to create a Telegram chat."))
                 return
 
             await puppet.intent.join_room(room)
@@ -71,9 +71,9 @@ class MatrixHandler:
                     await puppet.intent.invite(portal.mxid, inviter.mxid)
                     await puppet.intent.send_notice(room, text=None, html=(
                         "You already have a private chat with me: "
-                        + f"<a href='https://matrix.to/#/{portal.mxid}'>"
-                        + "Link to room"
-                        + "</a>"))
+                        f"<a href='https://matrix.to/#/{portal.mxid}'>"
+                        "Link to room"
+                        "</a>"))
                     await puppet.intent.leave_room(room)
                     return
                 except MatrixRequestError:
@@ -88,11 +88,19 @@ class MatrixHandler:
                                                   "Telegram chat is created for this room.")
 
     async def handle_invite(self, room, user, inviter):
-        inviter = User.get_by_mxid(inviter)
-        if not inviter.whitelisted:
-            return
-        elif user == self.az.bot_mxid:
+        self.log.debug(f"{inviter} invited {user} to {room}")
+        inviter = await User.get_by_mxid(inviter).ensure_started()
+        if user == self.az.bot_mxid:
             await self.az.intent.join_room(room)
+            if not inviter.whitelisted:
+                await self.az.intent.send_notice(
+                    room, text=None,
+                    html="You are not whitelisted to use this bridge.<br/><br/>"
+                         "If you are the owner of this bridge, see the "
+                         "<code>bridge.permissions</code> section in your config file.")
+                await self.az.intent.leave_room(room)
+            return
+        elif not inviter.whitelisted:
             return
 
         puppet = Puppet.get_by_mxid(user)
@@ -101,38 +109,44 @@ class MatrixHandler:
             return
 
         user = User.get_by_mxid(user, create=False)
+        if not user:
+            return
+        await user.ensure_started()
         portal = Portal.get_by_mxid(room)
         if user and user.has_full_access and portal:
             await portal.invite_telegram(inviter, user)
             return
 
         # The rest can probably be ignored
-        self.log.debug(f"{inviter} invited {user} to {room}")
 
-    async def handle_join(self, room, user):
-        user = User.get_by_mxid(user)
+    async def handle_join(self, room, user, event_id):
+        user = await User.get_by_mxid(user).ensure_started()
 
         portal = Portal.get_by_mxid(room)
         if not portal:
             return
 
-        if not user.whitelisted:
+        if not user.relaybot_whitelisted:
             await portal.main_intent.kick(room, user.mxid,
                                           "You are not whitelisted on this Telegram bridge.")
             return
-        elif not user.logged_in:
-            # TODO[waiting-for-bots] once we have bot support, this won't be needed.
+        elif not user.logged_in and not portal.has_bot:
             await portal.main_intent.kick(room, user.mxid,
-                                          "You are not logged into this Telegram bridge.")
+                                          "This chat does not have a bot relaying "
+                                          "messages for unauthenticated users.")
             return
 
         self.log.debug(f"{user} joined {room}")
-        # TODO join Telegram chat if applicable
+        if user.logged_in or portal.has_bot:
+            await portal.join_matrix(user, event_id)
 
-    async def handle_part(self, room, user, sender):
+    async def handle_part(self, room, user, sender, event_id):
         self.log.debug(f"{user} left {room}")
 
         sender = User.get_by_mxid(sender, create=False)
+        if not sender:
+            return
+        await sender.ensure_started()
 
         portal = Portal.get_by_mxid(room)
         if not portal:
@@ -143,8 +157,11 @@ class MatrixHandler:
             await portal.leave_matrix(puppet, sender)
 
         user = User.get_by_mxid(user, create=False)
-        if user and user.logged_in:
-            await portal.leave_matrix(user, sender)
+        if not user:
+            return
+        await user.ensure_started()
+        if user.logged_in or portal.has_bot:
+            await portal.leave_matrix(user, sender, event_id)
 
     def is_command(self, message):
         text = message.get("body", "")
@@ -158,14 +175,16 @@ class MatrixHandler:
         self.log.debug(f"{sender} sent {message} to ${room}")
 
         is_command, text = self.is_command(message)
-        sender = User.get_by_mxid(sender)
+        sender = await User.get_by_mxid(sender).ensure_started()
+        if not sender.relaybot_whitelisted:
+            return
 
         portal = Portal.get_by_mxid(room)
-        if sender.has_full_access and portal and not is_command:
+        if not is_command and portal and (sender.logged_in or portal.has_bot):
             await portal.handle_matrix_message(sender, message, event_id)
             return
 
-        if message["msgtype"] != "m.text":
+        if not sender.whitelisted or message["msgtype"] != "m.text":
             return
 
         try:
@@ -186,20 +205,25 @@ class MatrixHandler:
                                        is_portal=portal is not None)
 
     async def handle_redaction(self, room, sender, event_id):
+        sender = await User.get_by_mxid(sender).ensure_started()
+        if not sender.relaybot_whitelisted:
+            return
+
         portal = Portal.get_by_mxid(room)
-        sender = User.get_by_mxid(sender)
-        if sender.has_full_access and portal:
-            await portal.handle_matrix_deletion(sender, event_id)
+        if not portal:
+            return
+
+        await portal.handle_matrix_deletion(sender, event_id)
 
     async def handle_power_levels(self, room, sender, new, old):
         portal = Portal.get_by_mxid(room)
-        sender = User.get_by_mxid(sender)
+        sender = await User.get_by_mxid(sender).ensure_started()
         if sender.has_full_access and portal:
             await portal.handle_matrix_power_levels(sender, new["users"], old["users"])
 
     async def handle_room_meta(self, type, room, sender, content):
         portal = Portal.get_by_mxid(room)
-        sender = User.get_by_mxid(sender)
+        sender = await User.get_by_mxid(sender).ensure_started()
         if sender.has_full_access and portal:
             handler, content_key = {
                 "m.room.name": (portal.handle_matrix_title, "name"),
@@ -207,9 +231,20 @@ class MatrixHandler:
                 "m.room.avatar": (portal.handle_matrix_avatar, "url"),
             }[type]
             if content_key not in content:
-                # FIXME handle
-                pass
-                await handler(sender, content[content_key])
+                return
+            await handler(sender, content[content_key])
+
+    async def handle_room_pin(self, room, sender, new_events, old_events):
+        portal = Portal.get_by_mxid(room)
+        sender = await User.get_by_mxid(sender).ensure_started()
+        if sender.has_full_access and portal:
+            events = new_events - old_events
+            if len(events) > 0:
+                # New event pinned, set that as pinned in Telegram.
+                await portal.handle_matrix_pin(sender, events.pop())
+            elif len(new_events) == 0:
+                # All pinned events removed, remove pinned event in Telegram.
+                await portal.handle_matrix_pin(sender, None)
 
     def filter_matrix_event(self, event):
         return (event["sender"] == self.az.bot_mxid
@@ -222,19 +257,34 @@ class MatrixHandler:
         type = evt["type"]
         content = evt.get("content", {})
         if type == "m.room.member":
+            prev_content = evt.get("unsigned", {}).get("prev_content", {})
             membership = content.get("membership", "")
-            if membership == "invite":
+            prev_membership = prev_content.get("membership", "leave")
+            if membership == prev_membership:
+                # TODO handle displayname/avatar changes
+                pass
+            elif membership == "invite":
                 await self.handle_invite(evt["room_id"], evt["state_key"], evt["sender"])
-            elif membership == "leave":
-                await self.handle_part(evt["room_id"], evt["state_key"], evt["sender"])
+            elif prev_membership == "join" and membership == "leave":
+                await self.handle_part(evt["room_id"], evt["state_key"], evt["sender"],
+                                       evt["event_id"])
             elif membership == "join":
-                await self.handle_join(evt["room_id"], evt["state_key"])
-        elif type == "m.room.message":
+                await self.handle_join(evt["room_id"], evt["state_key"], evt["event_id"])
+        elif type in ("m.room.message", "m.sticker"):
+            if type != "m.room.message":
+                content["msgtype"] = type
             await self.handle_message(evt["room_id"], evt["sender"], content, evt["event_id"])
         elif type == "m.room.redaction":
             await self.handle_redaction(evt["room_id"], evt["sender"], evt["redacts"])
         elif type == "m.room.power_levels":
             await self.handle_power_levels(evt["room_id"], evt["sender"], evt["content"],
                                            evt["prev_content"])
-        elif type == "m.room.name" or type == "m.room.avatar" or type == "m.room.topic":
+        elif type in ("m.room.name", "m.room.avatar", "m.room.topic"):
             await self.handle_room_meta(type, evt["room_id"], evt["sender"], evt["content"])
+        elif type == "m.room.pinned_events":
+            new_events = set(evt["content"]["pinned"])
+            try:
+                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+            except KeyError:
+                old_events = set()
+            await self.handle_room_pin(evt["room_id"], evt["sender"], new_events, old_events)

mautrix_telegram/public/__init__.py

@@ -0,0 +1,166 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+from mako.template import Template
+import asyncio
+import pkg_resources
+import logging
+
+from telethon.errors import *
+
+from ..user import User
+from ..commands.auth import enter_password
+from ..util import format_duration
+
+
+class PublicBridgeWebsite:
+    log = logging.getLogger("mau.public")
+
+    def __init__(self, loop):
+        self.loop = loop
+
+        self.login = Template(
+            pkg_resources.resource_string("mautrix_telegram", "public/login.html.mako"))
+
+        self.app = web.Application(loop=loop)
+        self.app.router.add_route("GET", "/login", self.get_login)
+        self.app.router.add_route("POST", "/login", self.post_login)
+        self.app.router.add_static("/",
+                                   pkg_resources.resource_filename("mautrix_telegram", "public/"))
+
+    async def get_login(self, request):
+        user = (User.get_by_mxid(request.rel_url.query["mxid"], create=False)
+                if "mxid" in request.rel_url.query else None)
+        if not user:
+            return self.render_login(
+                mxid=request.rel_url.query["mxid"] if "mxid" in request.rel_url.query else None,
+                state="request")
+        elif not user.whitelisted:
+            return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403)
+        await user.ensure_started()
+        if not user.logged_in:
+            return self.render_login(mxid=user.mxid, state="request")
+
+        return self.render_login(mxid=user.mxid, username=user.username)
+
+    def render_login(self, status=200, username="", state="", error="", message="", mxid=""):
+        return web.Response(status=status, content_type="text/html",
+                            text=self.login.render(username=username, state=state, error=error,
+                                                   message=message, mxid=mxid))
+
+    async def post_login_phone(self, user, phone):
+        try:
+            await user.client.sign_in(phone or "+123")
+            return self.render_login(mxid=user.mxid, state="code", status=200,
+                                     message="Code requested successfully.")
+        except PhoneNumberInvalidError:
+            return self.render_login(mxid=user.mxid, state="request", status=400,
+                                     error="Invalid phone number.")
+        except PhoneNumberUnoccupiedError:
+            return self.render_login(mxid=user.mxid, state="request", status=404,
+                                     error="That phone number has not been registered.")
+        except PhoneNumberFloodError:
+            return self.render_login(
+                mxid=user.mxid, state="request", status=429,
+                error="Your phone number has been temporarily blocked for flooding. "
+                      "The ban is usually applied for around a day.")
+        except FloodWaitError as e:
+            return self.render_login(
+                mxid=user.mxid, state="request", status=429,
+                error="Your phone number has been temporarily blocked for flooding. "
+                      f"Please wait for {format_duration(e.seconds)} before trying again.")
+        except PhoneNumberBannedError:
+            return self.render_login(mxid=user.mxid, state="request", status=401,
+                                     error="Your phone number is banned from Telegram.")
+        except PhoneNumberAppSignupForbiddenError:
+            return self.render_login(mxid=user.mxid, state="request", status=401,
+                                     error="You have disabled 3rd party apps on your account.")
+        except Exception:
+            self.log.exception("Error requesting phone code")
+            return self.render_login(mxid=user.mxid, state="request", status=500,
+                                     error="Internal server error while requesting code.")
+
+    async def post_login_code(self, user, code, password_in_data):
+        try:
+            user_info = await user.client.sign_in(code=code)
+            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+            if user.command_status and user.command_status["action"] == "Login":
+                user.command_status = None
+            return self.render_login(mxid=user.mxid, state="logged-in", status=200,
+                                     username=user_info.username)
+        except PhoneCodeInvalidError:
+            return self.render_login(mxid=user.mxid, state="code", status=403,
+                                     error="Incorrect phone code.")
+        except PhoneCodeExpiredError:
+            return self.render_login(mxid=user.mxid, state="code", status=403,
+                                     error="Phone code expired.")
+        except SessionPasswordNeededError:
+            if not password_in_data:
+                if user.command_status and user.command_status["action"] == "Login":
+                    user.command_status = {
+                        "next": enter_password,
+                        "action": "Login (password entry)",
+                    }
+                return self.render_login(
+                    mxid=user.mxid, state="password", status=200,
+                    message="Code accepted, but you have 2-factor authentication is enabled.")
+            return None
+        except Exception:
+            self.log.exception("Error sending phone code")
+            return self.render_login(mxid=user.mxid, state="code", status=500,
+                                     error="Internal server error while sending code.")
+
+    async def post_login_password(self, user, password):
+        try:
+            user_info = await user.client.sign_in(password=password)
+            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+            if user.command_status and user.command_status["action"] == "Login (password entry)":
+                user.command_status = None
+            return self.render_login(mxid=user.mxid, state="logged-in", status=200,
+                                     username=user_info.username)
+        except (PasswordHashInvalidError, PasswordEmptyError):
+            return self.render_login(mxid=user.mxid, state="password", status=400,
+                                     error="Incorrect password.")
+        except Exception:
+            self.log.exception("Error sending password")
+            return self.render_login(mxid=user.mxid, state="password", status=500,
+                                     error="Internal server error while sending password.")
+
+    async def post_login(self, request):
+        data = await request.post()
+        if "mxid" not in data:
+            return self.render_login(error="Please enter your Matrix ID.", status=400)
+
+        user = await User.get_by_mxid(data["mxid"]).ensure_started(even_if_no_session=True)
+        if not user.whitelisted:
+            return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403)
+        elif user.logged_in:
+            return self.render_login(mxid=user.mxid, username=user.username)
+
+        if "phone" in data:
+            return await self.post_login_phone(user, data["phone"])
+        elif "code" in data:
+            resp = await self.post_login_code(user, data["code"],
+                                              password_in_data="password" in data)
+            if resp or "password" not in data:
+                return resp
+        elif "password" not in data:
+            return self.render_login(error="No data given.", status=400)
+
+        if "password" in data:
+            return await self.post_login_password(user, data["password"])
+        return self.render_login(error="This should never happen.", status=500)

mautrix_telegram/public/login.css

@@ -0,0 +1,50 @@
+/*
+ * mautrix-telegram - A Matrix-Telegram puppeting bridge
+ * Copyright (C) 2018 Tulir Asokan
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+form > div {
+	display: none;
+}
+
+form[data-status="request"]  > div.status-request,
+form[data-status="code"]     > div.status-code,
+form[data-status="password"] > div.status-password {
+	display: initial;
+}
+
+.container {
+	margin-top: 3rem;
+	max-width: 60rem;
+}
+
+.error, .message {
+	border-radius: .25rem;
+	padding: .5rem 1rem;
+	border: 1px solid transparent;
+	margin: .5rem 0;
+}
+
+.error {
+	border-color: #f5c6cb;
+	background-color: #f8d7da;
+	color: #721c24;
+}
+
+.message {
+	border-color: #c3e6cb;
+	background-color: #d4edda;
+	color: #155724;
+}

mautrix_telegram/public/login.html.mako

@@ -0,0 +1,97 @@
+<!--
+mautrix-telegram - A Matrix-Telegram puppeting bridge
+Copyright (C) 2018 Tulir Asokan
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-->
+<!DOCTYPE html>
+<html>
+<head>
+	<title>Mautrix-Telegram bridge</title>
+	<link rel="icon" type="image/png" href="favicon.png"/>
+	<meta property="og:title" content="Mautrix-Telegram bridge">
+	<meta property="og:description" content="A hybrid puppeting/relaybot Matrix-Telegram bridge">
+	<meta property="og:image" content="favicon.png">
+	<meta charset="utf-8">
+	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
+	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
+	<link rel="stylesheet"
+		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+	<link rel="stylesheet" href="login.css"/>
+</head>
+<body>
+<main class="container">
+	% if username:
+		% if state == "logged-in":
+			<h1>Logged in successfully!</h1>
+			<p>
+				Logged in as @${username}.
+				You can now close this page.
+				You should be invited to Telegram portals on Matrix momentarily.
+			</p>
+		% else:
+			<h1>You're already logged in!</h1>
+			<p>
+				You're logged in as @${username}.
+			</p>
+			<p>
+				If you want to log in with another account, log out using the <code>logout</code>
+				management command first.
+			</p>
+		% endif
+	% else:
+		<h1>Log in to Telegram</h1>
+	% if error:
+		<div class="error">${error}</div>
+	% endif
+	% if message:
+		<div class="message">${message}</div>
+	% endif
+		<form method="post">
+			<fieldset>
+				<label for="mxid">Matrix ID</label>
+				<input type="text" id="mxid" name="mxid" placeholder="Enter Matrix ID"
+					   value="${mxid}"/>
+				% if state == "request":
+					<label for="value">Phone number</label>
+					<input type="tel" id="value" name="phone" placeholder="Enter phone number"/>
+					<button type="submit">Request code</button>
+				% elif state == "code":
+					<label for="value">Phone code</label>
+					<input type="number" id="value" name="code" placeholder="Enter phone code"/>
+					<button type="submit">Sign in</button>
+					<div class="float-right">
+						<button class="button-clear" type="button"
+								onclick="location.replace(location.href)">
+							Go back
+						</button>
+					</div>
+				% elif state == "password":
+					<label for="value">Password</label>
+					<input type="password" id="value" name="password"
+						   placeholder="Enter password"/>
+					<button type="submit">Sign in</button>
+					<div class="float-right">
+						<button class="button-clear" type="button"
+								onclick="location.replace(location.href)">
+							Go back
+						</button>
+					</div>
+				% endif
+			</fieldset>
+		</form>
+	% endif
+</main>
+</body>
+</html>

mautrix_telegram/puppet.py

@@ -3,25 +3,26 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from difflib import SequenceMatcher
 import re
 import logging
 
-from telethon.tl.types import UserProfilePhoto, PeerUser
+from telethon.tl.types import UserProfilePhoto
 from telethon.errors.rpc_error_list import LocationInvalidError
 
 from .db import Puppet as DBPuppet
+from . import util
 
 config = None
 
@@ -31,19 +32,24 @@ class Puppet:
     db = None
     az = None
     mxid_regex = None
+    username_template = None
+    hs_domain = None
     cache = {}
 
-    def __init__(self, id=None, username=None, displayname=None, photo_id=None):
+    def __init__(self, id=None, username=None, displayname=None, displayname_source=None,
+                 photo_id=None, is_bot=None, db_instance=None):
         self.id = id
+        self.mxid = self.get_mxid_from_id(self.id)
 
-        self.localpart = config.get("bridge.username_template", "telegram_{userid}").format(
-            userid=self.id)
-        hs = config["homeserver"]["domain"]
-        self.mxid = f"@{self.localpart}:{hs}"
         self.username = username
         self.displayname = displayname
+        self.displayname_source = displayname_source
         self.photo_id = photo_id
+        self.is_bot = is_bot
+        self._db_instance = db_instance
+
         self.intent = self.az.intent.user(self.mxid)
+        self.logged_in = True
 
         self.cache[id] = self
 
@@ -51,17 +57,29 @@ class Puppet:
     def tgid(self):
         return self.id
 
-    def to_db(self):
-        return self.db.merge(
-            DBPuppet(id=self.id, username=self.username, displayname=self.displayname,
-                     photo_id=self.photo_id))
+    @property
+    def db_instance(self):
+        if not self._db_instance:
+            self._db_instance = self.new_db_instance()
+        return self._db_instance
+
+    def new_db_instance(self):
+        return DBPuppet(id=self.id, username=self.username, displayname=self.displayname,
+                        displayname_source=self.displayname_source, photo_id=self.photo_id,
+                        is_bot=self.is_bot)
 
     @classmethod
     def from_db(cls, db_puppet):
-        return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname, db_puppet.photo_id)
+        return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname,
+                      db_puppet.displayname_source, db_puppet.photo_id, db_puppet.is_bot,
+                      db_instance=db_puppet)
 
     def save(self):
-        self.to_db()
+        self.db_instance.username = self.username
+        self.db_instance.displayname = self.displayname
+        self.db_instance.displayname_source = self.displayname_source
+        self.db_instance.photo_id = self.photo_id
+        self.db_instance.is_bot = self.is_bot
         self.db.commit()
 
     def similarity(self, query):
@@ -89,7 +107,10 @@ class Puppet:
             name = data[preference]
             if name:
                 break
-        if not name:
+
+        if info.deleted:
+            name = f"Deleted account {info.id}"
+        elif not name:
             name = info.id
 
         if not format:
@@ -107,27 +128,36 @@ class Puppet:
         if isinstance(info.photo, UserProfilePhoto):
             changed = await self.update_avatar(source, info.photo.photo_big) or changed
 
+        self.is_bot = info.bot
+
         if changed:
             self.save()
 
     async def update_displayname(self, source, info):
+        ignore_source = (not source.is_relaybot
+                         and self.displayname_source is not None
+                         and self.displayname_source != source.tgid)
+        if ignore_source:
+            return
+
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
             await self.intent.set_display_name(displayname)
             self.displayname = displayname
+            self.displayname_source = source.tgid
+            return True
+        elif source.is_relaybot or self.displayname_source is None:
+            self.displayname_source = source.tgid
             return True
 
     async def update_avatar(self, source, photo):
         photo_id = f"{photo.volume_id}-{photo.local_id}"
         if self.photo_id != photo_id:
-            try:
-                file = await source.client.download_file_bytes(photo)
-            except LocationInvalidError:
-                return False
-            uploaded = await self.intent.upload_file(file)
-            await self.intent.set_avatar(uploaded["content_uri"])
-            self.photo_id = photo_id
-            return True
+            file = await util.transfer_file_to_matrix(self.db, source.client, self.intent, photo)
+            if file:
+                await self.intent.set_avatar(file.mxc)
+                self.photo_id = photo_id
+                return True
         return False
 
     @classmethod
@@ -143,7 +173,7 @@ class Puppet:
 
         if create:
             puppet = cls(id)
-            cls.db.add(puppet.to_db())
+            cls.db.add(puppet.db_instance)
             cls.db.commit()
             return puppet
 
@@ -161,10 +191,17 @@ class Puppet:
             return int(match.group(1))
         return None
 
+    @classmethod
+    def get_mxid_from_id(cls, id):
+        return f"@{cls.username_template.format(userid=id)}:{cls.hs_domain}"
+
     @classmethod
     def find_by_username(cls, username):
+        if not username:
+            return None
+
         for _, puppet in cls.cache.items():
-            if puppet.username == username:
+            if puppet.username and puppet.username.lower() == username.lower():
                 return puppet
 
         puppet = DBPuppet.query.filter(DBPuppet.username == username).one_or_none()
@@ -173,10 +210,26 @@ class Puppet:
 
         return None
 
+    @classmethod
+    def find_by_displayname(cls, displayname):
+        if not displayname:
+            return None
+
+        for _, puppet in cls.cache.items():
+            if puppet.displayname and puppet.displayname == displayname:
+                return puppet
+
+        puppet = DBPuppet.query.filter(DBPuppet.displayname == displayname).one_or_none()
+        if puppet:
+            return cls.from_db(puppet)
+
+        return None
+
 
 def init(context):
     global config
-    Puppet.az, Puppet.db, config, _ = context
-    localpart = config.get("bridge.username_template", "telegram_{userid}").format(userid="(.+)")
-    hs = config["homeserver"]["domain"]
-    Puppet.mxid_regex = re.compile(f"@{localpart}:{hs}")
+    Puppet.az, Puppet.db, config, _, _ = context
+    Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
+    Puppet.hs_domain = config["homeserver"]["domain"]
+    localpart = Puppet.username_template.format(userid="(.+)")
+    Puppet.mxid_regex = re.compile(f"@{localpart}:{Puppet.hs_domain}")

mautrix_telegram/tgclient.py

@@ -3,28 +3,33 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from io import BytesIO
 
 from telethon import TelegramClient
 from telethon.tl.functions.messages import SendMessageRequest, SendMediaRequest
 from telethon.tl.types import *
+from telethon.extensions.markdown import parse as parse_md
 
 
 class MautrixTelegramClient(TelegramClient):
-    async def send_message(self, entity, message, reply_to=None, entities=None, link_preview=True):
+    async def send_message(self, entity, message, reply_to=None, entities=None, markdown=False,
+                           link_preview=True):
         entity = await self.get_input_entity(entity)
 
+        if markdown:
+            message, entities = parse_md(message)
+
         request = SendMessageRequest(
             peer=entity,
             message=message,
@@ -46,26 +51,25 @@ class MautrixTelegramClient(TelegramClient):
 
         return self._get_response_message(request, result)
 
-    async def send_file(self, entity, file, mime_type=None, caption=None, attributes=None,
-                        file_name=None, reply_to=None, **kwargs):
-        entity = await self.get_input_entity(entity)
-        reply_to = self._get_message_id(reply_to)
+    async def upload_file(self, file, mime_type=None, attributes=None, file_name=None):
+        file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)
 
-        file_handle = await self.upload_file(file, file_name=file_name, use_cache=False)
-
-        if mime_type == "image/png":
-            media = InputMediaUploadedPhoto(file_handle, caption or "")
+        if mime_type == "image/png" or mime_type == "image/jpeg":
+            return InputMediaUploadedPhoto(file_handle)
         else:
             attributes = attributes or []
             attr_dict = {type(attr): attr for attr in attributes}
 
-            media = InputMediaUploadedDocument(
+            return InputMediaUploadedDocument(
                 file=file_handle,
                 mime_type=mime_type or "application/octet-stream",
-                attributes=list(attr_dict.values()),
-                caption=caption or "")
+                attributes=list(attr_dict.values()))
 
-        request = SendMediaRequest(entity, media, reply_to_msg_id=reply_to)
+    async def send_media(self, entity, media, caption=None, entities=None, reply_to=None):
+        entity = await self.get_input_entity(entity)
+        reply_to = self._get_message_id(reply_to)
+        request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],
+                                   reply_to_msg_id=reply_to)
         return self._get_response_message(request, await self(request))
 
     async def download_file_bytes(self, location):

mautrix_telegram/user.py

@@ -3,44 +3,41 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import logging
 import asyncio
-import platform
+import re
 
 from telethon.tl.types import *
 from telethon.tl.types.contacts import ContactsNotModified
-from telethon.tl.types import User as TLUser
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from mautrix_appservice import MatrixRequestError
 
-from .db import User as DBUser, Message as DBMessage, Contact as DBContact
-from .tgclient import MautrixTelegramClient
-from . import portal as po, puppet as pu, __version__
+from .db import User as DBUser, Contact as DBContact
+from .abstract_user import AbstractUser
+from . import portal as po, puppet as pu
 
 config = None
 
 
-class User:
-    loop = None
+class User(AbstractUser):
     log = logging.getLogger("mau.user")
-    db = None
-    az = None
     by_mxid = {}
     by_tgid = {}
 
     def __init__(self, mxid, tgid=None, username=None, db_contacts=None, saved_contacts=0,
-                 db_portals=None):
+                 db_portals=None, db_instance=None):
+        super().__init__()
         self.mxid = mxid
         self.tgid = tgid
         self.username = username
@@ -49,31 +46,27 @@ class User:
         self.db_contacts = db_contacts
         self.portals = {}
         self.db_portals = db_portals
+        self._db_instance = db_instance
 
         self.command_status = None
-        self.connected = False
-        self.client = None
-        self._init_client()
 
-        self.is_admin = self.mxid in config.get("bridge.admins", [])
-
-        whitelist = config.get("bridge.whitelist", None) or [self.mxid]
-        self.whitelisted = not whitelist or self.mxid in whitelist
-        if not self.whitelisted:
-            homeserver = self.mxid[self.mxid.index(":") + 1:]
-            self.whitelisted = homeserver in whitelist
+        (self.relaybot_whitelisted,
+         self.whitelisted,
+         self.is_admin) = config.get_permissions(self.mxid)
 
         self.by_mxid[mxid] = self
         if tgid:
             self.by_tgid[tgid] = self
 
     @property
-    def logged_in(self):
-        return self.client.is_user_authorized()
+    def name(self):
+        return self.mxid
 
     @property
-    def has_full_access(self):
-        return self.logged_in and self.whitelisted
+    def displayname(self):
+        # TODO show better username
+        match = re.compile("@(.+):(.+)").match(self.mxid)
+        return match.group(1)
 
     @property
     def db_contacts(self):
@@ -89,7 +82,7 @@ class User:
 
     @property
     def db_portals(self):
-        return [portal.to_db(merge=False) for _, portal in self.portals.items()]
+        return [portal.db_instance for portal in self.portals.values()]
 
     @db_portals.setter
     def db_portals(self, portals):
@@ -102,14 +95,23 @@ class User:
 
     # region Database conversion
 
-    def to_db(self):
-        return self.db.merge(
-            DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                   contacts=self.db_contacts, saved_contacts=self.saved_contacts,
-                   portals=self.db_portals))
+    @property
+    def db_instance(self):
+        if not self._db_instance:
+            self._db_instance = self.new_db_instance()
+        return self._db_instance
+
+    def new_db_instance(self):
+        return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
+                      contacts=self.db_contacts, saved_contacts=self.saved_contacts,
+                      portals=self.db_portals)
 
     def save(self):
-        self.to_db()
+        self.db_instance.tgid = self.tgid
+        self.db_instance.username = self.username
+        self.db_instance.contacts = self.db_contacts
+        self.db_instance.saved_contacts = self.saved_contacts
+        self.db_instance.portals = self.db_portals
         self.db.commit()
 
     def delete(self):
@@ -118,37 +120,28 @@ class User:
             del self.by_tgid[self.tgid]
         except KeyError:
             pass
-        self.db.delete(self.to_db())
-        self.db.commit()
+        if self._db_instance:
+            self.db.delete(self._db_instance)
+            self.db.commit()
 
     @classmethod
     def from_db(cls, db_user):
         return User(db_user.mxid, db_user.tgid, db_user.tg_username, db_user.contacts,
-                    db_user.saved_contacts, db_user.portals)
+                    db_user.saved_contacts, db_user.portals, db_instance=db_user)
 
     # endregion
     # region Telegram connection management
 
-    def _init_client(self):
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
-        self.client = MautrixTelegramClient(self.mxid,
-                                            config["telegram.api_id"],
-                                            config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device)
-        self.client.add_update_handler(self.update_catch)
-
     async def start(self, delete_unless_authenticated=False):
-        self.connected = await self.client.connect()
+        await super().start()
         if self.logged_in:
+            self.log.debug(f"Ensuring post_login() for {self.name}")
             asyncio.ensure_future(self.post_login(), loop=self.loop)
         elif delete_unless_authenticated:
+            self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting...")
             # User not logged in -> forget user
             self.client.disconnect()
-            self.client.session.delete()
+            # self.client.session.delete()
             self.delete()
         return self
 
@@ -157,14 +150,11 @@ class User:
             await self.update_info(info)
             await self.sync_dialogs()
             await self.sync_contacts()
+            if config["bridge.catch_up"]:
+                await self.client.catch_up()
         except Exception:
             self.log.exception("Failed to run post-login functions")
 
-    def stop(self):
-        self.client.disconnect()
-        self.client = None
-        self.connected = False
-
     # endregion
     # region Telegram actions that need custom methods
 
@@ -182,6 +172,8 @@ class User:
 
     async def log_out(self):
         for _, portal in self.portals.items():
+            if portal.has_bot:
+                continue
             try:
                 await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
             except MatrixRequestError:
@@ -233,18 +225,14 @@ class User:
 
         return await self._search_remote(query), True
 
-    async def sync_dialogs(self):
-        dialogs = await self.client.get_dialogs(limit=30)
+    async def sync_dialogs(self, synchronous_create=False):
         creators = []
-        for dialog in dialogs:
-            entity = dialog.entity
-            invalid = (isinstance(entity, (TLUser, ChatForbidden, ChannelForbidden))
-                       or (isinstance(entity, Chat) and (entity.deactivated or entity.left)))
-            if invalid:
-                continue
+        for entity in await self._get_dialogs(limit=30):
             portal = po.Portal.get_by_entity(entity)
             self.portals[portal.tgid_full] = portal
-            creators.append(portal.create_matrix_room(self, entity, invites=[self.mxid]))
+            creators.append(
+                portal.create_matrix_room(self, entity, invites=[self.mxid],
+                                          synchronous=synchronous_create))
         self.save()
         await asyncio.gather(*creators, loop=self.loop)
 
@@ -283,135 +271,6 @@ class User:
             self.contacts.append(puppet)
         self.save()
 
-    # endregion
-    # region Telegram update handling
-
-    async def update_catch(self, update):
-        try:
-            await self.update(update)
-        except Exception:
-            self.log.exception("Failed to handle Telegram update")
-
-    async def update(self, update):
-        if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
-                               UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
-            await self.update_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
-            await self.update_typing(update)
-        elif isinstance(update, UpdateUserStatus):
-            await self.update_status(update)
-        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
-            await self.update_admin(update)
-        elif isinstance(update, UpdateChatParticipants):
-            portal = po.Portal.get_by_tgid(update.participants.chat_id)
-            if portal and portal.mxid:
-                await portal.update_telegram_participants(update.participants.participants)
-        elif isinstance(update, UpdateChannelPinnedMessage):
-            portal = po.Portal.get_by_tgid(update.channel_id)
-            if portal and portal.mxid:
-                await portal.update_telegram_pin(self, update.id)
-        elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
-            await self.update_others_info(update)
-        elif isinstance(update, UpdateReadHistoryOutbox):
-            await self.update_read_receipt(update)
-        else:
-            self.log.debug("Unhandled update: %s", update)
-
-    async def update_read_receipt(self, update):
-        if not isinstance(update.peer, PeerUser):
-            self.log.debug("Unexpected read receipt peer: %s", update.peer)
-            return
-
-        portal = po.Portal.get_by_tgid(update.peer.user_id, self.tgid)
-        if not portal or not portal.mxid:
-            return
-
-        # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.query.get((update.max_id, self.tgid))
-        if not message:
-            return
-
-        puppet = pu.Puppet.get(update.peer.user_id)
-        await puppet.intent.mark_read(portal.mxid, message.mxid)
-
-    async def update_admin(self, update):
-        portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        if isinstance(update, UpdateChatAdmins):
-            await portal.set_telegram_admins_enabled(update.enabled)
-        elif isinstance(update, UpdateChatParticipantAdmin):
-            puppet = pu.Puppet.get(update.user_id)
-            user = User.get_by_tgid(update.user_id)
-            await portal.set_telegram_admin(puppet, user)
-
-    async def update_typing(self, update):
-        if isinstance(update, UpdateUserTyping):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
-        else:
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        sender = pu.Puppet.get(update.user_id)
-        await portal.handle_telegram_typing(sender, update)
-
-    async def update_others_info(self, update):
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update, UpdateUserName):
-            if await puppet.update_displayname(self, update):
-                puppet.save()
-        elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
-                puppet.save()
-
-    async def update_status(self, update):
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update.status, UserStatusOnline):
-            await puppet.intent.set_presence("online")
-        elif isinstance(update.status, UserStatusOffline):
-            await puppet.intent.set_presence("offline")
-        else:
-            self.log.warning("Unexpected user status update: %s", update)
-        return
-
-    def get_message_details(self, update):
-        if isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-            sender = pu.Puppet.get(update.from_id)
-        elif isinstance(update, UpdateShortMessage):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
-            sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
-        elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
-                                 UpdateEditMessage, UpdateEditChannelMessage)):
-            update = update.message
-            if isinstance(update.to_id, PeerUser) and not update.out:
-                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
-            else:
-                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
-            sender = pu.Puppet.get(update.from_id) if update.from_id else None
-        else:
-            self.log.warning(
-                f"Unexpected message type in User#get_message_details: {type(update)}")
-            return update, None, None
-        return update, sender, portal
-
-    def update_message(self, original_update):
-        update, sender, portal = self.get_message_details(original_update)
-
-        if isinstance(update, MessageService):
-            if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.debug(f"Ignoring action %s to %s by %d", update.action, portal.tgid_log,
-                               sender.id)
-                return
-            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
-                           sender.id)
-            return portal.handle_telegram_action(self, sender, update.action)
-
-        user = sender.tgid if sender else "admin"
-        if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-            return portal.handle_telegram_edit(self, sender, update)
-
-        self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
-        return portal.handle_telegram_message(self, sender, update)
-
     # endregion
     # region Class instance lookup
 
@@ -425,14 +284,12 @@ class User:
         user = DBUser.query.get(mxid)
         if user:
             user = cls.from_db(user)
-            asyncio.ensure_future(user.start(), loop=cls.loop)
             return user
 
         if create:
             user = cls(mxid)
-            cls.db.add(user.to_db())
+            cls.db.add(user.db_instance)
             cls.db.commit()
-            asyncio.ensure_future(user.start(), loop=cls.loop)
             return user
 
         return None
@@ -447,15 +304,17 @@ class User:
         user = DBUser.query.filter(DBUser.tgid == tgid).one_or_none()
         if user:
             user = cls.from_db(user)
-            asyncio.ensure_future(user.start(), loop=cls.loop)
             return user
 
         return None
 
     @classmethod
     def find_by_username(cls, username):
+        if not username:
+            return None
+
         for _, user in cls.by_tgid.items():
-            if user.username == username:
+            if user.username and user.username.lower() == username.lower():
                 return user
 
         puppet = DBUser.query.filter(DBUser.tg_username == username).one_or_none()
@@ -468,7 +327,7 @@ class User:
 
 def init(context):
     global config
-    User.az, User.db, config, User.loop = context
+    config = context.config
 
     users = [User.from_db(user) for user in DBUser.query.all()]
     return [user.start(delete_unless_authenticated=True) for user in users]

mautrix_telegram/util/__init__.py

@@ -0,0 +1,2 @@
+from .file_transfer import transfer_file_to_matrix, convert_image
+from .format_duration import format_duration

mautrix_telegram/util/file_transfer.py

@@ -0,0 +1,197 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from io import BytesIO
+import time
+import logging
+import asyncio
+
+import magic
+from sqlalchemy.exc import IntegrityError, InvalidRequestError
+from sqlalchemy.orm.exc import FlushError
+
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+try:
+    from moviepy.editor import VideoFileClip
+    import random
+    import string
+    import os
+    import mimetypes
+except ImportError:
+    VideoFileClip = random = string = os = mimetypes = None
+
+from telethon.tl.types import (Document, FileLocation, InputFileLocation,
+                               InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
+from telethon.errors import LocationInvalidError
+
+from ..db import TelegramFile as DBTelegramFile
+
+log = logging.getLogger("mau.util")
+
+
+def convert_image(file, source_mime="image/webp", target_type="png", thumbnail_to=None):
+    if not Image:
+        return source_mime, file, None, None
+    try:
+        image = Image.open(BytesIO(file)).convert("RGBA")
+        if thumbnail_to:
+            image.thumbnail(thumbnail_to, Image.ANTIALIAS)
+        new_file = BytesIO()
+        image.save(new_file, target_type)
+        w, h = image.size
+        return f"image/{target_type}", new_file.getvalue(), w, h
+    except Exception:
+        log.exception(f"Failed to convert {source_mime} to {target_type}")
+        return source_mime, file, None, None
+
+
+def _temp_file_name(ext):
+    return ("/tmp/mxtg-video-"
+            + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
+            + ext)
+
+
+def _read_video_thumbnail(data, video_ext="mp4", frame_ext="png", max_size=(1024, 720)):
+    # We don't have any way to read the video from memory, so save it to disk.
+    temp_file = _temp_file_name(video_ext)
+    with open(temp_file, "wb") as file:
+        file.write(data)
+
+    # Read temp file and get frame
+    clip = VideoFileClip(temp_file)
+    frame = clip.get_frame(0)
+
+    # Convert to png and save to BytesIO
+    image = Image.fromarray(frame).convert("RGBA")
+    thumbnail_file = BytesIO()
+    if max_size:
+        image.thumbnail(max_size, Image.ANTIALIAS)
+    image.save(thumbnail_file, frame_ext)
+
+    os.remove(temp_file)
+
+    w, h = image.size
+    return thumbnail_file.getvalue(), w, h
+
+
+def _location_to_id(location):
+    if isinstance(location, (Document, InputDocumentFileLocation)):
+        return f"{location.id}-{location.version}"
+    elif isinstance(location, (FileLocation, InputFileLocation)):
+        return f"{location.volume_id}-{location.local_id}"
+    else:
+        return None
+
+
+async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mime):
+    if not Image or not VideoFileClip:
+        return None
+
+    id = _location_to_id(thumbnail_loc)
+    if not id:
+        return None
+
+    video_ext = mimetypes.guess_extension(mime)
+    if VideoFileClip and video_ext:
+        try:
+            file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
+        except OSError:
+            return None
+        mime_type = "image/png"
+    else:
+        file = await client.download_file_bytes(thumbnail_loc)
+        width, height = None, None
+        mime_type = magic.from_buffer(file, mime=True)
+
+    uploaded = await intent.upload_file(file, mime_type)
+
+    return DBTelegramFile(id=id, mxc=uploaded["content_uri"], mime_type=mime_type,
+                          was_converted=False, timestamp=int(time.time()), size=len(file),
+                          width=width, height=height)
+
+
+transfer_locks = {}
+transfer_locks_lock = asyncio.Lock()
+
+
+async def transfer_file_to_matrix(db, client, intent, location, thumbnail=None, is_sticker=False):
+    id = _location_to_id(location)
+    if not id:
+        return None
+
+    db_file = DBTelegramFile.query.get(id)
+    if db_file:
+        return db_file
+
+    async with transfer_locks_lock:
+        try:
+            lock = transfer_locks[id]
+        except KeyError:
+            lock = asyncio.Lock()
+            transfer_locks[id] = lock
+    async with lock:
+        return await _unlocked_transfer_file_to_matrix(db, client, intent, id, location, thumbnail, is_sticker)
+
+
+async def _unlocked_transfer_file_to_matrix(db, client, intent, id, location, thumbnail, is_sticker):
+    db_file = DBTelegramFile.query.get(id)
+    if db_file:
+        return db_file
+
+    try:
+        file = await client.download_file_bytes(location)
+    except LocationInvalidError:
+        return None
+    width, height = None, None
+    mime_type = magic.from_buffer(file, mime=True)
+
+    image_converted = False
+    if mime_type == "image/webp":
+        new_mime_type, file, width, height = convert_image(file, source_mime="image/webp", target_type="png", thumbnail_to=(
+            256, 256) if is_sticker else None)
+        image_converted = new_mime_type != mime_type
+        mime_type = new_mime_type
+        thumbnail = None
+
+    uploaded = await intent.upload_file(file, mime_type)
+
+    db_file = DBTelegramFile(id=id, mxc=uploaded["content_uri"],
+                             mime_type=mime_type, was_converted=image_converted,
+                             timestamp=int(time.time()), size=len(file),
+                             width=width, height=height)
+    if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
+        if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
+            thumbnail = thumbnail.location
+        db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
+                                                               mime_type)
+
+    try:
+        db.add(db_file)
+        db.commit()
+    except FlushError as e:
+        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
+                      "This was probably caused by two simultaneous transfers of the same file, "
+                      "and should not cause any problems.")
+    except (IntegrityError, InvalidRequestError) as e:
+        db.rollback()
+        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
+                      "This was probably caused by two simultaneous transfers of the same file, "
+                      "and should not cause any problems.")
+
+    return db_file

mautrix_telegram/util/format_duration.py

@@ -0,0 +1,34 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+def format_duration(seconds):
+    def pluralize(count, singular): return singular if count == 1 else singular + "s"
+
+    def include(count, word): return f"{count} {pluralize(count, word)}" if count > 0 else ""
+
+    minutes, seconds = divmod(seconds, 60)
+    hours, minutes = divmod(minutes, 60)
+    days, hours = divmod(hours, 24)
+    parts = [a for a in [
+        include(days, "day"),
+        include(hours, "hour"),
+        include(minutes, "minute"),
+        include(seconds, "second")] if a]
+    if len(parts) > 2:
+        return "{} and {}".format(", ".join(parts[:-1]), parts[-1])
+    return " and ".join(parts)

optional-requirements.txt

@@ -0,0 +1,4 @@
+lxml
+cryptg
+Pillow
+moviepy

requirements.txt

@@ -1,8 +1,10 @@
 aiohttp
+mautrix-appservice
 ruamel.yaml
 python-magic
 SQLAlchemy
 alembic
 Markdown
-Pillow
 future-fstrings
+telethon-aio
+telethon-session-sqlalchemy

requirements/3.5.txt

@@ -1,2 +0,0 @@
--r base.txt
--e git+https://github.com/tulir/Telethon@asyncio-3.5#egg=Telethon

requirements/default.txt

@@ -1,2 +0,0 @@
--r base.txt
--e git+https://github.com/LonamiWebs/Telethon@asyncio#egg=Telethon

setup.py

@@ -1,7 +1,16 @@
 import setuptools
 import sys
+import glob
 import mautrix_telegram
 
+extras = {
+    "highlight_edits": ["lxml>=4.1.1,<5"],
+    "fast_crypto": ["cryptg>=0.1,<0.2"],
+    "webp_convert": ["Pillow>=5.0.0,<6"],
+    "hq_thumbnails": ["moviepy>=0.2,<0.3"],
+}
+extras["all"] = [deps[0] for deps in extras.values()]
+
 setuptools.setup(
     name="mautrix-telegram",
     version=mautrix_telegram.__version__,
@@ -10,30 +19,28 @@ setuptools.setup(
     author="Tulir Asokan",
     author_email="tulir@maunium.net",
 
-    description="A Matrix-Telegram puppeting bridge.",
+    description="A Matrix-Telegram hybrid puppeting/relaybot bridge.",
     long_description=open("README.md").read(),
 
     packages=setuptools.find_packages(),
 
     install_requires=[
-        "aiohttp>=2.3.10,<3",
-        "SQLAlchemy>=1.2.2,<2",
-        "alembic>=0.9.7",
+        "aiohttp>=3.0.1,<4",
+        "mautrix-appservice>=0.1.4,<0.2.0",
+        "SQLAlchemy>=1.2.3,<2",
+        "alembic>=0.9.8,<0.10",
         "Markdown>=2.6.11,<3",
         "ruamel.yaml>=0.15.35,<0.16",
-        "Pillow>=5.0.0,<6",
-        "future-fstrings>=0.4.1",
+        "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
+        "telethon-aio>=0.19.0,<0.19.1",
+        "telethon-session-sqlalchemy>=0.2.3,<0.3",
     ],
-    dependency_links=[
-        ("https://github.com/LonamiWebs/Telethon/tarball/6e854325a8e0e800a4f337257293d09006946162#egg=Telethon"
-         if sys.version_info > (3, 5)
-         else "https://github.com/tulir/Telethon/tarball/24dc21aea3305ef3bb8c7fcaef2025ae65d5c85e#egg=Telethon")
-    ],
+    extras_require=extras,
 
     classifiers=[
-        "Development Status :: 4 Beta",
-        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "Development Status :: 4 - Beta",
+        "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
         "Topic :: Communications :: Chat",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
@@ -44,4 +51,13 @@ setuptools.setup(
         [console_scripts]
         mautrix-telegram=mautrix_telegram.__main__:main
     """,
+    package_data={"mautrix_telegram": [
+        "public/*.mako", "public/*.png", "public/*.css",
+    ]},
+    data_files=[
+        (".", ["example-config.yaml", "alembic.ini"]),
+        ("alembic", ["alembic/env.py"]),
+        ("alembic/versions", glob.glob("alembic/versions/*.py"))
+    ],
 )
+