Update dependencies and bump version to 0.3.0rc2

Merge pull request #203 from turt2live/travis/patch-power-level-1
Fix a minor error regarding power level changes
2018-08-06 20:38:28 +03:00 · 2018-08-06 20:31:43 +03:00 · 2018-08-06 10:50:24 -06:00 · 2018-08-06 00:14:18 +03:00 · 2018-08-06 00:06:13 +03:00 · 2018-08-05 22:58:48 +03:00
83 changed files with 8779 additions and 2764 deletions
@@ -0,0 +1,8 @@
+engines:
+  sonar-python:
+    enabled: true
+    checks:
+      python:S107:
+        enabled: false
+exclude_patterns:
+- "alembic/"
@@ -0,0 +1,4 @@
+.editorconfig
+.codeclimate.yml
+*.png
+*.md
@@ -8,5 +8,8 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true

+[*.py]
+max_line_length = 99
+
 [*.{yaml,yml,py}]
 indent_style = space
@@ -7,6 +7,5 @@ __pycache__

 config.yaml
 registration.yaml
+*.log
 *.db
-*.session
-*.json
@@ -0,0 +1,29 @@
+FROM docker.io/alpine:3.8
+
+ENV UID=1337 \
+    GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
+
+COPY . /opt/mautrix-telegram
+WORKDIR /opt/mautrix-telegram
+RUN apk add --no-cache \
+      python3-dev \
+      build-base \
+      py3-virtualenv \
+      py3-pillow \
+      py3-aiohttp \
+      py3-lxml \
+      py3-magic \
+      py3-numpy \
+      py3-asn1crypto \
+      py3-sqlalchemy \
+      py3-markdown \
+      py3-psycopg2 \
+      ffmpeg \
+      ca-certificates \
+      su-exec \
+ && pip3 install -r requirements.txt -r optional-requirements.txt
+
+VOLUME /data
+
+CMD ["/opt/mautrix-telegram/docker-run.sh"]
@@ -1,23 +1,21 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007

- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
 Everyone is permitted to copy and distribute verbatim copies
 of this license document, but changing it is not allowed.

                            Preamble

-  The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.

  The licenses for most software and other practical works are designed
 to take away your freedom to share and change the works.  By contrast,
-the GNU General Public License is intended to guarantee your freedom to
+our General Public Licenses are intended to guarantee your freedom to
 share and change all versions of a program--to make sure it remains free
-software for all its users.  We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors.  You can apply it to
-your programs, too.
+software for all its users.

  When we speak of free software, we are referring to freedom, not
 price.  Our General Public Licenses are designed to make sure that you
@@ -26,44 +24,34 @@ them if you wish), that you receive source code or can get it if you
 want it, that you can change the software or use pieces of it in new
 free programs, and that you know you can do these things.

-  To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights.  Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.

-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received.  You must make sure that they, too, receive
-or can get the source code.  And you must show them these terms so they
-know their rights.
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.

-  Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.

-  For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software.  For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-  Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so.  This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software.  The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable.  Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products.  If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
-  Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary.  To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.

  The precise terms and conditions for copying, distribution and
 modification follow.
@@ -72,7 +60,7 @@ modification follow.

  0. Definitions.

-  "This License" refers to version 3 of the GNU General Public License.
+  "This License" refers to version 3 of the GNU Affero General Public License.

  "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
@@ -549,35 +537,45 @@ to collect a royalty for further conveying from those to whom you convey
 the Program, the only way you could satisfy both those terms and this
 License would be to refrain entirely from conveying the Program.

-  13. Use with the GNU Affero General Public License.
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.

  Notwithstanding any other provision of this License, you have
 permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
+under version 3 of the GNU General Public License into a single
 combined work, and to convey the resulting work.  The terms of this
 License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.

  14. Revised Versions of this License.

  The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
 address new problems or concerns.

  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU General
+Program specifies that a certain numbered version of the GNU Affero General
 Public License "or any later version" applies to it, you have the
 option of following the terms and conditions either of that numbered
 version or of any later version published by the Free Software
 Foundation.  If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
+GNU Affero General Public License, you may choose any version ever published
 by the Free Software Foundation.

  If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
+versions of the GNU Affero General Public License can be used, that proxy's
 public statement of acceptance of a version permanently authorizes you
 to choose that version for the Program.

@@ -635,40 +633,29 @@ the "copyright" line and a pointer to where the full notice is found.
    Copyright (C) <year>  <name of author>

    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
+    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
+    GNU Affero General Public License for more details.

-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.

 Also add information on how to contact you by electronic and paper mail.

-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.

  You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.
@@ -1,12 +1,14 @@
 # mautrix-telegram
-A Matrix-Telegram puppeting bridge.
+A Matrix-Telegram hybrid puppeting/relaybot bridge.

 ### [Wiki](https://github.com/tulir/mautrix-telegram/wiki)

-### [Features & Roadmap](ROADMAP.md)
+### [Features & Roadmap](https://github.com/tulir/mautrix-telegram/blob/master/ROADMAP.md)

 ## Discussion
 Matrix room: [`#telegram:maunium.net`](https://matrix.to/#/#telegram:maunium.net)

-A Telegram chat bridged to the Matrix room will be created once the bridge supports using a bot
-for unauthenticated users.
+Telegram chat: [`mautrix_telegram`](https://t.me/mautrix_telegram) (bridged to Matrix room)
+
+## Preview
+![Preview](https://raw.githubusercontent.com/tulir/mautrix-telegram/master/preview.png)
@@ -1,95 +1,53 @@
 # Features & roadmap

 * Matrix → Telegram
-  * [x] Plaintext messages
-  * [x] Formatted messages
-    * [ ] Bot commands (!command -> /command)
-    * [x] Mentions
-  * [x] Rich quotes
-  * [ ] Locations (not implemented in Riot)
-  * [x] Images
-  * [x] Files
+  * [x] Message content (text, formatting, files, etc..)
  * [x] Message redactions
+  * [ ] ‡ Message history
  * [ ] † Presence
  * [ ] † Typing notifications
  * [ ] † Read receipts
-  * [ ] Pinning messages
+  * [x] Pinning messages
  * [x] Power level
    * [x] Normal chats
      * [ ] Non-hardcoded PL requirements
 	* [x] Supergroups/channels
 	  * [ ] Precise bridging (non-hardcoded PL requirements, bridge specific permissions, etc..)
-  * [ ] Membership actions
-    * [x] Inviting
-      * [x] Puppets
-      * [x] Matrix users who have logged into Telegram
-    * [x] Kicking
-    * [ ] Joining
-      * [ ] Chat name as alias
-      * [ ] ‡ Chat invite link as alias
-    * [x] Leaving
+  * [x] Membership actions (invite/kick/join/leave)
  * [x] Room metadata changes (name, topic, avatar)
  * [x] Initial room metadata
  * [ ] User metadata
    * [ ] Initial displayname/username/avatar at register
    * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
-  * [x] Plaintext messages
-  * [x] Formatted messages
-    * [x] Bot commands (/command -> !command)
-    * [x] Mentions
-  * [x] Replies
-  * [x] Forwards
-  * [x] Images
-  * [x] Locations
-  * [x] Stickers
-  * [x] Audio messages
-  * [x] Video messages
-  * [x] Documents
-  * [ ] Message deletions (no way to tell difference between user-specific deletion and global deletion)
-  * [ ] Message edits (not supported in Matrix)
+  * [x] Message content (text, formatting, files, etc..)
+  * [x] Message deletions
+  * [x] Message edits
+  * [ ] Message history
  * [x] Avatars
  * [x] Presence
  * [x] Typing notifications
  * [x] Read receipts (private chat only)
  * [x] Pinning messages
  * [x] Admin/chat creator status
-  * [ ] Supergroup/channel permissions (precise per-user not supported in Matrix)
-  * [x] Membership actions
-    * [x] Inviting
-    * [x] Kicking
-    * [x] Joining/leaving
+  * [ ] Supergroup/channel permissions (precise per-user permissions not supported in Matrix)
+  * [x] Membership actions (invite/kick/join/leave)
  * [ ] Chat metadata changes
    * [x] Title
    * [x] Avatar
    * [ ] † About text
    * [ ] † Public channel username
  * [x] Initial chat metadata (about text missing)
-  * [x] User metadata
-    * [x] Initial displayname/avatar
-    * [x] Changes to displayname/avatar
+  * [x] User metadata (displayname/avatar)
  * [x] Supergroup upgrade
 * Misc
  * [x] Automatic portal creation
    * [x] At startup
    * [x] When receiving invite or message
  * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
-  * [ ] Option to use bot to relay messages for unauthenticated Matrix users
+  * [x] Option to use bot to relay messages for unauthenticated Matrix users
  * [ ] Option to use own Matrix account for messages sent from other Telegram clients
-* [Commands](https://github.com/tulir/mautrix-telegram/wiki/Management-commands)
-  * [x] Logging in and out (`login` + code entering)
-  * [x] Logging out
-  * [ ] Registering (`register`)
-  * [x] Searching for users (`search`)
-  * [x] Starting private chats (`pm`)
-  * [x] Joining chats with invite links (`join`)
-  * [x] Creating a Telegram chat for an existing Matrix room (`create`)
-  * [x] Upgrading the chat of a portal room into a supergroup (`upgrade`)
-  * [x] Change username of supergroup/channel (`groupname`)
-  * [x] Getting the Telegram invite link to a Matrix room (`invitelink`)
-  * Bridge administration
-    * [x] Clean up and forget a portal room (`deleteportal`)
-    * [ ] Setting Matrix-only power levels (`powerlevel`)
+  * [ ] ‡ Calls (hard, not yet supported by Telethon)

 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point
@@ -35,9 +35,6 @@ script_location = alembic
 # are written from script.py.mako
 # output_encoding = utf-8

-sqlalchemy.url = sqlite:///mautrix-telegram.db
-
-
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic
@@ -1,19 +1,36 @@
-from __future__ import with_statement
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig

 import sys
 from os.path import abspath, dirname
+
 sys.path.insert(0, dirname(dirname(abspath(__file__))))

 from mautrix_telegram.base import Base
+from mautrix_telegram.config import Config
+from alchemysession import AlchemySessionContainer
 import mautrix_telegram.db

 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config

+mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+mxtg_config = Config(mxtg_config_path, None, None)
+mxtg_config.load()
+config.set_main_option("sqlalchemy.url",
+                       mxtg_config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+
+
+class FakeDB:
+    @staticmethod
+    def query_property():
+        return None
+
+
+AlchemySessionContainer.create_table_classes(FakeDB(), "telethon_", Base)
+
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
@@ -24,6 +41,7 @@ fileConfig(config.config_file_name)
 # target_metadata = mymodel.Base.metadata
 target_metadata = Base.metadata

+
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
@@ -71,6 +89,7 @@ def run_migrations_online():
        with context.begin_transaction():
            context.run_migrations()

+
 if context.is_offline_mode():
    run_migrations_offline()
 else:
@@ -0,0 +1,28 @@
+"""Add TelegramFile table
+
+Revision ID: 1b241f7e8530
+Revises: 97d2a942bcf8
+Create Date: 2018-02-19 23:52:06.605741
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '1b241f7e8530'
+down_revision = '97d2a942bcf8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('telegram_file',
+                    sa.Column('id', sa.String(), nullable=False),
+                    sa.Column('mxc', sa.String(), nullable=True),
+                    sa.Column('mime_type', sa.String(), nullable=True),
+                    sa.Column('was_converted', sa.Boolean(), nullable=True),
+                    sa.PrimaryKeyConstraint('id'))
+
+
+def downgrade():
+    op.drop_table('telegram_file')
@@ -0,0 +1,24 @@
+"""Add is_bot field to puppets
+
+Revision ID: 1fa46383a9d3
+Revises: 30eca60587f1
+Create Date: 2018-04-29 23:44:40.102333
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1fa46383a9d3'
+down_revision = '30eca60587f1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('puppet', 'is_bot')
@@ -0,0 +1,41 @@
+"""Add cascade rules to UserPortal
+
+Revision ID: 2228d49c383f
+Revises: bcfefa1f1299
+Create Date: 2018-05-31 11:11:59.482112
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '2228d49c383f'
+down_revision = 'bcfefa1f1299'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+            batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+            batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+            batch_op.create_foreign_key("user_portal_user_fkey", "user", ["user"], ["tgid"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+            batch_op.create_foreign_key("user_portal_portal_fkey", "portal",
+                                        ["portal", "portal_receiver"], ["tgid", "tg_receiver"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+    except ValueError:
+        return
+
+
+def downgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+                batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+                batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+                batch_op.create_foreign_key("user_portal_user_fkey", "portal",
+                                            ["portal", "portal_receiver"], ["tgid", "tg_receiver"])
+                batch_op.create_foreign_key("user_portal_portal_fkey", "user", ["user"], ["tgid"])
+    except ValueError:
+        return
@@ -0,0 +1,23 @@
+"""Add megagroup field to portals
+
+Revision ID: 30eca60587f1
+Revises: cfc972368e50
+Create Date: 2018-04-29 15:51:04.656605
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '30eca60587f1'
+down_revision = 'cfc972368e50'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
+
+
+def downgrade():
+    op.drop_column('portal', 'megagroup')
@@ -0,0 +1,111 @@
+"""Move sessions to main database
+
+Revision ID: 501dad2868bc
+Revises: 7d47d84380b6
+Create Date: 2018-03-02 19:15:53.826985
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlite3
+import os
+
+# revision identifiers, used by Alembic.
+revision = '501dad2868bc'
+down_revision = '7d47d84380b6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    Session = op.create_table('telethon_sessions',
+                              sa.Column('session_id', sa.String, nullable=False),
+                              sa.Column('dc_id', sa.Integer, nullable=False),
+                              sa.Column('server_address', sa.String, nullable=True),
+                              sa.Column('port', sa.Integer, nullable=True),
+                              sa.Column('auth_key', sa.LargeBinary, nullable=True),
+                              sa.PrimaryKeyConstraint('session_id', 'dc_id'))
+    SentFile = op.create_table('telethon_sent_files',
+                               sa.Column('session_id', sa.String, nullable=False),
+                               sa.Column('md5_digest', sa.LargeBinary, nullable=False),
+                               sa.Column('file_size', sa.Integer, nullable=False),
+                               sa.Column('type', sa.Integer, nullable=False),
+                               sa.Column('id', sa.BigInteger, nullable=True),
+                               sa.Column('hash', sa.BigInteger, nullable=True),
+                               sa.PrimaryKeyConstraint('session_id', 'md5_digest', 'file_size',
+                                                       'type'))
+    Entity = op.create_table('telethon_entities',
+                             sa.Column('session_id', sa.String, nullable=False),
+                             sa.Column('id', sa.Integer, nullable=False),
+                             sa.Column('hash', sa.Integer, nullable=False),
+                             sa.Column('username', sa.String, nullable=True),
+                             sa.Column('phone', sa.Integer, nullable=True),
+                             sa.Column('name', sa.String, nullable=True),
+                             sa.PrimaryKeyConstraint('session_id', 'id'))
+    Version = op.create_table('telethon_version',
+                              sa.Column('version', sa.Integer, nullable=False),
+                              sa.PrimaryKeyConstraint('version'))
+    conn = op.get_bind()
+    sessions = [os.path.basename(f) for f in os.listdir(".") if f.endswith(".session")]
+    for session in sessions:
+        session_to_sqlalchemy(conn, session, Session, SentFile, Entity)
+
+
+def session_to_sqlalchemy(conn, path, Session, SentFile, Entity):
+    session_conn = sqlite3.connect(path)
+    session_id = os.path.splitext(path)[0]
+    c = session_conn.cursor()
+
+    auth_data_tuples = c.execute("SELECT * FROM sessions").fetchall()
+    auth_data_dicts = []
+    for row in auth_data_tuples:
+        dc_id, server_address, port, auth_key = row
+        auth_data_dicts.append({
+            "session_id": session_id,
+            "dc_id": dc_id,
+            "server_address": server_address,
+            "port": port,
+            "auth_key": auth_key,
+        })
+    if auth_data_dicts:
+        conn.execute(Session.insert().values(auth_data_dicts))
+
+    sent_file_tuples = c.execute("SELECT * FROM sent_files").fetchall()
+    sent_file_dicts = []
+    for row in sent_file_tuples:
+        md5_digest, file_size, type, id, hash = row
+        sent_file_dicts.append({
+            "session_id": session_id,
+            "md5_digest": md5_digest,
+            "file_size": file_size,
+            "type": type,
+            "id": id,
+            "hash": hash,
+        })
+    if sent_file_dicts:
+        conn.execute(SentFile.insert().values(sent_file_dicts))
+
+    entity_tuples = c.execute("SELECT * FROM entities").fetchall()
+    entity_dicts = []
+    for row in entity_tuples:
+        id, hash, username, phone, name = row
+        entity_dicts.append({
+            "session_id": session_id,
+            "id": id,
+            "hash": hash,
+            "username": username,
+            "phone": phone,
+            "name": name,
+        })
+    if entity_dicts:
+        conn.execute(Entity.insert().values(entity_dicts))
+
+    c.close()
+    session_conn.close()
+
+
+def downgrade():
+    op.drop_table('telethon_version')
+    op.drop_table('telethon_entities')
+    op.drop_table('telethon_sent_files')
+    op.drop_table('telethon_sessions')
@@ -0,0 +1,129 @@
+"""Move state store to main database
+
+Revision ID: 6ca3d74d51e4
+Revises: 2228d49c383f
+Create Date: 2018-06-26 21:31:26.911307
+
+"""
+from alembic import context, op
+import sqlalchemy.orm as orm
+import sqlalchemy as sa
+import json
+import re
+
+from mautrix_telegram.config import Config
+from mautrix_telegram.base import Base
+
+# revision identifiers, used by Alembic.
+revision = "6ca3d74d51e4"
+down_revision = "2228d49c383f"
+branch_labels = None
+depends_on = None
+
+
+class RoomState(Base):
+    query = None
+    __tablename__ = "mx_room_state"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    power_levels = sa.Column("power_levels", sa.Text, nullable=True)
+
+
+class UserProfile(Base):
+    query = None
+    __tablename__ = "mx_user_profile"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    user_id = sa.Column(sa.String, primary_key=True)
+    membership = sa.Column(sa.String, nullable=False, default="leave")
+    displayname = sa.Column(sa.String, nullable=True)
+    avatar_url = sa.Column(sa.String, nullable=True)
+
+
+class Puppet(Base):
+    query = None
+    __tablename__ = "puppet"
+    __table_args__ = {"extend_existing": True}
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    displayname = sa.Column(sa.String, nullable=True)
+    displayname_source = sa.Column(sa.Integer, nullable=True)
+    username = sa.Column(sa.String, nullable=True)
+    photo_id = sa.Column(sa.String, nullable=True)
+    is_bot = sa.Column(sa.Boolean, nullable=True)
+    matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False)
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+    op.create_table("mx_room_state",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("power_levels", sa.Text(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id"))
+    op.create_table("mx_user_profile",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("user_id", sa.String(), nullable=False),
+                    sa.Column("membership", sa.String(), nullable=False,
+                              default="leave"),
+                    sa.Column("displayname", sa.String(), nullable=True),
+                    sa.Column("avatar_url", sa.String(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id", "user_id"))
+
+    conn = op.get_bind()
+    session = orm.sessionmaker(bind=conn)
+    session = orm.scoping.scoped_session(session)
+    Puppet.query = session.query_property()
+
+    try:
+        with open("mx-state.json") as file:
+            data = json.load(file)
+    except FileNotFoundError:
+        return
+    if not data:
+        return
+    registrations = data.get("registrations", [])
+
+    mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+    mxtg_config = Config(mxtg_config_path, None, None)
+    mxtg_config.load()
+
+    username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}")
+    hs_domain = mxtg_config["homeserver.domain"]
+    localpart = username_template.format(userid="(.+)")
+    mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain))
+    for user in registrations:
+        match = mxid_regex.match(user)
+        if not match:
+            continue
+
+        puppet = Puppet.query.get(match.group(1))
+        if not puppet:
+            continue
+
+        puppet.matrix_registered = True
+        session.merge(puppet)
+    session.commit()
+
+    user_profiles = [UserProfile(room_id=room, user_id=user,
+                                 membership=member.get("membership", "leave"),
+                                 displayname=member.get("displayname", None),
+                                 avatar_url=member.get("avatar_url", None))
+                     for room, members in data.get("members", {}).items()
+                     for user, member in members.items()]
+    session.add_all(user_profiles)
+    session.commit()
+
+    room_state = [RoomState(room_id=room, power_levels=json.dumps(levels))
+                  for room, levels in data.get("power_levels", {}).items()]
+    session.add_all(room_state)
+    session.commit()
+
+
+def downgrade():
+    op.drop_table("mx_user_profile")
+    op.drop_table("mx_room_state")
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("matrix_registered")
@@ -0,0 +1,25 @@
+"""Add timestamp to TelegramFile
+
+Revision ID: 7d47d84380b6
+Revises: 1b241f7e8530
+Create Date: 2018-02-19 23:53:18.050871
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '7d47d84380b6'
+down_revision = '1b241f7e8530'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('telegram_file',
+                  sa.Column('timestamp', sa.BigInteger(), nullable=True, default=0,
+                            server_default="0"))
+
+
+def downgrade():
+    op.drop_column('telegram_file', 'timestamp')
@@ -1,14 +1,13 @@
 """initial revision

 Revision ID: 97d2a942bcf8
-Revises: 
+Revises:
 Create Date: 2018-02-11 18:40:55.483842

 """
 from alembic import op
 import sqlalchemy as sa

-
 # revision identifiers, used by Alembic.
 revision = '97d2a942bcf8'
 down_revision = None
@@ -17,12 +16,65 @@ depends_on = None


 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    pass
-    # ### end Alembic commands ###
+    op.create_table('portal',
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_receiver', sa.Integer),
+                    sa.Column('peer_type', sa.String, nullable=False, default=""),
+                    sa.Column('mxid', sa.String, nullable=True),
+                    sa.Column('username', sa.String, nullable=True),
+                    sa.Column('title', sa.String, nullable=True),
+                    sa.Column('about', sa.String, nullable=True),
+                    sa.Column('photo_id', sa.String, nullable=True),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_receiver'),
+                    sa.UniqueConstraint('mxid'))
+    op.create_table('user',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('tgid', sa.Integer, nullable=True, unique=True),
+                    sa.Column('tg_username', sa.String, nullable=True),
+                    sa.Column('saved_contacts', sa.Integer, nullable=False, default=0),
+                    sa.PrimaryKeyConstraint('mxid'))
+    op.create_table('puppet',
+                    sa.Column('id', sa.Integer),
+                    sa.Column('displayname', sa.String, nullable=True),
+                    sa.Column('username', sa.String, nullable=True),
+                    sa.Column('photo_id', sa.String, nullable=True),
+                    sa.PrimaryKeyConstraint('id'))
+    op.create_table('contact',
+                    sa.Column('user', sa.Integer),
+                    sa.Column('contact', sa.Integer),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
+                    sa.ForeignKeyConstraint(("contact",), ("puppet.id",)),
+                    sa.PrimaryKeyConstraint('user', 'contact'))
+    op.create_table('user_portal',
+                    sa.Column('user', sa.Integer),
+                    sa.Column('portal', sa.Integer),
+                    sa.Column('portal_receiver', sa.Integer),
+                    sa.PrimaryKeyConstraint('user', 'portal', 'portal_receiver'),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",),
+                                            name="user_portal_user_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"),
+                    sa.ForeignKeyConstraint(("portal", "portal_receiver"),
+                                            ("portal.tgid", "portal.tg_receiver"),
+                                            name="user_portal_portal_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"))
+    op.create_table('message',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    op.create_table('bot_chat',
+                    sa.Column('id', sa.Integer),
+                    sa.Column('type', sa.String, nullable=False, default=""),
+                    sa.PrimaryKeyConstraint('id'))


 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    pass
-    # ### end Alembic commands ###
+    op.drop_table('bot_chat')
+    op.drop_table('message')
+    op.drop_table('user_portal')
+    op.drop_table('contact')
+    op.drop_table('puppet')
+    op.drop_table('user')
+    op.drop_table('portal')
@@ -0,0 +1,23 @@
+"""Add displayname source fields for puppets
+
+Revision ID: bcfefa1f1299
+Revises: bdadd173ee02
+Create Date: 2018-05-19 17:00:21.078098
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'bcfefa1f1299'
+down_revision = 'bdadd173ee02'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('puppet', sa.Column('displayname_source', sa.Integer(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('puppet', 'displayname_source')
@@ -0,0 +1,43 @@
+"""Update telethon update state table
+
+Revision ID: bdadd173ee02
+Revises: eeaf0dae87ce
+Create Date: 2018-05-13 10:42:59.395597
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'bdadd173ee02'
+down_revision = 'eeaf0dae87ce'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column("id", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("hash", existing_type=sa.Integer, type_=sa.BigInteger)
+
+    with op.batch_alter_table("telethon_update_state") as batch_op:
+        batch_op.alter_column("entity_id", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("pts", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("qts", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("date", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.alter_column("seq", existing_type=sa.Integer, type_=sa.BigInteger)
+        batch_op.add_column(sa.Column("unread_count", sa.Integer))
+
+
+def downgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column("id", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("hash", existing_type=sa.BigInteger, type_=sa.Integer)
+
+    with op.batch_alter_table("telethon_update_state") as batch_op:
+        batch_op.alter_column("entity_id", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("pts", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("qts", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("date", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.alter_column("seq", existing_type=sa.BigInteger, type_=sa.Integer)
+        batch_op.drop_column("unread_count")
@@ -0,0 +1,35 @@
+"""Add metadata to TelegramFile
+
+Revision ID: cfc972368e50
+Revises: 501dad2868bc
+Create Date: 2018-03-09 16:07:01.236712
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'cfc972368e50'
+down_revision = '501dad2868bc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.add_column(sa.Column('size', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('width', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('height', sa.Integer(), nullable=True))
+        batch_op.add_column(sa.Column('thumbnail', sa.String(), nullable=True))
+        batch_op.create_foreign_key(constraint_name="fk_file_thumbnail",
+                                    referent_table="telegram_file",
+                                    local_cols=['thumbnail'],
+                                    remote_cols=['id'])
+
+
+def downgrade():
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column('size')
+        batch_op.drop_column('width')
+        batch_op.drop_column('height')
+        batch_op.drop_column('thumbnail')
@@ -0,0 +1,26 @@
+"""Add access_token and custom_mxid fields for puppets
+
+Revision ID: d5f7b8b4b456
+Revises: 6ca3d74d51e4
+Create Date: 2018-07-20 12:09:30.277960
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d5f7b8b4b456"
+down_revision = "6ca3d74d51e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
+    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("custom_mxid")
+        batch_op.drop_column("access_token")
@@ -0,0 +1,34 @@
+"""Add telethon update state table
+
+Revision ID: eeaf0dae87ce
+Revises: 1fa46383a9d3
+Create Date: 2018-04-30 17:30:59.610885
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'eeaf0dae87ce'
+down_revision = '1fa46383a9d3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column('phone', existing_type=sa.Integer, type_=sa.BigInteger)
+    op.create_table('telethon_update_state',
+                    sa.Column('session_id', sa.String, nullable=False),
+                    sa.Column('entity_id', sa.Integer, nullable=False),
+                    sa.Column('pts', sa.Integer, nullable=True),
+                    sa.Column('qts', sa.Integer, nullable=True),
+                    sa.Column('date', sa.Integer, nullable=True),
+                    sa.Column('seq', sa.Integer, nullable=True),
+                    sa.PrimaryKeyConstraint('session_id', 'entity_id'))
+
+
+def downgrade():
+    with op.batch_alter_table("telethon_entities") as batch_op:
+        batch_op.alter_column('phone', existing_type=sa.BigInteger, type_=sa.Integer)
+    op.drop_table('telethon_update_state')
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# Define functions.
+function fixperms {
+	chown -R $UID:$GID /data /opt/mautrix-telegram
+}
+
+cd /opt/mautrix-telegram
+
+# Replace database path in config.
+sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
+
+if [ -f /data/mx-state.json ]; then
+	ln -s /data/mx-state.json
+fi
+# Check that database is in the right state
+alembic -x config=/data/config.yaml upgrade head
+
+if [ ! -f /data/config.yaml ]; then
+	cp example-config.yaml /data/config.yaml
+	echo "Didn't find a config file."
+	echo "Copied default config file to /data/config.yaml"
+	echo "Modify that config file to your liking."
+	echo "Start the container again after that to generate the registration file."
+	fixperms
+	exit
+fi
+
+if [ ! -f /data/registration.yaml ]; then
+	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
+	echo "Didn't find a registration file."
+	echo "Generated one for you."
+	echo "Copy that over to synapses app service directory."
+	fixperms
+	exit
+fi
+
+fixperms
+exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml
@@ -1,27 +1,61 @@
 # Homeserver details
 homeserver:
+    # The address that this appservice can use to connect to the homeserver.
    address: https://matrix.org
+    # The domain of the homeserver (for MXIDs, etc).
    domain: matrix.org
+    # Whether or not to verify the SSL certificate of the homeserver.
+    # Only applies if address starts with https://
+    verify_ssl: true

 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
 appservice:
-    # The protocol the homeserver should use when connecting to this appservice.
-    # Usually "http" or "https".
-    protocol: http
+    # The address that the homeserver can use to connect to this appservice.
+    address: http://localhost:8080

-    # The hostname and port where the homeserver can find this appservice.
-    hostname: localhost
+    # The hostname and port where this appservice should listen.
+    hostname: 0.0.0.0
    port: 8080
+    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+    max_body_size: 1

-    # Whether or not to enable debug messages in the console.
-    debug: false
+    # The full URI to the database. SQLite and Postgres are fully supported.
+    # Other DBMSes supported by SQLAlchemy may or may not work.
+    database: sqlite:///mautrix-telegram.db
+
+    # Public part of web server for out-of-Matrix interaction with the bridge.
+    # Used for things like login if the user wants to make sure the 2FA password isn't stored in
+    # the HS database.
+    public:
+        # Whether or not the public-facing endpoints should be enabled.
+        enabled: true
+        # The prefix to use in the public-facing endpoints.
+        prefix: /public
+        # The base URL where the public-facing endpoints are available. The prefix is not added
+        # implicitly.
+        external: https://example.com/public
+
+    # Provisioning API part of the web server for automated portal creation and fetching information.
+    # Used by things like Dimension (https://dimension.t2bot.io/).
+    provisioning:
+        # Whether or not the provisioning API should be enabled.
+        enabled: true
+        # The prefix to use in the provisioning API endpoints.
+        prefix: /_matrix/provision/v1
+        # The shared secret to authorize users of the API.
+        # Set to "generate" to generate and save a new token.
+        shared_secret: generate

    # The unique ID of this appservice.
    id: telegram
    # Username of the appservice bot.
    bot_username: telegrambot
+    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+    # to leave display name/avatar as-is.
    bot_displayname: Telegram bridge bot
+    bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX

    # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
    as_token: "This value is generated when generating the registration"
@@ -58,32 +92,172 @@ bridge:
    - username
    - phone number

-    # Whether or not to use native Matrix replies. At the time of writing, only riot-web supports
-    # replies and the format of them is subject to change.
-    native_replies: True
-    # If native replies are disabled, should the custom replies contain a link to the message being
-    # replied to?
-    link_in_reply: False
    # Show message editing as a reply to the original message.
    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: False
+    edits_as_replies: false
+    # Highlight changed/added parts in edits. Requires lxml.
+    highlight_edits: false
+    # Whether or not Matrix bot messages (type m.notice) should be bridged.
+    bridge_notices: true
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    # Maximum number of members to sync per portal when starting up. Other members will be
+    # synced when they send messages. The maximum is 10000, after which the Telegram server
+    # will not send any more members.
+    # Defaults to no local limit (-> limited to 10000 by server)
+    max_initial_member_sync: -1
+    # Whether or not to sync the member list in channels.
+    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
+    # list regardless of this setting.
+    sync_channel_members: true
+    # The maximum number of simultaneous Telegram deletions to handle.
+    # A large number of simultaneous redactions could put strain on your homeserver.
+    max_telegram_delete: 10
+    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
+    # login website (see appservice.public config section)
+    allow_matrix_login: true
+    # Use inline images instead of m.image to make rich captions possible.
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    inline_images: false
+    # Whether or not to bridge plaintext highlights.
+    # Only enable this if your displayname_template has some static part that the bridge can use to
+    # reliably identify what is a plaintext highlight.
+    plaintext_highlights: false
+    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
+    public_portals: true
+    # Whether to send stickers as the new native m.sticker type or normal m.images.
+    # Old versions of Riot don't support the new type at all.
+    # Remember that proper sticker support always requires Pillow to convert webp into png.
+    native_stickers: true
+    # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
+    # WARNING: Probably buggy, might get stuck in infinite loop.
+    catch_up: false
+    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # your own Matrix account as the Matrix puppet for your Telegram account.
+    sync_with_custom_puppets: true
+
+    # The formats to use when sending messages to Telegram via the relay bot.
+    #
+    # Telegram doesn't have built-in emotes, so the m.emote format is also used for non-relaybot users.
+    #
+    # Available variables:
+    #   $sender_displayname    - The display name of the sender (e.g. Example User)
+    #   $sender_username       - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
+    #   $sender_mxid           - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $message               - The message content as HTML
+    message_formats:
+        m.text: "<b>$sender_displayname</b>: $message"
+        m.emote: "* <b>$sender_displayname</b> $message"
+        m.file: "<b>$sender_displayname</b> sent a file: $message"
+        m.image: "<b>$sender_displayname</b> sent an image: $message"
+        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "<b>$sender_displayname</b> sent a video: $message"
+        m.location: "<b>$sender_displayname</b> sent a location: $message"
+
+    # The formats to use when sending state events to Telegram via the relay bot.
+    #
+    # Variables from `message_formats` that have the `sender_` prefix are available without the prefix.
+    # In name_change events, `$prev_displayname` is the previous displayname.
+    #
+    # Set format to an empty string to disable the messages for that event.
+    state_event_formats:
+        join: "<b>$displayname</b> joined the room."
+        leave: "<b>$displayname</b> left the room."
+        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+
+    # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
+    # `filter-mode` management commands.
+    #
+    # Filters do not affect direct chats.
+    # An empty blacklist will essentially disable the filter.
+    filter:
+        # Filter mode to use. Either "blacklist" or "whitelist".
+        # If the mode is "blacklist", the listed chats will never be bridged.
+        # If the mode is "whitelist", only the listed chats can be bridged.
+        mode: blacklist
+        # The list of group/channel IDs to filter.
+        list: []

    # The prefix for commands. Only required in non-management rooms.
    command_prefix: "!tg"

-    # Whitelist of user IDs that are allowed to use this bridge. Leave empty to disable.
-    # You can enter a domain without the localpart to allow all users from that homeserver to use the bridge.
-    whitelist:
-    - "internal.example.com"
-    - "@user:public.example.com"
+    # Permissions for using the bridge.
+    # Permitted values:
+    #   relaybot - Only use the bridge via the relaybot, no access to commands.
+    #       user - Relaybot level + access to commands to create bridges.
+    #  puppeting - User level + logging in with a Telegram account.
+    #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
+    #      admin - Full access to use the bridge and some extra administration commands.
+    # Permitted keys:
+    #        * - All Matrix users
+    #   domain - All users on that homeserver
+    #     mxid - Specific user
+    permissions:
+        "*": "relaybot"
+        "public.example.com": "user"
+        "example.com": "full"
+        "@admin:example.com": "admin"

-    # Admins can do things like delete portal rooms. Here you must specify the exact MXID, domains
-    # are not accepted.
-    admins:
-    - "@admin:internal.example.com"
+    # Options related to the message relay Telegram bot.
+    relaybot:
+        # Whether or not to allow creating portals from Telegram.
+        authless_portals: true
+        # Whether or not to allow Telegram group admins to use the bot commands.
+        whitelist_group_admins: true
+        # Whether or not to ignore incoming events sent by the relay bot.
+        ignore_own_incoming_events: true
+        # List of usernames/user IDs who are also allowed to use the bot commands.
+        whitelist:
+        - myusername
+        - 12345678

 # Telegram config
 telegram:
    # Get your own API keys at https://my.telegram.org/apps
    api_id: 12345
    api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
+    # (Optional) Create your own bot at https://t.me/BotFather
+    bot_token: disabled
+    # Telethon proxy configuration.
+    # You must install PySocks from pip for proxies to work.
+    proxy:
+        # Allowed types: disabled, socks4, socks5, http
+        type: disabled
+        # Proxy IP address and port.
+        address: 127.0.0.1
+        port: 1080
+        # Whether or not to perform DNS resolving remotely.
+        rdns: true
+        # Proxy authentication (optional).
+        username: ""
+        password: ""
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        precise:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: precise
+            filename: ./mautrix-telegram.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: precise
+    loggers:
+        mau:
+            level: DEBUG
+        telethon:
+            level: DEBUG
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]
@@ -1,5 +0,0 @@
-from .appservice import AppService
-from .errors import MatrixError, MatrixRequestError, IntentError
-
-__version__ = "0.1.0"
-__author__ = "Tulir Asokan <tulir@maunium.net>"
@@ -1,180 +0,0 @@
-# -*- coding: future_fstrings -*-
-# matrix-appservice-python - A Matrix Application Service framework written in Python.
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# Partly based on github.com/Cadair/python-appservice-framework (MIT license)
-from functools import partial
-from contextlib import contextmanager
-from aiohttp import web
-import aiohttp
-import asyncio
-import logging
-
-from .intent_api import HTTPAPI
-from .state_store import StateStore
-
-
-class AppService:
-    def __init__(self, server, domain, as_token, hs_token, bot_localpart, loop=None, log=None,
-                 query_user=None, query_alias=None):
-        self.server = server
-        self.domain = domain
-        self.as_token = as_token
-        self.hs_token = hs_token
-        self.bot_mxid = f"@{bot_localpart}:{domain}"
-        self.state_store = StateStore(autosave_file="mx-state.json")
-        self.state_store.load("mx-state.json")
-
-        self.transactions = []
-
-        self._http_session = None
-        self._intent = None
-
-        self.loop = loop or asyncio.get_event_loop()
-        self.log = (logging.getLogger(log) if isinstance(log, str)
-                    else log or logging.getLogger("mautrix_appservice"))
-
-        def default_query_handler(_):
-            return None
-
-        self.query_user = query_user or default_query_handler
-        self.query_alias = query_alias or default_query_handler
-
-        self.event_handlers = []
-
-        self.app = web.Application(loop=self.loop)
-        self.app.router.add_route("PUT", "/transactions/{transaction_id}",
-                                  self._http_handle_transaction)
-        self.app.router.add_route("GET", "/rooms/{alias}", self._http_query_alias)
-        self.app.router.add_route("GET", "/users/{user_id}", self._http_query_user)
-
-        self.matrix_event_handler(self.update_state_store)
-
-    @property
-    def http_session(self):
-        if self._http_session is None:
-            raise AttributeError("the http_session attribute can only be used "
-                                 "from within the `AppService.run` context manager")
-        else:
-            return self._http_session
-
-    @property
-    def intent(self):
-        if self._intent is None:
-            raise AttributeError("the intent attribute can only be used from "
-                                 "within the `AppService.run` context manager")
-        else:
-            return self._intent
-
-    @contextmanager
-    def run(self, host="127.0.0.1", port=8080):
-        self._http_session = aiohttp.ClientSession(loop=self.loop)
-        self._intent = HTTPAPI(base_url=self.server, domain=self.domain, bot_mxid=self.bot_mxid,
-                               token=self.as_token, log=self.log, state_store=self.state_store,
-                               client_session=self._http_session).bot_intent()
-
-        yield self.loop.create_server(self.app.make_handler(), host, port)
-
-        self._intent = None
-        self._http_session.close()
-        self._http_session = None
-
-    def _check_token(self, request):
-        try:
-            token = request.rel_url.query["access_token"]
-        except KeyError:
-            return False
-
-        if token != self.hs_token:
-            return False
-
-        return True
-
-    async def _http_query_user(self, request):
-        if not self._check_token(request):
-            return web.Response(status=401)
-
-        user_id = request.match_info["userId"]
-
-        try:
-            response = await self.query_user(user_id)
-        except Exception:
-            self.log.exception("Exception in user query handler")
-            return web.Response(status=500)
-
-        if not response:
-            return web.Response(status=404)
-        return web.json_response(response)
-
-    async def _http_query_alias(self, request):
-        if not self._check_token(request):
-            return web.Response(status=401)
-
-        alias = request.match_info["alias"]
-
-        try:
-            response = await self.query_alias(alias)
-        except Exception:
-            self.log.exception("Exception in alias query handler")
-            return web.Response(status=500)
-
-        if not response:
-            return web.Response(status=404)
-        return web.json_response(response)
-
-    async def _http_handle_transaction(self, request):
-        if not self._check_token(request):
-            return web.Response(status=401)
-
-        transaction_id = request.match_info["transaction_id"]
-        if transaction_id in self.transactions:
-            return web.Response(status=200)
-
-        json = await request.json()
-
-        try:
-            events = json["events"]
-        except KeyError:
-            return web.Response(status=400)
-
-        for event in events:
-            self.handle_matrix_event(event)
-
-        self.transactions.append(transaction_id)
-
-        return web.json_response({})
-
-    async def update_state_store(self, event):
-        event_type = event["type"]
-        if event_type == "m.room.power_levels":
-            self.state_store.set_power_levels(event["room_id"], event["content"])
-        elif event_type == "m.room.member":
-            self.state_store.set_membership(event["room_id"], event["state_key"],
-                                            event["content"]["membership"])
-
-    def handle_matrix_event(self, event):
-        async def try_handle(handler):
-            try:
-                await handler(event)
-            except Exception:
-                self.log.exception("Exception in Matrix event handler")
-
-        for handler in self.event_handlers:
-            asyncio.ensure_future(try_handle(handler), loop=self.loop)
-
-    def matrix_event_handler(self, func):
-        self.event_handlers.append(func)
-        return func
@@ -1,38 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-
-class MatrixError(Exception):
-    """A generic Matrix error. Specific errors will subclass this."""
-    pass
-
-
-class IntentError(MatrixError):
-    def __init__(self, message, source):
-        super().__init__(message)
-        self.source = source
-
-
-class MatrixRequestError(MatrixError):
-    """ The home server returned an error response. """
-
-    def __init__(self, code=0, text="", errcode=None, message=None):
-        super().__init__("%d: %s" % (code, text))
-        self.code = code
-        self.text = text
-        self.errcode = errcode
-        self.message = message
@@ -1,521 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from urllib.parse import quote
-from time import time
-from json.decoder import JSONDecodeError
-from aiohttp.client_exceptions import ContentTypeError
-import re
-import json
-import magic
-import asyncio
-
-from .errors import MatrixError, MatrixRequestError, IntentError
-
-
-class HTTPAPI:
-    def __init__(self, base_url, domain=None, bot_mxid=None, token=None, identity=None, log=None,
-                 state_store=None, client_session=None, child=False):
-        self.base_url = base_url
-        self.token = token
-        self.identity = identity
-        self.validate_cert = True
-        self.session = client_session
-
-        self.domain = domain
-        self.bot_mxid = bot_mxid
-        self._bot_intent = None
-        self.state_store = state_store
-
-        if child:
-            self.log = log
-        else:
-            self.intent_log = log.getChild("intent")
-            self.log = log.getChild("api")
-            self.txn_id = 0
-            self.children = {}
-
-    def user(self, user):
-        try:
-            return self.children[user]
-        except KeyError:
-            child = ChildHTTPAPI(user, self)
-            self.children[user] = child
-            return child
-
-    def bot_intent(self):
-        if self._bot_intent:
-            return self._bot_intent
-        return IntentAPI(self.bot_mxid, self, state_store=self.state_store, log=self.intent_log)
-
-    def intent(self, user):
-        return IntentAPI(user, self.user(user), self.bot_intent(), self.state_store,
-                         self.intent_log)
-
-    async def _send(self, method, endpoint, content, query_params, headers):
-        while True:
-            query_params["access_token"] = self.token
-            request = self.session.request(method, endpoint, params=query_params,
-                                           data=content, headers=headers)
-            async with request as response:
-                if response.status < 200 or response.status >= 300:
-                    errcode = message = None
-                    try:
-                        response_data = await response.json()
-                        errcode = response_data["errcode"]
-                        message = response_data["error"]
-                    except (JSONDecodeError, ContentTypeError, KeyError):
-                        pass
-                    raise MatrixRequestError(code=response.status, text=await response.text(),
-                                             errcode=errcode, message=message)
-
-                if response.status == 429:
-                    await asyncio.sleep(response.json()["retry_after_ms"] / 1000)
-                else:
-                    return await response.json()
-
-    def _log_request(self, method, path, content, query_params):
-        log_content = content if not isinstance(content, bytes) else f"<{len(content)} bytes>"
-        log_content = log_content or "(No content)"
-        query_identity = query_params["user_id"] if "user_id" in query_params else "No identity"
-        self.log.debug("%s %s %s as user %s", method, path, log_content, query_identity)
-
-    def request(self, method, path, content=None, query_params=None, headers=None,
-                api_path="/_matrix/client/r0"):
-        content = content or {}
-        query_params = query_params or {}
-        headers = headers or {}
-
-        method = method.upper()
-        if method not in ["GET", "PUT", "DELETE", "POST"]:
-            raise MatrixError("Unsupported HTTP method: %s" % method)
-
-        if "Content-Type" not in headers:
-            headers["Content-Type"] = "application/json"
-        if headers["Content-Type"] == "application/json":
-            content = json.dumps(content)
-
-        if self.identity:
-            query_params["user_id"] = self.identity
-
-        self._log_request(method, path, content, query_params)
-
-        endpoint = self.base_url + api_path + path
-        return self._send(method, endpoint, content, query_params, headers or {})
-
-    def get_download_url(self, mxcurl):
-        if mxcurl.startswith('mxc://'):
-            return f"{self.base_url}/_matrix/media/r0/download/{mxcurl[6:]}"
-        else:
-            raise ValueError("MXC URL did not begin with 'mxc://'")
-
-    async def get_display_name(self, user_id):
-        content = await self.request("GET", f"/profile/{user_id}/displayname")
-        return content.get('displayname', None)
-
-    async def get_avatar_url(self, user_id):
-        content = await self.request("GET", f"/profile/{user_id}/avatar_url")
-        return content.get('avatar_url', None)
-
-    async def get_room_id(self, room_alias):
-        content = await self.request("GET", f"/directory/room/{quote(room_alias)}")
-        return content.get("room_id", None)
-
-    def set_typing(self, room_id, is_typing=True, timeout=5000, user=None):
-        content = {
-            "typing": is_typing
-        }
-        if is_typing:
-            content["timeout"] = timeout
-        user = user or self.identity
-        return self.request("PUT", f"/rooms/{room_id}/typing/{user}", content)
-
-
-class ChildHTTPAPI(HTTPAPI):
-    def __init__(self, user, parent):
-        super().__init__(parent.base_url, parent.domain, parent.bot_mxid, parent.token, user,
-                         parent.log, parent.state_store, parent.session, child=True)
-        self.parent = parent
-
-    @property
-    def txn_id(self):
-        return self.parent.txn_id
-
-    @txn_id.setter
-    def txn_id(self, value):
-        self.parent.txn_id = value
-
-
-class IntentAPI:
-    mxid_regex = re.compile("@(.+):(.+)")
-
-    def __init__(self, mxid, client, bot=None, state_store=None, log=None):
-        self.client = client
-        self.bot = bot
-        self.mxid = mxid
-        self.log = log
-
-        results = self.mxid_regex.search(mxid)
-        if not results:
-            raise ValueError("invalid MXID")
-        self.localpart = results.group(1)
-
-        self.state_store = state_store
-
-    def user(self, user):
-        if not self.bot:
-            return self.client.intent(user)
-        else:
-            self.log.warning("Called IntentAPI#user() of child intent object.")
-            return self.bot.client.intent(user)
-
-    # region User actions
-
-    async def get_joined_rooms(self):
-        await self.ensure_registered()
-        response = await self.client.request("GET", "/joined_rooms")
-        return response["joined_rooms"]
-
-    async def set_display_name(self, name):
-        await self.ensure_registered()
-        content = {"displayname": name}
-        return await self.client.request("PUT", f"/profile/{self.mxid}/displayname", content)
-
-    async def set_presence(self, status="online"):
-        await self.ensure_registered()
-        content = {
-            "presence": status
-        }
-        return await self.client.request("PUT", f"/presence/{self.mxid}/status", content)
-
-    async def set_avatar(self, url):
-        await self.ensure_registered()
-        content = {"avatar_url": url}
-        return await self.client.request("PUT", f"/profile/{self.mxid}/avatar_url", content)
-
-    async def upload_file(self, data, mime_type=None):
-        await self.ensure_registered()
-        mime_type = mime_type or magic.from_buffer(data, mime=True)
-        return await self.client.request("POST", "", content=data,
-                                         headers={"Content-Type": mime_type},
-                                         api_path="/_matrix/media/r0/upload")
-
-    async def download_file(self, url):
-        await self.ensure_registered()
-        url = self.client.get_download_url(url)
-        async with self.client.session.get(url) as response:
-            return await response.read()
-
-    # endregion
-    # region Room actions
-
-    async def create_room(self, alias=None, is_public=False, name=None, topic=None,
-                          is_direct=False, invitees=None, initial_state=None):
-        await self.ensure_registered()
-        content = {
-            "visibility": "public" if is_public else "private",
-            "is_direct": is_direct,
-        }
-        if alias:
-            content["room_alias_name"] = alias
-        if invitees:
-            content["invite"] = invitees
-        if name:
-            content["name"] = name
-        if topic:
-            content["topic"] = topic
-        if initial_state:
-            content["initial_state"] = initial_state
-
-        return await self.client.request("POST", "/createRoom", content)
-
-    def _invite_direct(self, room_id, user_id):
-        content = {"user_id": user_id}
-        return self.client.request("POST", "/rooms/" + room_id + "/invite", content)
-
-    async def invite(self, room_id, user_id, check_cache=False):
-        await self.ensure_joined(room_id)
-        try:
-            ok_states = {"invite", "join"}
-            do_invite = (not check_cache
-                         or self.state_store.get_membership(room_id, user_id) not in ok_states)
-            if do_invite:
-                response = await self._invite_direct(room_id, user_id)
-                self.state_store.invited(room_id, user_id)
-                return response
-        except MatrixRequestError as e:
-            if e.errcode != "M_FORBIDDEN":
-                raise IntentError(f"Failed to invite {user_id} to {room_id}", e)
-            if "is already in the room" in e.message:
-                self.state_store.joined(room_id, user_id)
-
-    def set_room_avatar(self, room_id, avatar_url, info=None):
-        content = {
-            "url": avatar_url,
-        }
-        if info:
-            content["info"] = info
-        return self.send_state_event(room_id, "m.room.avatar", content)
-
-    async def add_room_alias(self, room_id, localpart):
-        await self.ensure_registered()
-        content = {"room_id": room_id}
-        alias = f"#{localpart}:{self.client.domain}"
-        return await self.client.request("PUT", f"/directory/room/{quote(alias)}", content)
-
-    async def remove_room_alias(self, localpart):
-        await self.ensure_registered()
-        alias = f"#{localpart}:{self.client.domain}"
-        return await self.client.request("DELETE", f"/directory/room/{quote(alias)}")
-
-    def set_room_name(self, room_id, name):
-        body = {"name": name}
-        return self.send_state_event(room_id, "m.room.name", body)
-
-    async def get_power_levels(self, room_id, ignore_cache=False):
-        await self.ensure_joined(room_id)
-        if not ignore_cache:
-            try:
-                return self.state_store.get_power_levels(room_id)
-            except KeyError:
-                pass
-        levels = await self.client.request("GET",
-                                           f"/rooms/{quote(room_id)}/state/m.room.power_levels")
-        self.state_store.set_power_levels(room_id, levels)
-        return levels
-
-    async def set_power_levels(self, room_id, content):
-        if "events" not in content:
-            content["events"] = {}
-        response = await self.send_state_event(room_id, "m.room.power_levels", content)
-        self.state_store.set_power_levels(room_id, content)
-        return response
-
-    async def get_pinned_messages(self, room_id):
-        await self.ensure_joined(room_id)
-        response = await self.client.request("GET", f"/rooms/{room_id}/state/m.room.pinned_events")
-        return response["content"]["pinned"]
-
-    def set_pinned_messages(self, room_id, events):
-        return self.send_state_event(room_id, "m.room.pinned_events", {
-            "pinned": events
-        })
-
-    async def pin_message(self, room_id, event_id):
-        events = await self.get_pinned_messages(room_id)
-        if event_id not in events:
-            events.append(event_id)
-            await self.set_pinned_messages(room_id, events)
-
-    async def unpin_message(self, room_id, event_id):
-        events = await self.get_pinned_messages(room_id)
-        if event_id in events:
-            events.remove(event_id)
-            await self.set_pinned_messages(room_id, events)
-
-    async def get_event(self, room_id, event_id):
-        await self.ensure_joined(room_id)
-        return await self.client.request("GET", f"/rooms/{room_id}/event/{event_id}")
-
-    async def set_typing(self, room_id, is_typing=True, timeout=5000):
-        await self.ensure_joined(room_id)
-        content = {
-            "typing": is_typing
-        }
-        if is_typing:
-            content["timeout"] = timeout
-        return await self.client.request("PUT", f"/rooms/{room_id}/typing/{self.mxid}", content)
-
-    async def mark_read(self, room_id, event_id):
-        await self.ensure_joined(room_id)
-        return await self.client.request("POST", f"/rooms/{room_id}/receipt/m.read/{event_id}",
-                                         content={})
-
-    def send_notice(self, room_id, text, html=None, relates_to=None):
-        return self.send_text(room_id, text, html, "m.notice", relates_to)
-
-    def send_emote(self, room_id, text, html=None, relates_to=None):
-        return self.send_text(room_id, text, html, "m.emote", relates_to)
-
-    def send_image(self, room_id, url, info=None, text=None, relates_to=None):
-        return self.send_file(room_id, url, info or {}, text, "m.image", relates_to)
-
-    def send_file(self, room_id, url, info=None, text=None, file_type="m.file", relates_to=None):
-        return self.send_message(room_id, {
-            "msgtype": file_type,
-            "url": url,
-            "body": text or "Uploaded file",
-            "info": info or {},
-            "m.relates_to": relates_to or None,
-        })
-
-    def send_text(self, room_id, text, html=None, msgtype="m.text", relates_to=None):
-        if html:
-            if not text:
-                text = html
-            return self.send_message(room_id, {
-                "body": text,
-                "msgtype": msgtype,
-                "format": "org.matrix.custom.html",
-                "formatted_body": html or text,
-                "m.relates_to": relates_to or None,
-            })
-        else:
-            return self.send_message(room_id, {
-                "body": text,
-                "msgtype": msgtype,
-                "m.relates_to": relates_to or None,
-            })
-
-    def send_message(self, room_id, body):
-        return self.send_event(room_id, "m.room.message", body)
-
-    async def error_and_leave(self, room_id, text, html=None):
-        await self.ensure_joined(room_id)
-        await self.send_notice(room_id, text, html=html)
-        await self.leave_room(room_id)
-
-    def kick(self, room_id, user_id, message):
-        return self.set_membership(room_id, user_id, "leave", message)
-
-    def get_membership(self, room_id, user_id):
-        return self.get_state_event(room_id, "m.room.member", state_key=user_id)
-
-    def set_membership(self, room_id, user_id, membership, reason="", profile=None):
-        body = {
-            "membership": membership,
-            "reason": reason
-        }
-        profile = profile or {}
-        if "displayname" in profile:
-            body["displayname"] = profile["displayname"]
-        if "avatar_url" in profile:
-            body["avatar_url"] = profile["avatar_url"]
-
-        return self.send_state_event(room_id, "m.room.member", body, state_key=user_id)
-
-    @staticmethod
-    def _get_event_url(room_id, event_type, txn_id):
-        return f"/rooms/{quote(room_id)}/send/{quote(event_type)}/{quote(txn_id)}"
-
-    async def send_event(self, room_id, event_type, content, txn_id=None):
-        await self.ensure_joined(room_id)
-        await self._ensure_has_power_level_for(room_id, event_type)
-
-        txn_id = txn_id or str(self.client.txn_id) + str(int(time() * 1000))
-        self.client.txn_id += 1
-
-        url = self._get_event_url(room_id, event_type, txn_id)
-
-        return await self.client.request("PUT", url, content)
-
-    @staticmethod
-    def _get_state_url(room_id, event_type, state_key=""):
-        url = f"/rooms/{quote(room_id)}/state/{quote(event_type)}"
-        if state_key:
-            url += f"/{quote(state_key)}"
-        return url
-
-    async def send_state_event(self, room_id, event_type, content, state_key=""):
-        await self.ensure_joined(room_id)
-        await self._ensure_has_power_level_for(room_id, event_type)
-        url = self._get_state_url(room_id, event_type, state_key)
-        return await self.client.request("PUT", url, content)
-
-    async def get_state_event(self, room_id, event_type, state_key=""):
-        await self.ensure_joined(room_id)
-        url = self._get_state_url(room_id, event_type, state_key)
-        return await self.client.request("GET", url)
-
-    def join_room(self, room_id):
-        return self.ensure_joined(room_id, ignore_cache=True)
-
-    def _join_room_direct(self, room):
-        return self.client.request("POST", f"/join/{quote(room)}")
-
-    def leave_room(self, room_id):
-        try:
-            self.state_store.left(room_id, self.mxid)
-            return self.client.request("POST", f"/rooms/{quote(room_id)}/leave")
-        except MatrixRequestError as e:
-            if "not in room" not in e.message:
-                raise
-
-    def get_room_memberships(self, room_id):
-        return self.client.request("GET", f"/rooms/{quote(room_id)}/members")
-
-    async def get_room_members(self, room_id, allowed_memberships=("join",)):
-        memberships = await self.get_room_memberships(room_id)
-        return [membership["state_key"] for membership in memberships["chunk"] if
-                membership["content"]["membership"] in allowed_memberships]
-
-    async def get_room_state(self, room_id):
-        await self.ensure_joined(room_id)
-        state = await self.client.request("GET", f"/rooms/{quote(room_id)}/state")
-        # TODO update values based on state?
-        return state
-
-    # endregion
-    # region Ensure functions
-
-    async def ensure_joined(self, room_id, ignore_cache=False):
-        if not ignore_cache and self.state_store.is_joined(room_id, self.mxid):
-            return
-        await self.ensure_registered()
-        try:
-            await self._join_room_direct(room_id)
-            self.state_store.joined(room_id, self.mxid)
-        except MatrixRequestError as e:
-            if e.errcode != "M_FORBIDDEN" or not self.bot:
-                raise IntentError(f"Failed to join room {room_id} as {self.mxid}", e)
-            try:
-                await self.bot.invite(room_id, self.mxid)
-                await self._join_room_direct(room_id)
-                self.state_store.joined(room_id, self.mxid)
-            except MatrixRequestError as e2:
-                raise IntentError(f"Failed to join room {room_id} as {self.mxid}", e2)
-
-    def _register(self):
-        content = {"username": self.localpart}
-        query_params = {"kind": "user"}
-        return self.client.request("POST", "/register", content, query_params)
-
-    async def ensure_registered(self):
-        if self.state_store.is_registered(self.mxid):
-            return
-        try:
-            await self._register()
-        except MatrixRequestError as e:
-            if e.errcode != "M_USER_IN_USE":
-                self.log.exception(f"Failed to register {self.mxid}!")
-                # raise IntentError(f"Failed to register {self.mxid}", e)
-                return
-        self.state_store.registered(self.mxid)
-
-    async def _ensure_has_power_level_for(self, room_id, event_type):
-        if not self.state_store.has_power_levels(room_id):
-            await self.get_power_levels(room_id)
-        if self.state_store.has_power_level(room_id, self.mxid, event_type):
-            return
-        elif not self.bot:
-            self.log.warning(
-                f"Power level of {self.mxid} is not enough for {event_type} in {room_id}")
-            # raise IntentError(f"Power level of {self.mxid} is not enough"
-            #                  + f"for {event_type} in {room_id}")
-            return
-        # TODO implement
-
-    # endregion
@@ -1,123 +0,0 @@
-# -*- coding: future_fstrings -*-
-# matrix-appservice-python - A Matrix Application Service framework written in Python.
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-import json
-
-
-class StateStore:
-    def __init__(self, autosave_file=None):
-        self.registrations = set()
-        self.memberships = {}
-        self.power_levels = {}
-        self.autosave_file = autosave_file
-
-    def save(self, file):
-        if isinstance(file, str):
-            output = open(file, "w")
-        else:
-            output = file
-
-        json.dump({
-            "registrations": list(self.registrations),
-            "memberships": self.memberships,
-            "power_levels": self.power_levels,
-        }, output)
-
-        if isinstance(file, str):
-            output.close()
-
-    def load(self, file):
-        if isinstance(file, str):
-            try:
-                input_source = open(file, "r")
-            except FileNotFoundError:
-                return
-        else:
-            input_source = file
-
-        data = json.load(input_source)
-        if "registrations" in data:
-            self.registrations = set(data["registrations"])
-        if "memberships" in data:
-            self.memberships = data["memberships"]
-        if "power_levels" in data:
-            self.power_levels = data["power_levels"]
-
-        if isinstance(file, str):
-            input_source.close()
-
-    def _autosave(self):
-        if self.autosave_file:
-            self.save(self.autosave_file)
-
-    def is_registered(self, user):
-        return user in self.registrations
-
-    def registered(self, user):
-        self.registrations.add(user)
-        self._autosave()
-
-    def get_membership(self, room, user):
-        return self.memberships.get(room, {}).get(user, "left")
-
-    def is_joined(self, room, user):
-        return self.get_membership(room, user) == "join"
-
-    def set_membership(self, room, user, membership):
-        if room not in self.memberships:
-            self.memberships[room] = {}
-        self.memberships[room][user] = membership
-        self._autosave()
-
-    def joined(self, room, user):
-        return self.set_membership(room, user, "join")
-
-    def invited(self, room, user):
-        return self.set_membership(room, user, "invite")
-
-    def left(self, room, user):
-        return self.set_membership(room, user, "left")
-
-    def has_power_levels(self, room):
-        return room in self.power_levels
-
-    def get_power_levels(self, room):
-        return self.power_levels[room]
-
-    def has_power_level(self, room, user, event):
-        room_levels = self.power_levels.get(room, {})
-        required = room_levels.get("events", {}).get(event, 95)
-        has = room_levels.get("users", {}).get(user, 0)
-        return has >= required
-
-    def set_power_level(self, room, user, level):
-        if room not in self.power_levels:
-            self.power_levels[room] = {
-                "users": {},
-                "events": {},
-            }
-        elif "users" not in self.power_levels[room]:
-            self.power_levels[room]["users"] = {}
-        self.power_levels[room]["users"][user] = level
-        self._autosave()
-
-    def set_power_levels(self, room, content):
-        if "events" not in content:
-            content["events"] = {}
-        if "users" not in content:
-            content["users"] = {}
-        self.power_levels[room] = content
-        self._autosave()
@@ -1,2 +1,2 @@
-__version__ = "0.1.1"
+__version__ = "0.3.0rc2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"
@@ -3,55 +3,62 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional
 import argparse
-import sys
-import logging
 import asyncio
+import logging.config
+import sys

-import sqlalchemy as sql
 from sqlalchemy import orm
+import sqlalchemy as sql

 from mautrix_appservice import AppService
+from alchemysession import AlchemySessionContainer

+from .web.provisioning import ProvisioningAPI
+from .web.public import PublicBridgeWebsite
+from .abstract_user import init as init_abstract_user
 from .base import Base
+from .bot import init as init_bot
 from .config import Config
-from .matrix import MatrixHandler
-
+from .context import Context
 from .db import init as init_db
-from .user import init as init_user, User
+from .formatter import init as init_formatter
+from .matrix import MatrixHandler
 from .portal import init as init_portal
 from .puppet import init as init_puppet
-
-log = logging.getLogger("mau")
-time_formatter = logging.Formatter("[%(asctime)s] [%(levelname)s@%(name)s] %(message)s")
-handler = logging.StreamHandler()
-handler.setFormatter(time_formatter)
-log.addHandler(handler)
+from .sqlstatestore import SQLStateStore
+from .user import User, init as init_user
+from . import __version__

 parser = argparse.ArgumentParser(
    description="A Matrix-Telegram puppeting bridge.",
    prog="python -m mautrix-telegram")
 parser.add_argument("-c", "--config", type=str, default="config.yaml",
                    metavar="<path>", help="the path to your config file")
+parser.add_argument("-b", "--base-config", type=str, default="example-config.yaml",
+                    metavar="<path>", help="the path to the example config "
+                                           "(for automatic config updates)")
 parser.add_argument("-g", "--generate-registration", action="store_true",
                    help="generate registration and quit")
 parser.add_argument("-r", "--registration", type=str, default="registration.yaml",
                    metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()

-config = Config(args.config, args.registration)
+config = Config(args.config, args.registration, args.base_config)
 config.load()
+config.update()

 if args.generate_registration:
    config.generate_registration()
@@ -59,35 +66,68 @@ if args.generate_registration:
    print(f"Registration generated and saved to {config.registration_path}")
    sys.exit(0)

-if config["appservice.debug"]:
-    telethon_log = logging.getLogger("telethon")
-    telethon_log.addHandler(handler)
-    telethon_log.setLevel(logging.DEBUG)
-    log.setLevel(logging.DEBUG)
-    log.debug("Debug messages enabled.")
+logging.config.dictConfig(config["logging"])
+log = logging.getLogger("mau.init")  # type: logging.Logger
+log.debug(f"Initializing mautrix-telegram {__version__}")

-db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
 db_factory = orm.sessionmaker(bind=db_engine)
 db_session = orm.scoping.scoped_session(db_factory)
 Base.metadata.bind = db_engine
-Base.metadata.create_all()

-loop = asyncio.get_event_loop()
+session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
+                                            table_base=Base, table_prefix="telethon_",
+                                            manage_tables=False)
+
+loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
+
+state_store = SQLStateStore(db_session)
+mebibyte = 1024 ** 2
 appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                     config["appservice.as_token"], config["appservice.hs_token"],
-                     config["appservice.bot_username"], log="mau.as", loop=loop)
-context = (appserv, db_session, config, loop)
+                     config["appservice.bot_username"], log="mau.as", loop=loop,
+                     verify_ssl=config["homeserver.verify_ssl"], state_store=state_store,
+                     real_user_content_key="net.maunium.telegram.puppet",
+                     aiohttp_params={
+                         "client_max_size": config["appservice.max_body_size"] * mebibyte
+                     })
+
+context = Context(appserv, db_session, config, loop, session_container)
+
+if config["appservice.public.enabled"]:
+    public_website = PublicBridgeWebsite(loop)
+    appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public_website.app)
+    context.public_website = public_website
+
+if config["appservice.provisioning.enabled"]:
+    provisioning_api = ProvisioningAPI(context)
+    appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning",
+                           provisioning_api.app)
+    context.provisioning_api = provisioning_api

 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
-    MatrixHandler(context)
    init_db(db_session)
+    init_abstract_user(context)
+    context.bot = init_bot(context)
+    context.mx = MatrixHandler(context)
+    init_formatter(context)
    init_portal(context)
-    init_puppet(context)
-    startup_actions = init_user(context) + [start]
+    startup_actions = (init_puppet(context) +
+                       init_user(context) +
+                       [start,
+                        context.mx.init_as_bot()])
+
+    if context.bot:
+        startup_actions.append(context.bot.start())
+
    try:
+        log.debug("Initialization complete, running startup actions")
        loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
+        log.debug("Startup actions complete, now running forever")
        loop.run_forever()
    except KeyboardInterrupt:
-        for user in User.by_tgid.values():
-            user.stop()
+        log.debug("Keyboard interrupt received, stopping clients")
+        loop.run_until_complete(
+            asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
+        log.debug("Clients stopped, shutting down")
        sys.exit(0)
@@ -0,0 +1,377 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Optional, List, Union, TYPE_CHECKING
+from abc import ABC, abstractmethod
+import asyncio
+import logging
+import platform
+
+from sqlalchemy import orm
+from telethon.tl.types import Channel, ChannelForbidden, Chat, ChatForbidden, Message, \
+    MessageActionChannelMigrateFrom, MessageService, PeerUser, TypeUpdate, \
+    UpdateChannelPinnedMessage, UpdateChatAdmins, UpdateChatParticipantAdmin, \
+    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, \
+    UpdateDeleteMessages, UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, \
+    UpdateNewMessage, UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, \
+    UpdateUserName, UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, \
+    UserStatusOnline
+
+from mautrix_appservice import MatrixRequestError, AppService
+from alchemysession import AlchemySessionContainer
+
+from . import portal as po, puppet as pu, __version__
+from .db import Message as DBMessage
+from .tgclient import MautrixTelegramClient
+
+if TYPE_CHECKING:
+    from .context import Context
+    from .config import Config
+    from .bot import Bot
+
+config = None  # type: Config
+# Value updated from config in init()
+MAX_DELETIONS = 10  # type: int
+
+UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                      UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
+UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
+
+
+class AbstractUser(ABC):
+    session_container = None  # type: AlchemySessionContainer
+    loop = None  # type: asyncio.AbstractEventLoop
+    log = None  # type: logging.Logger
+    db = None  # type: orm.Session
+    az = None  # type: AppService
+    bot = None  # type: Bot
+    ignore_incoming_bot_events = True  # type: bool
+
+    def __init__(self):
+        self.is_admin = False  # type: bool
+        self.matrix_puppet_whitelisted = False  # type: bool
+        self.puppet_whitelisted = False  # type: bool
+        self.whitelisted = False  # type: bool
+        self.relaybot_whitelisted = False  # type: bool
+        self.client = None  # type: MautrixTelegramClient
+        self.tgid = None  # type: int
+        self.mxid = None  # type: str
+        self.is_relaybot = False  # type: bool
+        self.is_bot = False  # type: bool
+
+    @property
+    def connected(self) -> bool:
+        return self.client and self.client.is_connected()
+
+    @property
+    def _proxy_settings(self) -> Optional[Tuple[int, str, str, str, str, str]]:
+        proxy_type = config["telegram.proxy.type"].lower()
+        if proxy_type == "disabled":
+            return None
+        elif proxy_type == "socks4":
+            proxy_type = 1
+        elif proxy_type == "socks5":
+            proxy_type = 2
+        elif proxy_type == "http":
+            proxy_type = 3
+
+        return (proxy_type,
+                config["telegram.proxy.address"], config["telegram.proxy.port"],
+                config["telegram.proxy.rdns"],
+                config["telegram.proxy.username"], config["telegram.proxy.password"])
+
+    def _init_client(self):
+        self.log.debug(f"Initializing client for {self.name}")
+        device = f"{platform.system()} {platform.release()}"
+        sysversion = MautrixTelegramClient.__version__
+        self.session = self.session_container.new_session(self.name)
+        self.client = MautrixTelegramClient(session=self.session,
+                                            api_id=config["telegram.api_id"],
+                                            api_hash=config["telegram.api_hash"],
+                                            loop=self.loop,
+                                            app_version=__version__,
+                                            system_version=sysversion,
+                                            device_model=device,
+                                            timeout=120,
+                                            proxy=self._proxy_settings)
+        self.client.add_event_handler(self._update_catch)
+
+    @abstractmethod
+    async def update(self, update: TypeUpdate) -> bool:
+        return False
+
+    @abstractmethod
+    async def post_login(self):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def register_portal(self, portal: po.Portal):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def unregister_portal(self, portal: po.Portal):
+        raise NotImplementedError()
+
+    async def _update_catch(self, update: TypeUpdate):
+        try:
+            if not await self.update(update):
+                await self._update(update)
+        except Exception:
+            self.log.exception("Failed to handle Telegram update")
+
+    async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
+        if self.is_bot:
+            return []
+        dialogs = await self.client.get_dialogs(limit=limit)
+        return [dialog.entity for dialog in dialogs if (
+            not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden))
+            and not (isinstance(dialog.entity, Chat)
+                     and (dialog.entity.deactivated or dialog.entity.left)))]
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        raise NotImplementedError()
+
+    async def is_logged_in(self) -> bool:
+        return self.client and await self.client.is_user_authorized()
+
+    async def has_full_access(self, allow_bot: bool = False) -> bool:
+        return (self.puppet_whitelisted
+                and (not self.is_bot or allow_bot)
+                and await self.is_logged_in())
+
+    async def start(self, delete_unless_authenticated: bool = False) -> "AbstractUser":
+        if not self.client:
+            self._init_client()
+        await self.client.connect()
+        self.log.debug("%s connected: %s", self.mxid, self.connected)
+        return self
+
+    async def ensure_started(self, even_if_no_session=False) -> "AbstractUser":
+        if not self.puppet_whitelisted:
+            return self
+        self.log.debug("ensure_started(%s, connected=%s, even_if_no_session=%s, session_count=%s)",
+                       self.mxid, self.connected, even_if_no_session,
+                       self.session_container.Session.query.filter(
+                           self.session_container.Session.session_id == self.mxid).count())
+        should_connect = (even_if_no_session or
+                          self.session_container.Session.query.filter(
+                              self.session_container.Session.session_id == self.mxid).count() > 0)
+        if not self.connected and should_connect:
+            await self.start(delete_unless_authenticated=not even_if_no_session)
+        return self
+
+    async def stop(self):
+        await self.client.disconnect()
+        self.client = None
+
+    # region Telegram update handling
+
+    async def _update(self, update: TypeUpdate):
+        if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                               UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
+            await self.update_message(update)
+        elif isinstance(update, UpdateDeleteMessages):
+            await self.delete_message(update)
+        elif isinstance(update, UpdateDeleteChannelMessages):
+            await self.delete_channel_message(update)
+        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
+            await self.update_typing(update)
+        elif isinstance(update, UpdateUserStatus):
+            await self.update_status(update)
+        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
+            await self.update_admin(update)
+        elif isinstance(update, UpdateChatParticipants):
+            await self.update_participants(update)
+        elif isinstance(update, UpdateChannelPinnedMessage):
+            await self.update_pinned_messages(update)
+        elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
+            await self.update_others_info(update)
+        elif isinstance(update, UpdateReadHistoryOutbox):
+            await self.update_read_receipt(update)
+        else:
+            self.log.debug("Unhandled update: %s", update)
+
+    @staticmethod
+    async def update_pinned_messages(update: UpdateChannelPinnedMessage):
+        portal = po.Portal.get_by_tgid(update.channel_id)
+        if portal and portal.mxid:
+            await portal.receive_telegram_pin_id(update.id)
+
+    @staticmethod
+    async def update_participants(update: UpdateChatParticipants):
+        portal = po.Portal.get_by_tgid(update.participants.chat_id)
+        if portal and portal.mxid:
+            await portal.update_telegram_participants(update.participants.participants)
+
+    async def update_read_receipt(self, update: UpdateReadHistoryOutbox):
+        if not isinstance(update.peer, PeerUser):
+            self.log.debug("Unexpected read receipt peer: %s", update.peer)
+            return
+
+        portal = po.Portal.get_by_tgid(update.peer.user_id, self.tgid)
+        if not portal or not portal.mxid:
+            return
+
+        # We check that these are user read receipts, so tg_space is always the user ID.
+        message = DBMessage.query.get((update.max_id, self.tgid))
+        if not message:
+            return
+
+        puppet = pu.Puppet.get(update.peer.user_id)
+        await puppet.intent.mark_read(portal.mxid, message.mxid)
+
+    async def update_admin(self, update: Union[UpdateChatAdmins, UpdateChatParticipantAdmin]):
+        # TODO duplication not checked
+        portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+        if isinstance(update, UpdateChatAdmins):
+            await portal.set_telegram_admins_enabled(update.enabled)
+        elif isinstance(update, UpdateChatParticipantAdmin):
+            await portal.set_telegram_admin(update.user_id)
+        else:
+            self.log.warning("Unexpected admin status update: %s", update)
+
+    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]):
+        if isinstance(update, UpdateUserTyping):
+            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+        else:
+            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+        sender = pu.Puppet.get(update.user_id)
+        await portal.handle_telegram_typing(sender, update)
+
+    async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]):
+        # TODO duplication not checked
+        puppet = pu.Puppet.get(update.user_id)
+        if isinstance(update, UpdateUserName):
+            if await puppet.update_displayname(self, update):
+                puppet.save()
+        elif isinstance(update, UpdateUserPhoto):
+            if await puppet.update_avatar(self, update.photo.photo_big):
+                puppet.save()
+        else:
+            self.log.warning("Unexpected other user info update: %s", update)
+
+    async def update_status(self, update: UpdateUserStatus):
+        puppet = pu.Puppet.get(update.user_id)
+        if isinstance(update.status, UserStatusOnline):
+            await puppet.default_mxid_intent.set_presence("online")
+        elif isinstance(update.status, UserStatusOffline):
+            await puppet.default_mxid_intent.set_presence("offline")
+        else:
+            self.log.warning("Unexpected user status update: %s", update)
+        return
+
+    def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
+                                                                  Optional[pu.Puppet],
+                                                                  Optional[po.Portal]]:
+        if isinstance(update, UpdateShortChatMessage):
+            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+            sender = pu.Puppet.get(update.from_id)
+        elif isinstance(update, UpdateShortMessage):
+            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+            sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
+        elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
+                                 UpdateEditMessage, UpdateEditChannelMessage)):
+            update = update.message
+            if isinstance(update.to_id, PeerUser) and not update.out:
+                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
+            else:
+                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
+            sender = pu.Puppet.get(update.from_id) if update.from_id else None
+        else:
+            self.log.warning(
+                f"Unexpected message type in User#get_message_details: {type(update)}")
+            return update, None, None
+        return update, sender, portal
+
+    @staticmethod
+    async def _try_redact(portal: po.Portal, message: DBMessage):
+        if not portal:
+            return
+        try:
+            await portal.main_intent.redact(message.mx_room, message.mxid)
+        except MatrixRequestError:
+            pass
+
+    async def delete_message(self, update: UpdateDeleteMessages):
+        if len(update.messages) > MAX_DELETIONS:
+            return
+
+        for message in update.messages:
+            message = DBMessage.query.get((message, self.tgid))
+            if not message:
+                continue
+            self.db.delete(message)
+            number_left = DBMessage.query.filter(DBMessage.mxid == message.mxid,
+                                                 DBMessage.mx_room == message.mx_room).count()
+            if number_left == 0:
+                portal = po.Portal.get_by_mxid(message.mx_room)
+                await self._try_redact(portal, message)
+        self.db.commit()
+
+    async def delete_channel_message(self, update: UpdateDeleteChannelMessages):
+        if len(update.messages) > MAX_DELETIONS:
+            return
+
+        portal = po.Portal.get_by_tgid(update.channel_id)
+        if not portal:
+            return
+
+        for message in update.messages:
+            message = DBMessage.query.get((message, portal.tgid))
+            if not message:
+                continue
+            self.db.delete(message)
+            await self._try_redact(portal, message)
+        self.db.commit()
+
+    async def update_message(self, original_update: UpdateMessage):
+        update, sender, portal = self.get_message_details(original_update)
+        if self.ignore_incoming_bot_events and self.bot and sender.id == self.bot.tgid:
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
+            return
+
+        if isinstance(update, MessageService):
+            if isinstance(update.action, MessageActionChannelMigrateFrom):
+                self.log.debug(f"Ignoring action %s to %s by %d", update.action,
+                               portal.tgid_log,
+                               sender.id)
+                return
+            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
+                           sender.id)
+            return await portal.handle_telegram_action(self, sender, update)
+
+        user = sender.tgid if sender else "admin"
+        if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
+            if config["bridge.edits_as_replies"]:
+                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
+                return await portal.handle_telegram_edit(self, sender, update)
+            return
+
+        self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
+        return await portal.handle_telegram_message(self, sender, update)
+
+    # endregion
+
+
+def init(context: "Context"):
+    global config, MAX_DELETIONS
+    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, AbstractUser.relaybot = context
+    AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
+    AbstractUser.session_container = context.session_container
+    MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)
@@ -1,2 +1,2 @@
 from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()
+Base = declarative_base()  # type: declarative_base
@@ -0,0 +1,275 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Callable, Pattern, Dict, TYPE_CHECKING
+import logging
+import re
+
+from telethon.tl.types import *
+from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
+from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
+from telethon.errors import ChannelInvalidError, ChannelPrivateError
+
+from .abstract_user import AbstractUser
+from .db import BotChat
+from . import puppet as pu, portal as po, user as u
+
+if TYPE_CHECKING:
+    from .config import Config
+
+config = None  # type: Config
+
+ReplyFunc = Callable[[str], Awaitable[Message]]
+
+
+class Bot(AbstractUser):
+    log = logging.getLogger("mau.bot")  # type: logging.Logger
+    mxid_regex = re.compile("@.+:.+")  # type: Pattern
+
+    def __init__(self, token: str):
+        super().__init__()
+        self.token = token  # type: str
+        self.puppet_whitelisted = True  # type: bool
+        self.whitelisted = True  # type: bool
+        self.relaybot_whitelisted = True  # type: bool
+        self.username = None  # type: str
+        self.is_relaybot = True  # type: bool
+        self.is_bot = True  # type: bool
+        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}  # type: Dict[int, str]
+        self.tg_whitelist = []  # type: List[int]
+        self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
+                                       or False)  # type: bool
+
+    async def init_permissions(self):
+        whitelist = config["bridge.relaybot.whitelist"] or []
+        for id in whitelist:
+            if isinstance(id, str):
+                entity = await self.client.get_input_entity(id)
+                if isinstance(entity, InputUser):
+                    id = entity.user_id
+                else:
+                    id = None
+            if isinstance(id, int):
+                self.tg_whitelist.append(id)
+
+    async def start(self, delete_unless_authenticated: bool = False) -> "Bot":
+        await super().start(delete_unless_authenticated)
+        if not await self.is_logged_in():
+            await self.client.sign_in(bot_token=self.token)
+        await self.post_login()
+        return self
+
+    async def post_login(self):
+        await self.init_permissions()
+        info = await self.client.get_me()
+        self.tgid = info.id
+        self.username = info.username
+        self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
+
+        chat_ids = [id for id, type in self.chats.items() if type == "chat"]
+        response = await self.client(GetChatsRequest(chat_ids))
+        for chat in response.chats:
+            if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
+                self.remove_chat(chat.id)
+
+        channel_ids = [InputChannel(id, 0)
+                       for id, type in self.chats.items()
+                       if type == "channel"]
+        for id in channel_ids:
+            try:
+                await self.client(GetChannelsRequest([id]))
+            except (ChannelPrivateError, ChannelInvalidError):
+                self.remove_chat(id.channel_id)
+
+        if config["bridge.catch_up"]:
+            try:
+                await self.client.catch_up()
+            except Exception:
+                self.log.exception("Failed to run catch_up() for bot")
+
+    def register_portal(self, portal: po.Portal):
+        self.add_chat(portal.tgid, portal.peer_type)
+
+    def unregister_portal(self, portal: po.Portal):
+        self.remove_chat(portal.tgid)
+
+    def add_chat(self, id: int, type: str):
+        if id not in self.chats:
+            self.chats[id] = type
+            self.db.add(BotChat(id=id, type=type))
+            self.db.commit()
+
+    def remove_chat(self, id: int):
+        try:
+            del self.chats[id]
+        except KeyError:
+            pass
+        existing_chat = BotChat.query.get(id)
+        if existing_chat:
+            self.db.delete(existing_chat)
+            self.db.commit()
+
+    async def _can_use_commands(self, chat: TypePeer, tgid: int) -> bool:
+        if tgid in self.tg_whitelist:
+            return True
+
+        user = u.User.get_by_tgid(tgid)
+        if user and user.is_admin:
+            self.tg_whitelist.append(user.tgid)
+            return True
+
+        if self.whitelist_group_admins:
+            if isinstance(chat, PeerChannel):
+                p = await self.client(GetParticipantRequest(chat, tgid))
+                return isinstance(p, (ChannelParticipantCreator, ChannelParticipantAdmin))
+            elif isinstance(chat, PeerChat):
+                chat = await self.client(GetFullChatRequest(chat.chat_id))
+                participants = chat.full_chat.participants.participants
+                for p in participants:
+                    if p.user_id == tgid:
+                        return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+
+    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
+        if not await self._can_use_commands(event.to_id, event.from_id):
+            await reply("You do not have the permission to use that command.")
+            return False
+        return True
+
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc):
+        if not config["bridge.relaybot.authless_portals"]:
+            return await reply("This bridge doesn't allow portal creation from Telegram.")
+
+        if not portal.allow_bridging():
+            return await reply("This bridge doesn't allow bridging this chat.")
+
+        await portal.create_matrix_room(self)
+        if portal.mxid:
+            if portal.username:
+                return await reply(
+                    f"Portal is public: [{portal.alias}](https://matrix.to/#/{portal.alias})")
+            else:
+                return await reply(
+                    "Portal is not public. Use `/invite <mxid>` to get an invite.")
+
+    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc, mxid: str):
+        if len(mxid) == 0:
+            return await reply("Usage: `/invite <mxid>`")
+        elif not portal.mxid:
+            return await reply("Portal does not have Matrix room. "
+                               "Create one with /portal first.")
+        if not self.mxid_regex.match(mxid):
+            return await reply("That doesn't look like a Matrix ID.")
+        user = await u.User.get_by_mxid(mxid).ensure_started()
+        if not user.relaybot_whitelisted:
+            return await reply("That user is not whitelisted to use the bridge.")
+        elif await user.is_logged_in():
+            displayname = f"@{user.username}" if user.username else user.displayname
+            return await reply("That user seems to be logged in. "
+                               f"Just invite [{displayname}](tg://user?id={user.tgid})")
+        else:
+            await portal.main_intent.invite(portal.mxid, user.mxid)
+            return await reply(f"Invited `{user.mxid}` to the portal.")
+
+    def handle_command_id(self, message: Message, reply: ReplyFunc):
+        # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
+        # chat is a normal group or a supergroup/channel when using the ID.
+        if isinstance(message.to_id, PeerChannel):
+            return reply(f"-100{message.to_id.channel_id}")
+        return reply(str(-message.to_id.chat_id))
+
+    def match_command(self, text: str, command: str) -> bool:
+        text = text.lower()
+        command = f"/{command.lower()}"
+        command_targeted = f"{command}@{self.username.lower()}"
+
+        is_plain_command = text == command or text == command_targeted
+        if is_plain_command:
+            return True
+
+        is_arg_command = text.startswith(command + " ") or text.startswith(command_targeted + " ")
+        if is_arg_command:
+            return True
+
+        return False
+
+    async def handle_command(self, message: Message):
+        def reply(reply_text):
+            return self.client.send_message(message.to_id, reply_text, reply_to=message.id)
+
+        text = message.message
+
+        if self.match_command(text, "id"):
+            return await self.handle_command_id(message, reply)
+
+        portal = po.Portal.get_by_entity(message.to_id)
+
+        if self.match_command(text, "portal"):
+            if not await self.check_can_use_commands(message, reply):
+                return
+            await self.handle_command_portal(portal, reply)
+        elif self.match_command(text, "invite"):
+            if not await self.check_can_use_commands(message, reply):
+                return
+            try:
+                mxid = text[text.index(" ") + 1:]
+            except ValueError:
+                mxid = ""
+            await self.handle_command_invite(portal, reply, mxid=mxid)
+
+    def handle_service_message(self, message: MessageService):
+        to_id = message.to_id
+        if isinstance(to_id, PeerChannel):
+            to_id = to_id.channel_id
+            type = "channel"
+        elif isinstance(to_id, PeerChat):
+            to_id = to_id.chat_id
+            type = "chat"
+        else:
+            return
+
+        action = message.action
+        if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
+            self.add_chat(to_id, type)
+        elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
+            self.remove_chat(to_id)
+
+    async def update(self, update):
+        if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
+            return
+        if isinstance(update.message, MessageService):
+            return self.handle_service_message(update.message)
+
+        is_command = (isinstance(update.message, Message)
+                      and update.message.entities and len(update.message.entities) > 0
+                      and isinstance(update.message.entities[0], MessageEntityBotCommand))
+        if is_command:
+            return await self.handle_command(update.message)
+
+    def is_in_chat(self, peer_id) -> bool:
+        return peer_id in self.chats
+
+    @property
+    def name(self) -> str:
+        return "bot"
+
+
+def init(context) -> Optional[Bot]:
+    global config
+    config = context.config
+    token = config["telegram.bot_token"]
+    if token and not token.lower().startswith("disable"):
+        return Bot(token)
+    return None
@@ -1,2 +1,5 @@
-from .handler import command_handler, CommandHandler
-from . import clean_rooms, auth, meta, telegram
+from .handler import (command_handler, command_handlers as _command_handlers,
+                      CommandHandler, CommandProcessor, CommandEvent,
+                      SECTION_GENERAL, SECTION_AUTH, SECTION_CREATING_PORTALS,
+                      SECTION_PORTAL_MANAGEMENT, SECTION_MISC, SECTION_ADMIN)
+from . import clean_rooms, auth, meta, telegram, portal
@@ -3,117 +3,313 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
 import asyncio

 from telethon.errors import *

-from . import command_handler
+from . import command_handler, CommandEvent, SECTION_AUTH
+from .. import puppet as pu
+from ..util import format_duration


-@command_handler(needs_auth=False)
-async def ping(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
-    me = await evt.sender.client.get_me()
+@command_handler(needs_auth=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Check if you're logged into Telegram.")
+async def ping(evt: CommandEvent):
+    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
    if me:
        return await evt.reply(f"You're logged in as @{me.username}")
    else:
        return await evt.reply("You're not logged in.")


-@command_handler(needs_auth=False, management_only=True)
-def register(evt):
-    return evt.reply("Not yet implemented.")
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Get the info of the message relay Telegram bot.")
+async def ping_bot(evt: CommandEvent):
+    if not evt.tgbot:
+        return await evt.reply("Telegram message relay bot not configured.")
+    bot_info = await evt.tgbot.client.get_me()
+    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
+    displayname = bot_info.first_name
+    return await evt.reply("Telegram message relay bot is active: "
+                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
+                           "To use the bot, simply invite it to a portal room.")


-@command_handler(needs_auth=False, management_only=True)
-async def login(evt):
-    if evt.sender.logged_in:
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
+                           "account.")
+async def logout_matrix(evt: CommandEvent):
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
+
+
+@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account")
+async def login_matrix(evt: CommandEvent):
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        url = f"{prefix}/matrix-login?token={evt.public_website.make_token(evt.sender.mxid, '/matrix-login')}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+async def enter_matrix_token(evt: CommandEvent):
+    evt.sender.command_status = None
+
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+
+    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    if resp == 2:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    elif resp == 1:
+        return await evt.reply("Failed to verify access token.")
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")
+
+
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_phone_> <_full name_>",
+                 help_text="Register to Telegram")
+async def register(evt: CommandEvent):
+    if await evt.sender.is_logged_in():
        return await evt.reply("You are already logged in.")
-    elif len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp login <phone number>`")
+    elif len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
+
    phone_number = evt.args[0]
-    await evt.sender.client.sign_in(phone_number)
-    evt.sender.command_status = {
-        "next": enter_code,
-        "action": "Login",
-    }
-    return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
+    if len(evt.args) == 2:
+        full_name = evt.args[1], ""
+    else:
+        full_name = " ".join(evt.args[1:-1]), evt.args[-1]
+
+    await request_code(evt, phone_number, {
+        "next": enter_code_register,
+        "action": "Register",
+        "full_name": full_name,
+    })


-@command_handler(needs_auth=False)
-async def enter_code(evt):
+async def enter_code_register(evt: CommandEvent):
    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
-
+        return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
    try:
-        user = await evt.sender.client.sign_in(code=evt.args[0])
+        await evt.sender.ensure_started(even_if_no_session=True)
+        first_name, last_name = evt.sender.command_status["full_name"]
+        user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PhoneNumberUnoccupiedError:
-        return await evt.reply("That phone number has not been registered."
-                               "Please register with `$cmdprefix+sp register <phone>`.")
+        return await evt.reply(f"Successfully registered to Telegram.")
+    except PhoneNumberOccupiedError:
+        return await evt.reply("That phone number has already been registered. "
+                               "You can log in with `$cmdprefix+sp login`.")
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
    except PhoneCodeExpiredError:
        return await evt.reply(
-            "Phone code expired. Try again with `$cmdprefix+sp login <phone>`.")
+            "Phone code expired. Try again with `$cmdprefix+sp register <phone>`.")
    except PhoneCodeInvalidError:
        return await evt.reply("Invalid phone code.")
+    except Exception:
+        evt.log.exception("Error sending phone code")
+        return await evt.reply("Unhandled exception while sending code. "
+                               "Check console for more details.")
+
+
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Get instructions on how to log in.")
+async def login(evt: CommandEvent):
+    if await evt.sender.is_logged_in():
+        return await evt.reply("You are already logged in.")
+
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_phone_or_token,
+            "action": "Login",
+        }
+
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your phone number or bot "
+                "auth token here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended if you have two-factor authentication "
+                "enabled, because in-Matrix login would save your password in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your phone number or bot auth token here to start the login process.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+async def request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, str]):
+    ok = False
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        await evt.sender.client.sign_in(phone_number)
+        ok = True
+        return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
    except PhoneNumberAppSignupForbiddenError:
        return await evt.reply(
            "Your phone number does not allow 3rd party apps to sign in.")
    except PhoneNumberFloodError:
        return await evt.reply(
            "Your phone number has been temporarily blocked for flooding. "
-            "The block is usually applied for around a day.")
+            "The ban is usually applied for around a day.")
+    except FloodWaitError as e:
+        return await evt.reply(
+            "Your phone number has been temporarily blocked for flooding. "
+            f"Please wait for {format_duration(e.seconds)} before trying again.")
    except PhoneNumberBannedError:
-        return await evt.reply("Your phone number has been banned from Telegram.")
-    except SessionPasswordNeededError:
-        evt.sender.command_status = {
-            "next": enter_password,
-            "action": "Login (password entry)",
-        }
-        return await evt.reply("Your account has two-factor authentication."
-                               "Please send your password here.")
+        return await  evt.reply("Your phone number has been banned from Telegram.")
+    except PhoneNumberUnoccupiedError:
+        return await  evt.reply("That phone number has not been registered. "
+                                "Please register with `$cmdprefix+sp register <phone>`.")
+    except Exception:
+        evt.log.exception("Error requesting phone code")
+        return await evt.reply("Unhandled exception while requesting code. "
+                               "Check console for more details.")
+    finally:
+        evt.sender.command_status = next_status if ok else None
+
+
+@command_handler(needs_auth=False)
+async def enter_phone_or_token(evt: CommandEvent):
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+
+    # phone numbers don't contain colons but telegram bot auth tokens do
+    if evt.args[0].find(":") > 0:
+        try:
+            await sign_in(evt, bot_token=evt.args[0])
+        except Exception:
+            evt.log.exception("Error sending auth token")
+            return await evt.reply("Unhandled exception while sending auth token. "
+                                   "Check console for more details.")
+    else:
+        await request_code(evt, evt.args[0], {
+            "next": enter_code,
+            "action": "Login",
+        })
+
+
+@command_handler(needs_auth=False)
+async def enter_code(evt: CommandEvent):
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+    try:
+        await sign_in(evt, code=evt.args[0])
    except Exception:
        evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code."
+        return await evt.reply("Unhandled exception while sending code. "
                               "Check console for more details.")


@command_handler(needs_auth=False)
-async def enter_password(evt):
+async def enter_password(evt: CommandEvent):
    if len(evt.args) == 0:
        return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
-
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
    try:
-        user = await evt.sender.client.sign_in(password=evt.args[0])
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PasswordHashInvalidError:
-        return await evt.reply("Incorrect password.")
+        await sign_in(evt, password=" ".join(evt.args))
+    except AccessTokenInvalidError:
+        return await evt.reply("That bot token is not valid.")
+    except AccessTokenExpiredError:
+        return await evt.reply("That bot token has expired.")
    except Exception:
        evt.log.exception("Error sending password")
        return await evt.reply("Unhandled exception while sending password. "
                               "Check console for more details.")


-@command_handler(needs_auth=False)
-async def logout(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
+async def sign_in(evt: CommandEvent, **sign_in_info):
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        user = await evt.sender.client.sign_in(**sign_in_info)
+        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        evt.sender.command_status = None
+        return await evt.reply(f"Successfully logged in as @{user.username}")
+    except PhoneCodeExpiredError:
+        return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
+    except PasswordHashInvalidError:
+        return await evt.reply("Incorrect password.")
+    except SessionPasswordNeededError:
+        evt.sender.command_status = {
+            "next": enter_password,
+            "action": "Login (password entry)",
+        }
+        return await evt.reply("Your account has two-factor authentication. "
+                               "Please send your password here.")
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Log out from Telegram.")
+async def logout(evt: CommandEvent):
    if await evt.sender.log_out():
        return await evt.reply("Logged out successfully.")
    return await evt.reply("Failed to log out.")
@@ -3,28 +3,34 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from mautrix_appservice import MatrixRequestError
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, List

-from . import command_handler
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from . import command_handler, CommandEvent, SECTION_ADMIN
 from .. import puppet as pu, portal as po

+ManagementRoomList = List[Tuple[str, str]]
+RoomIDList = List[str]

-async def _find_rooms(intent):
-    management_rooms = []
-    unidentified_rooms = []
-    portals = []
-    empty_portals = []
+
+async def _find_rooms(intent: IntentAPI) -> Tuple[ManagementRoomList, RoomIDList,
+                                                  List["po.Portal"], List["po.Portal"]]:
+    management_rooms = []  # type: ManagementRoomList
+    unidentified_rooms = []  # type: RoomIDList
+    portals = []  # type: List[po.Portal]
+    empty_portals = []  # type: List[po.Portal]

    rooms = await intent.get_joined_rooms()
    for room in rooms:
@@ -52,12 +58,10 @@ async def _find_rooms(intent):
    return management_rooms, unidentified_rooms, portals, empty_portals


-@command_handler(needs_admin=True, name="clean-rooms")
-async def clean_rooms(evt):
-    if not evt.is_management:
-        return await evt.reply("`clean-rooms` is a particularly spammy command. Please don't "
-                               "run it in non-management rooms.")
-
+@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
+                 help_section=SECTION_ADMIN,
+                 help_text="Clean up unused portal/management rooms.")
+async def clean_rooms(evt: CommandEvent):
    management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)

    reply = ["#### Management rooms (M)"]
@@ -66,7 +70,7 @@ async def clean_rooms(evt):
              or ["No management rooms found."])
    reply.append("#### Active portal rooms (A)")
    reply += ([f"{n+1}. [P{n+1}](https://matrix.to/#/{portal.mxid}) "
-               + f"(to Telegram chat \"{portal.title}\")"
+               f"(to Telegram chat \"{portal.title}\")"
               for n, portal in enumerate(portals)]
              or ["No active portal rooms found."])
    reply.append("#### Unidentified rooms (U)")
@@ -75,7 +79,7 @@ async def clean_rooms(evt):
              or ["No unidentified rooms found."])
    reply.append("#### Inactive portal rooms (I)")
    reply += ([f"{n}. [E{n}](https://matrix.to/#/{portal.mxid}) "
-               + f"(to Telegram chat \"{portal.title}\")"
+               f"(to Telegram chat \"{portal.title}\")"
               for n, portal in enumerate(empty_portals)]
              or ["No inactive portal rooms found."])

@@ -90,7 +94,7 @@ async def clean_rooms(evt):
               "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
               "the group name."),
              "",
-              ("Please note that you will have to re-run `$cmdprefix+sp cleanrooms` "
+              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
               "between each use of the commands above.")]

    evt.sender.command_status = {
@@ -102,7 +106,9 @@ async def clean_rooms(evt):
    return await evt.reply("\n".join(reply))


-async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals, empty_portals):
+async def set_rooms_to_clean(evt, management_rooms: ManagementRoomList,
+                             unidentified_rooms: RoomIDList, portals: List["po.Portal"],
+                             empty_portals: List["po.Portal"]):
    command = evt.args[0]
    rooms_to_clean = []
    if command == "clean-recommended":
@@ -112,7 +118,7 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
            return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
        groups_to_clean = evt.args[1]
        if "M" in groups_to_clean:
-            rooms_to_clean += management_rooms
+            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
        if "A" in groups_to_clean:
            rooms_to_clean += portals
        if "U" in groups_to_clean:
@@ -126,7 +132,7 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
            start, end = range.split("-")
            start, end = int(start), int(end)
            if group == "M":
-                group = management_rooms
+                group = [room_id for (room_id, user_id) in management_rooms]
            elif group == "A":
                group = portals
            elif group == "U":
@@ -141,28 +147,28 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
                "**Usage:** `$cmdprefix+sp clean-groups <_M|A|U|I_><range>")
    else:
        return await evt.reply(f"Unknown room cleaning action `{command}`. "
-                               + "Use `$cmdprefix+sp cancel` to cancel room "
-                               + "cleaning.")
+                               "Use `$cmdprefix+sp cancel` to cancel room "
+                               "cleaning.")

    evt.sender.command_status = {
        "next": lambda confirm: execute_room_cleanup(confirm, rooms_to_clean),
        "action": "Room cleaning",
    }
    await evt.reply(f"To confirm cleaning up {len(rooms_to_clean)} rooms, type"
-                    + "`$cmdprefix+sp confirm-clean`.")
+                    "`$cmdprefix+sp confirm-clean`.")


 async def execute_room_cleanup(evt, rooms_to_clean):
    if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
        await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
-                        + "This might take a while.")
+                        "This might take a while.")
        cleaned = 0
        for room in rooms_to_clean:
            if isinstance(room, po.Portal):
                await room.cleanup_and_delete()
                cleaned += 1
            elif isinstance(room, str):
-                await po.Portal.cleanup_room(evt.az.intent, room, type="Room")
+                await po.Portal.cleanup_room(evt.az.intent, room, message="Room deleted")
                cleaned += 1
        evt.sender.command_status = None
        await evt.reply(f"{cleaned} rooms cleaned up successfully.")
@@ -3,49 +3,49 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Dict, Callable, Optional
+from collections import namedtuple
 import markdown
 import logging

 from telethon.errors import FloodWaitError

-command_handlers = {}
+from ..util import format_duration
+from .. import user as u, context as c

+command_handlers = {}  # type: Dict[str, CommandHandler]

-def command_handler(needs_auth=True, management_only=False, needs_admin=False, name=None):
-    def decorator(func):
-        def wrapper(evt):
-            if management_only and not evt.is_management:
-                return evt.reply(f"`{evt.command}` is a restricted command:"
-                                 + "you may only run it in management rooms.")
-            elif needs_auth and not evt.sender.logged_in:
-                return evt.reply("This command requires you to be logged in.")
-            elif needs_admin and not evt.sender.is_admin:
-                return evt.reply("This is command requires administrator privileges.")
-            return func(evt)
+HelpSection = namedtuple("HelpSection", "name order description")

-        command_handlers[name or func.__name__.replace("_", "-")] = wrapper
-        return wrapper
-
-    return decorator
+SECTION_GENERAL = HelpSection("General", 0, "")
+SECTION_AUTH = HelpSection("Authentication", 10, "")
+SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
+SECTION_PORTAL_MANAGEMENT = HelpSection("Portal management", 30, "")
+SECTION_MISC = HelpSection("Miscellaneous", 40, "")
+SECTION_ADMIN = HelpSection("Administration", 50, "")


 class CommandEvent:
-    def __init__(self, handler, room, sender, command, args, is_management, is_portal):
-        self.az = handler.az
-        self.log = handler.log
-        self.loop = handler.loop
-        self.command_prefix = handler.command_prefix
+    def __init__(self, processor: "CommandProcessor", room: str, sender: u.User, command: str,
+                 args: List[str], is_management: bool, is_portal: bool):
+        self.az = processor.az
+        self.log = processor.log
+        self.loop = processor.loop
+        self.tgbot = processor.tgbot
+        self.config = processor.config
+        self.public_website = processor.public_website
+        self.command_prefix = processor.command_prefix
        self.room_id = room
        self.sender = sender
        self.command = command
@@ -53,7 +53,7 @@ class CommandEvent:
        self.is_management = is_management
        self.is_portal = is_portal

-    def reply(self, message, allow_html=False, render_markdown=True):
+    def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True):
        message = message.replace("$cmdprefix+sp ",
                                  "" if self.is_management else f"{self.command_prefix} ")
        message = message.replace("$cmdprefix", self.command_prefix)
@@ -65,42 +65,94 @@ class CommandEvent:
        return self.az.intent.send_notice(self.room_id, message, html=html)


-def format_duration(seconds):
-    def pluralize(count, singular): return singular if count == 1 else singular + "s"
-
-    def include(count, word): return f"{count} {pluralize(count, word)}" if count > 0 else ""
-
-    minutes, seconds = divmod(seconds, 60)
-    hours, minutes = divmod(minutes, 60)
-    days, hours = divmod(hours, 24)
-    parts = [a for a in [
-        include(days, "day"),
-        include(hours, "hour"),
-        include(minutes, "minute"),
-        include(seconds, "second")] if a]
-    if len(parts) > 2:
-        return "{} and {}".format(", ".join(parts[:-1]), parts[-1])
-    return " and ".join(parts)
-
-
 class CommandHandler:
+    def __init__(self, handler: Callable[[CommandEvent], None], needs_auth: bool,
+                 needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
+                 management_only: bool, name: str, help_text: str, help_args: str,
+                 help_section: HelpSection):
+        self._handler = handler
+        self.needs_auth = needs_auth
+        self.needs_puppeting = needs_puppeting
+        self.needs_matrix_puppeting = needs_matrix_puppeting
+        self.needs_admin = needs_admin
+        self.management_only = management_only
+        self.name = name
+        self._help_text = help_text
+        self._help_args = help_args
+        self.help_section = help_section
+
+    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        if self.management_only and not evt.is_management:
+            return (f"`{evt.command}` is a restricted command: "
+                    "you may only run it in management rooms.")
+        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
+            return "This command requires puppeting privileges."
+        elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
+            return "This command requires Matrix puppeting privileges."
+        elif self.needs_admin and not evt.sender.is_admin:
+            return "This command requires administrator privileges."
+        elif self.needs_auth and not await evt.sender.is_logged_in():
+            return "This command requires you to be logged in."
+        return None
+
+    def has_permission(self, is_management: bool, puppet_whitelisted: bool,
+                       matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
+        return ((not self.management_only or is_management) and
+                (not self.needs_puppeting or puppet_whitelisted) and
+                (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
+                (not self.needs_admin or is_admin) and
+                (not self.needs_auth or is_logged_in))
+
+    async def __call__(self, evt: CommandEvent):
+        error = await self.get_permission_error(evt)
+        if error is not None:
+            return await evt.reply(error)
+        return await self._handler(evt)
+
+    @property
+    def has_help(self) -> bool:
+        return bool(self.help_section) and bool(self._help_text)
+
+    @property
+    def help(self) -> str:
+        return f"**{self.name}** {self._help_args} - {self._help_text}"
+
+
+def command_handler(_func: Optional[Callable[[CommandEvent], None]] = None, *, needs_auth=True,
+                    needs_puppeting=True, needs_matrix_puppeting=False, needs_admin=False,
+                    management_only=False, name=None, help_text="", help_args="",
+                    help_section=None):
+    input_name = name
+
+    def decorator(func: Callable[[CommandEvent], None]):
+        name = input_name or func.__name__.replace("_", "-")
+        handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
+                                 needs_admin, management_only, name, help_text, help_args,
+                                 help_section)
+        command_handlers[handler.name] = handler
+        return handler
+
+    return decorator if _func is None else decorator(_func)
+
+
+class CommandProcessor:
    log = logging.getLogger("mau.commands")

-    def __init__(self, context):
-        self.az, self.db, self.config, self.loop = context
+    def __init__(self, context: c.Context):
+        self.az, self.db, self.config, self.loop, self.tgbot = context
+        self.public_website = context.public_website
        self.command_prefix = self.config["bridge.command_prefix"]

-    # region Utility functions for handling commands
-
-    async def handle(self, room, sender, command, args, is_management, is_portal):
-        evt = CommandEvent(self, room, sender, command, args,
-                           is_management, is_portal)
+    async def handle(self, room: str, sender: u.User, command: str, args: List[str],
+                     is_management: bool, is_portal: bool):
+        evt = CommandEvent(self, room, sender, command, args, is_management, is_portal)
+        orig_command = command
        command = command.lower()
        try:
            command = command_handlers[command]
        except KeyError:
            if sender.command_status and "next" in sender.command_status:
-                args.insert(0, command)
+                args.insert(0, orig_command)
                evt.command = ""
                command = sender.command_status["next"]
            else:
@@ -108,8 +160,9 @@ class CommandHandler:
        try:
            await command(evt)
        except FloodWaitError as e:
-            return evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
+            return await evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
        except Exception:
-            self.log.exception(f"Fatal error handling command "
-                               + f"{evt.command} {' '.join(args)} from {sender.mxid}")
-            return evt.reply("Fatal error while handling command. Check logs for more details.")
+            self.log.exception("Unhandled error while handling command "
+                               f"{evt.command} {' '.join(args)} from {sender.mxid}")
+            return await evt.reply("Unhandled error while handling command. "
+                                   "Check logs for more details.")
@@ -3,22 +3,24 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from . import command_handler
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from . import command_handler, CommandEvent, _command_handlers, SECTION_GENERAL


-@command_handler()
-def cancel(evt):
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Cancel an ongoing action (such as login)")
+def cancel(evt: CommandEvent):
    if evt.sender.command_status:
        action = evt.sender.command_status["action"]
        evt.sender.command_status = None
@@ -27,50 +29,41 @@ def cancel(evt):
        return evt.reply("No ongoing command.")


-@command_handler()
-def unknown_command(evt):
+@command_handler(needs_auth=False, needs_puppeting=False)
+def unknown_command(evt: CommandEvent):
    return evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")


-@command_handler()
-def help(evt):
+help_cache = {}
+
+
+async def _get_help_text(evt: CommandEvent):
+    cache_key = (evt.is_management, evt.sender.puppet_whitelisted,
+                 evt.sender.matrix_puppet_whitelisted, evt.sender.is_admin,
+                 await evt.sender.is_logged_in())
+    if cache_key not in help_cache:
+        help = {}
+        for handler in _command_handlers.values():
+            if handler.has_help and handler.has_permission(*cache_key):
+                help.setdefault(handler.help_section, [])
+                help[handler.help_section].append(handler.help + "  ")
+        help = sorted(help.items(), key=lambda item: item[0].order)
+        help = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help]
+        help_cache[cache_key] = "\n".join(help)
+    return help_cache[cache_key]
+
+
+def _get_management_status(evt: CommandEvent):
    if evt.is_management:
-        management_status = ("This is a management room: prefixing commands "
-                             "with `$cmdprefix` is not required.\n")
+        return "This is a management room: prefixing commands with `$cmdprefix` is not required."
    elif evt.is_portal:
-        management_status = ("**This is a portal room**: you must always "
-                             "prefix commands with `$cmdprefix`.\n"
-                             "Management commands will not be sent to Telegram.")
-    else:
-        management_status = ("**This is not a management room**: you must "
-                             "prefix commands with `$cmdprefix`.\n")
-    help = """\n
-#### Generic bridge commands
-**help**   - Show this help message.  
-**cancel** - Cancel an ongoing action (such as login).
+        return ("**This is a portal room**: you must always prefix commands with `$cmdprefix`.\n"
+                "Management commands will not be sent to Telegram.")
+    return "**This is not a management room**: you must prefix commands with `$cmdprefix`."

-#### Authentication
-**login** <_phone_> - Request an authentication code.  
-**logout**          - Log out from Telegram.  
-**ping**            - Check if you're logged into Telegram.

-#### Initiating chats
-**search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users.  
-**pm** <_identifier_>                - Open a private chat with the given Telegram user. The
-                                       identifier is either the internal user ID, the username or
-                                       the phone number.  
-**join** <_link_>                    - Join a chat with an invite link.  
-**create** [_type_]                  - Create a Telegram chat of the given type for the current
-                                       Matrix room. The type is either `group`, `supergroup` or
-                                       `channel` (defaults to `group`).
-
-#### Portal management  
-**upgrade**                 - Upgrade a normal Telegram group to a supergroup.  
-**invite-link**             - Get a Telegram invite link to the current chat.  
-**delete-portal**           - Forget the current portal room. Only works for group chats; to delete
-                             a private chat portal, simply leave the room.  
-**group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash
-                             (`-`) as the name.  
-**clean-rooms**             - Clean up unused portal/management rooms.
-"""
-    return evt.reply(management_status + help)
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Show this help message.")
+async def help(evt: CommandEvent):
+    return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))
@@ -0,0 +1,490 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Callable
+import asyncio
+
+from telethon.errors import *
+from telethon.tl.types import ChatForbidden, ChannelForbidden
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from .. import portal as po, user as u
+from . import (command_handler, CommandEvent,
+               SECTION_ADMIN, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT)
+
+
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
+                 help_section=SECTION_ADMIN,
+                 help_args="<_level_> [_mxid_]",
+                 help_text="Set a temporary power level without affecting Telegram.")
+async def set_power_level(evt: CommandEvent):
+    try:
+        level = int(evt.args[0])
+    except KeyError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp set-power <level> [mxid]`")
+    except ValueError:
+        return await evt.reply("The level must be an integer.")
+    levels = await evt.az.intent.get_power_levels(evt.room_id)
+    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
+    levels["users"][mxid] = level
+    try:
+        await evt.az.intent.set_power_levels(evt.room_id, levels)
+    except MatrixRequestError:
+        evt.log.exception("Failed to set power level.")
+        return await evt.reply("Failed to set power level.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Get a Telegram invite link to the current chat.")
+async def invite_link(evt: CommandEvent):
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if portal.peer_type == "user":
+        return await evt.reply("You can't invite users to private chats.")
+
+    try:
+        link = await portal.get_invite_link(evt.sender)
+        return await evt.reply(f"Invite link to {portal.title}: {link}")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to create an invite link.")
+
+
+async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50):
+    if sender.is_admin:
+        return True
+    # Make sure the state store contains the power levels.
+    try:
+        await intent.get_power_levels(room)
+    except MatrixRequestError:
+        return False
+    return intent.state_store.has_power_level(room, sender.mxid,
+                                              event=f"net.maunium.telegram.{event}",
+                                              default=default)
+
+
+async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
+                                           action: Optional[str] = None):
+    room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if not portal:
+        that_this = "This" if room_id == evt.room_id else "That"
+        return await evt.reply(f"{that_this} is not a portal room."), False
+
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
+        action = action or f"{permission.replace('_', ' ')}s"
+        return await evt.reply(f"You do not have the permissions to {action} that portal."), False
+    return portal, True
+
+
+def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
+                                completed_message: str):
+    async def post_confirm(confirm):
+        confirm.sender.command_status = None
+        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
+            await function()
+            if confirm.room_id != room_id:
+                return await confirm.reply(completed_message)
+        else:
+            return await confirm.reply(f"{action} cancelled.")
+
+    return {
+        "next": post_confirm,
+        "action": action,
+    }
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove all users from the current portal room and forget the portal. "
+                           "Only works for group chats; to delete a private chat portal, simply "
+                           "leave the room.")
+async def delete_portal(evt: CommandEvent):
+    portal, ok = await _get_portal_and_check_permission(evt, "unbridge")
+    if not ok:
+        return
+
+    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
+                                                            portal.cleanup_and_delete, "delete",
+                                                            "Portal successfully deleted.")
+    return await evt.reply("Please confirm deletion of portal "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           f"to Telegram chat \"{portal.title}\" "
+                           "by typing `$cmdprefix+sp confirm-delete`"
+                           "\n\n"
+                           "**WARNING:** If the bridge bot has the power level to do so, **this "
+                           "will kick ALL users** in the room. If you just want to remove the "
+                           "bridge, use `$cmdprefix+sp unbridge` instead.")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove puppets from the current portal room and forget the portal.")
+async def unbridge(evt: CommandEvent):
+    portal, ok = await _get_portal_and_check_permission(evt, "unbridge")
+    if not ok:
+        return
+
+    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
+                                                            portal.unbridge, "unbridge",
+                                                            "Room successfully unbridged.")
+    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           "by typing `$cmdprefix+sp confirm-unbridge`")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="[_id_]",
+                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
+                           "ID. The ID must be the prefixed version that you get with the `/id` "
+                           "command of the Telegram-side bot.")
+async def bridge(evt: CommandEvent):
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** "
+                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    room_id = evt.args[1] if len(evt.args) > 1 else evt.room_id
+    that_this = "This" if room_id == evt.room_id else "That"
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if portal:
+        return await evt.reply(f"{that_this} room is already a portal room.")
+
+    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
+
+    # The /id bot command provides the prefixed ID, so we assume
+    tgid = evt.args[0]
+    if tgid.startswith("-100"):
+        tgid = int(tgid[4:])
+        peer_type = "channel"
+    elif tgid.startswith("-"):
+        tgid = -int(tgid)
+        peer_type = "chat"
+    else:
+        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
+                               "If you did not get the ID using the `/id` bot command, please "
+                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+                               "Bridging private chats to existing rooms is not allowed.")
+
+    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    if not portal.allow_bridging():
+        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
+                               "If you're the bridge admin, try "
+                               "`$cmdprefix+sp whitelist <Telegram chat ID>` first.")
+    if portal.mxid:
+        has_portal_message = (
+            "That Telegram chat already has a portal at "
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
+        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+            return await evt.reply(f"{has_portal_message}"
+                                   "Additionally, you do not have the permissions to unbridge "
+                                   "that room.")
+        evt.sender.command_status = {
+            "next": confirm_bridge,
+            "action": "Room bridging",
+            "mxid": portal.mxid,
+            "bridge_to_mxid": room_id,
+            "tgid": portal.tgid,
+            "peer_type": portal.peer_type,
+        }
+        return await evt.reply(f"{has_portal_message}"
+                               "However, you have the permissions to unbridge that room.\n\n"
+                               "To delete that portal completely and continue bridging, use "
+                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
+    evt.sender.command_status = {
+        "next": confirm_bridge,
+        "action": "Room bridging",
+        "bridge_to_mxid": room_id,
+        "tgid": portal.tgid,
+        "peer_type": portal.peer_type,
+    }
+    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
+                           "chat to this room, use `$cmdprefix+sp continue`")
+
+
+async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"):
+    if not portal.mxid:
+        await evt.reply("The portal seems to have lost its Matrix room between you"
+                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                        "Continuing without touching previous Matrix room...")
+        return True, None
+    elif evt.args[0] == "delete-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Portal deleted (moving to another room)")
+    elif evt.args[0] == "unbridge-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Room unbridged (portal moving to another room)",
+                                         puppets_only=True)
+    else:
+        await evt.reply(
+            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
+            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
+            "continue` to either delete or unbridge the existing room (respectively) and "
+            "continue with the bridging.\n\n"
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
+        return False, None
+
+
+async def confirm_bridge(evt: CommandEvent):
+    status = evt.sender.command_status
+    try:
+        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        bridge_to_mxid = status["bridge_to_mxid"]
+    except KeyError:
+        evt.sender.command_status = None
+        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
+                               "This shouldn't happen unless you're messing with the command "
+                               "handler code.")
+    if "mxid" in status:
+        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
+        if not ok:
+            return
+        elif coro:
+            asyncio.ensure_future(coro, loop=evt.loop)
+            await evt.reply("Cleaning up previous portal room...")
+    elif portal.mxid:
+        evt.sender.command_status = None
+        return await evt.reply("The portal seems to have created a Matrix room between you "
+                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                               "Please start over by calling the bridge command again.")
+    elif evt.args[0] != "continue":
+        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
+                               "`$cmdprefix+sp cancel` to cancel.")
+
+    is_logged_in = await evt.sender.is_logged_in()
+    user = evt.sender if is_logged_in else evt.tgbot
+    try:
+        entity = await user.client.get_entity(portal.peer)
+    except Exception:
+        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+        if is_logged_in:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You are logged in, are you in that chat?")
+        else:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You're not logged in, is the relay bot in the chat?")
+    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
+        if is_logged_in:
+            return await evt.reply("You don't seem to be in that chat.")
+        else:
+            return await evt.reply("The bot doesn't seem to be in that chat.")
+
+    direct = False
+
+    portal.mxid = bridge_to_mxid
+    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    portal.photo_id = ""
+    portal.save()
+
+    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
+                          loop=evt.loop)
+
+    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")
+
+
+async def get_initial_state(intent: IntentAPI, room_id: str):
+    state = await intent.get_room_state(room_id)
+    title = None
+    about = None
+    levels = None
+    for event in state:
+        try:
+            if event["type"] == "m.room.name":
+                title = event["content"]["name"]
+            elif event["type"] == "m.room.topic":
+                about = event["content"]["topic"]
+            elif event["type"] == "m.room.power_levels":
+                levels = event["content"]
+            elif event["type"] == "m.room.canonical_alias":
+                title = title or event["content"]["alias"]
+        except KeyError:
+            # Some state event probably has empty content
+            pass
+    return title, about, levels
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_type_]",
+                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
+                           "The type is either `group`, `supergroup` or `channel` (defaults to "
+                           "`group`).")
+async def create(evt: CommandEvent):
+    type = evt.args[0] if len(evt.args) > 0 else "group"
+    if type not in {"chat", "group", "supergroup", "channel"}:
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
+
+    if po.Portal.get_by_mxid(evt.room_id):
+        return await evt.reply("This is already a portal room.")
+
+    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply("You do not have the permissions to bridge this room.")
+
+    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    if not title:
+        return await evt.reply("Please set a title before creating a Telegram chat.")
+
+    supergroup = type == "supergroup"
+    type = {
+        "supergroup": "channel",
+        "channel": "channel",
+        "chat": "chat",
+        "group": "chat",
+    }[type]
+
+    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
+    try:
+        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+    except ValueError as e:
+        portal.delete()
+        return await evt.reply(e.args[0])
+    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Upgrade a normal Telegram group to a supergroup.")
+async def upgrade(evt: CommandEvent):
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type == "channel":
+        return await evt.reply("This is already a supergroup or a channel.")
+    elif portal.peer_type == "user":
+        return await evt.reply("You can't upgrade private chats.")
+
+    try:
+        await portal.upgrade_telegram_chat(evt.sender)
+        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to upgrade this group.")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="<_name_|`-`>",
+                 help_text="Change the username of a supergroup/channel. "
+                           "To disable, use a dash (`-`) as the name.")
+async def group_name(evt: CommandEvent):
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type != "channel":
+        return await evt.reply("Only channels and supergroups have usernames.")
+
+    try:
+        await portal.set_telegram_username(evt.sender,
+                                           evt.args[0] if evt.args[0] != "-" else "")
+        if portal.username:
+            return await evt.reply(f"Username of channel changed to {portal.username}.")
+        else:
+            return await evt.reply(f"Channel is now private.")
+    except ChatAdminRequiredError:
+        return await evt.reply(
+            "You don't have the permission to set the username of this channel.")
+    except UsernameNotModifiedError:
+        if portal.username:
+            return await evt.reply("That is already the username of this channel.")
+        else:
+            return await evt.reply("This channel is already private")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username")
+
+
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`>",
+                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
+                           "default.")
+async def filter_mode(evt: CommandEvent):
+    try:
+        mode = evt.args[0]
+        if mode not in ("whitelist", "blacklist"):
+            raise ValueError()
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
+
+    evt.config["bridge.filter.mode"] = mode
+    evt.config.save()
+    po.Portal.filter_mode = mode
+    if mode == "whitelist":
+        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
+                               "To allow bridging a specific chat, use"
+                               "`!filter whitelist <chat ID>`.")
+    else:
+        return await evt.reply("The bridge will now allow bridging chats by default.\n"
+                               "To disallow bridging a specific chat, use"
+                               "`!filter blacklist <chat ID>`.")
+
+
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+                 help_text="Allow or disallow bridging a specific chat.")
+async def filter(evt: CommandEvent):
+    try:
+        action = evt.args[0]
+        if action not in ("whitelist", "blacklist", "add", "remove"):
+            raise ValueError()
+
+        id = evt.args[1]
+        if id.startswith("-100"):
+            id = int(id[4:])
+        elif id.startswith("-"):
+            id = int(id[1:])
+        else:
+            id = int(id)
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
+
+    mode = evt.config["bridge.filter.mode"]
+    if mode not in ("blacklist", "whitelist"):
+        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
+
+    list = evt.config["bridge.filter.list"]
+
+    if action in ("blacklist", "whitelist"):
+        action = "add" if mode == action else "remove"
+
+    def save():
+        evt.config["bridge.filter.list"] = list
+        evt.config.save()
+        po.Portal.filter_list = list
+
+    if action == "add":
+        if id in list:
+            return await evt.reply(f"That chat is already {mode}ed.")
+        list.append(id)
+        save()
+        return await evt.reply(f"Chat ID added to {mode}.")
+    elif action == "remove":
+        if id not in list:
+            return await evt.reply(f"That chat is not {mode}ed.")
+        list.remove(id)
+        save()
+        return await evt.reply(f"Chat ID removed from {mode}.")
@@ -3,28 +3,30 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from telethon.errors import *
 from telethon.tl.types import User as TLUser
 from telethon.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest
 from telethon.tl.functions.channels import JoinChannelRequest

 from .. import puppet as pu, portal as po
-from . import command_handler
+from . import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS


-@command_handler()
-async def search(evt):
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[_-r|--remote_] <_query_>",
+                 help_text="Search your contacts or the Telegram servers for users.")
+async def search(evt: CommandEvent):
    if len(evt.args) == 0:
        return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")

@@ -51,7 +53,7 @@ async def search(evt):
    else:
        reply += ["**Results in contacts:**", ""]
    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
-               + f"{puppet.id} ({similarity}% match)")
+               f"{puppet.id} ({similarity}% match)")
              for puppet, similarity in results]

    # TODO somehow show remote channel results when joining by alias is possible?
@@ -59,70 +61,56 @@ async def search(evt):
    return await evt.reply("\n".join(reply))


-@command_handler()
-async def pm(evt):
+@command_handler(name="pm",
+                 help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_identifier_>",
+                 help_text="Open a private chat with the given Telegram user. The identifier is "
+                           "either the internal user ID, the username or the phone number. "
+                           "**N.B.** The phone numbers you start chats with must already be in "
+                           "your contacts.")
+async def private_message(evt: CommandEvent):
    if len(evt.args) == 0:
        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")

-    user = await evt.sender.client.get_entity(evt.args[0])
+    try:
+        user = await evt.sender.client.get_entity(evt.args[0])
+    except ValueError:
+        return await evt.reply("Invalid user identifier or user not found.")
+
    if not user:
        return await evt.reply("User not found.")
    elif not isinstance(user, TLUser):
        return await evt.reply("That doesn't seem to be a user.")
    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
    await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply(
-        f"Created private chat room with {pu.Puppet.get_displayname(user, False)}")
+    return await evt.reply("Created private chat room with "
+                           f"{pu.Puppet.get_displayname(user, False)}")


-@command_handler()
-async def invite_link(evt):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
-        return await evt.reply("You can't invite users to private chats.")
-
-    try:
-        link = await portal.get_invite_link(evt.sender)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to create an invite link.")
+async def _join(evt: CommandEvent, arg: str):
+    if arg.startswith("joinchat/"):
+        invite_hash = arg[len("joinchat/"):]
+        try:
+            await evt.sender.client(CheckChatInviteRequest(invite_hash))
+        except InviteHashInvalidError:
+            return None, await evt.reply("Invalid invite link.")
+        except InviteHashExpiredError:
+            return None, await evt.reply("Invite link expired.")
+        try:
+            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
+        except UserAlreadyParticipantError:
+            return None, await evt.reply("You are already in that chat.")
+    else:
+        channel = await evt.sender.client.get_entity(arg)
+        if not channel:
+            return None, await evt.reply("Channel/supergroup not found.")
+        return await evt.sender.client(JoinChannelRequest(channel)), None


-@command_handler(needs_admin=True)
-async def delete_portal(evt):
-    room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if not portal:
-        that_this = "This" if room_id == evt.room_id else "That"
-        return await evt.reply(f"{that_this} is not a portal room.")
-
-    async def post_confirm(confirm):
-        evt.sender.command_status = None
-        if len(confirm.args) > 0 and confirm.args[0] == "confirm-delete":
-            await portal.cleanup_and_delete()
-            if confirm.room_id != room_id:
-                return await confirm.reply("Portal successfully deleted.")
-        else:
-            return await confirm.reply("Portal deletion cancelled.")
-
-    evt.sender.command_status = {
-        "next": post_confirm,
-        "action": "Portal deletion",
-    }
-    return await evt.reply("Please confirm deletion of portal "
-                           + f"[{room_id}](https://matrix.to/#/{room_id}) "
-                           + f"to Telegram chat \"{portal.title}\" "
-                           + "by typing `$cmdprefix+sp confirm-delete`")
-
-
-@command_handler()
-async def join(evt):
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_link_>",
+                 help_text="Join a chat with an invite link.")
+async def join(evt: CommandEvent):
    if len(evt.args) == 0:
        return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")

@@ -130,132 +118,37 @@ async def join(evt):
    arg = regex.match(evt.args[0])
    if not arg:
        return await evt.reply("That doesn't look like a Telegram invite link.")
-    arg = arg.group(1)
-    if arg.startswith("joinchat/"):
-        invite_hash = arg[len("joinchat/"):]
-        try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
-        except InviteHashInvalidError:
-            return await evt.reply("Invalid invite link.")
-        except InviteHashExpiredError:
-            return await evt.reply("Invite link expired.")
-        try:
-            updates = evt.sender.client(ImportChatInviteRequest(invite_hash))
-        except UserAlreadyParticipantError:
-            return await evt.reply("You are already in that chat.")
-    else:
-        channel = await evt.sender.client.get_entity(arg)
-        if not channel:
-            return await evt.reply("Channel/supergroup not found.")
-        updates = await evt.sender.client(JoinChannelRequest(channel))
+
+    updates, _ = await _join(evt, arg.group(1))
+    if not updates:
+        return
+
    for chat in updates.chats:
        portal = po.Portal.get_by_entity(chat)
        if portal.mxid:
+            await portal.invite_to_matrix([evt.sender.mxid])
+            return await evt.reply(f"Invited you to portal of {portal.title}")
+        else:
+            await evt.reply(f"Creating room for {chat.title}... This might take a while.")
            await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
            return await evt.reply(f"Created room for {portal.title}")
-        else:
-            await portal.invite_matrix([evt.sender.mxid])
-            return await evt.reply(f"Invited you to portal of {portal.title}")


-@command_handler()
-async def create(evt):
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
-
-    if po.Portal.get_by_mxid(evt.room_id):
-        return await evt.reply("This is already a portal room.")
-
-    state = await evt.az.intent.get_room_state(evt.room_id)
-    title = None
-    about = None
-    levels = None
-    for event in state:
-        if event["type"] == "m.room.name":
-            title = event["content"]["name"]
-        elif event["type"] == "m.room.topic":
-            about = event["content"]["topic"]
-        elif event["type"] == "m.room.power_levels":
-            levels = event["content"]
-    if not title:
-        return await evt.reply("Please set a title before creating a Telegram chat.")
-    elif (not levels or not levels["users"] or evt.az.intent.mxid not in levels["users"] or
-          levels["users"][evt.az.intent.mxid] < 100):
-        return await evt.reply(f"Please give "
-                               + f"[the bridge bot](https://matrix.to/#/{evt.az.intent.mxid})"
-                               + f" a power level of 100 before creating a Telegram chat.")
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[`chats`|`contacts`|`me`]",
+                 help_text="Synchronize your chat portals, contacts and/or own info.")
+async def sync(evt: CommandEvent):
+    if len(evt.args) > 0:
+        sync_only = evt.args[0]
+        if sync_only not in ("chats", "contacts", "me"):
+            return await evt.reply("**Usage:** `$cmdprefix+sp sync [chats|contacts|me]`")
    else:
-        for user, level in levels["users"].items():
-            if level >= 100 and user != evt.az.intent.mxid:
-                return await evt.reply(
-                    f"Please make sure only the bridge bot has power level above"
-                    + f"99 before creating a Telegram chat.\n\n"
-                    + f"Use power level 95 instead of 100 for admins.")
+        sync_only = None

-    supergroup = type == "supergroup"
-    type = {
-        "supergroup": "channel",
-        "channel": "channel",
-        "chat": "chat",
-        "group": "chat",
-    }[type]
-
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
-    try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
-
-
-@command_handler()
-async def upgrade(evt):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type == "channel":
-        return await evt.reply("This is already a supergroup or a channel.")
-    elif portal.peer_type == "user":
-        return await evt.reply("You can't upgrade private chats.")
-
-    try:
-        await portal.upgrade_telegram_chat(evt.sender)
-        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to upgrade this group.")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-
-
-@command_handler()
-async def group_name(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type != "channel":
-        return await evt.reply("Only channels and supergroups have usernames.")
-
-    try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
-        if portal.username:
-            return await evt.reply(f"Username of channel changed to {portal.username}.")
-        else:
-            return await evt.reply(f"Channel is now private.")
-    except ChatAdminRequiredError:
-        return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
-    except UsernameNotModifiedError:
-        if portal.username:
-            return await evt.reply("That is already the username of this channel.")
-        else:
-            return await evt.reply("This channel is already private")
-    except UsernameOccupiedError:
-        return await evt.reply("That username is already in use.")
-    except UsernameInvalidError:
-        return await evt.reply("Invalid username")
+    if not sync_only or sync_only == "chats":
+        await evt.sender.sync_dialogs(synchronous_create=True)
+    if not sync_only or sync_only == "contacts":
+        await evt.sender.sync_contacts()
+    if not sync_only or sync_only == "me":
+        await evt.sender.update_info()
+    return await evt.reply("Synchronization complete.")
@@ -3,74 +3,116 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Any, Optional
 from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
 import random
 import string

 yaml = YAML()
+yaml.indent(4)


 class DictWithRecursion:
-    def __init__(self, data=None):
-        self._data = data or {}
+    def __init__(self, data: CommentedMap = None):
+        self._data = data or CommentedMap()  # type: CommentedMap

-    def _recursive_get(self, data, key, default_value):
+    def _recursive_get(self, data: CommentedMap, key: str, default_value: Any) -> Any:
        if '.' in key:
            key, next_key = key.split('.', 1)
-            next_data = data.get(key, {})
+            next_data = data.get(key, CommentedMap())
            return self._recursive_get(next_data, next_key, default_value)
        return data.get(key, default_value)

-    def get(self, key, default_value, allow_recursion=True):
+    def get(self, key: str, default_value: Any, allow_recursion: bool = True) -> Any:
        if allow_recursion and '.' in key:
            return self._recursive_get(self._data, key, default_value)
        return self._data.get(key, default_value)

-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> Any:
        return self.get(key, None)

-    def _recursive_set(self, data, key, value):
+    def __contains__(self, key: str) -> bool:
+        return self[key] is not None
+
+    def _recursive_set(self, data: CommentedMap, key: str, value: Any):
        if '.' in key:
            key, next_key = key.split('.', 1)
            if key not in data:
-                data[key] = {}
-            next_data = data.get(key, {})
+                data[key] = CommentedMap()
+            next_data = data.get(key, CommentedMap())
            self._recursive_set(next_data, next_key, value)
            return
        data[key] = value

-    def set(self, key, value, allow_recursion=True):
+    def set(self, key: str, value: Any, allow_recursion: bool = True):
        if allow_recursion and '.' in key:
            self._recursive_set(self._data, key, value)
            return
        self._data[key] = value

-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value: Any):
        self.set(key, value)

+    def _recursive_del(self, data: CommentedMap, key: str):
+        if '.' in key:
+            key, next_key = key.split('.', 1)
+            if key not in data:
+                return
+            next_data = data[key]
+            self._recursive_del(next_data, next_key)
+            return
+        try:
+            del data[key]
+            del data.ca.items[key]
+        except KeyError:
+            pass
+
+    def delete(self, key: str, allow_recursion: bool = True):
+        if allow_recursion and '.' in key:
+            self._recursive_del(self._data, key)
+            return
+        try:
+            del self._data[key]
+            del self._data.ca.items[key]
+        except KeyError:
+            pass
+
+    def __delitem__(self, key: str):
+        self.delete(key)
+

 class Config(DictWithRecursion):
-    def __init__(self, path, registration_path):
+    def __init__(self, path: str, registration_path: str, base_path: str):
        super().__init__()
-        self.path = path
-        self.registration_path = registration_path
-        self._registration = None
+        self.path = path  # type: str
+        self.registration_path = registration_path  # type: str
+        self.base_path = base_path  # type: str
+        self._registration = None  # type: dict

    def load(self):
        with open(self.path, 'r') as stream:
            self._data = yaml.load(stream)

+    def load_base(self) -> Optional[DictWithRecursion]:
+        try:
+            with open(self.base_path, 'r') as stream:
+                return DictWithRecursion(yaml.load(stream))
+        except OSError:
+            pass
+        return None
+
    def save(self):
        with open(self.path, 'w') as stream:
            yaml.dump(self._data, stream)
@@ -79,8 +121,154 @@ class Config(DictWithRecursion):
                yaml.dump(self._registration, stream)

    @staticmethod
-    def _new_token():
-        return "".join(random.choices(string.ascii_lowercase + string.digits, k=64))
+    def _new_token() -> str:
+        return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
+
+    def update(self):
+        base = self.load_base()
+        if not base:
+            return
+
+        def copy(from_path, to_path=None):
+            if from_path in self:
+                base[to_path or from_path] = self[from_path]
+
+        def copy_dict(from_path, to_path=None, override_existing_map=True):
+            if from_path in self:
+                to_path = to_path or from_path
+                if override_existing_map or to_path not in base:
+                    base[to_path] = CommentedMap()
+                for key, value in self[from_path].items():
+                    base[to_path][key] = value
+
+        copy("homeserver.address")
+        copy("homeserver.verify_ssl")
+        copy("homeserver.domain")
+
+        if "appservice.protocol" in self and "appservice.address" not in self:
+            protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
+                                        self["appservice.port"])
+            base["appservice.address"] = f"{protocol}://{hostname}:{port}"
+        else:
+            copy("appservice.address")
+        copy("appservice.hostname")
+        copy("appservice.port")
+        copy("appservice.max_body_size")
+
+        copy("appservice.database")
+
+        copy("appservice.public.enabled")
+        copy("appservice.public.prefix")
+        copy("appservice.public.external")
+
+        copy("appservice.provisioning.enabled")
+        copy("appservice.provisioning.prefix")
+        copy("appservice.provisioning.shared_secret")
+        if base["appservice.provisioning.shared_secret"] == "generate":
+            base["appservice.provisioning.shared_secret"] = self._new_token()
+
+        copy("appservice.id")
+        copy("appservice.bot_username")
+        copy("appservice.bot_displayname")
+        copy("appservice.bot_avatar")
+
+        copy("appservice.as_token")
+        copy("appservice.hs_token")
+
+        copy("bridge.username_template")
+        copy("bridge.alias_template")
+        copy("bridge.displayname_template")
+
+        copy("bridge.displayname_preference")
+
+        copy("bridge.edits_as_replies")
+        copy("bridge.highlight_edits")
+        copy("bridge.bridge_notices")
+        copy("bridge.bot_messages_as_notices")
+        copy("bridge.max_initial_member_sync")
+        copy("bridge.sync_channel_members")
+        copy("bridge.max_telegram_delete")
+        copy("bridge.allow_matrix_login")
+        copy("bridge.inline_images")
+        copy("bridge.plaintext_highlights")
+        copy("bridge.public_portals")
+        copy("bridge.native_stickers")
+        copy("bridge.catch_up")
+        copy("bridge.sync_with_custom_puppets")
+
+        if "bridge.message_formats.m_text" in self:
+            del self["bridge.message_formats"]
+        copy_dict("bridge.message_formats", override_existing_map=False)
+        copy("bridge.state_event_formats.join")
+        copy("bridge.state_event_formats.leave")
+        copy("bridge.state_event_formats.name_change")
+
+        copy("bridge.filter.mode")
+        copy("bridge.filter.list")
+
+        copy("bridge.command_prefix")
+
+        migrate_permissions = ("bridge.permissions" not in self
+                               or "bridge.whitelist" in self
+                               or "bridge.admins" in self)
+        if migrate_permissions:
+            permissions = self["bridge.permissions"] or CommentedMap()
+            for entry in self["bridge.whitelist"] or []:
+                permissions[entry] = "full"
+            for entry in self["bridge.admins"] or []:
+                permissions[entry] = "admin"
+            base["bridge.permissions"] = permissions
+        else:
+            copy_dict("bridge.permissions")
+
+        if "bridge.relaybot" not in self:
+            copy("bridge.authless_relaybot_portals", "bridge.relaybot.authless_portals")
+        else:
+            copy("bridge.relaybot.authless_portals")
+            copy("bridge.relaybot.whitelist_group_admins")
+            copy("bridge.relaybot.whitelist")
+            copy("bridge.relaybot.ignore_own_incoming_events")
+
+        copy("telegram.api_id")
+        copy("telegram.api_hash")
+        copy("telegram.bot_token")
+        copy("telegram.proxy.type")
+        copy("telegram.proxy.address")
+        copy("telegram.proxy.port")
+        copy("telegram.proxy.rdns")
+        copy("telegram.proxy.username")
+        copy("telegram.proxy.password")
+
+        if "appservice.debug" in self and "logging" not in self:
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
+            base["logging.root.level"] = level
+            base["logging.loggers.mau.level"] = level
+            base["logging.loggers.telethon.level"] = level
+        else:
+            copy("logging")
+
+        self._data = base._data
+        self.save()
+
+    def _get_permissions(self, key: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
+        level = self["bridge.permissions"].get(key, "")
+        admin = level == "admin"
+        matrix_puppeting = level == "full" or admin
+        puppeting = level == "puppeting" or matrix_puppeting
+        user = level == "user" or puppeting
+        relaybot = level == "relaybot" or user
+        return relaybot, user, puppeting, matrix_puppeting, admin, level
+
+    def get_permissions(self, mxid: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
+        permissions = self["bridge.permissions"] or {}
+        if mxid in permissions:
+            return self._get_permissions(mxid)
+
+        homeserver = mxid[mxid.index(":") + 1:]
+        if homeserver in permissions:
+            return self._get_permissions(homeserver)
+
+        return self._get_permissions("*")

    def generate_registration(self):
        homeserver = self["homeserver.domain"]
@@ -93,10 +281,8 @@ class Config(DictWithRecursion):
        self.set("appservice.as_token", self._new_token())
        self.set("appservice.hs_token", self._new_token())

-        url = (f"{self['appservice.protocol']}://"
-               + f"{self['appservice.hostname']}:{self['appservice.port']}")
        self._registration = {
-            "id": self.get("appservice.id", "telegram"),
+            "id": self["appservice.id"] or "telegram",
            "as_token": self["appservice.as_token"],
            "hs_token": self["appservice.hs_token"],
            "namespaces": {
@@ -109,7 +295,7 @@ class Config(DictWithRecursion):
                    "regex": f"#{alias_format}:{homeserver}"
                }]
            },
-            "url": url,
+            "url": self["appservice.address"],
            "sender_localpart": self["appservice.bot_username"],
            "rate_limited": False
        }
@@ -0,0 +1,51 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    import asyncio
+
+    from sqlalchemy.orm import scoped_session
+
+    from alchemysession import AlchemySessionContainer
+    from mautrix_appservice import AppService
+
+    from .web import PublicBridgeWebsite, ProvisioningAPI
+    from .config import Config
+    from .bot import Bot
+    from .matrix import MatrixHandler
+
+
+class Context:
+    def __init__(self, az: "AppService", db: "scoped_session", config: "Config",
+                 loop: "asyncio.AbstractEventLoop", session_container: "AlchemySessionContainer"):
+        self.az = az  # type: AppService
+        self.db = db  # type: scoped_session
+        self.config = config  # type: Config
+        self.loop = loop  # type: asyncio.AbstractEventLoop
+        self.bot = None  # type: Optional[Bot]
+        self.mx = None  # type: MatrixHandler
+        self.session_container = session_container  # type: AlchemySessionContainer
+        self.public_website = None  # type: PublicBridgeWebsite
+        self.provisioning_api = None  # type: ProvisioningAPI
+
+    def __iter__(self):
+        yield self.az
+        yield self.db
+        yield self.config
+        yield self.loop
+        yield self.bot
@@ -3,31 +3,35 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from sqlalchemy import Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer, String
-from sqlalchemy.orm import relationship
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
+                        BigInteger, String, Boolean, Text)
+from sqlalchemy.sql import expression
+from sqlalchemy.orm import relationship, Query
+import json

 from .base import Base


 class Portal(Base):
-    query = None
+    query = None  # type: Query
    __tablename__ = "portal"

    # Telegram chat information
    tgid = Column(Integer, primary_key=True)
    tg_receiver = Column(Integer, primary_key=True)
-    peer_type = Column(String)
+    peer_type = Column(String, nullable=False)
+    megagroup = Column(Boolean)

    # Matrix portal information
    mxid = Column(String, unique=True, nullable=True)
@@ -40,7 +44,7 @@ class Portal(Base):


 class Message(Base):
-    query = None
+    query = None  # type: Query
    __tablename__ = "message"

    mxid = Column(String)
@@ -48,51 +52,123 @@ class Message(Base):
    tgid = Column(Integer, primary_key=True)
    tg_space = Column(Integer, primary_key=True)

-    __table_args__ = (UniqueConstraint('mxid', 'mx_room', 'tg_space', name='_mx_id_room'),)
+    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)


 class UserPortal(Base):
-    query = None
+    query = None  # type: Query
    __tablename__ = "user_portal"

-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
+    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
+                  primary_key=True)
    portal = Column(Integer, primary_key=True)
    portal_receiver = Column(Integer, primary_key=True)

    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
-                                           ("portal.tgid", "portal.tg_receiver")),)
+                                           ("portal.tgid", "portal.tg_receiver"),
+                                           onupdate="CASCADE", ondelete="CASCADE"),)


 class User(Base):
-    query = None
+    query = None  # type: Query
    __tablename__ = "user"

    mxid = Column(String, primary_key=True)
-    tgid = Column(Integer, nullable=True)
+    tgid = Column(Integer, nullable=True, unique=True)
    tg_username = Column(String, nullable=True)
-    saved_contacts = Column(Integer, default=0)
+    saved_contacts = Column(Integer, default=0, nullable=False)
    contacts = relationship("Contact", uselist=True,
                            cascade="save-update, merge, delete, delete-orphan")
-    portals = relationship("Portal", secondary="user_portal", single_parent=True,
-                           cascade="save-update, merge, delete, delete-orphan")
+    portals = relationship("Portal", secondary="user_portal")
+
+
+class RoomState(Base):
+    query = None  # type: Query
+    __tablename__ = "mx_room_state"
+
+    room_id = Column(String, primary_key=True)
+    _power_levels_text = Column("power_levels", Text, nullable=True)
+    _power_levels_json = None
+
+    @property
+    def has_power_levels(self):
+        return bool(self._power_levels_text)
+
+    @property
+    def power_levels(self):
+        if not self._power_levels_json and self._power_levels_text:
+            self._power_levels_json = json.loads(self._power_levels_text)
+        return self._power_levels_json or {}
+
+    @power_levels.setter
+    def power_levels(self, val):
+        self._power_levels_json = val
+        self._power_levels_text = json.dumps(val)
+
+
+class UserProfile(Base):
+    query = None  # type: Query
+    __tablename__ = "mx_user_profile"
+
+    room_id = Column(String, primary_key=True)
+    user_id = Column(String, primary_key=True)
+    membership = Column(String, nullable=False, default="leave")
+    displayname = Column(String, nullable=True)
+    avatar_url = Column(String, nullable=True)
+
+    def dict(self):
+        return {
+            "membership": self.membership,
+            "displayname": self.displayname,
+            "avatar_url": self.avatar_url,
+        }


 class Contact(Base):
-    query = None
+    query = None  # type: Query
    __tablename__ = "contact"

-    user = Column("user", Integer, ForeignKey("user.tgid"), primary_key=True)
-    contact = Column("contact", Integer, ForeignKey("puppet.id"), primary_key=True)
+    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
+    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)


 class Puppet(Base):
-    query = None
+    query = None  # type: Query
    __tablename__ = "puppet"

    id = Column(Integer, primary_key=True)
+    custom_mxid = Column(String, nullable=True)
+    access_token = Column(String, nullable=True)
    displayname = Column(String, nullable=True)
+    displayname_source = Column(Integer, nullable=True)
    username = Column(String, nullable=True)
    photo_id = Column(String, nullable=True)
+    is_bot = Column(Boolean, nullable=True)
+    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
+
+
+# Fucking Telegram not telling bots what chats they are in 3:<
+class BotChat(Base):
+    query = None  # type: Query
+    __tablename__ = "bot_chat"
+    id = Column(Integer, primary_key=True)
+    type = Column(String, nullable=False)
+
+
+class TelegramFile(Base):
+    query = None  # type: Query
+    __tablename__ = "telegram_file"
+
+    id = Column(String, primary_key=True)
+    mxc = Column(String)
+    mime_type = Column(String)
+    was_converted = Column(Boolean)
+    timestamp = Column(BigInteger)
+    size = Column(Integer, nullable=True)
+    width = Column(Integer, nullable=True)
+    height = Column(Integer, nullable=True)
+    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+    thumbnail = relationship("TelegramFile", uselist=False)


 def init(db_session):
@@ -101,3 +177,7 @@ def init(db_session):
    UserPortal.query = db_session.query_property()
    User.query = db_session.query_property()
    Puppet.query = db_session.query_property()
+    BotChat.query = db_session.query_property()
+    TelegramFile.query = db_session.query_property()
+    UserProfile.query = db_session.query_property()
+    RoomState.query = db_session.query_property()
@@ -1,380 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from html import escape, unescape
-from html.parser import HTMLParser
-from collections import deque
-import re
-import logging
-
-from mautrix_appservice import MatrixRequestError
-
-from telethon.tl.types import *
-
-from . import user as u, puppet as p
-from .db import Message as DBMessage
-
-log = logging.getLogger("mau.formatter")
-
-# TEXT LEN EXPLANATION:
-# Telegram formatting counts two bytes in an UTF-16 string as one character.
-#
-# For Telegram -> Matrix formatting, we get the same counting mechanism by encoding the input
-# text as UTF-16 Little Endian and doubling all the offsets and lengths given by Telegram. With
-# those doubled values, we process the input entities and text. The text is converted back to
-# native str format before it's inserted into the output HTML.
-#
-# For Matrix -> Telegram formatting, do the same input encoding, but divide the length by two
-# instead of multiplying when generating the lengths and offsets of Telegram entities.
-#
-# The endianness doesn't matter, but it has to be specified to avoid the two BOM bits messing
-# everything up.
-TEMP_ENC = "utf-16-le"
-
-
-# region Matrix to Telegram
-
-
-class MatrixParser(HTMLParser):
-    mention_regex = re.compile("https://matrix.to/#/(@.+)")
-
-    def __init__(self):
-        super().__init__()
-        self.text = ""
-        self.entities = []
-        self._building_entities = {}
-        self._list_counter = 0
-        self._open_tags = deque()
-        self._open_tags_meta = deque()
-        self._previous_ended_line = True
-
-    def handle_starttag(self, tag, attrs):
-        self._open_tags.appendleft(tag)
-        self._open_tags_meta.appendleft(0)
-        attrs = dict(attrs)
-        entity_type = None
-        args = {}
-        if tag == "strong" or tag == "b":
-            entity_type = MessageEntityBold
-        elif tag == "em" or tag == "i":
-            entity_type = MessageEntityItalic
-        elif tag == "code":
-            try:
-                pre = self._building_entities["pre"]
-                try:
-                    pre.language = attrs["class"][len("language-"):]
-                except KeyError:
-                    pass
-            except KeyError:
-                entity_type = MessageEntityCode
-        elif tag == "pre":
-            entity_type = MessageEntityPre
-            args["language"] = ""
-        elif tag == "a":
-            try:
-                url = attrs["href"]
-            except KeyError:
-                return
-            mention = self.mention_regex.search(url)
-            if mention:
-                mxid = mention.group(1)
-                user = p.Puppet.get_by_mxid(mxid, create=False)
-                if not user:
-                    user = u.User.get_by_mxid(mxid, create=False)
-                    if not user:
-                        return
-                if user.username:
-                    entity_type = MessageEntityMention
-                    url = f"@{user.username}"
-                else:
-                    entity_type = MessageEntityMentionName
-                    args["user_id"] = user.tgid
-            elif url.startswith("mailto:"):
-                url = url[len("mailto:"):]
-                entity_type = MessageEntityEmail
-            else:
-                if self.get_starttag_text() == url:
-                    entity_type = MessageEntityUrl
-                else:
-                    entity_type = MessageEntityTextUrl
-                    args["url"] = url
-                    url = None
-            self._open_tags_meta.popleft()
-            self._open_tags_meta.appendleft(url)
-
-        if entity_type and tag not in self._building_entities:
-            # See "TEXT LEN EXPLANATION" near start of file
-            offset = int(len(self.text.encode(TEMP_ENC)) / 2)
-            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
-
-    def _list_depth(self):
-        depth = 0
-        for tag in self._open_tags:
-            if tag == "ol" or tag == "ul":
-                depth += 1
-        return depth
-
-    def handle_data(self, text):
-        text = unescape(text)
-        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
-        list_format_offset = 0
-        if previous_tag == "a":
-            url = self._open_tags_meta[0]
-            if url:
-                text = url
-        elif len(self._open_tags) > 1 and self._previous_ended_line and previous_tag == "li":
-            list_type = self._open_tags[1]
-            indent = (self._list_depth() - 1) * 4 * " "
-            text = text.strip("\n")
-            if len(text) == 0:
-                return
-            elif list_type == "ul":
-                text = f"{indent}* {text}"
-                list_format_offset = len(indent) + 2
-            elif list_type == "ol":
-                n = self._open_tags_meta[1]
-                n += 1
-                self._open_tags_meta[1] = n
-                text = f"{indent}{n}. {text}"
-                list_format_offset = len(indent) + 3
-        for tag, entity in self._building_entities.items():
-            # See "TEXT LEN EXPLANATION" near start of file
-            entity.length += int(len(text.strip("\n").encode(TEMP_ENC)) / 2)
-            entity.offset += list_format_offset
-
-        if text.endswith("\n"):
-            self._previous_ended_line = True
-        else:
-            self._previous_ended_line = False
-
-        self.text += text
-
-    def handle_endtag(self, tag):
-        try:
-            self._open_tags.popleft()
-            self._open_tags_meta.popleft()
-        except IndexError:
-            pass
-        if (tag == "ul" or tag == "ol") and self.text.endswith("\n"):
-            self.text = self.text[:-1]
-        entity = self._building_entities.pop(tag, None)
-        if entity:
-            self.entities.append(entity)
-
-
-def matrix_to_telegram(html):
-    try:
-        parser = MatrixParser()
-        parser.feed(html)
-        return parser.text, parser.entities
-    except Exception:
-        log.exception("Failed to convert Matrix format:\nhtml=%s", html)
-
-
-def matrix_reply_to_telegram(content, tg_space, room_id=None):
-    try:
-        reply = content["m.relates_to"]["m.in_reply_to"]
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-        message = DBMessage.query.filter(DBMessage.mxid == event_id and
-                                         DBMessage.tg_space == tg_space and
-                                         DBMessage.mx_room == room_id).one_or_none()
-        if message:
-            return message.tgid
-    except KeyError:
-        pass
-    return None
-
-
-# endregion
-# region Telegram to Matrix
-
-def telegram_reply_to_matrix(evt, source):
-    if evt.reply_to_msg_id:
-        space = (evt.to_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
-                 else source.tgid)
-        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
-        if msg:
-            return {
-                "m.in_reply_to": {
-                    "event_id": msg.mxid,
-                    "room_id": msg.mx_room,
-                }
-            }
-    return {}
-
-
-async def telegram_event_to_matrix(evt, source, native_replies=False, message_link_in_reply=False,
-                                   main_intent=None, reply_text="Reply"):
-    text = evt.message
-    html = telegram_to_matrix(evt.message, evt.entities) if evt.entities else None
-    relates_to = {}
-
-    if evt.fwd_from:
-        if not html:
-            html = escape(text)
-        from_id = evt.fwd_from.from_id
-        user = u.User.get_by_tgid(from_id)
-        if user:
-            fwd_from = f"<a href='https://matrix.to/#/{user.mxid}'>{user.mxid}</a>"
-        else:
-            puppet = p.Puppet.get(from_id, create=False)
-            if puppet and puppet.displayname:
-                fwd_from = f"<a href='https://matrix.to/#/{puppet.mxid}'>{puppet.displayname}</a>"
-            else:
-                user = await source.client.get_entity(from_id)
-                if user:
-                    fwd_from = p.Puppet.get_displayname(user, format=False)
-                else:
-                    fwd_from = None
-        if not fwd_from:
-            fwd_from = "Unknown user"
-        html = (f"Forwarded message from <b>{fwd_from}</b><br/>"
-                + f"<blockquote>{html}</blockquote>")
-
-    if evt.reply_to_msg_id:
-        space = (evt.to_id.channel_id
-                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
-                 else source.tgid)
-        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
-        if msg:
-            if native_replies:
-                relates_to["m.in_reply_to"] = {
-                    "event_id": msg.mxid,
-                    "room_id": msg.mx_room,
-                }
-                if reply_text == "Edit":
-                    html = "<u>Edit:</u> " + (html or escape(text))
-            else:
-                try:
-                    event = await main_intent.get_event(msg.mx_room, msg.mxid)
-                    content = event["content"]
-                    body = (content["formatted_body"]
-                            if "formatted_body" in content
-                            else content["body"])
-                    sender = event['sender']
-                    puppet = p.Puppet.get_by_mxid(sender, create=False)
-                    displayname = puppet.displayname if puppet else sender
-                    reply_to_user = f"<a href='https://matrix.to/#/{sender}'>{displayname}</a>"
-                    reply_to_msg = (("<a href='https://matrix.to/#/"
-                                     + f"{msg.mx_room}/{msg.mxid}'>{reply_text}</a>")
-                                    if message_link_in_reply else "Reply")
-                    quote = f"{reply_to_msg} to {reply_to_user}<blockquote>{body}</blockquote>"
-                except (ValueError, KeyError, MatrixRequestError):
-                    quote = "{reply_text} to unknown user <em>(Failed to fetch message)</em>:<br/>"
-                if html:
-                    html = quote + html
-                else:
-                    html = quote + escape(text)
-
-    if isinstance(evt, Message) and evt.post and evt.post_author:
-        if not html:
-            html = escape(text)
-        text += f"\n- {evt.post_author}"
-        html += f"<br/><i>- <u>{evt.post_author}</u></i>"
-
-    if html:
-        html = html.replace("\n", "<br/>")
-
-    return text, html, relates_to
-
-
-def telegram_to_matrix(text, entities):
-    try:
-        return _telegram_to_matrix(text, entities)
-    except Exception:
-        log.exception("Failed to convert Telegram format:\n"
-                      "message=%s\n"
-                      "entities=%s",
-                      text, entities)
-
-
-def _telegram_to_matrix(text, entities):
-    if not entities:
-        return text
-    # See "TEXT LEN EXPLANATION" near start of file
-    text = text.encode(TEMP_ENC)
-    html = []
-    last_offset = 0
-    for entity in entities:
-        entity.offset *= 2
-        entity.length *= 2
-        if entity.offset > last_offset:
-            html.append(escape(text[last_offset:entity.offset].decode(TEMP_ENC)))
-        elif entity.offset < last_offset:
-            continue
-
-        skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length].decode(TEMP_ENC))
-        entity_type = type(entity)
-
-        if entity_type == MessageEntityBold:
-            html.append(f"<strong>{entity_text}</strong>")
-        elif entity_type == MessageEntityItalic:
-            html.append(f"<em>{entity_text}</em>")
-        elif entity_type == MessageEntityCode:
-            html.append(f"<code>{entity_text}</code>")
-        elif entity_type == MessageEntityPre:
-            if entity.language:
-                html.append("<pre>"
-                            + f"<code class='language-{entity.language}'>{entity_text}</code>"
-                            + "</pre>")
-            else:
-                html.append(f"<pre><code>{entity_text}</code></pre>")
-        elif entity_type == MessageEntityMention:
-            username = entity_text[1:]
-
-            user = u.User.find_by_username(username)
-            if user:
-                mxid = user.mxid
-            else:
-                puppet = p.Puppet.find_by_username(username)
-                mxid = puppet.mxid if puppet else None
-            if mxid:
-                html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
-            else:
-                skip_entity = True
-        elif entity_type == MessageEntityMentionName:
-            user = u.User.get_by_tgid(entity.user_id)
-            if user:
-                mxid = user.mxid
-            else:
-                puppet = p.Puppet.get(entity.user_id, create=False)
-                mxid = puppet.mxid if puppet else None
-            if mxid:
-                html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
-            else:
-                skip_entity = True
-        elif entity_type == MessageEntityEmail:
-            html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
-        elif entity_type in {MessageEntityTextUrl, MessageEntityUrl}:
-            url = escape(entity.url) if entity_type == MessageEntityTextUrl else entity_text
-            if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
-                url = "http://" + url
-            html.append(f"<a href='{url}'>{entity_text}</a>")
-        elif entity_type == MessageEntityBotCommand:
-            html.append(f"<font color='blue'>!{entity_text[1:]}")
-        elif entity_type == MessageEntityHashtag:
-            html.append(f"<font color='blue'>{entity_text}</font>")
-        else:
-            skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:].decode(TEMP_ENC))
-
-    return "".join(html)
-
-# endregion
@@ -0,0 +1,9 @@
+from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
+                          init_mx)
+from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
+from .. import context as c
+
+
+def init(context: c.Context):
+    init_mx(context)
+    init_tg(context)
@@ -0,0 +1,156 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple, Callable, Pattern, Match, TYPE_CHECKING
+import re
+import logging
+
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
+                               TypeMessageEntity)
+
+from ... import puppet as pu
+from ...db import Message as DBMessage
+from ..util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                    trim_reply_fallback_text)
+from .parser_common import ParsedMessage
+
+try:
+    from mautrix_telegram.formatter.from_matrix.parser_lxml import parse_html
+except ImportError:
+    from mautrix_telegram.formatter.from_matrix.parser_htmlparser import parse_html
+
+if TYPE_CHECKING:
+    from ...context import Context
+
+log = logging.getLogger("mau.fmt.mx")  # type: logging.Logger
+should_bridge_plaintext_highlights = False  # type: bool
+
+command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
+not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
+plain_mention_regex = None  # type: Pattern
+
+
+def plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
+def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+    if len(message) > 4096:
+        message = message[0:4082] + " [message cut]"
+        new_entities = []
+        for entity in entities:
+            if entity.offset > 4082:
+                continue
+            if entity.offset + entity.length > 4082:
+                entity.length = 4082 - entity.offset
+            new_entities.append(entity)
+        new_entities.append(MessageEntityItalic(4082, len(" [message cut]")))
+        entities = new_entities
+    return message, entities
+
+
+class FormatError(Exception):
+    pass
+
+
+def matrix_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(plain_mention_to_html, html)
+
+        html = add_surrogates(html)
+        text, entities = parse_html(add_surrogates(html))
+        text = remove_surrogates(text.strip())
+        text, entities = cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def matrix_reply_to_telegram(content: dict, tg_space: int, room_id: Optional[str] = None
+                             ) -> Optional[int]:
+    try:
+        reply = content["m.relates_to"]["m.in_reply_to"]
+        room_id = room_id or reply["room_id"]
+        event_id = reply["event_id"]
+
+        try:
+            if content["format"] == "org.matrix.custom.html":
+                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
+        except KeyError:
+            pass
+        content["body"] = trim_reply_fallback_text(content["body"])
+
+        message = DBMessage.query.filter(DBMessage.mxid == event_id,
+                                         DBMessage.tg_space == tg_space,
+                                         DBMessage.mx_room == room_id).one_or_none()
+        if message:
+            return message.tgid
+    except KeyError:
+        pass
+    return None
+
+
+def matrix_text_to_telegram(text: str) -> ParsedMessage:
+    text = command_regex.sub(r"/\1", text)
+    text = text.replace("\t", " " * 4)
+    text = not_command_regex.sub(r"\1", text)
+    if should_bridge_plaintext_highlights:
+        entities, pmr_replacer = plain_mention_to_text()
+        text = plain_mention_regex.sub(pmr_replacer, text)
+    else:
+        entities = []
+    return text, entities
+
+
+def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
+    entities = []
+
+    def replacer(match) -> str:
+        puppet = pu.Puppet.find_by_displayname(match.group(2))
+        if puppet:
+            offset = match.start()
+            length = match.end() - offset
+            if puppet.username:
+                entity = MessageEntityMention(offset, length)
+                text = f"@{puppet.username}"
+            else:
+                entity = MessageEntityMentionName(offset, length, user_id=puppet.tgid)
+                text = puppet.displayname
+            entities.append(entity)
+            return text
+        return "".join(match.groups())
+
+    return entities, replacer
+
+
+def init_mx(context: "Context"):
+    global plain_mention_regex, should_bridge_plaintext_highlights
+    config = context.config
+    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
+    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
+    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
+    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False
@@ -0,0 +1,31 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import re
+from typing import List, Tuple, Pattern
+from telethon.tl.types import TypeMessageEntity
+
+
+class MatrixParserCommon:
+    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")  # type: Pattern
+    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")  # type: Pattern
+    block_tags = ("br", "p", "pre", "blockquote",
+                  "ol", "ul", "li",
+                  "h1", "h2", "h3", "h4", "h5", "h6",
+                  "div", "hr", "table")  # type: Tuple[str, ...]
+
+
+ParsedMessage = Tuple[str, List[TypeMessageEntity]]
@@ -0,0 +1,236 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import (Optional, List, Tuple, Type, Dict, Any, Deque, Match)
+from html import unescape
+from html.parser import HTMLParser
+from collections import deque
+import math
+
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityEmail,
+                               MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
+                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
+                               MessageEntityBotCommand, TypeMessageEntity)
+
+from ... import user as u, puppet as pu, portal as po
+from ..util import html_to_unicode
+from .parser_common import MatrixParserCommon, ParsedMessage
+
+
+def parse_html(html: str) -> ParsedMessage:
+    parser = MatrixParser()
+    parser.feed(html)
+    return parser.text, parser.entities
+
+
+class MatrixParser(HTMLParser, MatrixParserCommon):
+    def __init__(self):
+        super(MatrixParser, self).__init__()
+        self.text = ""  # type: str
+        self.entities = []  # type: List[TypeMessageEntity]
+        self._building_entities = {}  # type: Dict[str, TypeMessageEntity]
+        self._list_counter = 0  # type: int
+        self._open_tags = deque()  # type: Deque[str]
+        self._open_tags_meta = deque()  # type: Deque[Any]
+        self._line_is_new = True  # type: bool
+        self._list_entry_is_new = False  # type: bool
+
+    def _parse_url(self, url: str, args: Dict[str, Any]
+                   ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]:
+        mention = self.mention_regex.match(url)  # type: Match
+        if mention:
+            mxid = mention.group(1)
+            user = (pu.Puppet.get_by_mxid(mxid)
+                    or u.User.get_by_mxid(mxid, create=False))
+            if not user:
+                return None, None
+            if user.username:
+                return MessageEntityMention, f"@{user.username}"
+            elif user.tgid:
+                args["user_id"] = user.tgid
+                return MessageEntityMentionName, user.displayname or None
+            else:
+                return None, None
+
+        room = self.room_regex.match(url)  # type: Match
+        if room:
+            username = po.Portal.get_username_from_mx_alias(room.group(1))
+            portal = po.Portal.find_by_username(username)
+            if portal and portal.username:
+                return MessageEntityMention, f"@{portal.username}"
+
+        if url.startswith("mailto:"):
+            return MessageEntityEmail, url[len("mailto:"):]
+        elif self.get_starttag_text() == url:
+            return MessageEntityUrl, url
+        else:
+            args["url"] = url
+            return MessageEntityTextUrl, None
+
+    def handle_starttag(self, tag: str, attrs: List[Tuple[str, str]]):
+        self._open_tags.appendleft(tag)
+        self._open_tags_meta.appendleft(0)
+
+        attrs = dict(attrs)
+        entity_type = None  # type: type(TypeMessageEntity)
+        args = {}  # type: Dict[str, Any]
+        if tag in ("strong", "b"):
+            entity_type = MessageEntityBold
+        elif tag in ("em", "i"):
+            entity_type = MessageEntityItalic
+        elif tag == "code":
+            try:
+                pre = self._building_entities["pre"]
+                try:
+                    # Pre tag and language found, add language to MessageEntityPre
+                    pre.language = attrs["class"][len("language-"):]
+                except KeyError:
+                    # Pre tag found, but language not found, keep pre as-is
+                    pass
+            except KeyError:
+                # No pre tag found, this is inline code
+                entity_type = MessageEntityCode
+        elif tag == "pre":
+            entity_type = MessageEntityPre
+            args["language"] = ""
+        elif tag == "command":
+            entity_type = MessageEntityBotCommand
+        elif tag == "li":
+            self._list_entry_is_new = True
+        elif tag == "a":
+            try:
+                url = attrs["href"]
+            except KeyError:
+                return
+            entity_type, url = self._parse_url(url, args)
+            self._open_tags_meta.popleft()
+            self._open_tags_meta.appendleft(url)
+
+        if tag in self.block_tags and ("blockquote" not in self._open_tags or tag == "br"):
+            self._newline()
+
+        if entity_type and tag not in self._building_entities:
+            offset = len(self.text)
+            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
+
+    @property
+    def _list_indent(self) -> int:
+        indent = 0
+        first_skipped = False
+        for index, tag in enumerate(self._open_tags):
+            if not first_skipped and tag in ("ol", "ul"):
+                # The first list level isn't indented, so skip it.
+                first_skipped = True
+                continue
+            if tag == "ol":
+                n = self._open_tags_meta[index]
+                extra_length_for_long_index = (int(math.log(n, 10)) - 1) * 3
+                indent += 4 + extra_length_for_long_index
+            elif tag == "ul":
+                indent += 3
+        return indent
+
+    def _newline(self, allow_multi: bool = False):
+        if self._line_is_new and not allow_multi:
+            return
+        self.text += "\n"
+        self._line_is_new = True
+        for entity in self._building_entities.values():
+            entity.length += 1
+
+    def _handle_special_previous_tags(self, text: str) -> str:
+        if "pre" not in self._open_tags and "code" not in self._open_tags:
+            text = text.replace("\n", "")
+        else:
+            text = text.strip()
+
+        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
+        if previous_tag == "a":
+            url = self._open_tags_meta[0]
+            if url:
+                text = url
+        elif previous_tag == "command":
+            text = f"/{text}"
+        return text
+
+    def _html_to_unicode(self, text: str) -> str:
+        strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags
+        if strikethrough and underline:
+            text = html_to_unicode(text, "\u0336\u0332")
+        elif strikethrough:
+            text = html_to_unicode(text, "\u0336")
+        elif underline:
+            text = html_to_unicode(text, "\u0332")
+        return text
+
+    def _handle_tags_for_data(self, text: str) -> Tuple[str, int]:
+        extra_offset = 0
+        list_entry_handled_once = False
+        # In order to maintain order of things like blockquotes in lists or lists in blockquotes,
+        # we can't just have ifs/elses and we need to actually loop through the open tags in order.
+        for index, tag in enumerate(self._open_tags):
+            if tag == "blockquote" and self._line_is_new:
+                text = f"> {text}"
+                extra_offset += 2
+            elif tag == "li" and not list_entry_handled_once:
+                list_type_index = index + 1
+                list_type = self._open_tags[list_type_index]
+                indent = self._list_indent * " " if self._line_is_new else ""
+                if list_type == "ol":
+                    n = self._open_tags_meta[list_type_index]
+                    if self._list_entry_is_new:
+                        n += 1
+                        self._open_tags_meta[list_type_index] = n
+                        prefix = f"{n}. "
+                    else:
+                        prefix = int(math.log(n, 10)) * 3 * " " + 4 * " "
+                else:
+                    prefix = "* " if self._list_entry_is_new else 3 * " "
+                if not self._list_entry_is_new and not self._line_is_new:
+                    prefix = ""
+                extra_offset += len(indent) + len(prefix)
+                text = indent + prefix + text
+                self._list_entry_is_new = False
+                list_entry_handled_once = True
+        return text, extra_offset
+
+    def _extend_entities_in_construction(self, text: str, extra_offset: int):
+        for tag, entity in self._building_entities.items():
+            entity.length += len(text) - extra_offset
+            entity.offset += extra_offset
+
+    def handle_data(self, text: str):
+        text = unescape(text)
+        text = self._handle_special_previous_tags(text)
+        text = self._html_to_unicode(text)
+        text, extra_offset = self._handle_tags_for_data(text)
+        self._extend_entities_in_construction(text, extra_offset)
+        self._line_is_new = False
+        self.text += text
+
+    def handle_endtag(self, tag: str):
+        try:
+            self._open_tags.popleft()
+            self._open_tags_meta.popleft()
+        except IndexError:
+            pass
+
+        entity = self._building_entities.pop(tag, None)
+        if entity:
+            self.entities.append(entity)
+
+        if tag in self.block_tags and tag != "br" and "blockquote" not in self._open_tags:
+            self._newline(allow_multi=tag == "br")
@@ -0,0 +1,343 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple, Union, Callable
+from lxml import html
+
+from telethon.tl.types import (MessageEntityMention as Mention,
+                               MessageEntityMentionName as MentionName, MessageEntityEmail as Email,
+                               MessageEntityUrl as URL, MessageEntityTextUrl as TextURL,
+                               MessageEntityBold as Bold, MessageEntityItalic as Italic,
+                               MessageEntityCode as Code, MessageEntityPre as Pre,
+                               MessageEntityBotCommand as Command, TypeMessageEntity,
+                               InputMessageEntityMentionName as InputMentionName)
+
+from ... import user as u, puppet as pu, portal as po
+from ..util import html_to_unicode
+from .parser_common import MatrixParserCommon, ParsedMessage
+
+
+def parse_html(html: str) -> ParsedMessage:
+    return MatrixParser.parse(html)
+
+
+class Entity:
+    @staticmethod
+    def copy(entity: TypeMessageEntity) -> Optional[TypeMessageEntity]:
+        if not entity:
+            return None
+        kwargs = {
+            "offset": entity.offset,
+            "length": entity.length,
+        }
+        if isinstance(entity, Pre):
+            kwargs["language"] = entity.language
+        elif isinstance(entity, TextURL):
+            kwargs["url"] = entity.url
+        elif isinstance(entity, (MentionName, InputMentionName)):
+            kwargs["user_id"] = entity.user_id
+        return entity.__class__(**kwargs)
+
+    @classmethod
+    def adjust(cls, entity: Union[TypeMessageEntity, List[TypeMessageEntity]],
+               func: Callable[[TypeMessageEntity], None]
+               ) -> Union[Optional[TypeMessageEntity], List[TypeMessageEntity]]:
+        if isinstance(entity, list):
+            return [Entity.adjust(element, func) for element in entity if entity]
+        elif not entity:
+            return None
+        entity = cls.copy(entity)
+        func(entity)
+        if entity.offset < 0:
+            entity.length += entity.offset
+            entity.offset = 0
+        return entity
+
+
+def offset_diff(amount: int):
+    def func(entity: TypeMessageEntity):
+        entity.offset += amount
+
+    return func
+
+
+def offset_length_multiply(amount: int):
+    def func(entity: TypeMessageEntity):
+        entity.offset *= amount
+        entity.length *= amount
+
+    return func
+
+
+class TelegramMessage:
+    def __init__(self, text: str = "", entities: Optional[List[TypeMessageEntity]] = None):
+        self.text = text  # type: str
+        self.entities = entities or []  # type: List[TypeMessageEntity]
+
+    def offset_entities(self, offset: int) -> "TelegramMessage":
+        def apply_offset(entity: TypeMessageEntity, inner_offset: int
+                         ) -> Optional[TypeMessageEntity]:
+            entity = Entity.copy(entity)
+            entity.offset += inner_offset
+            if entity.offset < 0:
+                entity.offset = 0
+            elif entity.offset > len(self.text):
+                return None
+            elif entity.offset + entity.length > len(self.text):
+                entity.length = len(self.text) - entity.offset
+            return entity
+
+        self.entities = [apply_offset(entity, offset) for entity in self.entities if entity]
+        self.entities = [x for x in self.entities if x is not None]
+        return self
+
+    def append(self, *args: Union[str, "TelegramMessage"]) -> "TelegramMessage":
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities += Entity.adjust(msg.entities, offset_diff(len(self.text)))
+            self.text += msg.text
+        return self
+
+    def prepend(self, *args: Union[str, "TelegramMessage"]) -> "TelegramMessage":
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities = msg.entities + Entity.adjust(self.entities, offset_diff(len(msg.text)))
+            self.text = msg.text + self.text
+        return self
+
+    def format(self, entity_type: type(TypeMessageEntity), offset: int = None, length: int = None,
+               **kwargs) -> "TelegramMessage":
+        self.entities.append(entity_type(offset=offset or 0,
+                                         length=length if length is not None else len(self.text),
+                                         **kwargs))
+        return self
+
+    def concat(self, *args: Union[str, "TelegramMessage"]) -> "TelegramMessage":
+        return TelegramMessage().append(self, *args)
+
+    def trim(self) -> "TelegramMessage":
+        orig_len = len(self.text)
+        self.text = self.text.lstrip()
+        diff = orig_len - len(self.text)
+        self.text = self.text.rstrip()
+        self.offset_entities(-diff)
+        return self
+
+    def split(self, separator, max_items: int = 0) -> List["TelegramMessage"]:
+        text_parts = self.text.split(separator, max_items - 1)
+        output = []  # type: List[TelegramMessage]
+
+        offset = 0
+        for part in text_parts:
+            msg = TelegramMessage(part)
+            for entity in self.entities:
+                start_in_range = len(part) > entity.offset - offset >= 0
+                end_in_range = len(part) >= entity.offset - offset + entity.length > 0
+                if start_in_range and end_in_range:
+                    msg.entities.append(Entity.adjust(entity, offset_diff(-offset)))
+            output.append(msg)
+
+            offset += len(part)
+            offset += len(separator)
+
+        return output
+
+    @staticmethod
+    def join(items: List[Union[str, "TelegramMessage"]], separator: str = " ") -> "TelegramMessage":
+        main = TelegramMessage()
+        for msg in items:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            main.entities += Entity.adjust(msg.entities, offset_diff(len(main.text)))
+            main.text += msg.text + separator
+        main.text = main.text[:-len(separator)]
+        return main
+
+
+class MatrixParser(MatrixParserCommon):
+    @classmethod
+    def list_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        ordered = node.tag == "ol"
+        tagged_children = cls.node_to_tagged_tmessages(node, strip_linebreaks)
+        counter = 1
+        indent_length = 0
+        if ordered:
+            try:
+                counter = int(node.attrib.get("start", "1"))
+            except ValueError:
+                counter = 1
+
+            longest_index = counter - 1 + len(tagged_children)
+            indent_length = len(str(longest_index))
+        indent = (indent_length + 4) * " "
+        children = []  # type: List[TelegramMessage]
+        for child, tag in tagged_children:
+            if tag != "li":
+                continue
+
+            if ordered:
+                prefix = f"{counter}. "
+                counter += 1
+            else:
+                prefix = "● "
+            child = child.prepend(prefix)
+            parts = child.split("\n")
+            parts = parts[:1] + [part.prepend(indent) for part in parts[1:]]
+            child = TelegramMessage.join(parts, "\n")
+            children.append(child)
+        return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def blockquote_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, strip_linebreaks)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def header_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        children = cls.node_to_tmessages(node, strip_linebreaks)
+        length = int(node.tag[1])
+        prefix = "#" * length + " "
+        return TelegramMessage.join(children, "").prepend(prefix)
+
+    @classmethod
+    def basic_format_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, strip_linebreaks)
+        if node.tag in ("b", "strong"):
+            msg.format(Bold)
+        elif node.tag in ("i", "em"):
+            msg.format(Italic)
+        elif node.tag == "command":
+            msg.format(Command)
+        elif node.tag in ("s", "del"):
+            msg.text = html_to_unicode(msg.text, "\u0336")
+        elif node.tag in ("u", "ins"):
+            msg.text = html_to_unicode(msg.text, "\u0332")
+
+        if node.tag in ("s", "del", "u", "ins"):
+            msg.entities = Entity.adjust(msg.entities, offset_length_multiply(2))
+
+        return msg
+
+    @classmethod
+    def link_to_tstring(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, strip_linebreaks)
+        href = node.attrib.get("href", "")
+        if not href:
+            return msg
+
+        if href.startswith("mailto:"):
+            return TelegramMessage(href[len("mailto:"):]).format(Email)
+
+        mention = cls.mention_regex.match(href)
+        if mention:
+            mxid = mention.group(1)
+            user = (pu.Puppet.get_by_mxid(mxid)
+                    or u.User.get_by_mxid(mxid, create=False))
+            if not user:
+                return msg
+            if user.username:
+                return TelegramMessage(f"@{user.username}").format(Mention)
+            elif user.tgid:
+                return TelegramMessage(user.displayname or msg.text).format(MentionName,
+                                                                            user_id=user.tgid)
+            return msg
+
+        room = cls.room_regex.match(href)
+        if room:
+            username = po.Portal.get_username_from_mx_alias(room.group(1))
+            portal = po.Portal.find_by_username(username)
+            if portal and portal.username:
+                return TelegramMessage(f"@{portal.username}").format(Mention)
+
+        return (msg.format(URL)
+                if msg.text == href
+                else msg.format(TextURL, url=href))
+
+    @classmethod
+    def node_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        if node.tag == "blockquote":
+            return cls.blockquote_to_tmessage(node, strip_linebreaks)
+        elif node.tag in ("ol", "ul"):
+            return cls.list_to_tmessage(node, strip_linebreaks)
+        elif node.tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            return cls.header_to_tmessage(node, strip_linebreaks)
+        elif node.tag == "br":
+            return TelegramMessage("\n")
+        elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
+            return cls.basic_format_to_tmessage(node, strip_linebreaks)
+        elif node.tag == "a":
+            return cls.link_to_tstring(node, strip_linebreaks)
+        elif node.tag == "p":
+            return cls.tag_aware_parse_node(node, strip_linebreaks).append("\n")
+        elif node.tag == "pre":
+            lang = ""
+            try:
+                if node[0].tag == "code":
+                    lang = node[0].attrib["class"][len("language-"):]
+                    node = node[0]
+            except (IndexError, KeyError):
+                pass
+            return cls.parse_node(node, strip_linebreaks=False).format(Pre, language=lang)
+        elif node.tag == "code":
+            return cls.parse_node(node, strip_linebreaks=False).format(Code)
+        return cls.tag_aware_parse_node(node, strip_linebreaks)
+
+    @staticmethod
+    def text_to_tmessage(text: str, strip_linebreaks: bool = True) -> TelegramMessage:
+        if strip_linebreaks:
+            text = text.replace("\n", "")
+        return TelegramMessage(text)
+
+    @classmethod
+    def node_to_tagged_tmessages(cls, node: html.HtmlElement, strip_linebreaks: bool = True
+                                 ) -> List[Tuple[TelegramMessage, str]]:
+        output = []
+
+        if node.text:
+            output.append((cls.text_to_tmessage(node.text, strip_linebreaks), "text"))
+        for child in node:
+            output.append((cls.node_to_tmessage(child, strip_linebreaks), child.tag))
+            if child.tail:
+                output.append((cls.text_to_tmessage(child.tail, strip_linebreaks), "text"))
+        return output
+
+    @classmethod
+    def node_to_tmessages(cls, node: html.HtmlElement, strip_linebreaks) -> List[TelegramMessage]:
+        return [msg for (msg, tag) in cls.node_to_tagged_tmessages(node, strip_linebreaks)]
+
+    @classmethod
+    def tag_aware_parse_node(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        msgs = cls.node_to_tagged_tmessages(node, strip_linebreaks)
+        output = TelegramMessage()
+        for msg, tag in msgs:
+            if tag in cls.block_tags:
+                msg = msg.append("\n").prepend("\n")
+            output = output.append(msg)
+        return output.trim()
+
+    @classmethod
+    def parse_node(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        return TelegramMessage.join(cls.node_to_tmessages(node, strip_linebreaks))
+
+    @classmethod
+    def parse(cls, data: str) -> ParsedMessage:
+        document = html.fromstring(f"<html>{data}</html>")
+        msg = cls.parse_node(document, strip_linebreaks=True)
+        return msg.text, msg.entities
@@ -0,0 +1,333 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple, TYPE_CHECKING
+from html import escape
+import logging
+import re
+
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName,
+                               MessageEntityEmail, MessageEntityUrl, MessageEntityTextUrl,
+                               MessageEntityBold, MessageEntityItalic, MessageEntityCode,
+                               MessageEntityPre, MessageEntityBotCommand, Message, PeerChannel,
+                               MessageEntityHashtag, TypeMessageEntity, MessageFwdHeader, PeerUser)
+
+from mautrix_appservice import MatrixRequestError
+from mautrix_appservice.intent_api import IntentAPI
+
+from .. import user as u, puppet as pu, portal as po
+from ..db import Message as DBMessage
+from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                   trim_reply_fallback_text, unicode_to_html)
+
+if TYPE_CHECKING:
+    from ..abstract_user import AbstractUser
+    from ..context import Context
+
+try:
+    from lxml.html.diff import htmldiff
+except ImportError:
+    htmldiff = None  # type: function
+
+
+log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
+should_highlight_edits = False  # type: bool
+
+
+def telegram_reply_to_matrix(evt: Message, source: "AbstractUser") -> dict:
+    if evt.reply_to_msg_id:
+        space = (evt.to_id.channel_id
+                 if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+                 else source.tgid)
+        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+        if msg:
+            return {
+                "m.in_reply_to": {
+                    "event_id": msg.mxid,
+                    "room_id": msg.mx_room,
+                }
+            }
+    return {}
+
+
+async def _add_forward_header(source, text: str, html: Optional[str],
+                              fwd_from: MessageFwdHeader) -> Tuple[str, str]:
+    if not html:
+        html = escape(text)
+    fwd_from_html, fwd_from_text = None, None
+    if fwd_from.from_id:
+        user = u.User.get_by_tgid(fwd_from.from_id)
+        if user:
+            fwd_from_text = user.displayname or user.mxid
+            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
+
+        if not fwd_from_text:
+            puppet = pu.Puppet.get(fwd_from.from_id, create=False)
+            if puppet and puppet.displayname:
+                fwd_from_text = puppet.displayname or puppet.mxid
+                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
+
+        if not fwd_from_text:
+            user = await source.client.get_entity(PeerUser(fwd_from.from_id))
+            if user:
+                fwd_from_text = pu.Puppet.get_displayname(user, False)
+                fwd_from_html = f"<b>{fwd_from_text}</b>"
+
+    if not fwd_from_text:
+        if fwd_from.from_id:
+            fwd_from_text = "Unknown user"
+        else:
+            fwd_from_text = "Unknown source"
+        fwd_from_html = f"<b>{fwd_from_text}</b>"
+
+    text = "\n".join([f"> {line}" for line in text.split("\n")])
+    text = f"Forwarded from {fwd_from_text}:\n{text}"
+    html = (f"Forwarded message from {fwd_from_html}<br/>"
+            f"<tg-forward><blockquote>{html}</blockquote></tg-forward>")
+    return text, html
+
+
+def highlight_edits(new_html: str, old_html: str) -> str:
+    # Don't include `Edit:` text in diff.
+    if old_html.startswith("<u>Edit:</u> "):
+        old_html = old_html[len("<u>Edit:</u> "):]
+
+    # Generate diff with lxml
+    new_html = htmldiff(old_html, new_html)
+
+    # Replace <ins> with <u> since Riot doesn't allow <ins>
+    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
+    # Remove <del>s since we just want to hide deletions.
+    new_html = re.sub("<del>.+?</del>", "", new_html)
+    return new_html
+
+
+async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
+                            relates_to: dict, main_intent: IntentAPI, is_edit: bool
+                            ) -> Tuple[str, str]:
+    space = (evt.to_id.channel_id
+             if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
+             else source.tgid)
+
+    msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+    if not msg:
+        return text, html
+
+    relates_to["m.in_reply_to"] = {
+        "event_id": msg.mxid,
+        "room_id": msg.mx_room,
+    }
+
+    try:
+        event = await main_intent.get_event(msg.mx_room, msg.mxid)
+
+        content = event["content"]
+        r_sender = event["sender"]
+
+        r_text_body = trim_reply_fallback_text(content["body"])
+        r_html_body = trim_reply_fallback_html(content["formatted_body"]
+                                               if "formatted_body" in content
+                                               else escape(content["body"]))
+
+        puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
+        r_displayname = puppet.displayname if puppet else r_sender
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
+
+        if is_edit and should_highlight_edits:
+            html = highlight_edits(html or escape(text), r_html_body)
+    except (ValueError, KeyError, MatrixRequestError):
+        r_sender_link = "unknown user"
+        r_displayname = "unknown user"
+        r_text_body = "Failed to fetch message"
+        r_html_body = "<em>Failed to fetch message</em>"
+
+    if is_edit:
+        html = f"<u>Edit:</u> {html or escape(text)}"
+        text = f"Edit: {text}"
+
+    r_keyword = "In reply to" if not is_edit else "Edit to"
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
+    html = (
+        f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
+        + (html or escape(text)))
+
+    lines = r_text_body.strip().split("\n")
+    text_with_quote = f"> <{r_displayname}> {lines.pop(0)}"
+    for line in lines:
+        if line:
+            text_with_quote += f"\n> {line}"
+    text_with_quote += "\n\n"
+    text_with_quote += text
+    return text_with_quote, html
+
+
+async def telegram_to_matrix(evt: Message, source: "AbstractUser",
+                             main_intent: Optional[IntentAPI] = None,
+                             is_edit: bool = False, prefix_text: Optional[str] = None,
+                             prefix_html: Optional[str] = None) -> Tuple[str, str, dict]:
+    text = add_surrogates(evt.message)
+    html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None
+    relates_to = {}
+
+    if prefix_html:
+        html = prefix_html + (html or escape(text))
+    if prefix_text:
+        text = prefix_text + text
+
+    if evt.fwd_from:
+        text, html = await _add_forward_header(source, text, html, evt.fwd_from)
+
+    if evt.reply_to_msg_id:
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
+                                             is_edit)
+
+    if isinstance(evt, Message) and evt.post and evt.post_author:
+        if not html:
+            html = escape(text)
+        text += f"\n- {evt.post_author}"
+        html += f"<br/><i>- <u>{evt.post_author}</u></i>"
+
+    html = unicode_to_html(text, html, "\u0336", "del")
+    html = unicode_to_html(text, html, "\u0332", "u")
+
+    if html:
+        html = html.replace("\n", "<br/>")
+
+    return remove_surrogates(text), remove_surrogates(html), relates_to
+
+
+def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEntity]) -> str:
+    try:
+        return _telegram_entities_to_matrix(text, entities)
+    except Exception:
+        log.exception("Failed to convert Telegram format:\n"
+                      "message=%s\n"
+                      "entities=%s",
+                      text, entities)
+
+
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
+    if not entities:
+        return text
+    html = []
+    last_offset = 0
+    for entity in entities:
+        if entity.offset > last_offset:
+            html.append(escape(text[last_offset:entity.offset]))
+        elif entity.offset < last_offset:
+            continue
+
+        skip_entity = False
+        entity_text = escape(text[entity.offset:entity.offset + entity.length])
+        entity_type = type(entity)
+
+        if entity_type == MessageEntityBold:
+            html.append(f"<strong>{entity_text}</strong>")
+        elif entity_type == MessageEntityItalic:
+            html.append(f"<em>{entity_text}</em>")
+        elif entity_type == MessageEntityCode:
+            html.append(f"<code>{entity_text}</code>")
+        elif entity_type == MessageEntityPre:
+            skip_entity = _parse_pre(html, entity_text, entity.language)
+        elif entity_type == MessageEntityMention:
+            skip_entity = _parse_mention(html, entity_text)
+        elif entity_type == MessageEntityMentionName:
+            skip_entity = _parse_name_mention(html, entity_text, entity.user_id)
+        elif entity_type == MessageEntityEmail:
+            html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
+        elif entity_type in {MessageEntityTextUrl, MessageEntityUrl}:
+            skip_entity = _parse_url(html, entity_text,
+                                     entity.url if entity_type == MessageEntityTextUrl else None)
+        elif entity_type == MessageEntityBotCommand:
+            html.append(f"<font color='blue'>!{entity_text[1:]}</font>")
+        elif entity_type == MessageEntityHashtag:
+            html.append(f"<font color='blue'>{entity_text}</font>")
+        else:
+            skip_entity = True
+        last_offset = entity.offset + (0 if skip_entity else entity.length)
+    html.append(text[last_offset:])
+
+    return "".join(html)
+
+
+def _parse_pre(html: List[str], entity_text: str, language: str) -> bool:
+    if language:
+        html.append("<pre>"
+                    f"<code class='language-{language}'>{entity_text}</code>"
+                    "</pre>")
+    else:
+        html.append(f"<pre><code>{entity_text}</code></pre>")
+    return False
+
+
+def _parse_mention(html: List[str], entity_text: str) -> bool:
+    username = entity_text[1:]
+
+    user = u.User.find_by_username(username) or pu.Puppet.find_by_username(username)
+    if user:
+        mxid = user.mxid
+    else:
+        portal = po.Portal.find_by_username(username)
+        mxid = portal.alias or portal.mxid if portal else None
+
+    if mxid:
+        html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
+    else:
+        return True
+    return False
+
+
+def _parse_name_mention(html: List[str], entity_text: str, user_id: int) -> bool:
+    user = u.User.get_by_tgid(user_id)
+    if user:
+        mxid = user.mxid
+    else:
+        puppet = pu.Puppet.get(user_id, create=False)
+        mxid = puppet.mxid if puppet else None
+    if mxid:
+        html.append(f"<a href='https://matrix.to/#/{mxid}'>{entity_text}</a>")
+    else:
+        return True
+    return False
+
+
+message_link_regex = re.compile(
+    r"https?://t(?:elegram)?\.(?:me|dog)/([A-Za-z][A-Za-z0-9_]{3,}[A-Za-z0-9])/([0-9]{1,50})")
+
+
+def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
+    url = escape(url) if url else entity_text
+    if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
+        url = "http://" + url
+
+    message_link_match = message_link_regex.match(url)
+    if message_link_match:
+        group, msgid = message_link_match.groups()
+        msgid = int(msgid)
+
+        portal = po.Portal.find_by_username(group)
+        if portal:
+            message = DBMessage.query.get((msgid, portal.tgid))
+            if message:
+                url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
+
+    html.append(f"<a href='{url}'>{entity_text}</a>")
+    return False
+
+
+def init_tg(context: "Context"):
+    global should_highlight_edits
+    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]
@@ -0,0 +1,86 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Pattern
+from html import escape
+import struct
+import re
+
+
+# add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
+# Licensed under the MIT license.
+# https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py
+def add_surrogates(text: Optional[str]) -> Optional[str]:
+    if text is None:
+        return None
+    return "".join("".join(chr(y) for y in struct.unpack("<HH", x.encode("utf-16-le")))
+                   if (0x10000 <= ord(x) <= 0x10FFFF) else x for x in text)
+
+
+def remove_surrogates(text: Optional[str]) -> Optional[str]:
+    if text is None:
+        return None
+    return text.encode("utf-16", "surrogatepass").decode("utf-16")
+
+
+def trim_reply_fallback_text(text: str) -> str:
+    if not text.startswith("> ") or "\n" not in text:
+        return text
+    lines = text.split("\n")
+    while len(lines) > 0 and lines[0].startswith("> "):
+        lines.pop(0)
+    return "\n".join(lines)
+
+
+html_reply_fallback_regex = re.compile("^<mx-reply>"
+                                       r"[\s\S]+?"
+                                       "</mx-reply>")  # type: Pattern
+
+
+def trim_reply_fallback_html(html: str) -> str:
+    return html_reply_fallback_regex.sub("", html)
+
+
+def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
+    if ctrl not in text:
+        return html
+    if not html:
+        html = escape(text)
+    tag_start = f"<{tag}>"
+    tag_end = f"</{tag}>"
+    characters = html.split(ctrl)
+    html = ""
+    in_tag = False
+    for char in characters:
+        if not in_tag:
+            if len(char) > 1:
+                html += char[0:-1]
+                char = char[-1]
+            html += tag_start
+            in_tag = True
+            html += char
+        else:
+            if len(char) > 1:
+                html += tag_end
+                in_tag = False
+            html += char
+    if in_tag:
+        html += tag_end
+    return html
+
+
+def html_to_unicode(text: str, ctrl: str) -> str:
+    return ctrl.join(text) + ctrl
@@ -3,150 +3,198 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Dict, Tuple, Set, Match
 import logging
+import asyncio
+import re

-from mautrix_appservice import MatrixRequestError
+from mautrix_appservice import MatrixRequestError, IntentError

-from .user import User
-from .portal import Portal
-from .puppet import Puppet
-from .commands import CommandHandler
+from . import user as u, portal as po, puppet as pu, commands as com


 class MatrixHandler:
-    log = logging.getLogger("mau.mx")
+    log = logging.getLogger("mau.mx")  # type: logging.Logger

    def __init__(self, context):
-        self.az, self.db, self.config, _ = context
-        self.commands = CommandHandler(context)
+        self.az, self.db, self.config, _, self.tgbot = context
+        self.commands = com.CommandProcessor(context)  # type: com.CommandProcessor
+        self.previously_typing = []  # type: List[str]

        self.az.matrix_event_handler(self.handle_event)

    async def init_as_bot(self):
-        self.az.intent.set_display_name(
-            self.config.get("appservice.bot_displayname", "Telegram bridge bot"))
+        displayname = self.config["appservice.bot_displayname"]
+        if displayname:
+            try:
+                await self.az.intent.set_display_name(
+                    displayname if displayname != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set displayname")

-    async def handle_puppet_invite(self, room, puppet, inviter):
-        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}")
-        if not inviter.logged_in:
-            await puppet.intent.error_and_leave(
-                room, text="Please log in before inviting Telegram puppets.")
+        avatar = self.config["appservice.bot_avatar"]
+        if avatar:
+            try:
+                await self.az.intent.set_avatar(avatar if avatar != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set avatar")
+
+    async def handle_puppet_invite(self, room_id, puppet: pu.Puppet, inviter: u.User):
+        intent = puppet.default_mxid_intent
+        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}")
+        if not await inviter.is_logged_in():
+            await intent.error_and_leave(
+                room_id, text="Please log in before inviting Telegram puppets.")
            return
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
        if portal:
            if portal.peer_type == "user":
-                await puppet.intent.error_and_leave(
-                    room, text="You can not invite additional users to private chats.")
+                await intent.error_and_leave(
+                    room_id, text="You can not invite additional users to private chats.")
                return
            await portal.invite_telegram(inviter, puppet)
-            await puppet.intent.join_room(room)
+            await intent.join_room(room_id)
            return
        try:
-            members = await self.az.intent.get_room_members(room)
+            members = await self.az.intent.get_room_members(room_id)
        except MatrixRequestError:
            members = []
-        if self.az.intent.mxid not in members:
+        if self.az.bot_mxid not in members:
            if len(members) > 1:
-                await puppet.intent.error_and_leave(room, text=None, html=(
+                await intent.error_and_leave(room_id, text=None, html=(
                    f"Please invite "
-                    + f"<a href='https://matrix.to/#/{self.az.intent.mxid}'>the bridge bot</a> "
-                    + f"first if you want to create a Telegram chat."))
+                    f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
+                    f"first if you want to create a Telegram chat."))
                return

-            await puppet.intent.join_room(room)
-            portal = Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
+            await intent.join_room(room_id)
+            portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
            if portal.mxid:
                try:
-                    await puppet.intent.invite(portal.mxid, inviter.mxid)
-                    await puppet.intent.send_notice(room, text=None, html=(
+                    await intent.invite(portal.mxid, inviter.mxid)
+                    await intent.send_notice(room_id, text=None, html=(
                        "You already have a private chat with me: "
-                        + f"<a href='https://matrix.to/#/{portal.mxid}'>"
-                        + "Link to room"
-                        + "</a>"))
-                    await puppet.intent.leave_room(room)
+                        f"<a href='https://matrix.to/#/{portal.mxid}'>"
+                        "Link to room"
+                        "</a>"))
+                    await intent.leave_room(room_id)
                    return
                except MatrixRequestError:
                    pass
-            portal.mxid = room
+            portal.mxid = room_id
            portal.save()
            inviter.register_portal(portal)
-            await puppet.intent.send_notice(room, "Portal to private chat created.")
+            await intent.send_notice(room_id, "po.Portal to private chat created.")
        else:
-            await puppet.intent.join_room(room)
-            await puppet.intent.send_notice(room, "This puppet will remain inactive until a "
-                                                  "Telegram chat is created for this room.")
+            await intent.join_room(room_id)
+            await intent.send_notice(room_id, "This puppet will remain inactive until a "
+                                              "Telegram chat is created for this room.")
+
+    async def accept_bot_invite(self, room_id: str, inviter: u.User):
+        tries = 0
+        while tries < 5:
+            try:
+                await self.az.intent.join_room(room_id)
+                break
+            except (IntentError, MatrixRequestError):
+                tries += 1
+                wait_for_seconds = (tries + 1) * 10
+                if tries < 5:
+                    self.log.exception(f"Failed to join room {room_id} with bridge bot, "
+                                       f"retrying in {wait_for_seconds} seconds...")
+                    await asyncio.sleep(wait_for_seconds)
+                else:
+                    self.log.exception("Failed to join room {room}, giving up.")
+                    return

-    async def handle_invite(self, room, user, inviter):
-        inviter = User.get_by_mxid(inviter)
        if not inviter.whitelisted:
-            return
-        elif user == self.az.bot_mxid:
-            await self.az.intent.join_room(room)
+            await self.az.intent.send_notice(
+                room_id, text=None,
+                html="You are not whitelisted to use this bridge.<br/><br/>"
+                     "If you are the owner of this bridge, see the "
+                     "<code>bridge.permissions</code> section in your config file.")
+            await self.az.intent.leave_room(room_id)
+
+    async def handle_invite(self, room_id: str, user_id: str, inviter_mxid: str):
+        self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
+        inviter = await u.User.get_by_mxid(inviter_mxid).ensure_started()
+        if user_id == self.az.bot_mxid:
+            return await self.accept_bot_invite(room_id, inviter)
+        elif not inviter.whitelisted:
            return

-        puppet = Puppet.get_by_mxid(user)
+        puppet = pu.Puppet.get_by_mxid(user_id)
        if puppet:
-            await self.handle_puppet_invite(room, puppet, inviter)
+            await self.handle_puppet_invite(room_id, puppet, inviter)
            return

-        user = User.get_by_mxid(user, create=False)
-        portal = Portal.get_by_mxid(room)
-        if user and user.has_full_access and portal:
+        user = u.User.get_by_mxid(user_id, create=False)
+        if not user:
+            return
+        await user.ensure_started()
+        portal = po.Portal.get_by_mxid(room_id)
+        if user and await user.has_full_access(allow_bot=True) and portal:
            await portal.invite_telegram(inviter, user)
            return

        # The rest can probably be ignored
-        self.log.debug(f"{inviter} invited {user} to {room}")

-    async def handle_join(self, room, user):
-        user = User.get_by_mxid(user)
+    async def handle_join(self, room_id: str, user_id: str, event_id: str):
+        user = await u.User.get_by_mxid(user_id).ensure_started()

-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
        if not portal:
            return

-        if not user.whitelisted:
-            await portal.main_intent.kick(room, user.mxid,
+        if not user.relaybot_whitelisted:
+            await portal.main_intent.kick(room_id, user.mxid,
                                          "You are not whitelisted on this Telegram bridge.")
            return
-        elif not user.logged_in:
-            # TODO[waiting-for-bots] once we have bot support, this won't be needed.
-            await portal.main_intent.kick(room, user.mxid,
-                                          "You are not logged into this Telegram bridge.")
+        elif not await user.is_logged_in() and not portal.has_bot:
+            await portal.main_intent.kick(room_id, user.mxid,
+                                          "This chat does not have a bot relaying "
+                                          "messages for unauthenticated users.")
            return

-        self.log.debug(f"{user} joined {room}")
-        # TODO join Telegram chat if applicable
+        self.log.debug(f"{user} joined {room_id}")
+        if await user.is_logged_in() or portal.has_bot:
+            await portal.join_matrix(user, event_id)

-    async def handle_part(self, room, user, sender):
-        self.log.debug(f"{user} left {room}")
+    async def handle_part(self, room_id: str, user_id, sender_mxid: str, event_id: str):
+        self.log.debug(f"{user_id} left {room_id}")

-        sender = User.get_by_mxid(sender, create=False)
+        sender = u.User.get_by_mxid(sender_mxid, create=False)
+        if not sender:
+            return
+        await sender.ensure_started()

-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
        if not portal:
            return

-        puppet = Puppet.get_by_mxid(user)
+        puppet = pu.Puppet.get_by_mxid(user_id)
        if sender and puppet:
-            await portal.leave_matrix(puppet, sender)
+            await portal.leave_matrix(puppet, sender, event_id)

-        user = User.get_by_mxid(user, create=False)
-        if user and user.logged_in:
-            await portal.leave_matrix(user, sender)
+        user = u.User.get_by_mxid(user_id, create=False)
+        if not user:
+            return
+        await user.ensure_started()
+        if await user.is_logged_in() or portal.has_bot:
+            await portal.leave_matrix(user, sender, event_id)

-    def is_command(self, message):
+    def is_command(self, message: dict) -> Tuple[bool, str]:
        text = message.get("body", "")
        prefix = self.config["bridge.command_prefix"]
        is_command = text.startswith(prefix)
@@ -155,17 +203,20 @@ class MatrixHandler:
        return is_command, text

    async def handle_message(self, room, sender, message, event_id):
-        self.log.debug(f"{sender} sent {message} to ${room}")
-
        is_command, text = self.is_command(message)
-        sender = User.get_by_mxid(sender)
+        sender = await u.User.get_by_mxid(sender).ensure_started()
+        if not sender.relaybot_whitelisted:
+            self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:"
+                           " u.User is not whitelisted.")
+            return
+        self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}")

-        portal = Portal.get_by_mxid(room)
-        if sender.has_full_access and portal and not is_command:
+        portal = po.Portal.get_by_mxid(room)
+        if not is_command and portal and (await sender.is_logged_in() or portal.has_bot):
            await portal.handle_matrix_message(sender, message, event_id)
            return

-        if message["msgtype"] != "m.text":
+        if not sender.whitelisted or message.get("msgtype", "m.unknown") != "m.text":
            return

        try:
@@ -185,56 +236,170 @@ class MatrixHandler:
            await self.commands.handle(room, sender, command, args, is_management,
                                       is_portal=portal is not None)

-    async def handle_redaction(self, room, sender, event_id):
-        portal = Portal.get_by_mxid(room)
-        sender = User.get_by_mxid(sender)
-        if sender.has_full_access and portal:
-            await portal.handle_matrix_deletion(sender, event_id)
+    @staticmethod
+    async def handle_redaction(room_id: str, sender_mxid: str, event_id: str):
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if not sender.relaybot_whitelisted:
+            return

-    async def handle_power_levels(self, room, sender, new, old):
-        portal = Portal.get_by_mxid(room)
-        sender = User.get_by_mxid(sender)
-        if sender.has_full_access and portal:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        await portal.handle_matrix_deletion(sender, event_id)
+
+    @staticmethod
+    async def handle_power_levels(room_id: str, sender_mxid: str, new: dict, old: dict):
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
            await portal.handle_matrix_power_levels(sender, new["users"], old["users"])

-    async def handle_room_meta(self, type, room, sender, content):
-        portal = Portal.get_by_mxid(room)
-        sender = User.get_by_mxid(sender)
-        if sender.has_full_access and portal:
+    @staticmethod
+    async def handle_room_meta(evt_type: str, room_id: str, sender_mxid: str, content: dict):
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
            handler, content_key = {
                "m.room.name": (portal.handle_matrix_title, "name"),
                "m.room.topic": (portal.handle_matrix_about, "topic"),
                "m.room.avatar": (portal.handle_matrix_avatar, "url"),
-            }[type]
+            }[evt_type]
            if content_key not in content:
-                # FIXME handle
-                pass
-                await handler(sender, content[content_key])
+                return
+            await handler(sender, content[content_key])

-    def filter_matrix_event(self, event):
-        return (event["sender"] == self.az.bot_mxid
-                or Puppet.get_id_from_mxid(event["sender"]) is not None)
+    @staticmethod
+    async def handle_room_pin(room_id: str, sender_mxid: str, new_events: Set[str],
+                              old_events: Set[str]):
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
+            events = new_events - old_events
+            if len(events) > 0:
+                # New event pinned, set that as pinned in Telegram.
+                await portal.handle_matrix_pin(sender, events.pop())
+            elif len(new_events) == 0:
+                # All pinned events removed, remove pinned event in Telegram.
+                await portal.handle_matrix_pin(sender, None)

-    async def handle_event(self, evt):
+    @staticmethod
+    async def handle_name_change(room_id: str, user_id: str, displayname: str,
+                                 prev_displayname: str, event_id: str):
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.has_bot:
+            return
+
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if await user.needs_relaybot(portal):
+            await portal.name_change_matrix(user, displayname, prev_displayname, event_id)
+
+    @staticmethod
+    def parse_read_receipts(content: dict) -> Dict[str, str]:
+        return {user_id: event_id
+                for event_id, receipts in content.items()
+                for user_id in receipts.get("m.read", {})}
+
+    @staticmethod
+    async def handle_read_receipts(room_id: str, receipts: Dict[str, str]):
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id, event_id in receipts.items():
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+            await portal.mark_read(user, event_id)
+
+    @staticmethod
+    async def handle_presence(user_id: str, presence: str):
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if not await user.is_logged_in():
+            return
+        await user.set_presence(presence == "online")
+
+    async def handle_typing(self, room_id: str, now_typing: List[str]):
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id in set(self.previously_typing + now_typing):
+            is_typing = user_id in now_typing
+            was_typing = user_id in self.previously_typing
+            if is_typing and was_typing:
+                continue
+
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+
+            await portal.set_typing(user, is_typing)
+
+        self.previously_typing = now_typing
+
+    def filter_matrix_event(self, event: dict):
+        sender = event.get("sender", None)
+        if not sender:
+            return False
+        return (sender == self.az.bot_mxid
+                or pu.Puppet.get_id_from_mxid(sender) is not None)
+
+    async def try_handle_event(self, evt: dict):
+        try:
+            await self.handle_event(evt)
+        except Exception:
+            self.log.exception("Error handling manually received Matrix event")
+
+    async def handle_event(self, evt: dict):
        if self.filter_matrix_event(evt):
            return
        self.log.debug("Received event: %s", evt)
-        type = evt["type"]
-        content = evt.get("content", {})
-        if type == "m.room.member":
-            membership = content.get("membership", "")
-            if membership == "invite":
-                await self.handle_invite(evt["room_id"], evt["state_key"], evt["sender"])
-            elif membership == "leave":
-                await self.handle_part(evt["room_id"], evt["state_key"], evt["sender"])
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: str
+        event_id = evt.get("event_id", None)  # type: str
+        sender = evt.get("sender", None)  # type: str
+        content = evt.get("content", {})  # type: dict
+        if evt_type == "m.room.member":
+            state_key = evt["state_key"]  # type: str
+            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: dict
+            membership = content.get("membership", "")  # type: str
+            prev_membership = prev_content.get("membership", "leave")  # type: str
+            if membership == prev_membership:
+                match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
+                localpart = match.group(1)  # type: str
+                displayname = content.get("displayname", localpart)  # type: str
+                prev_displayname = prev_content.get("displayname", localpart)  # type: str
+                if displayname != prev_displayname:
+                    await self.handle_name_change(room_id, state_key, displayname,
+                                                  prev_displayname, event_id)
+            elif membership == "invite":
+                await self.handle_invite(room_id, state_key, sender)
+            elif prev_membership == "join" and membership == "leave":
+                await self.handle_part(room_id, state_key, sender, event_id)
            elif membership == "join":
-                await self.handle_join(evt["room_id"], evt["state_key"])
-        elif type == "m.room.message":
-            await self.handle_message(evt["room_id"], evt["sender"], content, evt["event_id"])
-        elif type == "m.room.redaction":
-            await self.handle_redaction(evt["room_id"], evt["sender"], evt["redacts"])
-        elif type == "m.room.power_levels":
-            await self.handle_power_levels(evt["room_id"], evt["sender"], evt["content"],
-                                           evt["prev_content"])
-        elif type == "m.room.name" or type == "m.room.avatar" or type == "m.room.topic":
-            await self.handle_room_meta(type, evt["room_id"], evt["sender"], evt["content"])
+                await self.handle_join(room_id, state_key, event_id)
+        elif evt_type in ("m.room.message", "m.sticker"):
+            if evt_type != "m.room.message":
+                content["msgtype"] = evt_type
+            await self.handle_message(room_id, sender, content, event_id)
+        elif evt_type == "m.room.redaction":
+            await self.handle_redaction(room_id, sender, evt["redacts"])
+        elif evt_type == "m.room.power_levels":
+            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: dict
+            await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
+        elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
+            await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
+        elif evt_type == "m.room.pinned_events":
+            new_events = set(evt["content"]["pinned"])
+            try:
+                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+            except KeyError:
+                old_events = set()
+            await self.handle_room_pin(room_id, sender, new_events, old_events)
+        elif evt_type == "m.receipt":
+            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
+        elif evt_type == "m.presence":
+            await self.handle_presence(sender, content.get("presence", "offline"))
+        elif evt_type == "m.typing":
+            await self.handle_typing(room_id, content.get("user_ids", []))
@@ -3,67 +3,275 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Awaitable, Pattern, Dict, List, TYPE_CHECKING
 from difflib import SequenceMatcher
 import re
 import logging
+import asyncio

-from telethon.tl.types import UserProfilePhoto, PeerUser
-from telethon.errors.rpc_error_list import LocationInvalidError
+from sqlalchemy import orm
+
+from telethon.tl.types import UserProfilePhoto
+from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError

 from .db import Puppet as DBPuppet
+from . import util

-config = None
+if TYPE_CHECKING:
+    from .matrix import MatrixHandler
+    from .config import Config
+    from .context import Context
+
+config = None  # type: Config


 class Puppet:
-    log = logging.getLogger("mau.puppet")
-    db = None
-    az = None
-    mxid_regex = None
-    cache = {}
+    log = logging.getLogger("mau.puppet")  # type: logging.Logger
+    db = None  # type: orm.Session
+    az = None  # type: AppService
+    mx = None  # type: MatrixHandler
+    loop = None  # type: asyncio.AbstractEventLoop
+    mxid_regex = None  # type: Pattern
+    username_template = None  # type: str
+    hs_domain = None  # type: str
+    cache = {}  # type: Dict[str, Puppet]
+    by_custom_mxid = {}  # type: Dict[str, Puppet]

-    def __init__(self, id=None, username=None, displayname=None, photo_id=None):
+    def __init__(self, id=None, access_token=None, custom_mxid=None, username=None,
+                 displayname=None, displayname_source=None, photo_id=None, is_bot=None,
+                 is_registered=False, db_instance=None):
        self.id = id
+        self.access_token = access_token
+        self.custom_mxid = custom_mxid
+        self.is_real_user = self.custom_mxid and self.access_token
+        self.default_mxid = self.get_mxid_from_id(self.id)
+        self.mxid = self.custom_mxid or self.default_mxid

-        self.localpart = config.get("bridge.username_template", "telegram_{userid}").format(
-            userid=self.id)
-        hs = config["homeserver"]["domain"]
-        self.mxid = f"@{self.localpart}:{hs}"
        self.username = username
        self.displayname = displayname
+        self.displayname_source = displayname_source
        self.photo_id = photo_id
-        self.intent = self.az.intent.user(self.mxid)
+        self.is_bot = is_bot
+        self.is_registered = is_registered
+        self._db_instance = db_instance
+
+        self.default_mxid_intent = self.az.intent.user(self.default_mxid)
+        self.intent = None  # type: IntentAPI
+        self.refresh_intents()

        self.cache[id] = self
+        if self.custom_mxid:
+            self.by_custom_mxid[self.custom_mxid] = self

    @property
    def tgid(self):
        return self.id

-    def to_db(self):
-        return self.db.merge(
-            DBPuppet(id=self.id, username=self.username, displayname=self.displayname,
-                     photo_id=self.photo_id))
+    @staticmethod
+    async def is_logged_in():
+        return True
+
+    # region Custom puppet management
+    def refresh_intents(self):
+        self.is_real_user = self.custom_mxid and self.access_token
+        self.intent = (self.az.intent.user(self.custom_mxid, self.access_token)
+                       if self.is_real_user else self.default_mxid_intent)
+
+    async def switch_mxid(self, access_token, mxid):
+        prev_mxid = self.custom_mxid
+        self.custom_mxid = mxid
+        self.access_token = access_token
+        self.refresh_intents()
+
+        err = await self.init_custom_mxid()
+        if err != 0:
+            return err
+
+        try:
+            del self.by_custom_mxid[prev_mxid]
+        except KeyError:
+            pass
+        self.mxid = self.custom_mxid or self.default_mxid
+        if self.mxid != self.default_mxid:
+            self.by_custom_mxid[self.mxid] = self
+            await self.leave_rooms_with_default_user()
+        self.save()
+        return 0
+
+    async def init_custom_mxid(self):
+        if not self.is_real_user:
+            return 0
+
+        mxid = await self.intent.whoami()
+        if not mxid or mxid != self.custom_mxid:
+            self.custom_mxid = None
+            self.access_token = None
+            self.refresh_intents()
+            if mxid != self.custom_mxid:
+                return 2
+            return 1
+        if config["bridge.sync_with_custom_puppets"]:
+            asyncio.ensure_future(self.sync(), loop=self.loop)
+        return 0
+
+    async def leave_rooms_with_default_user(self):
+        for room_id in await self.default_mxid_intent.get_joined_rooms():
+            try:
+                await self.default_mxid_intent.leave_room(room_id)
+                await self.intent.ensure_joined(room_id)
+            except (IntentError, MatrixRequestError):
+                pass
+
+    def create_sync_filter(self) -> Awaitable[str]:
+        return self.intent.client.create_filter(self.custom_mxid, {
+            "room": {
+                "include_leave": False,
+                "state": {
+                    "types": []
+                },
+                "timeline": {
+                    "types": [],
+                },
+                "ephemeral": {
+                    "types": ["m.typing", "m.receipt"],
+                },
+                "account_data": {
+                    "types": []
+                }
+            },
+            "account_data": {
+                "types": [],
+            },
+            "presence": {
+                "types": ["m.presence"],
+                "senders": [self.custom_mxid],
+            },
+        })
+
+    def filter_events(self, events):
+        new_events = []
+        for event in events:
+            evt_type = event.get("type", None)
+            event.setdefault("content", {})
+            if evt_type == "m.typing":
+                is_typing = self.custom_mxid in event["content"].get("user_ids", [])
+                event["content"]["user_ids"] = [self.custom_mxid] if is_typing else []
+            elif evt_type == "m.receipt":
+                val = None
+                evt = None
+                for event_id in event["content"]:
+                    try:
+                        val = event["content"][event_id]["m.read"][self.custom_mxid]
+                        evt = event_id
+                        break
+                    except KeyError:
+                        pass
+                if val and evt:
+                    event["content"] = {evt: {"m.read": {
+                        self.custom_mxid: val
+                    }}}
+                else:
+                    continue
+            new_events.append(event)
+        return new_events
+
+    def handle_sync(self, presence, ephemeral):
+        presence = [self.mx.try_handle_event(event) for event in presence]
+
+        for room_id, events in ephemeral.items():
+            for event in events:
+                event["room_id"] = room_id
+
+        ephemeral = [self.mx.try_handle_event(event)
+                     for events in ephemeral.values()
+                     for event in self.filter_events(events)]
+
+        events = ephemeral + presence
+        coro = asyncio.gather(*events, loop=self.loop)
+        asyncio.ensure_future(coro, loop=self.loop)
+
+    async def sync(self):
+        try:
+            await self._sync()
+        except Exception:
+            self.log.exception("Fatal error syncing")
+
+    async def _sync(self):
+        if not self.is_real_user:
+            self.log.warning("Called sync() for non-custom puppet.")
+            return
+        custom_mxid = self.custom_mxid
+        access_token_at_start = self.access_token
+        errors = 0
+        next_batch = None
+        filter_id = await self.create_sync_filter()
+        self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.")
+        while access_token_at_start == self.access_token:
+            try:
+                sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch,
+                                                          set_presence="offline")
+                errors = 0
+                if next_batch is not None:
+                    presence = sync_resp.get("presence", {}).get("events", [])
+                    ephemeral = {room: data.get("ephemeral", {}).get("events", [])
+                                 for room, data
+                                 in sync_resp.get("rooms", {}).get("join", {}).items()}
+                    self.handle_sync(presence, ephemeral)
+                next_batch = sync_resp.get("next_batch", None)
+            except MatrixRequestError as e:
+                wait = min(errors, 11) ** 2
+                self.log.warning(f"Syncer for {custom_mxid} errored: {e}. "
+                                 f"Waiting for {wait} seconds...")
+                errors += 1
+                await asyncio.sleep(wait)
+        self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.")
+
+    # endregion
+    # region DB conversion
+
+    @property
+    def db_instance(self):
+        if not self._db_instance:
+            self._db_instance = self.new_db_instance()
+        return self._db_instance
+
+    def new_db_instance(self):
+        return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
+                        username=self.username, displayname=self.displayname,
+                        displayname_source=self.displayname_source, photo_id=self.photo_id,
+                        is_bot=self.is_bot, matrix_registered=self.is_registered)

    @classmethod
    def from_db(cls, db_puppet):
-        return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname, db_puppet.photo_id)
+        return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
+                      db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
+                      db_instance=db_puppet)

    def save(self):
-        self.to_db()
+        self.db_instance.access_token = self.access_token
+        self.db_instance.custom_mxid = self.custom_mxid
+        self.db_instance.username = self.username
+        self.db_instance.displayname = self.displayname
+        self.db_instance.displayname_source = self.displayname_source
+        self.db_instance.photo_id = self.photo_id
+        self.db_instance.is_bot = self.is_bot
+        self.db_instance.matrix_registered = self.is_registered
        self.db.commit()

+    # endregion
+    # region Info updating
    def similarity(self, query):
        username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                               if self.username else 0)
@@ -73,7 +281,7 @@ class Puppet:
        return round(similarity * 1000) / 10

    @staticmethod
-    def get_displayname(info, format=True):
+    def get_displayname(info, enable_format=True):
        data = {
            "phone number": info.phone if hasattr(info, "phone") else None,
            "username": info.username,
@@ -89,10 +297,13 @@ class Puppet:
            name = data[preference]
            if name:
                break
-        if not name:
+
+        if info.deleted:
+            name = f"Deleted account {info.id}"
+        elif not name:
            name = info.id

-        if not format:
+        if not enable_format:
            return name
        return config.get("bridge.displayname_template", "{displayname} (Telegram)").format(
            displayname=name)
@@ -107,53 +318,90 @@ class Puppet:
        if isinstance(info.photo, UserProfilePhoto):
            changed = await self.update_avatar(source, info.photo.photo_big) or changed

+        self.is_bot = info.bot
+
        if changed:
            self.save()

    async def update_displayname(self, source, info):
+        ignore_source = (not source.is_relaybot
+                         and self.displayname_source is not None
+                         and self.displayname_source != source.tgid)
+        if ignore_source:
+            return
+
        displayname = self.get_displayname(info)
        if displayname != self.displayname:
-            await self.intent.set_display_name(displayname)
+            await self.default_mxid_intent.set_display_name(displayname)
            self.displayname = displayname
+            self.displayname_source = source.tgid
+            return True
+        elif source.is_relaybot or self.displayname_source is None:
+            self.displayname_source = source.tgid
            return True

    async def update_avatar(self, source, photo):
        photo_id = f"{photo.volume_id}-{photo.local_id}"
        if self.photo_id != photo_id:
-            try:
-                file = await source.client.download_file_bytes(photo)
-            except LocationInvalidError:
-                return False
-            uploaded = await self.intent.upload_file(file)
-            await self.intent.set_avatar(uploaded["content_uri"])
-            self.photo_id = photo_id
-            return True
+            file = await util.transfer_file_to_matrix(self.db, source.client,
+                                                      self.default_mxid_intent, photo)
+            if file:
+                await self.default_mxid_intent.set_avatar(file.mxc)
+                self.photo_id = photo_id
+                return True
        return False

+    # endregion
+    # region Getters
+
    @classmethod
-    def get(cls, id, create=True):
+    def get(cls, tgid, create=True) -> "Optional[Puppet]":
        try:
-            return cls.cache[id]
+            return cls.cache[tgid]
        except KeyError:
            pass

-        puppet = DBPuppet.query.get(id)
+        puppet = DBPuppet.query.get(tgid)
        if puppet:
            return cls.from_db(puppet)

        if create:
-            puppet = cls(id)
-            cls.db.add(puppet.to_db())
+            puppet = cls(tgid)
+            cls.db.add(puppet.db_instance)
            cls.db.commit()
            return puppet

        return None

    @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid, create=True) -> "Optional[Puppet]":
        tgid = cls.get_id_from_mxid(mxid)
        return cls.get(tgid, create) if tgid else None

+    @classmethod
+    def get_by_custom_mxid(cls, mxid):
+        if not mxid:
+            raise ValueError("Matrix ID can't be empty")
+
+        try:
+            return cls.by_custom_mxid[mxid]
+        except KeyError:
+            pass
+
+        puppet = DBPuppet.query.filter(DBPuppet.custom_mxid == mxid).one_or_none()
+        if puppet:
+            puppet = cls.from_db(puppet)
+            return puppet
+
+        return None
+
+    @classmethod
+    def get_all_with_custom_mxid(cls):
+        return [cls.by_custom_mxid[puppet.mxid]
+                if puppet.custom_mxid in cls.by_custom_mxid
+                else cls.from_db(puppet)
+                for puppet in DBPuppet.query.filter(DBPuppet.custom_mxid is not None).all()]
+
    @classmethod
    def get_id_from_mxid(cls, mxid):
        match = cls.mxid_regex.match(mxid)
@@ -162,9 +410,16 @@ class Puppet:
        return None

    @classmethod
-    def find_by_username(cls, username):
+    def get_mxid_from_id(cls, tgid):
+        return f"@{cls.username_template.format(userid=tgid)}:{cls.hs_domain}"
+
+    @classmethod
+    def find_by_username(cls, username) -> "Optional[Puppet]":
+        if not username:
+            return None
+
        for _, puppet in cls.cache.items():
-            if puppet.username == username:
+            if puppet.username and puppet.username.lower() == username.lower():
                return puppet

        puppet = DBPuppet.query.filter(DBPuppet.username == username).one_or_none()
@@ -173,10 +428,29 @@ class Puppet:

        return None

+    @classmethod
+    def find_by_displayname(cls, displayname) -> "Optional[Puppet]":
+        if not displayname:
+            return None

-def init(context):
+        for _, puppet in cls.cache.items():
+            if puppet.displayname and puppet.displayname == displayname:
+                return puppet
+
+        puppet = DBPuppet.query.filter(DBPuppet.displayname == displayname).one_or_none()
+        if puppet:
+            return cls.from_db(puppet)
+
+        return None
+    # endregion
+
+
+def init(context: "Context") -> List[Awaitable[int]]:
    global config
-    Puppet.az, Puppet.db, config, _ = context
-    localpart = config.get("bridge.username_template", "telegram_{userid}").format(userid="(.+)")
-    hs = config["homeserver"]["domain"]
-    Puppet.mxid_regex = re.compile(f"@{localpart}:{hs}")
+    Puppet.az, Puppet.db, config, Puppet.loop, _ = context
+    Puppet.mx = context.mx
+    Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
+    Puppet.hs_domain = config["homeserver"]["domain"]
+    Puppet.mxid_regex = re.compile(
+        f"@{Puppet.username_template.format(userid='(.+)')}:{Puppet.hs_domain}")
+    return [puppet.init_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()]
@@ -0,0 +1,59 @@
+import argparse
+import sqlalchemy as sql
+from sqlalchemy import orm
+from sqlalchemy.ext.declarative import declarative_base
+
+from alchemysession import AlchemySessionContainer
+
+parser = argparse.ArgumentParser(description="mautrix-telegram dbms migration script",
+                                 prog="python -m mautrix_telegram.scripts.dbms_migrate")
+parser.add_argument("-f", "--from-url", type=str, required=True, metavar="<url>",
+                    help="the old database path")
+parser.add_argument("-t", "--to-url", type=str, required=True, metavar="<url>",
+                    help="the new database path")
+args = parser.parse_args()
+
+
+def connect(to):
+    import mautrix_telegram.base as base
+    base.Base = declarative_base()
+    from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile,
+                                     Contact, Puppet, BotChat, TelegramFile)
+    db_engine = sql.create_engine(to)
+    db_factory = orm.sessionmaker(bind=db_engine)
+    db_session = orm.scoped_session(db_factory)  # type: orm.Session
+    base.Base.metadata.bind = db_engine
+    session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
+                                                table_base=base.Base, table_prefix="telethon_",
+                                                manage_tables=False)
+
+    return db_session, {
+        "Version": session_container.Version,
+        "Session": session_container.Session,
+        "Entity": session_container.Entity,
+        "SentFile": session_container.SentFile,
+        "UpdateState": session_container.UpdateState,
+        "Portal": Portal,
+        "Message": Message,
+        "Puppet": Puppet,
+        "User": User,
+        "UserPortal": UserPortal,
+        "RoomState": RoomState,
+        "UserProfile": UserProfile,
+        "Contact": Contact,
+        "BotChat": BotChat,
+        "TelegramFile": TelegramFile,
+    }
+
+
+session, tables = connect(args.from_url)
+
+data = {}
+for name, table in tables.items():
+    data[name] = session.query(table).all()
+
+session, tables = connect(args.to_url)
+for name, table in tables.items():
+    for row in data[name]:
+        session.merge(row)
+session.commit()
@@ -0,0 +1,91 @@
+import argparse
+import sqlalchemy as sql
+from sqlalchemy import orm
+
+from mautrix_telegram.base import Base
+from mautrix_telegram.config import Config
+from mautrix_telegram.db import Portal, Message, Puppet, BotChat
+from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as TelematrixBase
+
+parser = argparse.ArgumentParser(
+    description="mautrix-telegram telematrix import script",
+    prog="python -m mautrix_telegram.scripts.telematrix_import")
+parser.add_argument("-c", "--config", type=str, default="config.yaml",
+                    metavar="<path>", help="the path to your mautrix-telegram config file")
+parser.add_argument("-b", "--bot-id", type=int, required=True,
+                    metavar="<id>", help="the telegram user ID of your relay bot")
+parser.add_argument("-t", "--telematrix-database", type=str, default="sqlite:///database.db",
+                    metavar="<url>", help="your telematrix database URL")
+args = parser.parse_args()
+
+config = Config(args.config, None, None)
+config.load()
+
+mxtg_db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+mxtg = orm.sessionmaker(bind=mxtg_db_engine)()
+Base.metadata.bind = mxtg_db_engine
+
+telematrix_db_engine = sql.create_engine(args.telematrix_database)
+telematrix = orm.sessionmaker(bind=telematrix_db_engine)()
+TelematrixBase.metadata.bind = telematrix_db_engine
+
+chat_links = telematrix.query(ChatLink).all()
+tg_users = telematrix.query(TgUser).all()
+mx_users = telematrix.query(MatrixUser).all()
+messages = telematrix.query(TMMessage).all()
+
+telematrix.close()
+telematrix_db_engine.dispose()
+
+portals = {}
+chats = {}
+messages = {}
+puppets = {}
+
+for chat_link in chat_links:
+    if type(chat_link.tg_room) is str:
+        print("Expected tg_room to be a number, got a string. Ignoring %s" % chat_link.tg_room)
+        continue
+    if chat_link.tg_room >= 0:
+        print("Unexpected unprefixed telegram chat ID: %s, ignoring..." % chat_link.tg_room)
+        continue
+    tgid = str(chat_link.tg_room)
+    if tgid.startswith("-100"):
+        tgid = int(tgid[4:])
+        peer_type = "channel"
+        megagroup = True
+    else:
+        tgid = -chat_link.tg_room
+        peer_type = "chat"
+        megagroup = False
+
+    portal = Portal(tgid=tgid, tg_receiver=tgid, peer_type=peer_type, megagroup=megagroup,
+                    mxid=chat_link.matrix_room)
+    bot_chat = BotChat(id=tgid, type=peer_type)
+    portals[chat_link.tg_room] = portal
+    chats[tgid] = bot_chat
+
+for tm_msg in messages:
+    try:
+        portal = portals[tm_msg.tg_group_id]
+    except KeyError:
+        print("Found message entry %d in unlinked chat %d, ignoring..." % (tm_msg.tg_message_id, tm_msg.tg_group_id))
+        continue
+    tg_space = portal.tgid if portal.peer_type == "channel" else args.bot_id
+    message = Message(mxid=tm_msg.matrix_event_id, mx_room=tm_msg.matrix_room_id,
+                      tgid=tm_msg.tg_message_id, tg_space=tg_space)
+    messages[tm_msg.matrix_event_id] = message
+
+for user in tg_users:
+    puppets[user.tg_id] = Puppet(id=user.tg_id, displayname=user.name, displayname_source=args.bot_id)
+
+for k, v in portals.items():
+    mxtg.add(v)
+for k, v in chats.items():
+    mxtg.add(v)
+for k, v in messages.items():
+    mxtg.add(v)
+for k, v in puppets.items():
+    mxtg.add(v)
+
+mxtg.commit()
@@ -0,0 +1,44 @@
+import sqlalchemy as sa
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+
+class ChatLink(Base):
+    __tablename__ = 'chat_link'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    matrix_room = sa.Column(sa.String)
+    tg_room = sa.Column(sa.BigInteger)
+    active = sa.Column(sa.Boolean)
+
+
+class TgUser(Base):
+    __tablename__ = 'tg_user'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    tg_id = sa.Column(sa.BigInteger)
+    name = sa.Column(sa.String)
+    profile_pic_id = sa.Column(sa.String, nullable=True)
+
+
+class MatrixUser(Base):
+    __tablename__ = 'matrix_user'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    matrix_id = sa.Column(sa.String)
+    name = sa.Column(sa.String)
+
+
+class Message(Base):
+    """Describes a message in a room bridged between Telegram and Matrix"""
+    __tablename__ = "message"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    tg_group_id = sa.Column(sa.BigInteger)
+    tg_message_id = sa.Column(sa.BigInteger)
+
+    matrix_room_id = sa.Column(sa.String)
+    matrix_event_id = sa.Column(sa.String)
+
+    displayname = sa.Column(sa.String)
@@ -0,0 +1,120 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Tuple
+
+from sqlalchemy import orm
+
+from mautrix_appservice import StateStore
+
+from . import puppet as pu
+from .db import RoomState, UserProfile
+
+
+class SQLStateStore(StateStore):
+    def __init__(self, db):
+        super().__init__()
+        self.db = db  # type: orm.Session
+        self.profile_cache = {}  # type: Dict[Tuple[str, str], UserProfile]
+        self.room_state_cache = {}  # type: Dict[str, RoomState]
+
+    @staticmethod
+    def is_registered(user: str) -> bool:
+        puppet = pu.Puppet.get_by_mxid(user)
+        return puppet.is_registered if puppet else False
+
+    @staticmethod
+    def registered(user: str):
+        puppet = pu.Puppet.get_by_mxid(user)
+        if puppet:
+            puppet.is_registered = True
+            puppet.save()
+
+    def update_state(self, event: dict):
+        event_type = event["type"]
+        if event_type == "m.room.power_levels":
+            self.set_power_levels(event["room_id"], event["content"])
+        elif event_type == "m.room.member":
+            self.set_member(event["room_id"], event["state_key"], event["content"])
+
+    def _get_user_profile(self, room_id: str, user_id: str, create: bool = True) -> UserProfile:
+        key = (room_id, user_id)
+        try:
+            return self.profile_cache[key]
+        except KeyError:
+            pass
+
+        profile = UserProfile.query.get(key)
+        if profile:
+            self.profile_cache[key] = profile
+        elif create:
+            profile = UserProfile(room_id=room_id, user_id=user_id)
+            self.db.add(profile)
+            self.db.commit()
+            self.profile_cache[key] = profile
+        return profile
+
+    def get_member(self, room: str, user: str) -> dict:
+        return self._get_user_profile(room, user).dict()
+
+    def set_member(self, room: str, user: str, member: dict):
+        profile = self._get_user_profile(room, user)
+        profile.membership = member.get("membership", profile.membership or "leave")
+        profile.displayname = member.get("displayname", profile.displayname)
+        profile.avatar_url = member.get("avatar_url", profile.avatar_url)
+        self.db.commit()
+
+    def set_membership(self, room: str, user: str, membership: str):
+        self.set_member(room, user, {
+            "membership": membership,
+        })
+
+    def _get_room_state(self, room_id: str, create: bool = True) -> RoomState:
+        try:
+            return self.room_state_cache[room_id]
+        except KeyError:
+            pass
+
+        room = RoomState.query.get(room_id)
+        if room:
+            self.room_state_cache[room_id] = room
+        elif create:
+            room = RoomState(room_id=room_id)
+            self.room_state_cache[room_id] = room
+        return room
+
+    def has_power_levels(self, room: str) -> bool:
+        return self._get_room_state(room).has_power_levels
+
+    def get_power_levels(self, room: str) -> dict:
+        return self._get_room_state(room).power_levels
+
+    def set_power_level(self, room: str, user: str, level: int):
+        room_state = self._get_room_state(room)
+        power_levels = room_state.power_levels
+        if not power_levels:
+            power_levels = {
+                "users": {},
+                "events": {},
+            }
+        power_levels[room]["users"][user] = level
+        room_state.power_levels = power_levels
+        self.db.commit()
+
+    def set_power_levels(self, room: str, content: dict):
+        state = self._get_room_state(room)
+        state.power_levels = content
+        self.db.commit()
@@ -3,82 +3,47 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-from io import BytesIO
-
-from telethon import TelegramClient
-from telethon.tl.functions.messages import SendMessageRequest, SendMediaRequest
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from telethon import TelegramClient, utils
+from telethon.tl.functions.messages import SendMediaRequest
 from telethon.tl.types import *
+from telethon.tl import custom


 class MautrixTelegramClient(TelegramClient):
-    async def send_message(self, entity, message, reply_to=None, entities=None, link_preview=True):
-        entity = await self.get_input_entity(entity)
+    async def upload_file_direct(self, file: bytes, mime_type: str = None,
+                                 attributes: List[TypeDocumentAttribute] = None,
+                                 file_name: str = None
+                                 ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
+        file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)

-        request = SendMessageRequest(
-            peer=entity,
-            message=message,
-            entities=entities,
-            no_webpage=not link_preview,
-            reply_to_msg_id=self._get_message_id(reply_to)
-        )
-        result = await self(request)
-        if isinstance(result, UpdateShortSentMessage):
-            return Message(
-                id=result.id,
-                to_id=entity,
-                message=message,
-                date=result.date,
-                out=result.out,
-                media=result.media,
-                entities=result.entities
-            )
-
-        return self._get_response_message(request, result)
-
-    async def send_file(self, entity, file, mime_type=None, caption=None, attributes=None,
-                        file_name=None, reply_to=None, **kwargs):
-        entity = await self.get_input_entity(entity)
-        reply_to = self._get_message_id(reply_to)
-
-        file_handle = await self.upload_file(file, file_name=file_name, use_cache=False)
-
-        if mime_type == "image/png":
-            media = InputMediaUploadedPhoto(file_handle, caption or "")
+        if mime_type == "image/png" or mime_type == "image/jpeg":
+            return InputMediaUploadedPhoto(file_handle)
        else:
            attributes = attributes or []
            attr_dict = {type(attr): attr for attr in attributes}

-            media = InputMediaUploadedDocument(
+            return InputMediaUploadedDocument(
                file=file_handle,
                mime_type=mime_type or "application/octet-stream",
-                attributes=list(attr_dict.values()),
-                caption=caption or "")
+                attributes=list(attr_dict.values()))

-        request = SendMediaRequest(entity, media, reply_to_msg_id=reply_to)
-        return self._get_response_message(request, await self(request))
-
-    async def download_file_bytes(self, location):
-        if isinstance(location, Document):
-            location = InputDocumentFileLocation(location.id, location.access_hash,
-                                                 location.version)
-        elif not isinstance(location, (InputFileLocation, InputDocumentFileLocation)):
-            location = InputFileLocation(location.volume_id, location.local_id, location.secret)
-
-        file = BytesIO()
-
-        await self.download_file(location, file)
-
-        data = file.getvalue()
-        file.close()
-        return data
+    async def send_media(self, entity: Union[TypeInputPeer, TypePeer],
+                         media: Union[TypeInputMedia, TypeMessageMedia],
+                         caption: str = None, entities: List[TypeMessageEntity] = None,
+                         reply_to: int = None) -> Optional[custom.Message]:
+        entity = await self.get_input_entity(entity)
+        reply_to = utils.get_message_id(reply_to)
+        request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],
+                                   reply_to_msg_id=reply_to)
+        return self._get_response_message(request, await self(request), entity)
@@ -3,113 +3,128 @@
 # Copyright (C) 2018 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Awaitable, Optional, Match, Tuple, TYPE_CHECKING
 import logging
 import asyncio
-import platform
+import re

 from telethon.tl.types import *
-from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.types import User as TLUser
+from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
+from telethon.tl.functions.account import UpdateStatusRequest
 from mautrix_appservice import MatrixRequestError

-from .db import User as DBUser, Message as DBMessage, Contact as DBContact
-from .tgclient import MautrixTelegramClient
-from . import portal as po, puppet as pu, __version__
+from .db import User as DBUser, Contact as DBContact, Portal as DBPortal
+from .abstract_user import AbstractUser
+from . import portal as po, puppet as pu

-config = None
+if TYPE_CHECKING:
+    from .config import Config
+    from .context import Context
+
+config = None  # type: Config
+
+SearchResults = List[Tuple["pu.Puppet", int]]


-class User:
-    loop = None
-    log = logging.getLogger("mau.user")
-    db = None
-    az = None
-    by_mxid = {}
-    by_tgid = {}
+class User(AbstractUser):
+    log = logging.getLogger("mau.user")  # type: logging.Logger
+    by_mxid = {}  # type: Dict[str, User]
+    by_tgid = {}  # type: Dict[int, User]

-    def __init__(self, mxid, tgid=None, username=None, db_contacts=None, saved_contacts=0,
-                 db_portals=None):
-        self.mxid = mxid
-        self.tgid = tgid
-        self.username = username
-        self.contacts = []
-        self.saved_contacts = saved_contacts
-        self.db_contacts = db_contacts
-        self.portals = {}
-        self.db_portals = db_portals
+    def __init__(self, mxid: str, tgid: Optional[int] = None, username: Optional[str] = None,
+                 db_contacts: Optional[List[DBContact]] = None, saved_contacts: int = 0,
+                 is_bot: bool = False, db_portals: Optional[List[DBPortal]] = None,
+                 db_instance: Optional[DBUser] = None):
+        super().__init__()
+        self.mxid = mxid  # type: str
+        self.tgid = tgid  # type: int
+        self.is_bot = is_bot  # type: bool
+        self.username = username  # type: str
+        self.contacts = []  # type: List[pu.Puppet]
+        self.saved_contacts = saved_contacts  # type: int
+        self.db_contacts = db_contacts  # type: List[DBContact]
+        self.portals = {}  # type: Dict[Tuple[int, int], po.Portal]
+        self.db_portals = db_portals  # type: List[DBPortal]
+        self._db_instance = db_instance  # type: DBUser

-        self.command_status = None
-        self.connected = False
-        self.client = None
-        self._init_client()
+        self.command_status = None  # type: dict

-        self.is_admin = self.mxid in config.get("bridge.admins", [])
-
-        whitelist = config.get("bridge.whitelist", None) or [self.mxid]
-        self.whitelisted = not whitelist or self.mxid in whitelist
-        if not self.whitelisted:
-            homeserver = self.mxid[self.mxid.index(":") + 1:]
-            self.whitelisted = homeserver in whitelist
+        (self.relaybot_whitelisted,
+         self.whitelisted,
+         self.puppet_whitelisted,
+         self.matrix_puppet_whitelisted,
+         self.is_admin,
+         self.permissions) = config.get_permissions(self.mxid)

        self.by_mxid[mxid] = self
        if tgid:
            self.by_tgid[tgid] = self

    @property
-    def logged_in(self):
-        return self.client.is_user_authorized()
+    def name(self) -> str:
+        return self.mxid

    @property
-    def has_full_access(self):
-        return self.logged_in and self.whitelisted
+    def mxid_localpart(self) -> str:
+        match = re.compile("@(.+):(.+)").match(self.mxid)  # type: Match
+        return match.group(1)
+
+    # TODO replace with proper displayname getting everywhere
+    @property
+    def displayname(self) -> str:
+        return self.mxid_localpart

    @property
-    def db_contacts(self):
+    def db_contacts(self) -> List[DBContact]:
        return [self.db.merge(DBContact(user=self.tgid, contact=puppet.id))
                for puppet in self.contacts]

    @db_contacts.setter
-    def db_contacts(self, contacts):
-        if contacts:
-            self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts]
-        else:
-            self.contacts = []
+    def db_contacts(self, contacts: List[DBContact]):
+        self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts] if contacts else []

    @property
-    def db_portals(self):
-        return [portal.to_db(merge=False) for _, portal in self.portals.items()]
+    def db_portals(self) -> List[DBPortal]:
+        return [portal.db_instance for portal in self.portals.values() if not portal.deleted]

    @db_portals.setter
-    def db_portals(self, portals):
-        if portals:
-            self.portals = {(portal.tgid, portal.tg_receiver):
-                                po.Portal.get_by_tgid(portal.tgid, portal.tg_receiver)
-                            for portal in portals}
-        else:
-            self.portals = {}
+    def db_portals(self, portals: List[DBPortal]):
+        self.portals = {(portal.tgid, portal.tg_receiver):
+                            po.Portal.get_by_tgid(portal.tgid, portal.tg_receiver)
+                        for portal in portals} if portals else {}

    # region Database conversion

-    def to_db(self):
-        return self.db.merge(
-            DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                   contacts=self.db_contacts, saved_contacts=self.saved_contacts,
-                   portals=self.db_portals))
+    @property
+    def db_instance(self) -> DBUser:
+        if not self._db_instance:
+            self._db_instance = self.new_db_instance()
+        return self._db_instance
+
+    def new_db_instance(self) -> DBUser:
+        return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
+                      contacts=self.db_contacts, saved_contacts=self.saved_contacts or 0,
+                      portals=self.db_portals)

    def save(self):
-        self.to_db()
+        self.db_instance.tgid = self.tgid
+        self.db_instance.username = self.username
+        self.db_instance.contacts = self.db_contacts
+        self.db_instance.saved_contacts = self.saved_contacts or 0
+        self.db_instance.portals = self.db_portals
        self.db.commit()

    def delete(self):
@@ -118,59 +133,77 @@ class User:
            del self.by_tgid[self.tgid]
        except KeyError:
            pass
-        self.db.delete(self.to_db())
-        self.db.commit()
+        if self._db_instance:
+            self.db.delete(self._db_instance)
+            self.db.commit()

    @classmethod
-    def from_db(cls, db_user):
+    def from_db(cls, db_user: DBUser) -> "User":
        return User(db_user.mxid, db_user.tgid, db_user.tg_username, db_user.contacts,
-                    db_user.saved_contacts, db_user.portals)
+                    False, db_user.saved_contacts, db_user.portals, db_instance=db_user)

    # endregion
    # region Telegram connection management

-    def _init_client(self):
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
-        self.client = MautrixTelegramClient(self.mxid,
-                                            config["telegram.api_id"],
-                                            config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device)
-        self.client.add_update_handler(self.update_catch)
-
-    async def start(self, delete_unless_authenticated=False):
-        self.connected = await self.client.connect()
-        if self.logged_in:
+    async def start(self, delete_unless_authenticated: bool = False) -> "User":
+        await super().start()
+        if await self.is_logged_in():
+            self.log.debug(f"Ensuring post_login() for {self.name}")
            asyncio.ensure_future(self.post_login(), loop=self.loop)
        elif delete_unless_authenticated:
-            # User not logged in -> forget user
-            self.client.disconnect()
+            self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
+            await self.client.disconnect()
            self.client.session.delete()
-            self.delete()
        return self

-    async def post_login(self, info=None):
+    async def post_login(self, info: TLUser = None):
        try:
            await self.update_info(info)
-            await self.sync_dialogs()
-            await self.sync_contacts()
+            if not self.is_bot:
+                await self.sync_dialogs()
+                await self.sync_contacts()
+            if config["bridge.catch_up"]:
+                await self.client.catch_up()
        except Exception:
-            self.log.exception("Failed to run post-login functions")
+            self.log.exception("Failed to run post-login functions for %s", self.mxid)

-    def stop(self):
-        self.client.disconnect()
-        self.client = None
-        self.connected = False
+    async def update(self, update: TypeUpdate):
+        if not self.is_bot:
+            return
+
+        if isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
+            message = update.message
+            if isinstance(message.to_id, PeerUser) and not message.out:
+                portal = po.Portal.get_by_tgid(message.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
+            else:
+                portal = po.Portal.get_by_entity(message.to_id, receiver_id=self.tgid)
+        elif isinstance(update, UpdateShortChatMessage):
+            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+        elif isinstance(update, UpdateShortMessage):
+            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+        else:
+            return
+
+        self.register_portal(portal)

    # endregion
    # region Telegram actions that need custom methods

-    async def update_info(self, info=None):
+    def ensure_started(self, even_if_no_session: bool = False) -> "Awaitable[User]":
+        return super().ensure_started(even_if_no_session)
+
+    def set_presence(self, online: bool = True):
+        if self.is_bot:
+            return
+        return self.client(UpdateStatusRequest(offline=not online))
+
+    async def update_info(self, info: TLUser = None):
        info = info or await self.client.get_me()
        changed = False
+        if self.is_bot != info.bot:
+            self.is_bot = info.bot
+            changed = True
        if self.username != info.username:
            self.username = info.username
            changed = True
@@ -181,7 +214,12 @@ class User:
            self.save()

    async def log_out(self):
+        puppet = pu.Puppet.get(self.tgid)
+        if puppet.is_real_user:
+            await puppet.switch_mxid(None, None)
        for _, portal in self.portals.items():
+            if not portal.mxid or portal.has_bot:
+                continue
            try:
                await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
            except MatrixRequestError:
@@ -202,8 +240,9 @@ class User:
        self.delete()
        return True

-    def _search_local(self, query, max_results=5, min_similarity=45):
-        results = []
+    def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
+                      ) -> SearchResults:
+        results = []  # type: SearchResults
        for contact in self.contacts:
            similarity = contact.similarity(query)
            if similarity >= min_similarity:
@@ -211,11 +250,11 @@ class User:
        results.sort(key=lambda tup: tup[1], reverse=True)
        return results[0:max_results]

-    async def _search_remote(self, query, max_results=5):
+    async def _search_remote(self, query: str, max_results: int = 5) -> SearchResults:
        if len(query) < 5:
            return []
        server_results = await self.client(SearchRequest(q=query, limit=max_results))
-        results = []
+        results = []  # type: SearchResults
        for user in server_results.users:
            puppet = pu.Puppet.get(user.id)
            await puppet.update_info(self, user)
@@ -223,7 +262,7 @@ class User:
        results.sort(key=lambda tup: tup[1], reverse=True)
        return results[0:max_results]

-    async def search(self, query, force_remote=False):
+    async def search(self, query: str, force_remote: bool = False) -> Tuple[SearchResults, bool]:
        if force_remote:
            return await self._search_remote(query), True

@@ -233,22 +272,18 @@ class User:

        return await self._search_remote(query), True

-    async def sync_dialogs(self):
-        dialogs = await self.client.get_dialogs(limit=30)
+    async def sync_dialogs(self, synchronous_create: bool = False):
        creators = []
-        for dialog in dialogs:
-            entity = dialog.entity
-            invalid = (isinstance(entity, (TLUser, ChatForbidden, ChannelForbidden))
-                       or (isinstance(entity, Chat) and (entity.deactivated or entity.left)))
-            if invalid:
-                continue
+        for entity in await self.get_dialogs(limit=30):
            portal = po.Portal.get_by_entity(entity)
            self.portals[portal.tgid_full] = portal
-            creators.append(portal.create_matrix_room(self, entity, invites=[self.mxid]))
+            creators.append(
+                portal.create_matrix_room(self, entity, invites=[self.mxid],
+                                          synchronous=synchronous_create))
        self.save()
        await asyncio.gather(*creators, loop=self.loop)

-    def register_portal(self, portal):
+    def register_portal(self, portal: po.Portal):
        try:
            if self.portals[portal.tgid_full] == portal:
                return
@@ -257,14 +292,18 @@ class User:
        self.portals[portal.tgid_full] = portal
        self.save()

-    def unregister_portal(self, portal):
+    def unregister_portal(self, portal: po.Portal):
        try:
            del self.portals[portal.tgid_full]
            self.save()
        except KeyError:
            pass

-    def _hash_contacts(self):
+    async def needs_relaybot(self, portal: po.Portal) -> bool:
+        return not await self.is_logged_in() or (
+            self.is_bot and portal.tgid_full not in self.portals)
+
+    def _hash_contacts(self) -> int:
        acc = 0
        for id in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
            acc = (acc * 20261 + id) & 0xffffffff
@@ -283,140 +322,14 @@ class User:
            self.contacts.append(puppet)
        self.save()

-    # endregion
-    # region Telegram update handling
-
-    async def update_catch(self, update):
-        try:
-            await self.update(update)
-        except Exception:
-            self.log.exception("Failed to handle Telegram update")
-
-    async def update(self, update):
-        if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
-                               UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
-            await self.update_message(update)
-        elif isinstance(update, (UpdateChatUserTyping, UpdateUserTyping)):
-            await self.update_typing(update)
-        elif isinstance(update, UpdateUserStatus):
-            await self.update_status(update)
-        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
-            await self.update_admin(update)
-        elif isinstance(update, UpdateChatParticipants):
-            portal = po.Portal.get_by_tgid(update.participants.chat_id)
-            if portal and portal.mxid:
-                await portal.update_telegram_participants(update.participants.participants)
-        elif isinstance(update, UpdateChannelPinnedMessage):
-            portal = po.Portal.get_by_tgid(update.channel_id)
-            if portal and portal.mxid:
-                await portal.update_telegram_pin(self, update.id)
-        elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
-            await self.update_others_info(update)
-        elif isinstance(update, UpdateReadHistoryOutbox):
-            await self.update_read_receipt(update)
-        else:
-            self.log.debug("Unhandled update: %s", update)
-
-    async def update_read_receipt(self, update):
-        if not isinstance(update.peer, PeerUser):
-            self.log.debug("Unexpected read receipt peer: %s", update.peer)
-            return
-
-        portal = po.Portal.get_by_tgid(update.peer.user_id, self.tgid)
-        if not portal or not portal.mxid:
-            return
-
-        # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.query.get((update.max_id, self.tgid))
-        if not message:
-            return
-
-        puppet = pu.Puppet.get(update.peer.user_id)
-        await puppet.intent.mark_read(portal.mxid, message.mxid)
-
-    async def update_admin(self, update):
-        portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        if isinstance(update, UpdateChatAdmins):
-            await portal.set_telegram_admins_enabled(update.enabled)
-        elif isinstance(update, UpdateChatParticipantAdmin):
-            puppet = pu.Puppet.get(update.user_id)
-            user = User.get_by_tgid(update.user_id)
-            await portal.set_telegram_admin(puppet, user)
-
-    async def update_typing(self, update):
-        if isinstance(update, UpdateUserTyping):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
-        else:
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        sender = pu.Puppet.get(update.user_id)
-        await portal.handle_telegram_typing(sender, update)
-
-    async def update_others_info(self, update):
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update, UpdateUserName):
-            if await puppet.update_displayname(self, update):
-                puppet.save()
-        elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
-                puppet.save()
-
-    async def update_status(self, update):
-        puppet = pu.Puppet.get(update.user_id)
-        if isinstance(update.status, UserStatusOnline):
-            await puppet.intent.set_presence("online")
-        elif isinstance(update.status, UserStatusOffline):
-            await puppet.intent.set_presence("offline")
-        else:
-            self.log.warning("Unexpected user status update: %s", update)
-        return
-
-    def get_message_details(self, update):
-        if isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-            sender = pu.Puppet.get(update.from_id)
-        elif isinstance(update, UpdateShortMessage):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
-            sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
-        elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
-                                 UpdateEditMessage, UpdateEditChannelMessage)):
-            update = update.message
-            if isinstance(update.to_id, PeerUser) and not update.out:
-                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
-            else:
-                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
-            sender = pu.Puppet.get(update.from_id) if update.from_id else None
-        else:
-            self.log.warning(
-                f"Unexpected message type in User#get_message_details: {type(update)}")
-            return update, None, None
-        return update, sender, portal
-
-    def update_message(self, original_update):
-        update, sender, portal = self.get_message_details(original_update)
-
-        if isinstance(update, MessageService):
-            if isinstance(update.action, MessageActionChannelMigrateFrom):
-                self.log.debug(f"Ignoring action %s to %s by %d", update.action, portal.tgid_log,
-                               sender.id)
-                return
-            self.log.debug("Handling action %s to %s by %d", update.action, portal.tgid_log,
-                           sender.id)
-            return portal.handle_telegram_action(self, sender, update.action)
-
-        user = sender.tgid if sender else "admin"
-        if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-            return portal.handle_telegram_edit(self, sender, update)
-
-        self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
-        return portal.handle_telegram_message(self, sender, update)
-
    # endregion
    # region Class instance lookup

    @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid: str, create: bool=True) -> "Optional[User]":
+        if not mxid:
+            raise ValueError("Matrix ID can't be empty")
+
        try:
            return cls.by_mxid[mxid]
        except KeyError:
@@ -425,20 +338,18 @@ class User:
        user = DBUser.query.get(mxid)
        if user:
            user = cls.from_db(user)
-            asyncio.ensure_future(user.start(), loop=cls.loop)
            return user

        if create:
            user = cls(mxid)
-            cls.db.add(user.to_db())
+            cls.db.add(user.db_instance)
            cls.db.commit()
-            asyncio.ensure_future(user.start(), loop=cls.loop)
            return user

        return None

    @classmethod
-    def get_by_tgid(cls, tgid):
+    def get_by_tgid(cls, tgid: int) -> "Optional[User]":
        try:
            return cls.by_tgid[tgid]
        except KeyError:
@@ -447,15 +358,17 @@ class User:
        user = DBUser.query.filter(DBUser.tgid == tgid).one_or_none()
        if user:
            user = cls.from_db(user)
-            asyncio.ensure_future(user.start(), loop=cls.loop)
            return user

        return None

    @classmethod
-    def find_by_username(cls, username):
+    def find_by_username(cls, username: str) -> "Optional[User]":
+        if not username:
+            return None
+
        for _, user in cls.by_tgid.items():
-            if user.username == username:
+            if user.username and user.username.lower() == username.lower():
                return user

        puppet = DBUser.query.filter(DBUser.tg_username == username).one_or_none()
@@ -466,9 +379,9 @@ class User:
    # endregion


-def init(context):
+def init(context: "Context") -> List[Awaitable[User]]:
    global config
-    User.az, User.db, config, User.loop = context
+    config = context.config

    users = [User.from_db(user) for user in DBUser.query.all()]
-    return [user.start(delete_unless_authenticated=True) for user in users]
+    return [user.ensure_started() for user in users]
@@ -0,0 +1,3 @@
+from .file_transfer import transfer_file_to_matrix, convert_image
+from .format_duration import format_duration
+from .signed_token import sign_token, verify_token
@@ -0,0 +1,215 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Tuple, Union, Dict
+from io import BytesIO
+import time
+import logging
+import asyncio
+
+import magic
+from sqlalchemy import orm
+from sqlalchemy.exc import IntegrityError, InvalidRequestError
+from sqlalchemy.orm.exc import FlushError
+
+from telethon.tl.types import (Document, FileLocation, InputFileLocation,
+                               InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
+from telethon.errors import *
+from mautrix_appservice import IntentAPI
+
+from ..tgclient import MautrixTelegramClient
+from ..db import TelegramFile as DBTelegramFile
+
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+try:
+    from moviepy.editor import VideoFileClip
+    import random
+    import string
+    import os
+    import mimetypes
+except ImportError:
+    VideoFileClip = random = string = os = mimetypes = None
+
+log = logging.getLogger("mau.util")  # type: logging.Logger
+
+TypeLocation = Union[Document, InputDocumentFileLocation, FileLocation, InputFileLocation]
+
+
+def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
+                  thumbnail_to: Optional[Tuple[int, int]] = None
+                  ) -> Tuple[str, bytes, Optional[int], Optional[int]]:
+    if not Image:
+        return source_mime, file, None, None
+    try:
+        image = Image.open(BytesIO(file)).convert("RGBA")  # type: Image.Image
+        if thumbnail_to:
+            image.thumbnail(thumbnail_to, Image.ANTIALIAS)
+        new_file = BytesIO()
+        image.save(new_file, target_type)
+        w, h = image.size
+        return f"image/{target_type}", new_file.getvalue(), w, h
+    except Exception:
+        log.exception(f"Failed to convert {source_mime} to {target_type}")
+        return source_mime, file, None, None
+
+
+def _temp_file_name(ext: str) -> str:
+    return ("/tmp/mxtg-video-"
+            + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
+            + ext)
+
+
+def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str = "png",
+                          max_size: Tuple[int, int] = (1024, 720)) -> Tuple[bytes, int, int]:
+    # We don't have any way to read the video from memory, so save it to disk.
+    temp_file = _temp_file_name(video_ext)
+    with open(temp_file, "wb") as file:
+        file.write(data)
+
+    # Read temp file and get frame
+    clip = VideoFileClip(temp_file)
+    frame = clip.get_frame(0)
+
+    # Convert to png and save to BytesIO
+    image = Image.fromarray(frame).convert("RGBA")
+    thumbnail_file = BytesIO()
+    if max_size:
+        image.thumbnail(max_size, Image.ANTIALIAS)
+    image.save(thumbnail_file, frame_ext)
+
+    os.remove(temp_file)
+
+    w, h = image.size
+    return thumbnail_file.getvalue(), w, h
+
+
+def _location_to_id(location: TypeLocation) -> str:
+    if isinstance(location, (Document, InputDocumentFileLocation)):
+        return f"{location.id}-{location.version}"
+    elif isinstance(location, (FileLocation, InputFileLocation)):
+        return f"{location.volume_id}-{location.local_id}"
+
+
+async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                       thumbnail_loc: TypeLocation, video: bytes,
+                                       mime: str) -> Optional[DBTelegramFile]:
+    if not Image or not VideoFileClip:
+        return None
+
+    loc_id = _location_to_id(thumbnail_loc)
+    if not loc_id:
+        return None
+
+    video_ext = mimetypes.guess_extension(mime)
+    if VideoFileClip and video_ext:
+        try:
+            file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
+        except OSError:
+            return None
+        mime_type = "image/png"
+    else:
+        file = await client.download_file(thumbnail_loc)
+        width, height = None, None
+        mime_type = magic.from_buffer(file, mime=True)
+
+    content_uri = await intent.upload_file(file, mime_type)
+
+    return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
+                          was_converted=False, timestamp=int(time.time()), size=len(file),
+                          width=width, height=height)
+
+
+transfer_locks = {}  # type: Dict[str, asyncio.Lock]
+
+
+async def transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient, intent: IntentAPI,
+                                  location: TypeLocation, thumbnail: Optional[TypeLocation] = None,
+                                  is_sticker: bool = False) -> Optional[DBTelegramFile]:
+    location_id = _location_to_id(location)
+    if not location_id:
+        return None
+
+    db_file = DBTelegramFile.query.get(location_id)
+    if db_file:
+        return db_file
+
+    try:
+        lock = transfer_locks[location_id]
+    except KeyError:
+        lock = asyncio.Lock()
+        transfer_locks[location_id] = lock
+    async with lock:
+        return await _unlocked_transfer_file_to_matrix(db, client, intent, location_id, location,
+                                                       thumbnail, is_sticker)
+
+
+async def _unlocked_transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient,
+                                            intent: IntentAPI, loc_id: str, location: TypeLocation,
+                                            thumbnail: Optional[TypeLocation],
+                                            is_sticker: bool) -> Optional[DBTelegramFile]:
+    db_file = DBTelegramFile.query.get(loc_id)
+    if db_file:
+        return db_file
+
+    try:
+        file = await client.download_file(location)
+    except LocationInvalidError:
+        return None
+    except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
+        log.exception(f"{e.__class__.__name__} while downloading a file.")
+        return None
+
+    width, height = None, None
+    mime_type = magic.from_buffer(file, mime=True)
+
+    image_converted = False
+    if mime_type == "image/webp":
+        new_mime_type, file, width, height = convert_image(
+            file, source_mime="image/webp", target_type="png",
+            thumbnail_to=(256, 256) if is_sticker else None)
+        image_converted = new_mime_type != mime_type
+        mime_type = new_mime_type
+        thumbnail = None
+
+    content_uri = await intent.upload_file(file, mime_type)
+
+    db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
+                             mime_type=mime_type, was_converted=image_converted,
+                             timestamp=int(time.time()), size=len(file),
+                             width=width, height=height)
+    if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
+        if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
+            thumbnail = thumbnail.location
+        db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
+                                                               mime_type)
+
+    try:
+        db.add(db_file)
+        db.commit()
+    except FlushError as e:
+        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
+                      "This was probably caused by two simultaneous transfers of the same file, "
+                      "and should not cause any problems.")
+    except (IntegrityError, InvalidRequestError) as e:
+        db.rollback()
+        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
+                      "This was probably caused by two simultaneous transfers of the same file, "
+                      "and should not cause any problems.")
+
+    return db_file
@@ -0,0 +1,36 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+def format_duration(seconds: int) -> str:
+    def pluralize(count, singular):
+        return singular if count == 1 else singular + "s"
+
+    def include(count, word):
+        return f"{count} {pluralize(count, word)}" if count > 0 else ""
+
+    minutes, seconds = divmod(seconds, 60)
+    hours, minutes = divmod(minutes, 60)
+    days, hours = divmod(hours, 24)
+    parts = [a for a in [
+        include(days, "day"),
+        include(hours, "hour"),
+        include(minutes, "minute"),
+        include(seconds, "second")] if a]
+    if len(parts) > 2:
+        return "{} and {}".format(", ".join(parts[:-1]), parts[-1])
+    return " and ".join(parts)
@@ -0,0 +1,53 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional
+import json
+import base64
+import hashlib
+
+
+def _get_checksum(key: str, payload: bytes) -> str:
+    hasher = hashlib.sha256()
+    hasher.update(payload)
+    hasher.update(key.encode("utf-8"))
+    checksum = hasher.hexdigest()
+    return checksum
+
+
+def sign_token(key: str, payload: dict) -> str:
+    payload = base64.urlsafe_b64encode(json.dumps(payload).encode("utf-8"))
+    checksum = _get_checksum(key, payload)
+    return f"{checksum}:{payload.decode('utf-8')}"
+
+
+def verify_token(key: str, data: str) -> Optional[dict]:
+    if not data:
+        return None
+
+    try:
+        checksum, payload = data.split(":", 1)
+    except ValueError:
+        return None
+
+    if checksum != _get_checksum(key, payload.encode("utf-8")):
+        return None
+
+    payload = base64.urlsafe_b64decode(payload).decode("utf-8")
+    try:
+        return json.loads(payload)
+    except json.JSONDecodeError:
+        return None
@@ -0,0 +1,2 @@
+from .provisioning import ProvisioningAPI
+from .public import PublicBridgeWebsite
@@ -0,0 +1 @@
+from .auth_api import AuthAPI
@@ -0,0 +1,178 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from abc import abstractmethod
+import abc
+import asyncio
+import logging
+
+from telethon.errors import *
+
+from ...commands.auth import enter_password
+from ...util import format_duration
+from ...puppet import Puppet
+from ...user import User
+
+
+class AuthAPI(abc.ABC):
+    log = logging.getLogger("mau.web.auth")
+
+    def __init__(self, loop):
+        self.loop = loop  # type: asyncio.AbstractEventLoop
+
+    @abstractmethod
+    def get_login_response(self, status=200, state="", username="", mxid="", message="", error="",
+                           errcode=""):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_mx_login_response(self, status=200, state="", username="", mxid="", message="",
+                              error="", errcode=""):
+        raise NotImplementedError()
+
+    async def post_matrix_token(self, user: User, token):
+        puppet = Puppet.get(user.tgid)
+        if puppet.is_real_user:
+            return self.get_mx_login_response(state="already-logged-in", status=409,
+                                              error="You have already logged in with your Matrix "
+                                                    "account.", errcode="already-logged-in")
+
+        resp = await puppet.switch_mxid(token, user.mxid)
+        if resp == 2:
+            return self.get_mx_login_response(status=403, errcode="only-login-self",
+                                              error="You can only log in as your own Matrix user.")
+        elif resp == 1:
+            return self.get_mx_login_response(status=401, errcode="invalid-access-token",
+                                              error="Failed to verify access token.")
+
+        return self.get_mx_login_response(mxid=user.mxid, status=200, state="logged-in")
+
+    async def post_matrix_password(self, user, password):
+        return self.get_mx_login_response(mxid=user.mxid, status=501, error="Not yet implemented",
+                                          errcode="not-yet-implemented")
+
+    async def post_login_phone(self, user, phone):
+        try:
+            await user.client.sign_in(phone or "+123")
+            return self.get_login_response(mxid=user.mxid, state="code", status=200,
+                                           message="Code requested successfully.")
+        except PhoneNumberInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=400,
+                                           errcode="phone_number_invalid",
+                                           error="Invalid phone number.")
+        except PhoneNumberBannedError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=403,
+                                           errcode="phone_number_banned",
+                                           error="Your phone number is banned from Telegram.")
+        except PhoneNumberAppSignupForbiddenError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=403,
+                                           errcode="phone_number_app_signup_forbidden",
+                                           error="You have disabled 3rd party apps on your account.")
+        except PhoneNumberUnoccupiedError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=404,
+                                           errcode="phone_number_unoccupied",
+                                           error="That phone number has not been registered.")
+        except PhoneNumberFloodError:
+            return self.get_login_response(
+                mxid=user.mxid, state="request", status=429, errcode="phone_number_flood",
+                error="Your phone number has been temporarily blocked for flooding. "
+                      "The ban is usually applied for around a day.")
+        except FloodWaitError as e:
+            return self.get_login_response(
+                mxid=user.mxid, state="request", status=429, errcode="flood_wait",
+                error="Your phone number has been temporarily blocked for flooding. "
+                      f"Please wait for {format_duration(e.seconds)} before trying again.")
+        except Exception:
+            self.log.exception("Error requesting phone code")
+            return self.get_login_response(mxid=user.mxid, state="request", status=500,
+                                           errcode="unknown_error",
+                                           error="Internal server error while requesting code.")
+
+    async def post_login_token(self, user, token):
+        try:
+            user_info = await user.client.sign_in(bot_token=token)
+            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+            if user.command_status and user.command_status["action"] == "Login":
+                user.command_status = None
+            return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
+                                           username=user_info.username)
+        except AccessTokenInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="token", status=401,
+                                           errcode="bot_token_invalid",
+                                           error="Bot token invalid.")
+        except AccessTokenExpiredError:
+            return self.get_login_response(mxid=user.mxid, state="token", status=403,
+                                           errcode="bot_token_expired",
+                                           error="Bot token expired.")
+        except Exception:
+            self.log.exception("Error sending bot token")
+            return self.get_login_response(mxid=user.mxid, state="token", status=500,
+                                           error="Internal server error while sending token.")
+
+    async def post_login_code(self, user, code, password_in_data):
+        try:
+            user_info = await user.client.sign_in(code=code)
+            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+            if user.command_status and user.command_status["action"] == "Login":
+                user.command_status = None
+            return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
+                                           username=user_info.username)
+        except PhoneCodeInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="code", status=401,
+                                           errcode="phone_code_invalid",
+                                           error="Incorrect phone code.")
+        except PhoneCodeExpiredError:
+            return self.get_login_response(mxid=user.mxid, state="code", status=403,
+                                           errcode="phone_code_expired",
+                                           error="Phone code expired.")
+        except SessionPasswordNeededError:
+            if not password_in_data:
+                if user.command_status and user.command_status["action"] == "Login":
+                    user.command_status = {
+                        "next": enter_password,
+                        "action": "Login (password entry)",
+                    }
+                return self.get_login_response(
+                    mxid=user.mxid, state="password", status=202,
+                    message="Code accepted, but you have 2-factor authentication is enabled.")
+            return None
+        except Exception:
+            self.log.exception("Error sending phone code")
+            return self.get_login_response(mxid=user.mxid, state="code", status=500,
+                                           errcode="unknown_error",
+                                           error="Internal server error while sending code.")
+
+    async def post_login_password(self, user, password):
+        try:
+            user_info = await user.client.sign_in(password=password)
+            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+            if user.command_status and user.command_status["action"] == "Login (password entry)":
+                user.command_status = None
+            return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
+                                           username=user_info.username)
+        except PasswordEmptyError:
+            return self.get_login_response(mxid=user.mxid, state="password", status=400,
+                                           errcode="password_empty",
+                                           error="Empty password.")
+        except PasswordHashInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="password", status=401,
+                                           errcode="password_invalid",
+                                           error="Incorrect password.")
+        except Exception:
+            self.log.exception("Error sending password")
+            return self.get_login_response(mxid=user.mxid, state="password", status=500,
+                                           errcode="unknown_error",
+                                           error="Internal server error while sending password.")
@@ -0,0 +1,459 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+from typing import Tuple, Optional, Callable, Awaitable, TYPE_CHECKING
+import asyncio
+import logging
+import json
+
+from telethon.utils import get_peer_id, resolve_id
+from telethon.tl.types import ChatForbidden, ChannelForbidden, TypeChat
+from mautrix_appservice import AppService, MatrixRequestError, IntentError
+
+from ...user import User
+from ...portal import Portal
+from ...commands.portal import user_has_power_level, get_initial_state
+from ..common import AuthAPI
+
+if TYPE_CHECKING:
+    from ...context import Context
+
+
+class ProvisioningAPI(AuthAPI):
+    log = logging.getLogger("mau.web.provisioning")
+
+    def __init__(self, context: "Context"):
+        super().__init__(context.loop)
+        self.secret = context.config["appservice.provisioning.shared_secret"]
+        self.az = context.az  # type: AppService
+        self.context = context  # type: Context
+
+        self.app = web.Application(loop=context.loop, middlewares=[self.error_middleware])
+
+        portal_prefix = "/portal/{mxid:![^/]+}"
+        self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal_by_mxid)
+        self.app.router.add_route("GET", "/portal/{tgid:-[0-9]+}", self.get_portal_by_tgid)
+        self.app.router.add_route("POST", portal_prefix + "/connect/{chat_id:[0-9]+}",
+                                  self.connect_chat)
+        self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat)
+        self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat)
+
+        user_prefix = "/user/{mxid:@[^:]*:[^/]+}"
+        self.app.router.add_route("GET", f"{user_prefix}", self.get_user_info)
+        self.app.router.add_route("GET", f"{user_prefix}/chats", self.get_chats)
+
+        self.app.router.add_route("POST", f"{user_prefix}/logout", self.logout)
+        self.app.router.add_route("POST", f"{user_prefix}/login/bot_token", self.send_bot_token)
+        self.app.router.add_route("POST", f"{user_prefix}/login/request_code", self.request_code)
+        self.app.router.add_route("POST", f"{user_prefix}/login/send_code", self.send_code)
+        self.app.router.add_route("POST", f"{user_prefix}/login/send_password", self.send_password)
+
+    async def get_portal_by_mxid(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        mxid = request.match_info["mxid"]
+        portal = Portal.get_by_mxid(mxid)
+        if not portal:
+            return self.get_error_response(404, "portal_not_found",
+                                           "Portal with given Matrix ID not found.")
+        return web.json_response({
+            "mxid": portal.mxid,
+            "chat_id": get_peer_id(portal.peer),
+            "peer_type": portal.peer_type,
+            "title": portal.title,
+            "about": portal.about,
+            "username": portal.username,
+            "megagroup": portal.megagroup,
+        })
+
+    async def get_portal_by_tgid(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        try:
+            tgid, _ = resolve_id(int(request.match_info["tgid"]))
+        except ValueError:
+            return self.get_error_response(400, "tgid_invalid",
+                                           "Given chat ID is not valid.")
+        portal = Portal.get_by_tgid(tgid)
+        if not portal:
+            return self.get_error_response(404, "portal_not_found",
+                                           "Portal to given Telegram chat not found.")
+        return web.json_response({
+            "mxid": portal.mxid,
+            "chat_id": get_peer_id(portal.peer),
+            "peer_type": portal.peer_type,
+            "title": portal.title,
+            "about": portal.about,
+            "username": portal.username,
+            "megagroup": portal.megagroup,
+        })
+
+    async def connect_chat(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        room_id = request.match_info["mxid"]
+        if Portal.get_by_mxid(room_id):
+            return self.get_error_response(409, "room_already_bridged",
+                                           "Room is already bridged to another Telegram chat.")
+
+        chat_id = request.match_info["chat_id"]
+        if chat_id.startswith("-100"):
+            tgid = int(chat_id[4:])
+            peer_type = "channel"
+        elif chat_id.startswith("-"):
+            tgid = -int(chat_id)
+            peer_type = "chat"
+        else:
+            return self.get_error_response(400, "tgid_invalid", "Invalid Telegram chat ID.")
+
+        user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                        require_puppeting=False)
+        if err is not None:
+            return err
+        elif user and not await user_has_power_level(room_id, self.az.intent, user, "bridge"):
+            return self.get_error_response(403, "not_enough_permissions",
+                                           "You do not have the permissions to bridge that room.")
+
+        portal = Portal.get_by_tgid(tgid, peer_type=peer_type)
+        if portal.mxid == room_id:
+            return self.get_error_response(200, "bridge_exists",
+                                           "Telegram chat is already bridged to that Matrix room.")
+        elif portal.mxid:
+            force = request.query.get("force", None)
+            if force in ("delete", "unbridge"):
+                delete = force == "delete"
+                await portal.cleanup_room(portal.main_intent, portal.mxid, puppets_only=not delete,
+                                          message=("Portal deleted (moving to another room)"
+                                                   if delete
+                                                   else "Room unbridged (portal moving to another "
+                                                        "room)"))
+            else:
+                return self.get_error_response(409, "chat_already_bridged",
+                                               "Telegram chat is already bridged to another "
+                                               "Matrix room.")
+
+        is_logged_in = user is not None and await user.is_logged_in()
+        user = user if is_logged_in else self.context.bot
+        if not user:
+            return self.get_login_response(status=403, errcode="not_logged_in",
+                                           error="You are not logged in and there is no relay bot.")
+
+        entity = None  # type: Optional[TypeChat]
+        try:
+            entity = await user.client.get_entity(portal.peer)
+        except Exception:
+            self.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+
+        if not entity or isinstance(entity, (ChatForbidden, ChannelForbidden)):
+            if is_logged_in:
+                return self.get_error_response(403, "user_not_in_chat",
+                                               "Failed to get info of Telegram chat. "
+                                               "Are you in the chat?")
+            return self.get_error_response(403, "bot_not_in_chat",
+                                           "Failed to get info of Telegram chat. "
+                                           "Is the relay bot in the chat?")
+
+        direct = False
+
+        portal.mxid = room_id
+        portal.title, portal.about, levels = await get_initial_state(self.az.intent, room_id)
+        portal.photo_id = ""
+        portal.save()
+
+        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
+                              loop=self.loop)
+
+        return web.Response(status=202, body="{}")
+
+    async def create_chat(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        data = await self.get_data(request)
+        if not data:
+            return self.get_error_response(400, "json_invalid", "Invalid JSON.")
+
+        room_id = request.match_info["mxid"]
+        if Portal.get_by_mxid(room_id):
+            return self.get_error_response(409, "room_already_bridged",
+                                           "Room is already bridged to another Telegram chat.")
+
+        user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                        require_puppeting=False)
+        if err is not None:
+            return err
+        elif not await user.is_logged_in() or user.is_bot:
+            return self.get_error_response(403, "not_logged_in_real_account",
+                                           "You are not logged in with a real account.")
+        elif not await user_has_power_level(room_id, self.az.intent, user, "bridge"):
+            return self.get_error_response(403, "not_enough_permissions",
+                                           "You do not have the permissions to bridge that room.")
+
+        try:
+            title, about, _ = await get_initial_state(self.az.intent, room_id)
+        except (MatrixRequestError, IntentError):
+            return self.get_error_response(403, "bot_not_in_room",
+                                           "The bridge bot is not in the given room.")
+
+        about = data.get("about", about)
+
+        title = data.get("title", title)
+        if len(title) == 0:
+            return self.get_error_response(400, "body_value_invalid", "Title can not be empty.")
+
+        type = data.get("type", "")
+        if type not in ("group", "chat", "supergroup", "channel"):
+            return self.get_error_response(400, "body_value_invalid",
+                                           "Given chat type is not valid.")
+
+        supergroup = type == "supergroup"
+        type = {
+            "supergroup": "channel",
+            "channel": "channel",
+            "chat": "chat",
+            "group": "chat",
+        }[type]
+
+        portal = Portal(tgid=None, mxid=room_id, title=title, about=about, peer_type=type)
+        try:
+            await portal.create_telegram_chat(user, supergroup=supergroup)
+        except ValueError as e:
+            portal.delete()
+            return self.get_error_response(500, "unknown_error", e.args[0])
+
+        return web.json_response({
+            "chat_id": portal.tgid,
+        }, status=201)
+
+    async def disconnect_chat(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        portal = Portal.get_by_mxid(request.match_info["mxid"])
+        if not portal or not portal.tgid:
+            return self.get_error_response(404, "portal_not_found",
+                                           "Room is not a portal.")
+
+        user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                        require_puppeting=False, require_user=False)
+        if err is not None:
+            return err
+        elif user and not await user_has_power_level(portal.mxid, self.az.intent, user, "unbridge"):
+            return self.get_error_response(403, "not_enough_permissions",
+                                           "You do not have the permissions to unbridge that room.")
+
+        delete = request.query.get("delete", "").lower() in ("true", "t", "1", "yes", "y")
+        sync = request.query.get("delete", "").lower() in ("true", "t", "1", "yes", "y")
+
+        coro = portal.cleanup_and_delete() if delete else portal.unbridge()
+        if sync:
+            try:
+                await coro
+            except Exception:
+                self.log.exception("Failed to disconnect chat")
+                return self.get_error_response(500, "exception", "Failed to disconnect chat")
+        else:
+            asyncio.ensure_future(coro, loop=self.loop)
+        return web.json_response({}, status=200 if sync else 202)
+
+    async def get_user_info(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request, expect_logged_in=None,
+                                                           require_puppeting=False)
+        if err is not None:
+            return err
+
+        user_data = None
+        if await user.is_logged_in():
+            me = await user.client.get_me()
+            await user.update_info(me)
+            user_data = {
+                "id": user.tgid,
+                "username": user.username,
+                "first_name": me.first_name,
+                "last_name": me.last_name,
+                "phone": me.phone,
+                "is_bot": user.is_bot,
+            }
+        return web.json_response({
+            "telegram": user_data,
+            "mxid": user.mxid,
+            "permissions": user.permissions,
+        })
+
+    async def get_chats(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request, expect_logged_in=True)
+        if err is not None:
+            return err
+
+        if not user.is_bot:
+            chats = await user.get_dialogs()
+            return web.json_response([{
+                "id": get_peer_id(chat),
+                "title": chat.title,
+            } for chat in chats])
+        else:
+            return web.json_response([{
+                "id": get_peer_id(chat.peer),
+                "title": chat.title,
+            } for chat in user.portals.values() if chat.tgid])
+
+    async def send_bot_token(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_token(user, data.get("token", ""))
+
+    async def request_code(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_phone(user, data.get("phone", ""))
+
+    async def send_code(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_code(user, data.get("code", 0), password_in_data=False)
+
+    async def send_password(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_password(user, data.get("password", ""))
+
+    async def logout(self, request: web.Request) -> web.Response:
+        _, user, err = await self.get_user_request_info(request, expect_logged_in=True,
+                                                        require_puppeting=False,
+                                                        want_data=False)
+        if err is not None:
+            return err
+        await user.log_out()
+
+    @staticmethod
+    async def error_middleware(_, handler) -> Callable[[web.Request], Awaitable[web.Response]]:
+        async def middleware_handler(request: web.Request) -> web.Response:
+            try:
+                return await handler(request)
+            except web.HTTPException as ex:
+                return web.json_response({
+                    "error": f"Unhandled HTTP {ex.status}",
+                    "errcode": f"unhandled_http_{ex.status}",
+                }, status=ex.status)
+
+        return middleware_handler
+
+    @staticmethod
+    def get_error_response(status=200, errcode="", error="") -> web.Response:
+        return web.json_response({
+            "error": error,
+            "errcode": errcode,
+        }, status=status)
+
+    def get_mx_login_response(self, status=200, state="", username="", mxid="", message="",
+                              error="", errcode=""):
+        raise NotImplementedError()
+
+    def get_login_response(self, status=200, state="", username="", mxid="", message="", error="",
+                           errcode="") -> web.Response:
+        if username:
+            resp = {
+                "state": "logged-in",
+                "username": username,
+            }
+        elif message:
+            resp = {
+                "state": state,
+                "message": message,
+            }
+        else:
+            resp = {
+                "error": error,
+                "errcode": errcode,
+            }
+            if state:
+                resp["state"] = state
+        return web.json_response(resp, status=status)
+
+    def check_authorization(self, request: web.Request) -> Optional[web.Response]:
+        auth = request.headers.get("Authorization", "")
+        if auth != f"Bearer {self.secret}":
+            return self.get_error_response(error="Shared secret is not valid.",
+                                           errcode="shared_secret_invalid",
+                                           status=401)
+        return None
+
+    @staticmethod
+    async def get_data(request: web.Request) -> Optional[dict]:
+        try:
+            return await request.json()
+        except json.JSONDecodeError:
+            return None
+
+    async def get_user(self, mxid: str, expect_logged_in: Optional[bool] = False,
+                       require_puppeting: bool = True, require_user: bool = True
+                       ) -> Tuple[Optional[User], Optional[web.Response]]:
+        if not mxid:
+            if not require_user:
+                return None, None
+            return None, self.get_login_response(error="User ID not given.",
+                                                 errcode="mxid_empty", status=400)
+
+        user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True)
+        if require_puppeting and not user.puppet_whitelisted:
+            return user, self.get_login_response(error="You are not whitelisted.",
+                                                 errcode="mxid_not_whitelisted", status=403)
+        if expect_logged_in is not None:
+            logged_in = await user.is_logged_in()
+            if not expect_logged_in and logged_in:
+                return user, self.get_login_response(username=user.username, status=409,
+                                                     error="You are already logged in.",
+                                                     errcode="already_logged_in")
+            elif expect_logged_in and not logged_in:
+                return user, self.get_login_response(status=403, error="You are not logged in.",
+                                                     errcode="not_logged_in")
+        return user, None
+
+    async def get_user_request_info(self, request: web.Request,
+                                    expect_logged_in: Optional[bool] = False,
+                                    require_puppeting: bool = False,
+                                    want_data: bool = True,
+                                    ) -> (Tuple[Optional[dict],
+                                                Optional[User],
+                                                Optional[web.Response]]):
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        data = None
+        if want_data and (request.method == "POST" or request.method == "PUT"):
+            data = await self.get_data(request)
+            if not data:
+                return None, None, self.get_login_response(error="Invalid JSON.",
+                                                           errcode="json_invalid", status=400)
+
+        mxid = request.match_info["mxid"]
+        user, err = await self.get_user(mxid, expect_logged_in, require_puppeting)
+
+        return data, user, err
@@ -0,0 +1,861 @@
+swagger: "2.0"
+
+info:
+  title: Mautrix-Telegram provisioning
+  version: 0.3.0
+  description: The provisioning API for Mautrix-Telegram, the Matrix-Telegram puppeting/relaybot bridge.
+  license:
+    name: AGPLv3
+    url: https://github.com/tulir/mautrix-telegram/blob/master/LICENSE
+
+externalDocs:
+  description: Provisioning API wiki page on GitHub
+  url: https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API
+
+basePath: /_matrix/provision/v1
+
+schemes: [https]
+consumes: [application/json]
+produces: [application/json]
+
+tags:
+- name: User info
+- name: Authentication
+- name: Bridging
+
+paths:
+  /portal/{room_id}:
+    get:
+      operationId: get_portal
+      summary: Get the bridging status and info of the connected Telegram chat
+      tags: [Bridging]
+      responses:
+        200:
+          description: Room is bridged
+          schema:
+            $ref: "#/definitions/PortalInfo"
+        400:
+          $ref: "#/responses/BadRequest"
+        404:
+          description: Unknown portal
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - portal_not_found
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: string
+          pattern: "![^/]+"
+  /portal/{chat_id}:
+    get:
+      operationId: get_portal_by_tgid
+      summary: Get the bridging status and info of the connected Telegram chat
+      tags: [Bridging]
+      responses:
+        200:
+          description: Chat is bridged
+          schema:
+            $ref: "#/definitions/PortalInfo"
+        400:
+          description: Invalid Telegram chat ID
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - tgid_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        404:
+          description: Unknown portal
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - portal_not_found
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: chat_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: integer
+          pattern: "-[0-9]+"
+  /portal/{room_id}/connect/{chat_id}:
+    post:
+      operationId: connect_portal
+      summary: Connect an existing Telegram chat to the given room
+      tags: [Bridging]
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room to which the Telegram chat should be connected
+          required: true
+          type: string
+        - name: chat_id
+          in: path
+          description: The ID of the Telegram chat to connect
+          required: true
+          type: integer
+          pattern: "-[0-9]+"
+        - name: force
+          in: query
+          description: Set to force bridging by unbridging or deleting existing portal rooms.
+          required: false
+          type: string
+          enum:
+          - delete
+          - unbridge
+        - name: user_id
+          in: query
+          description: Optional Matrix user ID to check if the user has permissions to do the bridging.
+          required: false
+          type: string
+      responses:
+        200:
+          description: Telegram chat was already bridged to given room.
+        202:
+          description: Room bridging initiated
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          description: "Given user doesn't have permission to bridge the room, or the bridge bot is not in the room"
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_enough_permissions
+                - bot_not_in_room
+                - bot_not_in_chat
+                - not_logged_in
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          description: Matrix room or Telegram chat is already bridged to another chat/room
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: <room|chat>_already_bridged
+                enum:
+                - room_already_bridged
+                - chat_already_bridged
+              error:
+                $ref: "#/definitions/HumanReadableError"
+  /portal/{room_id}/create:
+    post:
+      operationId: create_portal
+      summary: Create a new Telegram chat for the given room
+      tags: [Bridging]
+      responses:
+        201:
+          description: Telegram chat created
+          schema:
+            type: object
+            properties:
+              chat_id:
+                type: integer
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          description: "Given user isn't logged in with a real account or doesn't have permission to bridge the room, or the bridge bot is not in the room"
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_logged_in_real_account
+                - not_enough_permissions
+                - bot_not_in_room
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          description: Room is already bridged
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - room_already_bridged
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            required: [type]
+            properties:
+              type:
+                description: The type of chat to create
+                type: string
+                example: supergroup
+                enum:
+                - chat
+                - supergroup
+                - channel
+              title:
+                description: Title for the new chat
+                type: string
+                example: Mautrix-Telegram Bridge
+              about:
+                description: About text for the new chat
+                type: string
+                example: Discussion about mautrix-telegram
+        - name: user_id
+          in: query
+          description: Matrix user to create the chat as.
+          required: true
+          type: string
+  /portal/{room_id}/disconnect:
+    post:
+      operationId: disconnect_portal
+      summary: Disconnect the Telegram chat from the room
+      tags: [Bridging]
+      responses:
+        202:
+          description: Room unbridging initiated
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          $ref: "#/responses/PermissionError"
+        404:
+          description: Unknown portal
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - portal_not_found
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: string
+        - name: user_id
+          in: query
+          description: Optional Matrix user ID to check if the user has permissions to do the bridging.
+          required: false
+          type: string
+        - name: delete
+          in: query
+          description: Whether or not to delete the room completely (kick all users instead of just Telegram puppets)
+          required: false
+          type: boolean
+          default: false
+        - name: sync
+          in: query
+          description: Whether or not to wait for the unbridging to be completed before responding. **Could cause timeouts in large rooms**
+          required: false
+          type: boolean
+          default: false
+
+  /user/{user_id}:
+    get:
+      operationId: get_me
+      summary: Get the info of the Telegram user the given Matrix user is logged in as
+      tags: [User info]
+      responses:
+        200:
+          description: User found
+          schema:
+            $ref: "#/definitions/UserInfo"
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          $ref: "#/responses/NotWhitelistedError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+  /user/{user_id}/chats:
+    get:
+      operationId: get_chats
+      summary: Get the list of Telegram chats the given Matrix user has access to
+      tags: [User info]
+      responses:
+        200:
+          description: User is logged in
+          schema:
+            $ref: "#/definitions/UserChats"
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          description: User is not logged in or not whitelisted
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_logged_in
+                - mxid_not_whitelisted
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+
+  /user/{user_id}/login/bot_token:
+    post:
+      operationId: post_bot_token
+      summary: Log in with a bot token
+      tags: [Authentication]
+      responses:
+        200:
+          description: Login successful
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          $ref: "#/responses/BadRequest"
+        401:
+          description: Invalid or expired bot token or invalid shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: bot_token_<error>
+                enum:
+                - bot_token_invalid
+                - bot_token_expired
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          $ref: "#/responses/NotWhitelistedError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              token:
+                type: string
+                description: The access token of the bot to log in as
+                example: "297900271:IXjeGEcAN61zHnjPgkWnYWyvVp9K4ulHBEv"
+  /user/{user_id}/login/request_code:
+    post:
+      operationId: post_login_phone
+      summary: Request a phone code from Telegram
+      tags: [Authentication]
+      responses:
+        200:
+          description: Code requested successfully
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          description: Invalid phone number or JSON
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: machine_readable_error
+                enum:
+                - phone_number_invalid
+                - json_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        401:
+          description: Invalid shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          description: Matrix ID is not whitelisted or phone number is banned or has forbidden 3rd party apps
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: machine_readable_error
+                enum:
+                - mxid_not_whitelisted
+                - phone_number_banned
+                - phone_number_app_signup_forbidden
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        404:
+          description: Unregistered phone number
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - phone_number_unoccupied
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        429:
+          description: Phone number has been temporarily blocked for flooding
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - flood_wait
+                - phone_number_flood
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              phone:
+                type: string
+                description: The phone number to log in as.
+                example: "+123456789"
+  /user/{user_id}/login/send_code:
+    post:
+      operationId: post_login_code
+      summary: Send the login code
+      tags: [Authentication]
+      responses:
+        200:
+          description: Login successful
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        202:
+          description: Correct code, but two-factor authentication is enabled
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          $ref: "#/responses/BadRequest"
+        401:
+          description: Invalid phone code or shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - phone_code_invalid
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          description: Matrix ID not whitelisted or phone code expired
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: machine_readable_error
+                enum:
+                - mxid_not_whitelisted
+                - phone_code_expired
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              code:
+                type: integer
+                description: The phone code from Telegram.
+                format: int32
+                example: 123456
+  /user/{user_id}/login/send_password:
+    post:
+      operationId: post_login_password
+      summary: Send the two-factor auth password
+      tags: [Authentication]
+      responses:
+        200:
+          description: Login successful
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          description: Missing password or invalid JSON
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: <field>_empty
+                enum:
+                - password_empty
+                - json_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        401:
+          description: Incorrect password or invalid shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - password_invalid
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          $ref: "#/responses/NotWhitelistedError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              password:
+                type: string
+                description: The two-factor auth password
+                format: password
+                example: hunter2
+  /user/{user_id}/logout:
+    post:
+      operationId: logout
+      summary: Log out
+      tags: [Authentication]
+      responses:
+        200:
+          description: Logout successful
+        403:
+          description: User was not logged in
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_logged_in
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log out as
+          required: true
+          type: string
+
+responses:
+  NotWhitelistedError:
+    description: Matrix ID not whitelisted for puppeting
+    schema:
+      type: object
+      title: Error
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          enum:
+          - mxid_not_whitelisted
+        error:
+          $ref: "#/definitions/HumanReadableError"
+  AlreadyLoggedInError:
+    description: The Matrix user is already logged in
+    schema:
+      type: object
+      properties:
+        state:
+          type: string
+          enum:
+          - logged-in
+        username:
+          type: string
+          description: The Telegram username the user is logged in as.
+  BadRequest:
+    description: Invalid JSON.
+    schema:
+      type: object
+      title: Error
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          enum:
+          - json_invalid
+          - mxid_empty
+          - body_value_missing
+          - body_value_invalid
+        error:
+          $ref: "#/definitions/HumanReadableError"
+  UnknownError:
+    description: Unknown error
+    schema:
+      type: object
+      title: UnknownError
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          enum:
+          - unknown_error
+          - unhandled_error
+        error:
+          type: string
+          title: Error
+          description: A human-readable description of the error
+          example: Internal server error while <action>.
+  PermissionError:
+    description: The given Matrix user doesn't have the permissions to do that.
+    schema:
+      type: object
+      title: Error
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          example: not_enough_permissions
+          enum:
+          - not_enough_permissions
+        error:
+          $ref: "#/definitions/HumanReadableError"
+
+definitions:
+  UserInfo:
+    type: object
+    properties:
+      mxid:
+        type: string
+        example: "@usern:example.com"
+      permissions:
+        type: string
+        example: user
+        enum:
+        - none
+        - relaybot
+        - user
+        - full
+        - admin
+      telegram:
+        type: object
+        properties:
+          id:
+            type: integer
+            example: 123456789
+          username:
+            type: string
+            example: username
+          first_name:
+            type: string
+            example: Usern
+          last_name:
+            type: string
+            example: A.
+          phone:
+            type: string
+            example: +123456789
+          is_bot:
+            type: boolean
+            example: false
+  UserChats:
+    type: array
+    items:
+      type: object
+      properties:
+        id:
+          type: integer
+          example: -123456789
+          description: A bot API style chat ID.
+        title:
+          type: string
+
+  PortalInfo:
+    type: object
+    properties:
+      mxid:
+        type: string
+        example: "!foo:example.com"
+      chat_id:
+        type: integer
+        example: -100123456789
+      peer_type:
+        type: string
+        enum:
+        - user
+        - chat
+        - channel
+      megagroup:
+        type: boolean
+      username:
+        type: string
+      title:
+        type: string
+      about:
+        type: string
+
+  AuthSuccess:
+    type: object
+    properties:
+      state:
+        type: string
+        description: The state/next step after the successful operation.
+        enum:
+        - code
+        - request
+        - password
+        - token
+        - logged-in
+      username:
+        type: string
+        description: The Telegram username the user is logged in as. Only applicable if state=logged-in
+
+  HumanReadableError:
+    type: string
+    description: A human-readable description of the error
+    example: A human-readable description of the error
+
+security:
+  - Bearer: []
+securityDefinitions:
+  Bearer:
+    description: Required authentication for all endpoints
+    name: Authorization
+    in: header
+    type: apiKey
@@ -0,0 +1,173 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+from mako.template import Template
+import pkg_resources
+import logging
+import random
+import string
+import time
+
+from ...util import sign_token, verify_token
+from ...user import User
+from ...puppet import Puppet
+from ..common import AuthAPI
+
+
+class PublicBridgeWebsite(AuthAPI):
+    log = logging.getLogger("mau.web.public")
+
+    def __init__(self, loop):
+        super().__init__(loop)
+        self.secret_key = "".join(
+            random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
+
+        self.login = Template(
+            pkg_resources.resource_string("mautrix_telegram", "web/public/login.html.mako"))
+
+        self.mx_login = Template(
+            pkg_resources.resource_string("mautrix_telegram", "web/public/matrix-login.html.mako"))
+
+        self.app = web.Application(loop=loop)
+        self.app.router.add_route("GET", "/login", self.get_login)
+        self.app.router.add_route("POST", "/login", self.post_login)
+        self.app.router.add_route("GET", "/matrix-login", self.get_matrix_login)
+        self.app.router.add_route("POST", "/matrix-login", self.post_matrix_login)
+        self.app.router.add_static("/", pkg_resources.resource_filename("mautrix_telegram",
+                                                                        "web/public/"))
+
+    def make_token(self, mxid, endpoint="/login", expires_in=900):
+        return sign_token(self.secret_key, {
+            "mxid": mxid,
+            "endpoint": endpoint,
+            "expiry": int(time.time()) + expires_in,
+        })
+
+    def verify_token(self, token, endpoint="/login"):
+        token = verify_token(self.secret_key, token)
+        if token and (token.get("expiry", 0) > int(time.time()) and
+                      token.get("endpoint", None) == endpoint):
+            return token.get("mxid", None)
+        return None
+
+    async def get_login(self, request):
+        state = "bot_token" if request.rel_url.query.get("mode", "") == "bot" else "request"
+
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login")
+        if not mxid:
+            return self.get_login_response(status=401, state="invalid-token")
+        user = User.get_by_mxid(mxid, create=False) if mxid else None
+
+        if not user:
+            return self.get_login_response(mxid=mxid, state=state)
+        elif not user.puppet_whitelisted:
+            return self.get_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                           status=403)
+        await user.ensure_started()
+        if not await user.is_logged_in():
+            return self.get_login_response(mxid=user.mxid, state=state)
+
+        return self.get_login_response(mxid=user.mxid, username=user.username)
+
+    async def get_matrix_login(self, request):
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        if not mxid:
+            return self.get_mx_login_response(status=401, state="invalid-token")
+        user = User.get_by_mxid(mxid, create=False) if mxid else None
+
+        if not user:
+            return self.get_mx_login_response(mxid=mxid)
+        elif not user.puppet_whitelisted:
+            return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                              status=403)
+        await user.ensure_started()
+        if not await user.is_logged_in():
+            return self.get_mx_login_response(mxid=user.mxid, status=403,
+                                              error="You are not logged in to Telegram.")
+
+        puppet = Puppet.get(user.tgid)
+        if puppet.is_real_user:
+            return self.get_mx_login_response(state="already-logged-in", status=409)
+
+        return self.get_mx_login_response(mxid=user.mxid)
+
+    def get_login_response(self, status=200, state="", username="", mxid="", message="", error="",
+                           errcode=""):
+        return web.Response(status=status, content_type="text/html",
+                            text=self.login.render(username=username, state=state, error=error,
+                                                   message=message, mxid=mxid))
+
+    def get_mx_login_response(self, status=200, state="", username="", mxid="", message="",
+                              error="", errcode=""):
+        return web.Response(status=status, content_type="text/html",
+                            text=self.mx_login.render(username=username, state=state, error=error,
+                                                      message=message, mxid=mxid))
+
+    async def post_matrix_login(self, request):
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        if not mxid:
+            return self.get_mx_login_response(status=401, state="invalid-token")
+
+        data = await request.post()
+
+        user = await User.get_by_mxid(mxid).ensure_started()
+        if not user.puppet_whitelisted:
+            return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                              status=403)
+        elif not await user.is_logged_in():
+            return self.get_mx_login_response(mxid=user.mxid, status=403,
+                                              error="You are not logged in to Telegram.")
+        mode = data.get("mode", "access_token")
+        if mode == "password":
+            return await self.post_matrix_password(user, data["value"])
+        elif mode == "access_token":
+            return await self.post_matrix_token(user, data["value"])
+        return self.get_mx_login_response(mxid=user.mxid, status=400,
+                                          error="You must provide an access token or "
+                                                "password.")
+
+    async def post_login(self, request):
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login")
+        if not mxid:
+            return self.get_login_response(status=401, state="invalid-token")
+
+        data = await request.post()
+
+        user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True)
+        if not user.puppet_whitelisted:
+            return self.get_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                           status=403)
+        elif await user.is_logged_in():
+            return self.get_login_response(mxid=user.mxid, username=user.username)
+
+        await user.ensure_started(even_if_no_session=True)
+
+        if "phone" in data:
+            return await self.post_login_phone(user, data["phone"])
+        elif "bot_token" in data:
+            return await self.post_login_token(user, data["bot_token"])
+        elif "code" in data:
+            resp = await self.post_login_code(user, data["code"],
+                                              password_in_data="password" in data)
+            if resp or "password" not in data:
+                return resp
+        elif "password" not in data:
+            return self.get_login_response(error="No data given.", status=400)
+
+        if "password" in data:
+            return await self.post_login_password(user, data["password"])
+        return self.get_login_response(error="This should never happen.", status=500)
@@ -0,0 +1,99 @@
+/*
+ * mautrix-telegram - A Matrix-Telegram puppeting bridge
+ * Copyright (C) 2018 Tulir Asokan
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+form > div {
+	display: none;
+}
+
+form[data-status="request"] > div.status-request,
+form[data-status="code"] > div.status-code,
+form[data-status="password"] > div.status-password {
+	display: initial;
+}
+
+.container {
+	margin-top: 3rem;
+	max-width: 60rem;
+}
+
+.error, .message {
+	border-radius: .25rem;
+	padding: .5rem 1rem;
+	border: 1px solid transparent;
+	margin: .5rem 0;
+}
+
+.error {
+	border-color: #f5c6cb;
+	background-color: #f8d7da;
+	color: #721c24;
+}
+
+.message {
+	border-color: #c3e6cb;
+	background-color: #d4edda;
+	color: #155724;
+}
+
+[type="checkbox"], [type="radio"] {
+	position: absolute;
+	opacity: 0;
+}
+
+[type="checkbox"] + label, [type="radio"] + label {
+	position: relative;
+	padding-left: 2.5rem;
+	cursor: pointer;
+	display: inline-block;
+}
+
+[type="checkbox"] + label:before, [type="radio"] + label:before {
+	content: '';
+	position: absolute;
+	left: 0;
+	top: 0.4rem;
+	width: 1.8rem;
+	height: 1.8rem;
+	border: 0.1rem solid #d1d1d1;
+}
+
+[type="radio"] + label:before, [type="radio"] + label:after {
+	border-radius: 50%;
+}
+
+[type="checkbox"]:checked + label:after,
+[type="radio"]:checked + label:after {
+	content: '';
+	width: 0.8rem;
+	height: 0.8rem;
+	background: #9b4dca;
+	position: absolute;
+	top: 0.9rem;
+	left: 0.5rem;
+}
+
+[type="radio"]:disabled + label:before, [type="checkbox"]:disabled + label:before {
+	background-color: #d1d1d1;
+}
+
+[type="radio"]:disabled + label, [type="checkbox"]:disabled + label {
+	color: #d1d1d1;
+}
+
+[type="radio"]:disabled:checked + label:after, [type="checkbox"]:disabled:checked + label:after {
+	background: #606c76;
+}
@@ -0,0 +1,127 @@
+<!--
+mautrix-telegram - A Matrix-Telegram puppeting bridge
+Copyright (C) 2018 Tulir Asokan
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-->
+<!DOCTYPE html>
+<html>
+<head>
+	<title>Login - Mautrix-Telegram bridge</title>
+	<link rel="icon" type="image/png" href="favicon.png"/>
+	<meta property="og:title" content="Login - Mautrix-Telegram bridge">
+	<meta property="og:description" content="A hybrid puppeting/relaybot Matrix-Telegram bridge">
+	<meta property="og:image" content="favicon.png">
+	<meta charset="utf-8">
+	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
+	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
+	<link rel="stylesheet"
+		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+	<link rel="stylesheet" href="login.css"/>
+
+	<script>
+		function switchToBotLogin() {
+			const params = new URLSearchParams(location.search.slice(1))
+			params.set("mode", "bot")
+			location.search = "?" + params.toString()
+			console.log(location.search)
+		}
+
+		function goBack() {
+			let params = new URLSearchParams(location.search.slice(1))
+			const token = params.get("token")
+			params = new URLSearchParams()
+			if (token) {
+				params.set("token", token)
+			}
+			location.replace(location.href.split("?")[0] + "?" + params.toString())
+		}
+	</script>
+</head>
+<body>
+<main class="container">
+	% if username:
+		% if state == "logged-in":
+			<h1>Logged in successfully!</h1>
+			<p>
+				Logged in as @${username}.
+				You can now close this page.
+				You should be invited to Telegram portals on Matrix momentarily.
+			</p>
+		% elif state == "bot-logged-in":
+			<h1>Logged in successfully!</h1>
+			<p>
+				Logged in as @${username}.
+				You can now close this page.
+				You should be invited to Telegram portals on Matrix momentarily.
+			</p>
+		% else:
+			<h1>You're already logged in!</h1>
+			<p>
+				You're logged in as @${username}.
+			</p>
+			<p>
+				If you want to log in with another account, log out using the <code>logout</code>
+				management command first.
+			</p>
+		% endif
+	% elif state == "invalid-token":
+		<h1>Invalid or expired token</h1>
+		<div class="error">Please ask the bridge bot for a new login link.</div>
+	% else:
+		<h1>Log in to Telegram</h1>
+	% if error:
+		<div class="error">${error}</div>
+	% endif
+	% if message:
+		<div class="message">${message}</div>
+	% endif
+		<form method="post">
+			<fieldset>
+				<label for="mxid">Matrix ID</label>
+				<input type="text" id="mxid" name="mxid" disabled value="${mxid}"/>
+				% if state == "request":
+					<label for="value">Phone number</label>
+					<input type="tel" id="value" name="phone" placeholder="Enter phone number"/>
+					<button type="submit">Request code</button>
+					<button class="button-clear" type="button" onclick="switchToBotLogin()">
+						Use bot token
+					</button>
+				% elif state == "bot_token":
+					<label for="value">Bot token</label>
+					<input type="text" id="value" name="bot_token" placeholder="Enter bot API token"/>
+					<button type="submit">Sign in</button>
+				% elif state == "code":
+					<label for="value">Phone code</label>
+					<input type="number" id="value" name="code" placeholder="Enter phone code"/>
+					<button type="submit">Sign in</button>
+				% elif state == "password":
+					<label for="value">Password</label>
+					<input type="password" id="value" name="password"
+						   placeholder="Enter password"/>
+					<button type="submit">Sign in</button>
+				% endif
+				% if state != "request":
+					<div class="float-right">
+						<button class="button-clear" type="button" onclick="goBack()">
+							Go back
+						</button>
+					</div>
+				% endif
+			</fieldset>
+		</form>
+	% endif
+</main>
+</body>
+</html>
@@ -0,0 +1,78 @@
+<!--
+mautrix-telegram - A Matrix-Telegram puppeting bridge
+Copyright (C) 2018 Tulir Asokan
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-->
+<!DOCTYPE html>
+<html>
+<head>
+	<title>Matrix login - Mautrix-Telegram bridge</title>
+	<link rel="icon" type="image/png" href="favicon.png"/>
+	<meta property="og:title" content="Matrix login - Mautrix-Telegram bridge">
+	<meta property="og:description" content="A hybrid puppeting/relaybot Matrix-Telegram bridge">
+	<meta property="og:image" content="favicon.png">
+	<meta charset="utf-8">
+	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
+	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
+	<link rel="stylesheet"
+		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+	<link rel="stylesheet" href="login.css"/>
+</head>
+<body>
+<main class="container">
+	% if state == "logged-in":
+		<h1>Logged in successfully!</h1>
+		<p>
+			Logged in as ${mxid}.
+			You can now close this page.
+		</p>
+	% elif state == "already-logged-in":
+		<h1>You're already logged in!</h1>
+		<p>
+			If you want to log in with another account, log out using the
+			<code>logout-matrix</code> management command first.
+		</p>
+	% elif state == "invalid-token":
+		<h1>Invalid or expired token</h1>
+		<div class="error">Please ask the bridge bot for a new login link.</div>
+	% else:
+		<h1>Log in to Matrix</h1>
+	% if error:
+		<div class="error">${error}</div>
+	% endif
+	% if message:
+		<div class="message">${message}</div>
+	% endif
+		<form method="post">
+			<fieldset>
+				<label for="mxid">Matrix ID</label>
+				<input type="text" id="mxid" name="mxid" disabled value="${mxid}"/>
+
+				<input id="access_token" type="radio" name="mode" value="access_token" checked>
+				<label for="access_token">Access token</label><br>
+				<input id="password" type="radio" name="mode" value="password" disabled>
+				<label for="password">Password</label><br>
+
+				<label for="value">Value</label>
+				<input type="text" id="value" name="value"
+					   placeholder="Enter Matrix access token or password"/>
+
+				<button type="submit">Sign in</button>
+			</fieldset>
+		</form>
+	% endif
+</main>
+</body>
+</html>
@@ -0,0 +1,4 @@
+lxml
+cryptg
+Pillow
+moviepy
@@ -1,8 +1,10 @@
 aiohttp
+mautrix-appservice
 ruamel.yaml
 python-magic
 SQLAlchemy
 alembic
 Markdown
-Pillow
 future-fstrings
+telethon
+telethon-session-sqlalchemy
@@ -1,2 +0,0 @@
-r base.txt
-e git+https://github.com/tulir/Telethon@asyncio-3.5#egg=Telethon
@@ -1,2 +0,0 @@
-r base.txt
-e git+https://github.com/LonamiWebs/Telethon@asyncio#egg=Telethon
@@ -1,7 +1,16 @@
 import setuptools
-import sys
+import glob
 import mautrix_telegram

+extras = {
+    "highlight_edits": ["lxml>=4.1.1,<5"],
+    "better_formatter": ["lxml>=4.1.1,<5"],
+    "fast_crypto": ["cryptg>=0.1,<0.2"],
+    "webp_convert": ["Pillow>=5.0.0,<6"],
+    "hq_thumbnails": ["moviepy>=0.2,<0.3"],
+}
+extras["all"] = list(set(deps[0] for deps in extras.values()))
+
 setuptools.setup(
    name="mautrix-telegram",
    version=mautrix_telegram.__version__,
@@ -10,30 +19,28 @@ setuptools.setup(
    author="Tulir Asokan",
    author_email="tulir@maunium.net",

-    description="A Matrix-Telegram puppeting bridge.",
+    description="A Matrix-Telegram hybrid puppeting/relaybot bridge.",
    long_description=open("README.md").read(),

    packages=setuptools.find_packages(),

    install_requires=[
-        "aiohttp>=2.3.10,<3",
-        "SQLAlchemy>=1.2.2,<2",
-        "alembic>=0.9.7",
+        "aiohttp>=3.0.1,<4",
+        "mautrix-appservice>=0.3.6,<0.4.0",
+        "SQLAlchemy>=1.2.3,<2",
+        "alembic>=1.0.0,<2",
        "Markdown>=2.6.11,<3",
        "ruamel.yaml>=0.15.35,<0.16",
-        "Pillow>=5.0.0,<6",
-        "future-fstrings>=0.4.1",
+        "future-fstrings>=0.4.2",
        "python-magic>=0.4.15,<0.5",
+        "telethon>=1.0,<1.2",
+        "telethon-session-sqlalchemy>=0.2.3,<0.3",
    ],
-    dependency_links=[
-        ("https://github.com/LonamiWebs/Telethon/tarball/6e854325a8e0e800a4f337257293d09006946162#egg=Telethon"
-         if sys.version_info > (3, 5)
-         else "https://github.com/tulir/Telethon/tarball/24dc21aea3305ef3bb8c7fcaef2025ae65d5c85e#egg=Telethon")
-    ],
+    extras_require=extras,

    classifiers=[
-        "Development Status :: 4 Beta",
-        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "Development Status :: 4 - Beta",
+        "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
        "Topic :: Communications :: Chat",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
@@ -44,4 +51,13 @@ setuptools.setup(
        [console_scripts]
        mautrix-telegram=mautrix_telegram.__main__:main
    """,
+    package_data={"mautrix_telegram": [
+        "web/public/*.mako", "web/public/*.png", "web/public/*.css",
+    ]},
+    data_files=[
+        (".", ["example-config.yaml", "alembic.ini"]),
+        ("alembic", ["alembic/env.py"]),
+        ("alembic/versions", glob.glob("alembic/versions/*.py"))
+    ],
 )
+