From 7515b3116463fa3ae9e7f8af514d666796b16eaf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 28 Jun 2018 00:20:01 +0300 Subject: [PATCH 01/65] Move Matrix state cache to main database. Fixes #159 --- ...d51e4_move_state_store_to_main_database.py | 126 ++++++++++++++++++ mautrix_telegram/__main__.py | 4 +- mautrix_telegram/db.py | 49 ++++++- mautrix_telegram/puppet.py | 8 +- mautrix_telegram/sqlstatestore.py | 99 ++++++++++++++ 5 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py create mode 100644 mautrix_telegram/sqlstatestore.py diff --git a/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py b/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py new file mode 100644 index 00000000..7e06de1f --- /dev/null +++ b/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py @@ -0,0 +1,126 @@ +"""Move state store to main database + +Revision ID: 6ca3d74d51e4 +Revises: 2228d49c383f +Create Date: 2018-06-26 21:31:26.911307 + +""" +from alembic import context, op +import sqlalchemy.orm as orm +import sqlalchemy as sa +import json +import re + +from mautrix_telegram.config import Config +from mautrix_telegram.base import Base + +# revision identifiers, used by Alembic. +revision = "6ca3d74d51e4" +down_revision = "2228d49c383f" +branch_labels = None +depends_on = None + + +class RoomState(Base): + query = None + __tablename__ = "mx_room_state" + __table_args__ = {"extend_existing": True} + + room_id = sa.Column(sa.String, primary_key=True) + power_levels = sa.Column("power_levels", sa.Text, nullable=True) + + +class UserProfile(Base): + query = None + __tablename__ = "mx_user_profile" + __table_args__ = {"extend_existing": True} + + room_id = sa.Column(sa.String, primary_key=True) + user_id = sa.Column(sa.String, primary_key=True) + membership = sa.Column(sa.String, nullable=False, default="leave") + displayname = sa.Column(sa.String, nullable=True) + avatar_url = sa.Column(sa.String, nullable=True) + + +class Puppet(Base): + query = None + __tablename__ = "puppet" + __table_args__ = {"extend_existing": True} + + id = sa.Column(sa.Integer, primary_key=True) + displayname = sa.Column(sa.String, nullable=True) + displayname_source = sa.Column(sa.Integer, nullable=True) + username = sa.Column(sa.String, nullable=True) + photo_id = sa.Column(sa.String, nullable=True) + is_bot = sa.Column(sa.Boolean, nullable=True) + matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False) + + +def upgrade(): + op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False, + server_default=sa.sql.expression.false())) + op.create_table("mx_room_state", + sa.Column("room_id", sa.String(), nullable=False), + sa.Column("power_levels", sa.Text(), nullable=True), + sa.PrimaryKeyConstraint("room_id")) + op.create_table("mx_user_profile", + sa.Column("room_id", sa.String(), nullable=False), + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("membership", sa.String(), nullable=False, + default="leave"), + sa.Column("displayname", sa.String(), nullable=True), + sa.Column("avatar_url", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("room_id", "user_id")) + + conn = op.get_bind() + session = orm.sessionmaker(bind=conn) + session = orm.scoping.scoped_session(session) + Puppet.query = session.query_property() + + with open("mx-state.json") as file: + data = json.load(file) + if not data: + return + registrations = data.get("registrations", []) + + mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml") + mxtg_config = Config(mxtg_config_path, None, None) + mxtg_config.load() + + username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}") + hs_domain = mxtg_config["homeserver.domain"] + localpart = username_template.format(userid="(.+)") + mxid_regex = re.compile(f"@{localpart}:{hs_domain}") + for user in registrations: + match = mxid_regex.match(user) + if not match: + continue + + puppet = Puppet.query.get(match.group(1)) + if not puppet: + continue + + puppet.matrix_registered = True + session.merge(puppet) + session.commit() + + user_profiles = [UserProfile(room_id=room, user_id=user, + membership=member.get("membership", "leave"), + displayname=member.get("displayname", None), + avatar_url=member.get("avatar_url", None)) + for room, members in data.get("members", {}).items() + for user, member in members.items()] + session.add_all(user_profiles) + session.commit() + + room_state = [RoomState(room_id=room, power_levels=json.dumps(levels)) + for room, levels in data.get("power_levels", {}).items()] + session.add_all(room_state) + session.commit() + + +def downgrade(): + op.drop_table("mx_user_profile") + op.drop_table("mx_room_state") + with op.batch_alter_table("puppet") as batch_op: + batch_op.drop_column("matrix_registered") diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index b8f29d58..f50bfc92 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -38,6 +38,7 @@ from .puppet import init as init_puppet from .formatter import init as init_formatter from .public import PublicBridgeWebsite from .context import Context +from .sqlstatestore import SQLStateStore log = logging.getLogger("mau") time_formatter = logging.Formatter("[%(asctime)s] [%(levelname)s@%(name)s] %(message)s") @@ -87,10 +88,11 @@ telethon_session_container = AlchemySessionContainer(engine=db_engine, session=d loop = asyncio.get_event_loop() +state_store = SQLStateStore(db_session) appserv = AppService(config["homeserver.address"], config["homeserver.domain"], config["appservice.as_token"], config["appservice.hs_token"], config["appservice.bot_username"], log="mau.as", loop=loop, - verify_ssl=config["homeserver.verify_ssl"]) + verify_ssl=config["homeserver.verify_ssl"], state_store=state_store) context = Context(appserv, db_session, config, loop, None, None, telethon_session_container) diff --git a/mautrix_telegram/db.py b/mautrix_telegram/db.py index 2ef9525e..4709fbff 100644 --- a/mautrix_telegram/db.py +++ b/mautrix_telegram/db.py @@ -15,8 +15,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer, - BigInteger, String, Boolean) + BigInteger, String, Boolean, Text) +from sqlalchemy.sql import expression from sqlalchemy.orm import relationship +import json from .base import Base @@ -80,6 +82,48 @@ class User(Base): portals = relationship("Portal", secondary="user_portal") +class RoomState(Base): + query = None + __tablename__ = "mx_room_state" + + room_id = Column(String, primary_key=True) + _power_levels_text = Column("power_levels", Text, nullable=True) + _power_levels_json = None + +# def __init__(self, *args, **kwargs): +# super().__init__(*args, **kwargs) +# self._power_levels_json = None + + @property + def power_levels(self): + if not self._power_levels_json and self._power_levels_text: + self._power_levels_json = json.loads(self._power_levels_text) + return self._power_levels_json or {} + + @power_levels.setter + def power_levels(self, val): + self._power_levels_json = val + self._power_levels_text = json.dumps(val) + + +class UserProfile(Base): + query = None + __tablename__ = "mx_user_profile" + + room_id = Column(String, primary_key=True) + user_id = Column(String, primary_key=True) + membership = Column(String, nullable=False, default="leave") + displayname = Column(String, nullable=True) + avatar_url = Column(String, nullable=True) + + def dict(self): + return { + "membership": self.membership, + "displayname": self.displayname, + "avatar_url": self.avatar_url, + } + + class Contact(Base): query = None __tablename__ = "contact" @@ -98,6 +142,7 @@ class Puppet(Base): username = Column(String, nullable=True) photo_id = Column(String, nullable=True) is_bot = Column(Boolean, nullable=True) + matrix_registered = Column(Boolean, nullable=False, server_default=expression.false()) # Fucking Telegram not telling bots what chats they are in 3:< @@ -132,3 +177,5 @@ def init(db_session): Puppet.query = db_session.query_property() BotChat.query = db_session.query_property() TelegramFile.query = db_session.query_property() + UserProfile.query = db_session.query_property() + RoomState.query = db_session.query_property() diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index ad0648bc..b0977036 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -36,7 +36,7 @@ class Puppet: cache = {} def __init__(self, id=None, username=None, displayname=None, displayname_source=None, - photo_id=None, is_bot=None, db_instance=None): + photo_id=None, is_bot=None, is_registered=False, db_instance=None): self.id = id self.mxid = self.get_mxid_from_id(self.id) @@ -45,6 +45,7 @@ class Puppet: self.displayname_source = displayname_source self.photo_id = photo_id self.is_bot = is_bot + self.is_registered = is_registered self._db_instance = db_instance self.intent = self.az.intent.user(self.mxid) @@ -67,13 +68,13 @@ class Puppet: def new_db_instance(self): return DBPuppet(id=self.id, username=self.username, displayname=self.displayname, displayname_source=self.displayname_source, photo_id=self.photo_id, - is_bot=self.is_bot) + is_bot=self.is_bot, matrix_registered=self.is_registered) @classmethod def from_db(cls, db_puppet): return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname, db_puppet.displayname_source, db_puppet.photo_id, db_puppet.is_bot, - db_instance=db_puppet) + db_puppet.matrix_registered, db_instance=db_puppet) def save(self): self.db_instance.username = self.username @@ -81,6 +82,7 @@ class Puppet: self.db_instance.displayname_source = self.displayname_source self.db_instance.photo_id = self.photo_id self.db_instance.is_bot = self.is_bot + self.db_instance.matrix_registered = self.is_registered self.db.commit() def similarity(self, query): diff --git a/mautrix_telegram/sqlstatestore.py b/mautrix_telegram/sqlstatestore.py new file mode 100644 index 00000000..1d9442e2 --- /dev/null +++ b/mautrix_telegram/sqlstatestore.py @@ -0,0 +1,99 @@ +# -*- coding: future_fstrings -*- +# mautrix-telegram - A Matrix-Telegram puppeting bridge +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import json + +from mautrix_appservice import StateStore + +from . import puppet as pu +from .db import RoomState, UserProfile + + +class SQLStateStore(StateStore): + def __init__(self, db): + super().__init__() + self.db = db + + def is_registered(self, user: str) -> bool: + puppet = pu.Puppet.get_by_mxid(user) + return puppet.is_registered if puppet else False + + def registered(self, user: str): + puppet = pu.Puppet.get_by_mxid(user) + if puppet: + puppet.is_registered = True + puppet.save() + + def update_state(self, event: dict): + event_type = event["type"] + if event_type == "m.room.power_levels": + self.set_power_levels(event["room_id"], event["content"]) + elif event_type == "m.room.member": + self.set_member(event["room_id"], event["state_key"], event["content"]) + + def get_member(self, room: str, user: str) -> dict: + profile = UserProfile.query.get((room, user)) + if profile: + return profile.dict() + return {} + + def set_member(self, room: str, user: str, member: dict): + profile = UserProfile(room_id=room, user_id=user, + membership=member.get("membership", "leave"), + displayname=member.get("displayname", None), + avatar_url=member.get("avatar_url", None)) + self.db.merge(profile) + self.db.commit() + + def set_membership(self, room: str, user: str, membership: str): + profile = UserProfile.query.get((room, user)) + if not profile: + profile = UserProfile(room_id=room, user_id=user, membership=membership) + self.db.add(profile) + else: + profile.membership = membership + self.db.commit() + + def has_power_levels(self, room: str) -> bool: + room = RoomState.query.get(room) + return room and room._power_levels_text + + def get_power_levels(self, room: str) -> dict: + return RoomState.query.get(room).power_levels + + def set_power_level(self, room: str, user: str, level: int): + room_state = RoomState.query.get(room) + if not room_state: + room_state = RoomState(room) + self.db.add(room_state) + + power_levels = room_state.power_levels + if not power_levels: + power_levels = { + "users": {}, + "events": {}, + } + power_levels[room]["users"][user] = level + room_state.power_levels = power_levels + self.db.commit() + + def set_power_levels(self, room: str, content: dict): + state = RoomState.query.get(room) + if not state: + state = RoomState(room_id=room) + self.db.add(state) + state.power_levels = content + self.db.commit() From 042d89cf65578a93c71730347ff716935864cf7f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 12 Jul 2018 22:47:40 +0300 Subject: [PATCH 02/65] Add full log config. Fixes #166 --- .gitignore | 1 + example-config.yaml | 31 ++++++++++++++++++++++++++++--- mautrix_telegram/__main__.py | 27 ++++++++++++--------------- mautrix_telegram/abstract_user.py | 4 ++-- mautrix_telegram/config.py | 10 ++++++++-- mautrix_telegram/user.py | 2 +- 6 files changed, 52 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index b7e3188b..7d5457da 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__ config.yaml registration.yaml +logs/ *.db *.session *.json diff --git a/example-config.yaml b/example-config.yaml index ea03b25f..0c88b987 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -34,9 +34,6 @@ appservice: # implicitly. external: https://example.com/public - # Whether or not to enable debug messages in the console. - debug: true - # The unique ID of this appservice. id: telegram # Username of the appservice bot. @@ -194,3 +191,31 @@ telegram: api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz # (Optional) Create your own bot at https://t.me/BotFather bot_token: disabled + +# Python logging configuration. +# +# See section 16.7.2 of the Python documentation for more info: +# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema +logging: + version: 1 + formatters: + precise: + format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" + handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: ./mautrix-telegram.log + maxBytes: 10485760 + backupCount: 10 + console: + class: logging.StreamHandler + formatter: precise + loggers: + mau: + level: DEBUG + telethon: + level: DEBUG + root: + level: DEBUG + handlers: [file, console] diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index b8f29d58..8fce6699 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -17,6 +17,7 @@ import argparse import sys import logging +import logging.config import asyncio import sqlalchemy as sql @@ -29,6 +30,7 @@ from .base import Base from .config import Config from .matrix import MatrixHandler +from . import __version__ from .db import init as init_db from .abstract_user import init as init_abstract_user from .user import init as init_user, User @@ -39,12 +41,6 @@ from .formatter import init as init_formatter from .public import PublicBridgeWebsite from .context import Context -log = logging.getLogger("mau") -time_formatter = logging.Formatter("[%(asctime)s] [%(levelname)s@%(name)s] %(message)s") -handler = logging.StreamHandler() -handler.setFormatter(time_formatter) -log.addHandler(handler) - parser = argparse.ArgumentParser( description="A Matrix-Telegram puppeting bridge.", prog="python -m mautrix-telegram") @@ -69,14 +65,11 @@ if args.generate_registration: print(f"Registration generated and saved to {config.registration_path}") sys.exit(0) -if config["appservice.debug"]: - telethon_log = logging.getLogger("telethon") - telethon_log.addHandler(handler) - telethon_log.setLevel(logging.DEBUG) - log.setLevel(logging.DEBUG) - log.debug("Debug messages enabled.") +logging.config.dictConfig(config["logging"]) +log = logging.getLogger("mau.init") +log.debug(f"Initializing mautrix-telegram {__version__}") -db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db")) +db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db") db_factory = orm.sessionmaker(bind=db_engine) db_session = orm.scoping.scoped_session(db_factory) Base.metadata.bind = db_engine @@ -112,9 +105,13 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st startup_actions.append(context.bot.start()) try: + log.debug("Initialization complete, running startup actions") loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop)) + log.debug("Startup actions complete, now running forever") loop.run_forever() except KeyboardInterrupt: - for user in User.by_tgid.values(): - user.stop() + log.debug("Keyboard interrupt received, stopping clients") + loop.run_until_complete( + asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop)) + log.debug("Clients stopped, shutting down") sys.exit(0) diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index 0658f904..722b4655 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -118,8 +118,8 @@ class AbstractUser: await self.start(delete_unless_authenticated=not even_if_no_session) return self - def stop(self): - self.client.disconnect() + async def stop(self): + await self.client.disconnect() self.client = None # region Telegram update handling diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 09476659..8c9f9410 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -154,8 +154,6 @@ class Config(DictWithRecursion): copy("appservice.public.prefix") copy("appservice.public.external") - copy("appservice.debug") - copy("appservice.id") copy("appservice.bot_username") copy("appservice.bot_displayname") @@ -218,6 +216,14 @@ class Config(DictWithRecursion): copy("telegram.api_hash") copy("telegram.bot_token") + if "appservice.debug" in self and "logging" not in self: + level = "DEBUG" if self["appservice.debug"] else "INFO" + base["logging.root.level"] = level + base["logging.loggers.mau.level"] = level + base["logging.loggers.telethon.level"] = level + else: + copy("logging") + self._data = base._data self.save() diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index 1a72fd0a..da62fad8 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -145,7 +145,7 @@ class User(AbstractUser): asyncio.ensure_future(self.post_login(), loop=self.loop) elif delete_unless_authenticated: self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...") - self.client.disconnect() + await self.client.disconnect() self.client.session.delete() return self From 1d9455f6398527bdee1375c63a8570dfa74104f6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 12 Jul 2018 22:59:17 +0300 Subject: [PATCH 03/65] Allow specifying address and listen host/port separately. Fixes #160 --- example-config.yaml | 9 ++++----- mautrix_telegram/config.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/example-config.yaml b/example-config.yaml index 0c88b987..82042e2d 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -11,12 +11,11 @@ homeserver: # Application service host/registration related details # Changing these values requires regeneration of the registration. appservice: - # The protocol the homeserver should use when connecting to this appservice. - # Usually "http" or "https". - protocol: http + # The address that the homeserver can use to connect to this appservice. + address: http://localhost:8080 - # The hostname and port where the homeserver can find this appservice. - hostname: localhost + # The hostname and port where this appservice should listen. + hostname: 0.0.0.0 port: 8080 # The full URI to the database. diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 8c9f9410..8b90eabd 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -144,7 +144,12 @@ class Config(DictWithRecursion): copy("homeserver.verify_ssl") copy("homeserver.domain") - copy("appservice.protocol") + if "appservice.protocol" in self and "appservice.address" not in self: + protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"], + self["appservice.port"]) + base["appservice.address"] = f"{protocol}://{hostname}:{port}" + else: + copy("appservice.address") copy("appservice.hostname") copy("appservice.port") @@ -257,10 +262,8 @@ class Config(DictWithRecursion): self.set("appservice.as_token", self._new_token()) self.set("appservice.hs_token", self._new_token()) - url = (f"{self['appservice.protocol']}://" - f"{self['appservice.hostname']}:{self['appservice.port']}") self._registration = { - "id": self.get("appservice.id", "telegram"), + "id": self["appservice.id"] or "telegram", "as_token": self["appservice.as_token"], "hs_token": self["appservice.hs_token"], "namespaces": { @@ -273,7 +276,7 @@ class Config(DictWithRecursion): "regex": f"#{alias_format}:{homeserver}" }] }, - "url": url, + "url": self["appservice.address"], "sender_localpart": self["appservice.bot_username"], "rate_limited": False } From 15fd394d5433c0df4570174ada2c3324b88b46f4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 12 Jul 2018 23:08:08 +0300 Subject: [PATCH 04/65] Add proxy config. Fixes #153 --- example-config.yaml | 13 +++++++++++++ mautrix_telegram/abstract_user.py | 24 ++++++++++++++++++++++-- mautrix_telegram/config.py | 6 ++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/example-config.yaml b/example-config.yaml index 82042e2d..27c64991 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -190,6 +190,19 @@ telegram: api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz # (Optional) Create your own bot at https://t.me/BotFather bot_token: disabled + # Telethon proxy configuration. + # You must install PySocks from pip for proxies to work. + proxy: + # Allowed types: disabled, socks4, socks5, http + type: disabled + # Proxy IP address and port. + address: 127.0.0.1 + port: 1080 + # Whether or not to perform DNS resolving remotely. + rdns: true + # Proxy authentication (optional). + username: "" + password: "" # Python logging configuration. # diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index 722b4655..dd1fe03b 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -50,6 +50,23 @@ class AbstractUser: def connected(self): return self.client and self.client.is_connected() + @property + def _proxy_settings(self): + type = config["telegram.proxy.type"].lower() + if type == "disabled": + return None + elif type == "socks4": + type = 1 + elif type == "socks5": + type = 2 + elif type == "http": + type = 3 + + return (type, + config["telegram.proxy.address"], config["telegram.proxy.port"], + config["telegram.proxy.rdns"], + config["telegram.proxy.username"], config["telegram.proxy.password"]) + def _init_client(self): self.log.debug(f"Initializing client for {self.name}") device = f"{platform.system()} {platform.release()}" @@ -62,7 +79,8 @@ class AbstractUser: app_version=__version__, system_version=sysversion, device_model=device, - timeout=120) + timeout=120, + proxy=self._proxy_settings) self.client.add_event_handler(self._update_catch) async def update(self, update): @@ -95,7 +113,9 @@ class AbstractUser: return self.client and await self.client.is_user_authorized() async def has_full_access(self, allow_bot=False): - return self.puppet_whitelisted and (not self.is_bot or allow_bot) and await self.is_logged_in() + return (self.puppet_whitelisted + and (not self.is_bot or allow_bot) + and await self.is_logged_in()) async def start(self, delete_unless_authenticated=False): if not self.client: diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 8b90eabd..fb2e14a2 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -220,6 +220,12 @@ class Config(DictWithRecursion): copy("telegram.api_id") copy("telegram.api_hash") copy("telegram.bot_token") + copy("telegram.proxy.type") + copy("telegram.proxy.address") + copy("telegram.proxy.port") + copy("telegram.proxy.rdns") + copy("telegram.proxy.username") + copy("telegram.proxy.password") if "appservice.debug" in self and "logging" not in self: level = "DEBUG" if self["appservice.debug"] else "INFO" From f6923a5e1b518ac5754ef825cec65dede9cb4dbf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 24 Jun 2018 21:22:12 +0300 Subject: [PATCH 05/65] Add provisioning API config (ref #154) --- example-config.yaml | 11 +++++++++++ mautrix_telegram/__main__.py | 16 +++++++++++----- mautrix_telegram/abstract_user.py | 2 +- mautrix_telegram/config.py | 4 ++++ mautrix_telegram/context.py | 4 ++-- mautrix_telegram/provisioning_api.py | 27 +++++++++++++++++++++++++++ 6 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 mautrix_telegram/provisioning_api.py diff --git a/example-config.yaml b/example-config.yaml index 27c64991..6f9e8df5 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -33,6 +33,17 @@ appservice: # implicitly. external: https://example.com/public + # Provisioning API part of the web server for automated portal creation and fetching information. + # Used by things like Dimension (https://dimension.t2bot.io/). + provisioning: + # Whether or not the provisioning API should be enabled. + enabled: true + # The prefix to use in the provisioning API endpoints. + prefix: /_matrix/provision + # The shared secret to authorize users of the API. + # You can generate a decent secret with `pwgen -snc 32 1` + shared_secret: "Very secret shared secret" + # The unique ID of this appservice. id: telegram # Username of the appservice bot. diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index 8fce6699..84d01698 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -39,6 +39,7 @@ from .portal import init as init_portal from .puppet import init as init_puppet from .formatter import init as init_formatter from .public import PublicBridgeWebsite +from .provisioning_api import ProvisioningAPI from .context import Context parser = argparse.ArgumentParser( @@ -74,9 +75,9 @@ db_factory = orm.sessionmaker(bind=db_engine) db_session = orm.scoping.scoped_session(db_factory) Base.metadata.bind = db_engine -telethon_session_container = AlchemySessionContainer(engine=db_engine, session=db_session, - table_base=Base, table_prefix="telethon_", - manage_tables=False) +session_container = AlchemySessionContainer(engine=db_engine, session=db_session, + table_base=Base, table_prefix="telethon_", + manage_tables=False) loop = asyncio.get_event_loop() @@ -85,11 +86,16 @@ appserv = AppService(config["homeserver.address"], config["homeserver.domain"], config["appservice.bot_username"], log="mau.as", loop=loop, verify_ssl=config["homeserver.verify_ssl"]) -context = Context(appserv, db_session, config, loop, None, None, telethon_session_container) +context = Context(appserv, db_session, config, loop, None, None, session_container) if config["appservice.public.enabled"]: public = PublicBridgeWebsite(loop) - appserv.app.add_subapp(config.get("appservice.public.prefix", "/public"), public.app) + appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public.app) + +if config["appservice.provisioning.enabled"]: + provisioning_api = ProvisioningAPI(loop) + appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning", + provisioning_api.app) with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start: init_db(db_session) diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index dd1fe03b..d15e9e66 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -328,5 +328,5 @@ class AbstractUser: def init(context): global config, MAX_DELETIONS AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, _ = context - AbstractUser.session_container = context.telethon_session_container + AbstractUser.session_container = context.session_container MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10) diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index fb2e14a2..07944b3c 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -159,6 +159,10 @@ class Config(DictWithRecursion): copy("appservice.public.prefix") copy("appservice.public.external") + copy("appservice.provisioning.enabled") + copy("appservice.provisioning.prefix") + copy("appservice.provisioning.shared_secret") + copy("appservice.id") copy("appservice.bot_username") copy("appservice.bot_displayname") diff --git a/mautrix_telegram/context.py b/mautrix_telegram/context.py index 530f1eed..dfb32b59 100644 --- a/mautrix_telegram/context.py +++ b/mautrix_telegram/context.py @@ -17,14 +17,14 @@ class Context: - def __init__(self, az, db, config, loop, bot, mx, telethon_session_container): + def __init__(self, az, db, config, loop, bot, mx, session_container): self.az = az self.db = db self.config = config self.loop = loop self.bot = bot self.mx = mx - self.telethon_session_container = telethon_session_container + self.session_container = session_container def __iter__(self): yield self.az diff --git a/mautrix_telegram/provisioning_api.py b/mautrix_telegram/provisioning_api.py new file mode 100644 index 00000000..db89c506 --- /dev/null +++ b/mautrix_telegram/provisioning_api.py @@ -0,0 +1,27 @@ +# -*- coding: future_fstrings -*- +# mautrix-telegram - A Matrix-Telegram puppeting bridge +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from aiohttp import web +import logging + + +class ProvisioningAPI: + log = logging.getLogger("mau.provisioning") + + def __init__(self, loop): + self.loop = loop + + self.app = web.Application(loop=loop) From 5d48040eb8bda89b41c7f304a1903df1541f9eba Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 24 Jun 2018 21:35:07 +0300 Subject: [PATCH 06/65] Separate auth methods from public API --- mautrix_telegram/public/__init__.py | 125 ++++++---------------------- mautrix_telegram/public/auth_api.py | 115 +++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 99 deletions(-) create mode 100644 mautrix_telegram/public/auth_api.py diff --git a/mautrix_telegram/public/__init__.py b/mautrix_telegram/public/__init__.py index 6a463f2e..60e22fd6 100644 --- a/mautrix_telegram/public/__init__.py +++ b/mautrix_telegram/public/__init__.py @@ -26,12 +26,14 @@ from ..user import User from ..commands.auth import enter_password from ..util import format_duration +from .auth_api import AuthAPI -class PublicBridgeWebsite: + +class PublicBridgeWebsite(AuthAPI): log = logging.getLogger("mau.public") def __init__(self, loop): - self.loop = loop + super(AuthAPI, self).__init__(loop) self.login = Template( pkg_resources.resource_string("mautrix_telegram", "public/login.html.mako")) @@ -43,22 +45,24 @@ class PublicBridgeWebsite: pkg_resources.resource_filename("mautrix_telegram", "public/")) async def get_login(self, request): - user = (User.get_by_mxid(request.rel_url.query["mxid"], create=False) - if "mxid" in request.rel_url.query else None) state = "token" if request.rel_url.query.get("mode", "") == "bot" else "request" + + mxid = request.rel_url.query.get("mxid", None) + user = User.get_by_mxid(mxid, create=False) if mxid else None + if not user: - return self.render_login( - mxid=request.rel_url.query["mxid"] if "mxid" in request.rel_url.query else None, - state=state) + return self.get_login_response(mxid=mxid, state=state) elif not user.puppet_whitelisted: - return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403) + return self.get_login_response(mxid=user.mxid, error="You are not whitelisted.", + status=403) await user.ensure_started() if not await user.is_logged_in(): - return self.render_login(mxid=user.mxid, state=state) + return self.get_login_response(mxid=user.mxid, state=state) - return self.render_login(mxid=user.mxid, username=user.username) + return self.get_login_response(mxid=user.mxid, username=user.username) - def render_login(self, status=200, username="", state="", error="", message="", mxid=""): + def get_login_response(self, status=200, state="", username="", mxid="", message="", error="", + errcode=""): return web.Response(status=status, content_type="text/html", text=self.login.render(username=username, state=state, error=error, message=message, mxid=mxid)) @@ -69,101 +73,24 @@ class PublicBridgeWebsite: asyncio.ensure_future(user.post_login(user_info), loop=self.loop) if user.command_status and user.command_status["action"] == "Login": user.command_status = None - return self.render_login(mxid=user.mxid, state="logged-in", status=200, - username=user_info.username) + return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, + username=user_info.username) except Exception: self.log.exception("Error sending bot token") - return self.render_login(mxid=user.mxid, state="token", status=500, - error="Internal server error while sending token.") - - async def post_login_phone(self, user, phone): - try: - await user.client.sign_in(phone or "+123") - return self.render_login(mxid=user.mxid, state="code", status=200, - message="Code requested successfully.") - except PhoneNumberInvalidError: - return self.render_login(mxid=user.mxid, state="request", status=400, - error="Invalid phone number.") - except PhoneNumberUnoccupiedError: - return self.render_login(mxid=user.mxid, state="request", status=404, - error="That phone number has not been registered.") - except PhoneNumberFloodError: - return self.render_login( - mxid=user.mxid, state="request", status=429, - error="Your phone number has been temporarily blocked for flooding. " - "The ban is usually applied for around a day.") - except FloodWaitError as e: - return self.render_login( - mxid=user.mxid, state="request", status=429, - error="Your phone number has been temporarily blocked for flooding. " - f"Please wait for {format_duration(e.seconds)} before trying again.") - except PhoneNumberBannedError: - return self.render_login(mxid=user.mxid, state="request", status=401, - error="Your phone number is banned from Telegram.") - except PhoneNumberAppSignupForbiddenError: - return self.render_login(mxid=user.mxid, state="request", status=401, - error="You have disabled 3rd party apps on your account.") - except Exception: - self.log.exception("Error requesting phone code") - return self.render_login(mxid=user.mxid, state="request", status=500, - error="Internal server error while requesting code.") - - async def post_login_code(self, user, code, password_in_data): - try: - user_info = await user.client.sign_in(code=code) - asyncio.ensure_future(user.post_login(user_info), loop=self.loop) - if user.command_status and user.command_status["action"] == "Login": - user.command_status = None - return self.render_login(mxid=user.mxid, state="logged-in", status=200, - username=user_info.username) - except PhoneCodeInvalidError: - return self.render_login(mxid=user.mxid, state="code", status=403, - error="Incorrect phone code.") - except PhoneCodeExpiredError: - return self.render_login(mxid=user.mxid, state="code", status=403, - error="Phone code expired.") - except SessionPasswordNeededError: - if not password_in_data: - if user.command_status and user.command_status["action"] == "Login": - user.command_status = { - "next": enter_password, - "action": "Login (password entry)", - } - return self.render_login( - mxid=user.mxid, state="password", status=200, - message="Code accepted, but you have 2-factor authentication is enabled.") - return None - except Exception: - self.log.exception("Error sending phone code") - return self.render_login(mxid=user.mxid, state="code", status=500, - error="Internal server error while sending code.") - - async def post_login_password(self, user, password): - try: - user_info = await user.client.sign_in(password=password) - asyncio.ensure_future(user.post_login(user_info), loop=self.loop) - if user.command_status and user.command_status["action"] == "Login (password entry)": - user.command_status = None - return self.render_login(mxid=user.mxid, state="logged-in", status=200, - username=user_info.username) - except (PasswordHashInvalidError, PasswordEmptyError): - return self.render_login(mxid=user.mxid, state="password", status=400, - error="Incorrect password.") - except Exception: - self.log.exception("Error sending password") - return self.render_login(mxid=user.mxid, state="password", status=500, - error="Internal server error while sending password.") + return self.get_login_response(mxid=user.mxid, state="token", status=500, + error="Internal server error while sending token.") async def post_login(self, request): data = await request.post() if "mxid" not in data: - return self.render_login(error="Please enter your Matrix ID.", status=400) + return self.get_login_response(error="Please enter your Matrix ID.", status=400) - user = await User.get_by_mxid(data["mxid"]).ensure_started() + user = await User.get_by_mxid(data["mxid"]).ensure_started(even_if_no_session=True) if not user.puppet_whitelisted: - return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403) + return self.get_login_response(mxid=user.mxid, error="You are not whitelisted.", + status=403) elif await user.is_logged_in(): - return self.render_login(mxid=user.mxid, username=user.username) + return self.get_login_response(mxid=user.mxid, username=user.username) await user.ensure_started(even_if_no_session=True) @@ -177,8 +104,8 @@ class PublicBridgeWebsite: if resp or "password" not in data: return resp elif "password" not in data: - return self.render_login(error="No data given.", status=400) + return self.get_login_response(error="No data given.", status=400) if "password" in data: return await self.post_login_password(user, data["password"]) - return self.render_login(error="This should never happen.", status=500) + return self.get_login_response(error="This should never happen.", status=500) diff --git a/mautrix_telegram/public/auth_api.py b/mautrix_telegram/public/auth_api.py new file mode 100644 index 00000000..ec763452 --- /dev/null +++ b/mautrix_telegram/public/auth_api.py @@ -0,0 +1,115 @@ +# -*- coding: future_fstrings -*- +# mautrix-telegram - A Matrix-Telegram puppeting bridge +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from abc import abstractmethod +import abc +import asyncio +import logging + +from telethon.errors import * + +from ..commands.auth import enter_password +from ..util import format_duration + + +class AuthAPI(abc.ABC): + log = logging.getLogger("mau.public.auth") + + def __init__(self, loop): + self.loop = loop + + @abstractmethod + def get_login_response(self, status=200, state="", username="", mxid="", message="", error="", + errcode=""): + raise NotImplementedError() + + async def post_login_phone(self, user, phone): + try: + await user.client.sign_in(phone or "+123") + return self.get_login_response(mxid=user.mxid, state="code", status=200, + message="Code requested successfully.") + except PhoneNumberInvalidError: + return self.get_login_response(mxid=user.mxid, state="request", status=400, + error="Invalid phone number.") + except PhoneNumberUnoccupiedError: + return self.get_login_response(mxid=user.mxid, state="request", status=404, + error="That phone number has not been registered.") + except PhoneNumberFloodError: + return self.get_login_response( + mxid=user.mxid, state="request", status=429, + error="Your phone number has been temporarily blocked for flooding. " + "The ban is usually applied for around a day.") + except FloodWaitError as e: + return self.get_login_response( + mxid=user.mxid, state="request", status=429, + error="Your phone number has been temporarily blocked for flooding. " + f"Please wait for {format_duration(e.seconds)} before trying again.") + except PhoneNumberBannedError: + return self.get_login_response(mxid=user.mxid, state="request", status=401, + error="Your phone number is banned from Telegram.") + except PhoneNumberAppSignupForbiddenError: + return self.get_login_response(mxid=user.mxid, state="request", status=401, + error="You have disabled 3rd party apps on your account.") + except Exception: + self.log.exception("Error requesting phone code") + return self.get_login_response(mxid=user.mxid, state="request", status=500, + error="Internal server error while requesting code.") + + async def post_login_code(self, user, code, password_in_data): + try: + user_info = await user.client.sign_in(code=code) + asyncio.ensure_future(user.post_login(user_info), loop=self.loop) + if user.command_status and user.command_status["action"] == "Login": + user.command_status = None + return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, + username=user_info.username) + except PhoneCodeInvalidError: + return self.get_login_response(mxid=user.mxid, state="code", status=403, + error="Incorrect phone code.") + except PhoneCodeExpiredError: + return self.get_login_response(mxid=user.mxid, state="code", status=403, + error="Phone code expired.") + except SessionPasswordNeededError: + if not password_in_data: + if user.command_status and user.command_status["action"] == "Login": + user.command_status = { + "next": enter_password, + "action": "Login (password entry)", + } + return self.get_login_response( + mxid=user.mxid, state="password", status=200, + message="Code accepted, but you have 2-factor authentication is enabled.") + return None + except Exception: + self.log.exception("Error sending phone code") + return self.get_login_response(mxid=user.mxid, state="code", status=500, + error="Internal server error while sending code.") + + async def post_login_password(self, user, password): + try: + user_info = await user.client.sign_in(password=password) + asyncio.ensure_future(user.post_login(user_info), loop=self.loop) + if user.command_status and user.command_status["action"] == "Login (password entry)": + user.command_status = None + return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, + username=user_info.username) + except (PasswordHashInvalidError, PasswordEmptyError): + return self.get_login_response(mxid=user.mxid, state="password", status=400, + error="Incorrect password.") + except Exception: + self.log.exception("Error sending password") + return self.get_login_response(mxid=user.mxid, state="password", status=500, + error="Internal server error while sending password.") From fa30cb5c1f6cdff7329be426701ef5ed3966b25b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 24 Jun 2018 21:39:01 +0300 Subject: [PATCH 07/65] Move web stuff to web package --- mautrix_telegram/__main__.py | 4 +-- mautrix_telegram/web/__init__.py | 2 ++ mautrix_telegram/web/common/__init__.py | 1 + .../{public => web/common}/auth_api.py | 32 +++++++++--------- .../provisioning/__init__.py} | 0 mautrix_telegram/web/provisioning/spec.yaml | 0 mautrix_telegram/{ => web}/public/__init__.py | 9 ++--- mautrix_telegram/{ => web}/public/favicon.png | Bin mautrix_telegram/{ => web}/public/login.css | 0 .../{ => web}/public/login.html.mako | 0 10 files changed, 23 insertions(+), 25 deletions(-) create mode 100644 mautrix_telegram/web/__init__.py create mode 100644 mautrix_telegram/web/common/__init__.py rename mautrix_telegram/{public => web/common}/auth_api.py (79%) rename mautrix_telegram/{provisioning_api.py => web/provisioning/__init__.py} (100%) create mode 100644 mautrix_telegram/web/provisioning/spec.yaml rename mautrix_telegram/{ => web}/public/__init__.py (96%) rename mautrix_telegram/{ => web}/public/favicon.png (100%) rename mautrix_telegram/{ => web}/public/login.css (100%) rename mautrix_telegram/{ => web}/public/login.html.mako (100%) diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index 84d01698..9702468f 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -38,8 +38,8 @@ from .bot import init as init_bot from .portal import init as init_portal from .puppet import init as init_puppet from .formatter import init as init_formatter -from .public import PublicBridgeWebsite -from .provisioning_api import ProvisioningAPI +from .web.public import PublicBridgeWebsite +from .web.provisioning import ProvisioningAPI from .context import Context parser = argparse.ArgumentParser( diff --git a/mautrix_telegram/web/__init__.py b/mautrix_telegram/web/__init__.py new file mode 100644 index 00000000..002510e8 --- /dev/null +++ b/mautrix_telegram/web/__init__.py @@ -0,0 +1,2 @@ +from .provisioning import ProvisioningAPI +from .public import PublicBridgeWebsite diff --git a/mautrix_telegram/web/common/__init__.py b/mautrix_telegram/web/common/__init__.py new file mode 100644 index 00000000..ccb0d922 --- /dev/null +++ b/mautrix_telegram/web/common/__init__.py @@ -0,0 +1 @@ +from .auth_api import AuthAPI diff --git a/mautrix_telegram/public/auth_api.py b/mautrix_telegram/web/common/auth_api.py similarity index 79% rename from mautrix_telegram/public/auth_api.py rename to mautrix_telegram/web/common/auth_api.py index ec763452..9502db53 100644 --- a/mautrix_telegram/public/auth_api.py +++ b/mautrix_telegram/web/common/auth_api.py @@ -21,8 +21,8 @@ import logging from telethon.errors import * -from ..commands.auth import enter_password -from ..util import format_duration +from mautrix_telegram.commands.auth import enter_password +from mautrix_telegram.util import format_duration class AuthAPI(abc.ABC): @@ -33,20 +33,20 @@ class AuthAPI(abc.ABC): @abstractmethod def get_login_response(self, status=200, state="", username="", mxid="", message="", error="", - errcode=""): + errcode=""): raise NotImplementedError() async def post_login_phone(self, user, phone): try: await user.client.sign_in(phone or "+123") return self.get_login_response(mxid=user.mxid, state="code", status=200, - message="Code requested successfully.") + message="Code requested successfully.") except PhoneNumberInvalidError: return self.get_login_response(mxid=user.mxid, state="request", status=400, - error="Invalid phone number.") + error="Invalid phone number.") except PhoneNumberUnoccupiedError: return self.get_login_response(mxid=user.mxid, state="request", status=404, - error="That phone number has not been registered.") + error="That phone number has not been registered.") except PhoneNumberFloodError: return self.get_login_response( mxid=user.mxid, state="request", status=429, @@ -59,14 +59,14 @@ class AuthAPI(abc.ABC): f"Please wait for {format_duration(e.seconds)} before trying again.") except PhoneNumberBannedError: return self.get_login_response(mxid=user.mxid, state="request", status=401, - error="Your phone number is banned from Telegram.") + error="Your phone number is banned from Telegram.") except PhoneNumberAppSignupForbiddenError: return self.get_login_response(mxid=user.mxid, state="request", status=401, - error="You have disabled 3rd party apps on your account.") + error="You have disabled 3rd party apps on your account.") except Exception: self.log.exception("Error requesting phone code") return self.get_login_response(mxid=user.mxid, state="request", status=500, - error="Internal server error while requesting code.") + error="Internal server error while requesting code.") async def post_login_code(self, user, code, password_in_data): try: @@ -75,13 +75,13 @@ class AuthAPI(abc.ABC): if user.command_status and user.command_status["action"] == "Login": user.command_status = None return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, - username=user_info.username) + username=user_info.username) except PhoneCodeInvalidError: return self.get_login_response(mxid=user.mxid, state="code", status=403, - error="Incorrect phone code.") + error="Incorrect phone code.") except PhoneCodeExpiredError: return self.get_login_response(mxid=user.mxid, state="code", status=403, - error="Phone code expired.") + error="Phone code expired.") except SessionPasswordNeededError: if not password_in_data: if user.command_status and user.command_status["action"] == "Login": @@ -96,7 +96,7 @@ class AuthAPI(abc.ABC): except Exception: self.log.exception("Error sending phone code") return self.get_login_response(mxid=user.mxid, state="code", status=500, - error="Internal server error while sending code.") + error="Internal server error while sending code.") async def post_login_password(self, user, password): try: @@ -105,11 +105,11 @@ class AuthAPI(abc.ABC): if user.command_status and user.command_status["action"] == "Login (password entry)": user.command_status = None return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, - username=user_info.username) + username=user_info.username) except (PasswordHashInvalidError, PasswordEmptyError): return self.get_login_response(mxid=user.mxid, state="password", status=400, - error="Incorrect password.") + error="Incorrect password.") except Exception: self.log.exception("Error sending password") return self.get_login_response(mxid=user.mxid, state="password", status=500, - error="Internal server error while sending password.") + error="Internal server error while sending password.") diff --git a/mautrix_telegram/provisioning_api.py b/mautrix_telegram/web/provisioning/__init__.py similarity index 100% rename from mautrix_telegram/provisioning_api.py rename to mautrix_telegram/web/provisioning/__init__.py diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml new file mode 100644 index 00000000..e69de29b diff --git a/mautrix_telegram/public/__init__.py b/mautrix_telegram/web/public/__init__.py similarity index 96% rename from mautrix_telegram/public/__init__.py rename to mautrix_telegram/web/public/__init__.py index 60e22fd6..c08b1bc6 100644 --- a/mautrix_telegram/public/__init__.py +++ b/mautrix_telegram/web/public/__init__.py @@ -20,13 +20,8 @@ import asyncio import pkg_resources import logging -from telethon.errors import * - -from ..user import User -from ..commands.auth import enter_password -from ..util import format_duration - -from .auth_api import AuthAPI +from ...user import User +from ..common import AuthAPI class PublicBridgeWebsite(AuthAPI): diff --git a/mautrix_telegram/public/favicon.png b/mautrix_telegram/web/public/favicon.png similarity index 100% rename from mautrix_telegram/public/favicon.png rename to mautrix_telegram/web/public/favicon.png diff --git a/mautrix_telegram/public/login.css b/mautrix_telegram/web/public/login.css similarity index 100% rename from mautrix_telegram/public/login.css rename to mautrix_telegram/web/public/login.css diff --git a/mautrix_telegram/public/login.html.mako b/mautrix_telegram/web/public/login.html.mako similarity index 100% rename from mautrix_telegram/public/login.html.mako rename to mautrix_telegram/web/public/login.html.mako From f07009d0d2d509b22398d77a5f13a9633658faaa Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 24 Jun 2018 23:19:29 +0300 Subject: [PATCH 08/65] Add initial parts of provisioning API spec --- mautrix_telegram/web/provisioning/spec.yaml | 70 +++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index e69de29b..7b0dc01f 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -0,0 +1,70 @@ +tags: + - + name: login + description: 'Authentication endpoints.' +definitions: + Error: + x-oad-type: object + type: object + title: Error + properties: + errcode: + x-oad-type: string + type: string + title: 'Error code' + description: 'A machine-readable error code' + error: + x-oad-type: string + type: string + title: Error + description: 'A human-readable description of the error' + status: + x-oad-type: integer + type: integer + title: Status + description: 'The HTTP status code' + format: int32 + AuthSuccess: + x-oad-type: object + type: object + properties: + state: + x-oad-type: string + type: string + enum: + - code + - request + - password + - token + - logged-in +security: + - + Bearer: [] +securityDefinitions: + Bearer: + description: 'Required authentication for all endpoints' + name: Authorization + in: header + type: apiKey +info: + title: 'mautrix-telegram provisioning' + version: 0.3.0 + description: 'The provisioning API for mautrix-telegram.' + contact: + name: 'Tulir Asokan' + email: tulir@maunium.net + url: 'https://maunium.net' + license: + name: AGPLv3 + url: 'https://github.com/tulir/mautrix-telegram/blob/master/LICENSE' +externalDocs: + description: 'Provisioning API wiki page on GitHub.' + url: 'https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API' +basePath: /_matrix/provisioning +schemes: + - https +consumes: + - application/json +produces: + - application/json +swagger: '2.0' From c0ceb1b2b007ad0f878a4c11ae993e7eb7f59b24 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 12 Jul 2018 23:45:15 +0300 Subject: [PATCH 09/65] Move post_login_token to common/auth_api --- mautrix_telegram/web/common/auth_api.py | 13 +++++++++++++ mautrix_telegram/web/public/__init__.py | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/mautrix_telegram/web/common/auth_api.py b/mautrix_telegram/web/common/auth_api.py index 9502db53..3ce557ba 100644 --- a/mautrix_telegram/web/common/auth_api.py +++ b/mautrix_telegram/web/common/auth_api.py @@ -68,6 +68,19 @@ class AuthAPI(abc.ABC): return self.get_login_response(mxid=user.mxid, state="request", status=500, error="Internal server error while requesting code.") + async def post_login_token(self, user, token): + try: + user_info = await user.client.sign_in(bot_token=token) + asyncio.ensure_future(user.post_login(user_info), loop=self.loop) + if user.command_status and user.command_status["action"] == "Login": + user.command_status = None + return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, + username=user_info.username) + except Exception: + self.log.exception("Error sending bot token") + return self.get_login_response(mxid=user.mxid, state="token", status=500, + error="Internal server error while sending token.") + async def post_login_code(self, user, code, password_in_data): try: user_info = await user.client.sign_in(code=code) diff --git a/mautrix_telegram/web/public/__init__.py b/mautrix_telegram/web/public/__init__.py index c08b1bc6..89a76fd7 100644 --- a/mautrix_telegram/web/public/__init__.py +++ b/mautrix_telegram/web/public/__init__.py @@ -62,19 +62,6 @@ class PublicBridgeWebsite(AuthAPI): text=self.login.render(username=username, state=state, error=error, message=message, mxid=mxid)) - async def post_login_token(self, user, token): - try: - user_info = await user.client.sign_in(bot_token=token) - asyncio.ensure_future(user.post_login(user_info), loop=self.loop) - if user.command_status and user.command_status["action"] == "Login": - user.command_status = None - return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, - username=user_info.username) - except Exception: - self.log.exception("Error sending bot token") - return self.get_login_response(mxid=user.mxid, state="token", status=500, - error="Internal server error while sending token.") - async def post_login(self, request): data = await request.post() if "mxid" not in data: From 1fd920255fc15856d82b96c7322febfd6848e81b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Jul 2018 21:25:51 +0300 Subject: [PATCH 10/65] Finish initial provisioning API spec and impl --- mautrix_telegram/web/common/auth_api.py | 44 +- mautrix_telegram/web/provisioning/__init__.py | 68 ++- mautrix_telegram/web/provisioning/spec.yaml | 437 +++++++++++++++--- mautrix_telegram/web/public/__init__.py | 1 - 4 files changed, 480 insertions(+), 70 deletions(-) diff --git a/mautrix_telegram/web/common/auth_api.py b/mautrix_telegram/web/common/auth_api.py index 3ce557ba..6980f1af 100644 --- a/mautrix_telegram/web/common/auth_api.py +++ b/mautrix_telegram/web/common/auth_api.py @@ -43,29 +43,34 @@ class AuthAPI(abc.ABC): message="Code requested successfully.") except PhoneNumberInvalidError: return self.get_login_response(mxid=user.mxid, state="request", status=400, + errcode="phone_number_invalid", error="Invalid phone number.") + except PhoneNumberBannedError: + return self.get_login_response(mxid=user.mxid, state="request", status=403, + errcode="phone_number_banned", + error="Your phone number is banned from Telegram.") + except PhoneNumberAppSignupForbiddenError: + return self.get_login_response(mxid=user.mxid, state="request", status=403, + errcode="phone_number_app_signup_forbidden", + error="You have disabled 3rd party apps on your account.") except PhoneNumberUnoccupiedError: return self.get_login_response(mxid=user.mxid, state="request", status=404, + errcode="phone_number_unoccupied", error="That phone number has not been registered.") except PhoneNumberFloodError: return self.get_login_response( - mxid=user.mxid, state="request", status=429, + mxid=user.mxid, state="request", status=429, errcode="phone_number_flood", error="Your phone number has been temporarily blocked for flooding. " "The ban is usually applied for around a day.") except FloodWaitError as e: return self.get_login_response( - mxid=user.mxid, state="request", status=429, + mxid=user.mxid, state="request", status=429, errcode="flood_wait", error="Your phone number has been temporarily blocked for flooding. " f"Please wait for {format_duration(e.seconds)} before trying again.") - except PhoneNumberBannedError: - return self.get_login_response(mxid=user.mxid, state="request", status=401, - error="Your phone number is banned from Telegram.") - except PhoneNumberAppSignupForbiddenError: - return self.get_login_response(mxid=user.mxid, state="request", status=401, - error="You have disabled 3rd party apps on your account.") except Exception: self.log.exception("Error requesting phone code") return self.get_login_response(mxid=user.mxid, state="request", status=500, + errcode="exception", error="Internal server error while requesting code.") async def post_login_token(self, user, token): @@ -76,6 +81,14 @@ class AuthAPI(abc.ABC): user.command_status = None return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, username=user_info.username) + except AccessTokenInvalidError: + return self.get_login_response(mxid=user.mxid, state="token", status=401, + errcode="bot_token_invalid", + error="Bot token invalid.") + except AccessTokenExpiredError: + return self.get_login_response(mxid=user.mxid, state="token", status=403, + errcode="bot_token_expired", + error="Bot token expired.") except Exception: self.log.exception("Error sending bot token") return self.get_login_response(mxid=user.mxid, state="token", status=500, @@ -90,10 +103,12 @@ class AuthAPI(abc.ABC): return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, username=user_info.username) except PhoneCodeInvalidError: - return self.get_login_response(mxid=user.mxid, state="code", status=403, + return self.get_login_response(mxid=user.mxid, state="code", status=401, + errcode="phone_code_invalid", error="Incorrect phone code.") except PhoneCodeExpiredError: return self.get_login_response(mxid=user.mxid, state="code", status=403, + errcode="phone_code_expired", error="Phone code expired.") except SessionPasswordNeededError: if not password_in_data: @@ -103,12 +118,13 @@ class AuthAPI(abc.ABC): "action": "Login (password entry)", } return self.get_login_response( - mxid=user.mxid, state="password", status=200, + mxid=user.mxid, state="password", status=202, message="Code accepted, but you have 2-factor authentication is enabled.") return None except Exception: self.log.exception("Error sending phone code") return self.get_login_response(mxid=user.mxid, state="code", status=500, + errcode="exception", error="Internal server error while sending code.") async def post_login_password(self, user, password): @@ -119,10 +135,16 @@ class AuthAPI(abc.ABC): user.command_status = None return self.get_login_response(mxid=user.mxid, state="logged-in", status=200, username=user_info.username) - except (PasswordHashInvalidError, PasswordEmptyError): + except PasswordEmptyError: return self.get_login_response(mxid=user.mxid, state="password", status=400, + errcode="password_empty", + error="Empty password.") + except PasswordHashInvalidError: + return self.get_login_response(mxid=user.mxid, state="password", status=401, + errcode="password_invalid", error="Incorrect password.") except Exception: self.log.exception("Error sending password") return self.get_login_response(mxid=user.mxid, state="password", status=500, + errcode="exception", error="Internal server error while sending password.") diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index db89c506..05c73816 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -17,11 +17,75 @@ from aiohttp import web import logging +from ..common import AuthAPI -class ProvisioningAPI: + +class ProvisioningAPI(AuthAPI): log = logging.getLogger("mau.provisioning") def __init__(self, loop): - self.loop = loop + super(AuthAPI, self).__init__(loop) self.app = web.Application(loop=loop) + + login_prefix = "/login/{mxid:@[^:]*:.+}" + self.app.router.add_route("POST", f"{login_prefix}/bot_token", self.send_bot_token) + self.app.router.add_route("POST", f"{login_prefix}/request_code", self.request_code) + self.app.router.add_route("POST", f"{login_prefix}/send_code", self.send_code) + self.app.router.add_route("POST", f"{login_prefix}/send_password", self.send_password) + + def get_login_response(self, status=200, state="", username="", mxid="", message="", error="", + errcode=""): + if username: + resp = { + "state": "logged-in", + "username": username, + } + elif message: + resp = { + "message": message + } + else: + resp = { + "error": error, + "errcode": errcode, + } + return web.json_response(resp, status=status) + + async def get_user(self, request: web.Request): + mxid = request.match_info["mxid"] + user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True) + if not user.puppet_whitelisted: + return user, self.get_login_response(mxid=user.mxid, error="You are not whitelisted.", + errcode="mxid_not_whitelisted", status=403) + elif await user.is_logged_in(): + return user, self.get_login_response(mxid=user.mxid, username=user.username, status=409) + return user, None + + async def send_bot_token(self, request: web.Request): + user, err = await self.get_user(request) + if err: + return err + data = await request.json() + return await self.post_login_token(user, data.get("token", "")) + + async def request_code(self, request: web.Request): + user, err = await self.get_user(request) + if err: + return err + data = await request.json() + return await self.post_login_phone(user, data.get("phone", "")) + + async def send_code(self, request: web.Request): + user, err = await self.get_user(request) + if err: + return err + data = await request.json() + return await self.post_login_code(user, data.get("code", 0), password_in_data=False) + + async def send_password(self, request: web.Request): + user, err = await self.get_user(request) + if err: + return err + data = await request.json() + return await self.post_login_password(user, data.get("password", "")) diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index 7b0dc01f..e8fe08f3 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -1,70 +1,395 @@ +swagger: "2.0" + +info: + title: mautrix-telegram provisioning + version: 0.3.0 + description: The provisioning API for mautrix-telegram. + contact: + name: Tulir Asokan + email: tulir@maunium.net + url: https://maunium.net + license: + name: AGPLv3 + url: https://github.com/tulir/mautrix-telegram/blob/master/LICENSE + +externalDocs: + description: Provisioning API wiki page on GitHub. + url: https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API + +basePath: /_matrix/provision + +schemes: [https] +consumes: [application/json] +produces: [application/json] + tags: - - - name: login - description: 'Authentication endpoints.' +- name: Authentication + +paths: + /login/{mxid}/bot_token: + post: + operationId: post_bot_token + summary: Log in with a bot token + tags: [Authentication] + responses: + 200: + description: Login successful + schema: + $ref: "#/definitions/AuthSuccess" + 400: + $ref: "#/responses/MissingMXIDError" + 401: + description: Invalid or expired bot token + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + example: bot_token_ + enum: + - bot_token_invalid + - bot_token_expired + error: + $ref: "#/definitions/HumanReadableError" + 403: + $ref: "#/responses/NotWhitelistedError" + 409: + $ref: "#/responses/AlreadyLoggedInError" + 500: + $ref: "#/responses/UnknownError" + parameters: + - name: mxid + in: path + description: The Matrix ID of the user who to log in as + required: true + type: string + - name: body + in: body + required: true + schema: + type: object + properties: + token: + type: string + description: The access token of the bot to log in as + example: "297900271:IXjeGEcAN61zHnjPgkWnYWyvVp9K4ulHBEv" + /login/{mxid}/request_code: + post: + operationId: post_login_phone + summary: Request a phone code from Telegram + tags: [Authentication] + responses: + 200: + description: Code requested successfully + schema: + $ref: "#/definitions/AuthSuccess" + 400: + description: Invalid phone number or missing Matrix ID + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + example: machine_readable_error + enum: + - phone_number_invalid + - mxid_empty + error: + $ref: "#/definitions/HumanReadableError" + 403: + description: Matrix ID is not whitelisted or phone number is banned or has forbidden 3rd party apps + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + example: machine_readable_error + enum: + - mxid_not_whitelisted + - phone_number_banned + - phone_number_app_signup_forbidden + error: + $ref: "#/definitions/HumanReadableError" + 404: + description: Unregistered phone number + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - phone_number_unoccupied + error: + $ref: "#/definitions/HumanReadableError" + 409: + $ref: "#/responses/AlreadyLoggedInError" + 429: + description: Phone number has been temporarily blocked for flooding + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - flood_wait + - phone_number_flood + error: + $ref: "#/definitions/HumanReadableError" + 500: + $ref: "#/responses/UnknownError" + parameters: + - name: mxid + in: path + description: The Matrix ID of the user who to log in as + required: true + type: string + - name: body + in: body + required: true + schema: + type: object + properties: + phone: + type: string + description: The phone number to log in as. + example: "+123456789" + /login/{mxid}/send_code: + post: + operationId: post_login_code + summary: Send the login code + tags: [Authentication] + responses: + 200: + description: Login successful + schema: + $ref: "#/definitions/AuthSuccess" + 202: + description: Correct code, but two-factor authentication is enabled + schema: + $ref: "#/definitions/AuthSuccess" + 400: + $ref: "#/responses/MissingMXIDError" + 401: + description: Invalid phone code + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - phone_code_invalid + error: + $ref: "#/definitions/HumanReadableError" + 403: + description: Matrix ID not whitelisted or phone code expired + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + example: machine_readable_error + enum: + - mxid_not_whitelisted + - phone_code_expired + error: + $ref: "#/definitions/HumanReadableError" + 409: + $ref: "#/responses/AlreadyLoggedInError" + 500: + $ref: "#/responses/UnknownError" + parameters: + - name: mxid + in: path + description: The Matrix ID of the user who to log in as + required: true + type: string + - name: body + in: body + required: true + schema: + type: object + properties: + code: + type: integer + description: The phone code from Telegram. + format: int32 + example: 123456 + /login/{mxid}/send_password: + post: + operationId: post_login_password + summary: Send the two-factor auth password + tags: [Authentication] + responses: + 200: + description: Login successful + schema: + $ref: "#/definitions/AuthSuccess" + 400: + description: Missing password or Matrix ID + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + example: _empty + enum: + - password_empty + - mxid_empty + error: + $ref: "#/definitions/HumanReadableError" + 401: + description: Incorrect password + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - password_invalid + error: + $ref: "#/definitions/HumanReadableError" + 403: + $ref: "#/responses/NotWhitelistedError" + 409: + $ref: "#/responses/AlreadyLoggedInError" + 500: + $ref: "#/responses/UnknownError" + parameters: + - name: mxid + in: path + description: The Matrix ID of the user who to log in as + required: true + type: string + - name: body + in: body + required: true + schema: + type: object + properties: + password: + type: string + description: The two-factor auth password + format: password + example: hunter2 + +responses: + NotWhitelistedError: + description: Matrix ID not whitelisted for puppeting + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - mxid_not_whitelisted + error: + $ref: "#/definitions/HumanReadableError" + AlreadyLoggedInError: + description: The Matrix user is already logged in + schema: + type: object + properties: + state: + type: string + enum: + - logged-in + username: + type: string + description: The Telegram username the user is logged in as. + MissingMXIDError: + description: Missing Matrix ID + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - mxid_empty + error: + $ref: "#/definitions/HumanReadableError" + UnknownError: + description: Unknown error + schema: + type: object + title: UnknownError + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - exception + error: + type: string + title: Error + description: A human-readable description of the error + example: Internal server error while . + enum: + - Internal server error while requesting code. + - Internal server error while sending code. + - Internal server error while sending password. + - Internal server error while sending token. + definitions: - Error: - x-oad-type: object - type: object - title: Error - properties: - errcode: - x-oad-type: string - type: string - title: 'Error code' - description: 'A machine-readable error code' - error: - x-oad-type: string - type: string - title: Error - description: 'A human-readable description of the error' - status: - x-oad-type: integer - type: integer - title: Status - description: 'The HTTP status code' - format: int32 + HumanReadableError: + type: string + description: A human-readable description of the error + example: A human-readable description of the error AuthSuccess: - x-oad-type: object type: object properties: state: - x-oad-type: string type: string + description: The state/next step after the successful operation. enum: - - code - - request - - password - - token - - logged-in + - code + - request + - password + - token + - logged-in + username: + type: string + description: The Telegram username the user is logged in as. Only applicable if state=logged-in + + security: - - - Bearer: [] + - Bearer: [] securityDefinitions: Bearer: - description: 'Required authentication for all endpoints' + description: Required authentication for all endpoints name: Authorization in: header type: apiKey -info: - title: 'mautrix-telegram provisioning' - version: 0.3.0 - description: 'The provisioning API for mautrix-telegram.' - contact: - name: 'Tulir Asokan' - email: tulir@maunium.net - url: 'https://maunium.net' - license: - name: AGPLv3 - url: 'https://github.com/tulir/mautrix-telegram/blob/master/LICENSE' -externalDocs: - description: 'Provisioning API wiki page on GitHub.' - url: 'https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API' -basePath: /_matrix/provisioning -schemes: - - https -consumes: - - application/json -produces: - - application/json -swagger: '2.0' diff --git a/mautrix_telegram/web/public/__init__.py b/mautrix_telegram/web/public/__init__.py index 89a76fd7..7a83d731 100644 --- a/mautrix_telegram/web/public/__init__.py +++ b/mautrix_telegram/web/public/__init__.py @@ -16,7 +16,6 @@ # along with this program. If not, see . from aiohttp import web from mako.template import Template -import asyncio import pkg_resources import logging From bc160e0593a3a2d26c6429b84a24631d47d07013 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Jul 2018 22:11:05 +0300 Subject: [PATCH 11/65] Update logger names --- mautrix_telegram/web/common/auth_api.py | 2 +- mautrix_telegram/web/provisioning/__init__.py | 2 +- mautrix_telegram/web/public/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix_telegram/web/common/auth_api.py b/mautrix_telegram/web/common/auth_api.py index 6980f1af..2ea50bca 100644 --- a/mautrix_telegram/web/common/auth_api.py +++ b/mautrix_telegram/web/common/auth_api.py @@ -26,7 +26,7 @@ from mautrix_telegram.util import format_duration class AuthAPI(abc.ABC): - log = logging.getLogger("mau.public.auth") + log = logging.getLogger("mau.web.auth") def __init__(self, loop): self.loop = loop diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index 05c73816..c2f087a9 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -21,7 +21,7 @@ from ..common import AuthAPI class ProvisioningAPI(AuthAPI): - log = logging.getLogger("mau.provisioning") + log = logging.getLogger("mau.web.provisioning") def __init__(self, loop): super(AuthAPI, self).__init__(loop) diff --git a/mautrix_telegram/web/public/__init__.py b/mautrix_telegram/web/public/__init__.py index 7a83d731..43738d8e 100644 --- a/mautrix_telegram/web/public/__init__.py +++ b/mautrix_telegram/web/public/__init__.py @@ -24,7 +24,7 @@ from ..common import AuthAPI class PublicBridgeWebsite(AuthAPI): - log = logging.getLogger("mau.public") + log = logging.getLogger("mau.web.public") def __init__(self, loop): super(AuthAPI, self).__init__(loop) From 48665acf1d16cc1e7abbfd17aa8b12790398508a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Jul 2018 22:14:04 +0300 Subject: [PATCH 12/65] Fix imports and other mistakes --- mautrix_telegram/web/provisioning/__init__.py | 3 ++- mautrix_telegram/web/public/__init__.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index c2f087a9..f5d56c5e 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -17,6 +17,7 @@ from aiohttp import web import logging +from ...user import User from ..common import AuthAPI @@ -24,7 +25,7 @@ class ProvisioningAPI(AuthAPI): log = logging.getLogger("mau.web.provisioning") def __init__(self, loop): - super(AuthAPI, self).__init__(loop) + super().__init__(loop) self.app = web.Application(loop=loop) diff --git a/mautrix_telegram/web/public/__init__.py b/mautrix_telegram/web/public/__init__.py index 43738d8e..eab1f5fe 100644 --- a/mautrix_telegram/web/public/__init__.py +++ b/mautrix_telegram/web/public/__init__.py @@ -27,16 +27,16 @@ class PublicBridgeWebsite(AuthAPI): log = logging.getLogger("mau.web.public") def __init__(self, loop): - super(AuthAPI, self).__init__(loop) + super().__init__(loop) self.login = Template( - pkg_resources.resource_string("mautrix_telegram", "public/login.html.mako")) + pkg_resources.resource_string("mautrix_telegram", "web/public/login.html.mako")) self.app = web.Application(loop=loop) self.app.router.add_route("GET", "/login", self.get_login) self.app.router.add_route("POST", "/login", self.post_login) - self.app.router.add_static("/", - pkg_resources.resource_filename("mautrix_telegram", "public/")) + self.app.router.add_static("/", pkg_resources.resource_filename("mautrix_telegram", + "web/public/")) async def get_login(self, request): state = "token" if request.rel_url.query.get("mode", "") == "bot" else "request" From 5082cd1c94308be66f7b4fa07389a13828de9509 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Jul 2018 22:28:43 +0300 Subject: [PATCH 13/65] Fix bad JSON handling and include state in all responses --- mautrix_telegram/web/provisioning/__init__.py | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index f5d56c5e..909725c6 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -44,49 +44,55 @@ class ProvisioningAPI(AuthAPI): } elif message: resp = { - "message": message + "state": state, + "message": message, } else: resp = { + "state": state, "error": error, "errcode": errcode, } return web.json_response(resp, status=status) - async def get_user(self, request: web.Request): + async def get_request_info(self, request: web.Request): mxid = request.match_info["mxid"] user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True) if not user.puppet_whitelisted: - return user, self.get_login_response(mxid=user.mxid, error="You are not whitelisted.", - errcode="mxid_not_whitelisted", status=403) + return None, user, self.get_login_response(mxid=user.mxid, + error="You are not whitelisted.", + errcode="mxid_not_whitelisted", status=403) elif await user.is_logged_in(): - return user, self.get_login_response(mxid=user.mxid, username=user.username, status=409) - return user, None + return None, user, self.get_login_response(mxid=user.mxid, username=user.username, + status=409) + + try: + data = await request.json() + except Exception: + return None, user, self.get_login_response(mxid=user.mxid, error="Invalid JSON.", + errcode="invalid_json", status=400) + return data, user, None async def send_bot_token(self, request: web.Request): - user, err = await self.get_user(request) + data, user, err = await self.get_request_info(request) if err: return err - data = await request.json() return await self.post_login_token(user, data.get("token", "")) async def request_code(self, request: web.Request): - user, err = await self.get_user(request) + data, user, err = await self.get_request_info(request) if err: return err - data = await request.json() return await self.post_login_phone(user, data.get("phone", "")) async def send_code(self, request: web.Request): - user, err = await self.get_user(request) + data, user, err = await self.get_request_info(request) if err: return err - data = await request.json() return await self.post_login_code(user, data.get("code", 0), password_in_data=False) async def send_password(self, request: web.Request): - user, err = await self.get_user(request) + data, user, err = await self.get_request_info(request) if err: return err - data = await request.json() return await self.post_login_password(user, data.get("password", "")) From 998e2fa19cb76ac34dc8cad83df94cf35838979b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Jul 2018 22:46:38 +0300 Subject: [PATCH 14/65] Enable aiohttp logging by default --- example-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/example-config.yaml b/example-config.yaml index 6f9e8df5..1d5cfdb7 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -239,6 +239,8 @@ logging: level: DEBUG telethon: level: DEBUG + aiohttp: + level: INFO root: level: DEBUG handlers: [file, console] From 94a2344f3bc8a1eadde4d04cd00e5923421afe40 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Jul 2018 22:47:09 +0300 Subject: [PATCH 15/65] Enable and spec authorization and json validation --- mautrix_telegram/__main__.py | 2 +- mautrix_telegram/web/provisioning/__init__.py | 39 ++++++++++++------- mautrix_telegram/web/provisioning/spec.yaml | 32 ++++++++++++--- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index 9702468f..c8865686 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -93,7 +93,7 @@ if config["appservice.public.enabled"]: appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public.app) if config["appservice.provisioning.enabled"]: - provisioning_api = ProvisioningAPI(loop) + provisioning_api = ProvisioningAPI(config, loop) appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning", provisioning_api.app) diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index 909725c6..1c609aaf 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -16,6 +16,7 @@ # along with this program. If not, see . from aiohttp import web import logging +import json from ...user import User from ..common import AuthAPI @@ -24,8 +25,9 @@ from ..common import AuthAPI class ProvisioningAPI(AuthAPI): log = logging.getLogger("mau.web.provisioning") - def __init__(self, loop): + def __init__(self, config, loop): super().__init__(loop) + self.secret = config["appservice.provisioning.shared_secret"] self.app = web.Application(loop=loop) @@ -56,43 +58,50 @@ class ProvisioningAPI(AuthAPI): return web.json_response(resp, status=status) async def get_request_info(self, request: web.Request): + auth = request.headers.get("Authorization", "") + if auth != f"Bearer {self.secret}": + return None, None, self.get_login_response(error="Shared secret is not valid.", + errcode="shared_secret_invalid", + status=401) + + data = None + try: + data = await request.json() + except json.JSONDecodeError: + pass + if not data: + return None, None, self.get_login_response(error="Invalid JSON.", + errcode="json_invalid", status=400) + mxid = request.match_info["mxid"] user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True) if not user.puppet_whitelisted: - return None, user, self.get_login_response(mxid=user.mxid, - error="You are not whitelisted.", + return None, user, self.get_login_response(error="You are not whitelisted.", errcode="mxid_not_whitelisted", status=403) elif await user.is_logged_in(): - return None, user, self.get_login_response(mxid=user.mxid, username=user.username, - status=409) - - try: - data = await request.json() - except Exception: - return None, user, self.get_login_response(mxid=user.mxid, error="Invalid JSON.", - errcode="invalid_json", status=400) + return None, user, self.get_login_response(username=user.username, status=409) return data, user, None async def send_bot_token(self, request: web.Request): data, user, err = await self.get_request_info(request) - if err: + if err is not None: return err return await self.post_login_token(user, data.get("token", "")) async def request_code(self, request: web.Request): data, user, err = await self.get_request_info(request) - if err: + if err is not None: return err return await self.post_login_phone(user, data.get("phone", "")) async def send_code(self, request: web.Request): data, user, err = await self.get_request_info(request) - if err: + if err is not None: return err return await self.post_login_code(user, data.get("code", 0), password_in_data=False) async def send_password(self, request: web.Request): data, user, err = await self.get_request_info(request) - if err: + if err is not None: return err return await self.post_login_password(user, data.get("password", "")) diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index e8fe08f3..aa911d46 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -39,7 +39,7 @@ paths: 400: $ref: "#/responses/MissingMXIDError" 401: - description: Invalid or expired bot token + description: Invalid or expired bot token or invalid shared secret schema: type: object title: Error @@ -52,6 +52,7 @@ paths: enum: - bot_token_invalid - bot_token_expired + - shared_secret_invalid error: $ref: "#/definitions/HumanReadableError" 403: @@ -87,7 +88,7 @@ paths: schema: $ref: "#/definitions/AuthSuccess" 400: - description: Invalid phone number or missing Matrix ID + description: Invalid phone number or JSON or missing Matrix ID schema: type: object title: Error @@ -100,6 +101,21 @@ paths: enum: - phone_number_invalid - mxid_empty + - json_invalid + error: + $ref: "#/definitions/HumanReadableError" + 401: + description: Invalid shared secret + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - shared_secret_invalid error: $ref: "#/definitions/HumanReadableError" 403: @@ -185,7 +201,7 @@ paths: 400: $ref: "#/responses/MissingMXIDError" 401: - description: Invalid phone code + description: Invalid phone code or shared secret schema: type: object title: Error @@ -196,6 +212,7 @@ paths: description: A machine-readable error code enum: - phone_code_invalid + - shared_secret_invalid error: $ref: "#/definitions/HumanReadableError" 403: @@ -246,7 +263,7 @@ paths: schema: $ref: "#/definitions/AuthSuccess" 400: - description: Missing password or Matrix ID + description: Missing password or Matrix ID or invalid JSON schema: type: object title: Error @@ -259,10 +276,11 @@ paths: enum: - password_empty - mxid_empty + - json_invalid error: $ref: "#/definitions/HumanReadableError" 401: - description: Incorrect password + description: Incorrect password or invalid shared secret schema: type: object title: Error @@ -273,6 +291,7 @@ paths: description: A machine-readable error code enum: - password_invalid + - shared_secret_invalid error: $ref: "#/definitions/HumanReadableError" 403: @@ -327,7 +346,7 @@ responses: type: string description: The Telegram username the user is logged in as. MissingMXIDError: - description: Missing Matrix ID + description: Missing Matrix ID or invalid JSON. schema: type: object title: Error @@ -338,6 +357,7 @@ responses: description: A machine-readable error code enum: - mxid_empty + - json_invalid error: $ref: "#/definitions/HumanReadableError" UnknownError: From ac4d7cc412440daa0a3d64287e63c8af7aad2814 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Jul 2018 22:58:07 +0300 Subject: [PATCH 16/65] Add /get_me endpoint --- mautrix_telegram/web/provisioning/__init__.py | 37 ++++++++--- mautrix_telegram/web/provisioning/spec.yaml | 64 +++++++++++++++++++ 2 files changed, 92 insertions(+), 9 deletions(-) diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index 1c609aaf..95919c58 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -31,6 +31,7 @@ class ProvisioningAPI(AuthAPI): self.app = web.Application(loop=loop) + self.app.router.add_route("GET", "/{mxid:@[^:]*:.+}/get_me", self.get_me) login_prefix = "/login/{mxid:@[^:]*:.+}" self.app.router.add_route("POST", f"{login_prefix}/bot_token", self.send_bot_token) self.app.router.add_route("POST", f"{login_prefix}/request_code", self.request_code) @@ -57,7 +58,7 @@ class ProvisioningAPI(AuthAPI): } return web.json_response(resp, status=status) - async def get_request_info(self, request: web.Request): + async def get_request_info(self, request: web.Request, get_data=True, fail_on_logged_in=True): auth = request.headers.get("Authorization", "") if auth != f"Bearer {self.secret}": return None, None, self.get_login_response(error="Shared secret is not valid.", @@ -65,23 +66,41 @@ class ProvisioningAPI(AuthAPI): status=401) data = None - try: - data = await request.json() - except json.JSONDecodeError: - pass - if not data: - return None, None, self.get_login_response(error="Invalid JSON.", - errcode="json_invalid", status=400) + if get_data: + try: + data = await request.json() + except json.JSONDecodeError: + pass + if not data: + return None, None, self.get_login_response(error="Invalid JSON.", + errcode="json_invalid", status=400) mxid = request.match_info["mxid"] user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True) if not user.puppet_whitelisted: return None, user, self.get_login_response(error="You are not whitelisted.", errcode="mxid_not_whitelisted", status=403) - elif await user.is_logged_in(): + elif fail_on_logged_in and await user.is_logged_in(): return None, user, self.get_login_response(username=user.username, status=409) return data, user, None + async def get_me(self, request: web.Request): + data, user, err = await self.get_request_info(request, get_data=False, + fail_on_logged_in=False) + if err is not None: + return err + if not await user.is_logged_in(): + return self.get_login_response(status=403, error="You are not logged in.", + errcode="not_logged_in") + me = await user.client.get_me() + return web.json_response({ + "username": me.username, + "first_name": me.first_name, + "last_name": me.last_name, + "phone": me.phone, + "is_bot": me.bot, + }) + async def send_bot_token(self, request: web.Request): data, user, err = await self.get_request_info(request) if err is not None: diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index aa911d46..5766af2e 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -26,6 +26,51 @@ tags: - name: Authentication paths: + /{mxid}/get_me: + get: + operationId: get_me + summary: Get the info of the Telegram user the given Matrix user is logged in as + tags: [Authentication] + responses: + 200: + description: User is logged in + schema: + $ref: "#/definitions/AuthInfo" + 400: + $ref: "#/responses/MissingMXIDError" + 403: + description: User is not logged in or not whitelisted + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - not_logged_in + - mxid_not_whitelisted + error: + $ref: "#/definitions/HumanReadableError" + 500: + $ref: "#/responses/UnknownError" + parameters: + - name: mxid + in: path + description: The Matrix ID of the user who to log in as + required: true + type: string + - name: body + in: body + required: true + schema: + type: object + properties: + token: + type: string + description: The access token of the bot to log in as + example: "297900271:IXjeGEcAN61zHnjPgkWnYWyvVp9K4ulHBEv" /login/{mxid}/bot_token: post: operationId: post_bot_token @@ -388,6 +433,25 @@ definitions: type: string description: A human-readable description of the error example: A human-readable description of the error + AuthInfo: + type: object + properties: + username: + type: string + example: username + first_name: + type: string + example: Usern + last_name: + type: string + example: A. + phone: + type: string + example: +123456789 + is_bot: + type: boolean + example: false + AuthSuccess: type: object properties: From f6fb37f5daa927a5b1465f34295462cd25d579b3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Jul 2018 22:59:26 +0300 Subject: [PATCH 17/65] Update endpoint paths --- mautrix_telegram/web/provisioning/__init__.py | 12 ++++++------ mautrix_telegram/web/provisioning/spec.yaml | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index 95919c58..a792720b 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -31,12 +31,12 @@ class ProvisioningAPI(AuthAPI): self.app = web.Application(loop=loop) - self.app.router.add_route("GET", "/{mxid:@[^:]*:.+}/get_me", self.get_me) - login_prefix = "/login/{mxid:@[^:]*:.+}" - self.app.router.add_route("POST", f"{login_prefix}/bot_token", self.send_bot_token) - self.app.router.add_route("POST", f"{login_prefix}/request_code", self.request_code) - self.app.router.add_route("POST", f"{login_prefix}/send_code", self.send_code) - self.app.router.add_route("POST", f"{login_prefix}/send_password", self.send_password) + auth_prefix = "/auth/{mxid:@[^:]*:.+}" + self.app.router.add_route("GET", f"{auth_prefix}/get_me", self.get_me) + self.app.router.add_route("POST", f"{auth_prefix}/send_bot_token", self.send_bot_token) + self.app.router.add_route("POST", f"{auth_prefix}/request_code", self.request_code) + self.app.router.add_route("POST", f"{auth_prefix}/send_code", self.send_code) + self.app.router.add_route("POST", f"{auth_prefix}/send_password", self.send_password) def get_login_response(self, status=200, state="", username="", mxid="", message="", error="", errcode=""): diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index 5766af2e..c0b5e3a6 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -26,7 +26,7 @@ tags: - name: Authentication paths: - /{mxid}/get_me: + /auth/{mxid}/get_me: get: operationId: get_me summary: Get the info of the Telegram user the given Matrix user is logged in as @@ -71,7 +71,7 @@ paths: type: string description: The access token of the bot to log in as example: "297900271:IXjeGEcAN61zHnjPgkWnYWyvVp9K4ulHBEv" - /login/{mxid}/bot_token: + /auth/{mxid}/send_bot_token: post: operationId: post_bot_token summary: Log in with a bot token @@ -122,7 +122,7 @@ paths: type: string description: The access token of the bot to log in as example: "297900271:IXjeGEcAN61zHnjPgkWnYWyvVp9K4ulHBEv" - /login/{mxid}/request_code: + /auth/{mxid}/request_code: post: operationId: post_login_phone summary: Request a phone code from Telegram @@ -229,7 +229,7 @@ paths: type: string description: The phone number to log in as. example: "+123456789" - /login/{mxid}/send_code: + /auth/{mxid}/send_code: post: operationId: post_login_code summary: Send the login code @@ -297,7 +297,7 @@ paths: description: The phone code from Telegram. format: int32 example: 123456 - /login/{mxid}/send_password: + /auth/{mxid}/send_password: post: operationId: post_login_password summary: Send the two-factor auth password From 90e7a09b7e83a78359559da0b31625e1efc71b9d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Jul 2018 23:03:34 +0300 Subject: [PATCH 18/65] Automatically generate provisioning shared secret if it has the default value --- example-config.yaml | 2 +- mautrix_telegram/config.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/example-config.yaml b/example-config.yaml index 1d5cfdb7..eb6c57d8 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -41,7 +41,7 @@ appservice: # The prefix to use in the provisioning API endpoints. prefix: /_matrix/provision # The shared secret to authorize users of the API. - # You can generate a decent secret with `pwgen -snc 32 1` + # If you leave the default token, a random token will be generated and saved at startup. shared_secret: "Very secret shared secret" # The unique ID of this appservice. diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 07944b3c..facb4806 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -162,6 +162,8 @@ class Config(DictWithRecursion): copy("appservice.provisioning.enabled") copy("appservice.provisioning.prefix") copy("appservice.provisioning.shared_secret") + if base["appservice.provisioning.shared_secret"] == "Very secret shared secret": + base["appservice.provisioning.shared_secret"] = self._new_token() copy("appservice.id") copy("appservice.bot_username") From 298e326de7c71488104cfd4ca6b29cf5c7896973 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jul 2018 14:39:49 +0300 Subject: [PATCH 19/65] Fix login command and add token login error handlers --- mautrix_telegram/commands/auth.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mautrix_telegram/commands/auth.py b/mautrix_telegram/commands/auth.py index 8caf21a5..4c6b135c 100644 --- a/mautrix_telegram/commands/auth.py +++ b/mautrix_telegram/commands/auth.py @@ -174,7 +174,7 @@ async def enter_phone_or_token(evt: CommandEvent): # phone numbers don't contain colons but telegram bot auth tokens do if evt.args[0].find(":") > 0: try: - await sign_in(bot_token=evt.args[0]) + await sign_in(evt, bot_token=evt.args[0]) except Exception: evt.log.exception("Error sending auth token") return await evt.reply("Unhandled exception while sending auth token. " @@ -194,7 +194,7 @@ async def enter_code(evt: CommandEvent): return await evt.reply("This bridge instance does not allow in-Matrix login. " "Please use `$cmdprefix+sp login` to get login instructions") try: - await sign_in(code=evt.args[0]) + await sign_in(evt, code=evt.args[0]) except Exception: evt.log.exception("Error sending phone code") return await evt.reply("Unhandled exception while sending code. " @@ -209,12 +209,17 @@ async def enter_password(evt: CommandEvent): return await evt.reply("This bridge instance does not allow in-Matrix login. " "Please use `$cmdprefix+sp login` to get login instructions") try: - await sign_in(password=" ".join(evt.args)) + await sign_in(evt, password=" ".join(evt.args)) + except AccessTokenInvalidError: + return await evt.reply("That bot token is not valid.") + except AccessTokenExpiredError: + return await evt.reply("That bot token has expired.") except Exception: evt.log.exception("Error sending password") return await evt.reply("Unhandled exception while sending password. " "Check console for more details.") + async def sign_in(evt: CommandEvent, **sign_in_info): try: await evt.sender.ensure_started(even_if_no_session=True) @@ -236,6 +241,7 @@ async def sign_in(evt: CommandEvent, **sign_in_info): return await evt.reply("Your account has two-factor authentication. " "Please send your password here.") + @command_handler(needs_auth=True, help_section=SECTION_AUTH, help_text="Log out from Telegram.") From d97281bcdc20ef9df26952430a495d7fed4a2117 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jul 2018 16:00:20 +0300 Subject: [PATCH 20/65] Require authentication for web login. Fixes #163 --- mautrix_telegram/__main__.py | 11 +++-- mautrix_telegram/commands/auth.py | 2 +- mautrix_telegram/commands/handler.py | 7 +-- mautrix_telegram/context.py | 29 +++++++---- mautrix_telegram/formatter/__init__.py | 4 +- mautrix_telegram/formatter/from_matrix.py | 5 +- mautrix_telegram/formatter/from_telegram.py | 5 +- mautrix_telegram/util/__init__.py | 1 + mautrix_telegram/util/signed_token.py | 53 +++++++++++++++++++++ mautrix_telegram/web/public/__init__.py | 40 ++++++++++++---- mautrix_telegram/web/public/login.html.mako | 10 ++-- 11 files changed, 130 insertions(+), 37 deletions(-) create mode 100644 mautrix_telegram/util/signed_token.py diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index c8865686..1969437b 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -85,18 +85,21 @@ appserv = AppService(config["homeserver.address"], config["homeserver.domain"], config["appservice.as_token"], config["appservice.hs_token"], config["appservice.bot_username"], log="mau.as", loop=loop, verify_ssl=config["homeserver.verify_ssl"]) - -context = Context(appserv, db_session, config, loop, None, None, session_container) +public_website = None +provisioning_api = None if config["appservice.public.enabled"]: - public = PublicBridgeWebsite(loop) - appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public.app) + public_website = PublicBridgeWebsite(loop) + appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public_website.app) if config["appservice.provisioning.enabled"]: provisioning_api = ProvisioningAPI(config, loop) appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning", provisioning_api.app) +context = Context(appserv, db_session, config, loop, None, None, session_container, public_website, + provisioning_api) + with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start: init_db(db_session) init_abstract_user(context) diff --git a/mautrix_telegram/commands/auth.py b/mautrix_telegram/commands/auth.py index 8caf21a5..b5eaf1d9 100644 --- a/mautrix_telegram/commands/auth.py +++ b/mautrix_telegram/commands/auth.py @@ -114,7 +114,7 @@ async def login(evt: CommandEvent): if evt.config["appservice.public.enabled"]: prefix = evt.config["appservice.public.external"] - url = f"{prefix}/login?mxid={evt.sender.mxid}" + url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid)}" if evt.config.get("bridge.allow_matrix_login", True): return await evt.reply( "This bridge instance allows you to log in inside or outside Matrix.\n\n" diff --git a/mautrix_telegram/commands/handler.py b/mautrix_telegram/commands/handler.py index 7bf4323d..53a71a4b 100644 --- a/mautrix_telegram/commands/handler.py +++ b/mautrix_telegram/commands/handler.py @@ -22,8 +22,7 @@ import logging from telethon.errors import FloodWaitError from ..util import format_duration -from ..context import Context -from .. import user as u +from .. import user as u, context as c command_handlers = {} # type: Dict[str, CommandHandler] @@ -45,6 +44,7 @@ class CommandEvent: self.loop = processor.loop self.tgbot = processor.tgbot self.config = processor.config + self.public_website = processor.public_website self.command_prefix = processor.command_prefix self.room_id = room self.sender = sender @@ -131,8 +131,9 @@ def command_handler(_func: Optional[Callable[[CommandEvent], None]] = None, *, n class CommandProcessor: log = logging.getLogger("mau.commands") - def __init__(self, context: Context): + def __init__(self, context: c.Context): self.az, self.db, self.config, self.loop, self.tgbot = context + self.public_website = context.public_website self.command_prefix = self.config["bridge.command_prefix"] async def handle(self, room: str, sender: u.User, command: str, args: List[str], diff --git a/mautrix_telegram/context.py b/mautrix_telegram/context.py index dfb32b59..ad48d7e4 100644 --- a/mautrix_telegram/context.py +++ b/mautrix_telegram/context.py @@ -14,17 +14,30 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import asyncio + +from sqlalchemy.orm import scoped_session +from alchemysession import AlchemySessionContainer +from mautrix_appservice import AppService class Context: - def __init__(self, az, db, config, loop, bot, mx, session_container): - self.az = az - self.db = db - self.config = config - self.loop = loop - self.bot = bot - self.mx = mx - self.session_container = session_container + def __init__(self, az, db, config, loop, bot, mx, session_container, public_website, + provisioning_api): + from .web import PublicBridgeWebsite, ProvisioningAPI + from .config import Config + from .bot import Bot + from .matrix import MatrixHandler + + self.az = az # type: AppService + self.db = db # type: scoped_session + self.config = config # type: Config + self.loop = loop # type: asyncio.AbstractEventLoop + self.bot = bot # type: Bot + self.mx = mx # type: MatrixHandler + self.session_container = session_container # type: AlchemySessionContainer + self.public_website = public_website # type: PublicBridgeWebsite + self.provisioning_api = provisioning_api # type: ProvisioningAPI def __iter__(self): yield self.az diff --git a/mautrix_telegram/formatter/__init__.py b/mautrix_telegram/formatter/__init__.py index 7cb102f7..51802ebb 100644 --- a/mautrix_telegram/formatter/__init__.py +++ b/mautrix_telegram/formatter/__init__.py @@ -1,9 +1,9 @@ from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram, init_mx) from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg) -from ..context import Context +from .. import context as c -def init(context: Context): +def init(context: c.Context): init_mx(context) init_tg(context) diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index 145177fc..f98d3ad5 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -27,8 +27,7 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, M MessageEntityItalic, MessageEntityCode, MessageEntityPre, MessageEntityBotCommand, TypeMessageEntity) -from ..context import Context -from .. import user as u, puppet as pu, portal as po +from .. import user as u, puppet as pu, portal as po, context as c from ..db import Message as DBMessage from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, trim_reply_fallback_text, html_to_unicode) @@ -352,7 +351,7 @@ def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], st return entities, replacer -def init_mx(context: Context): +def init_mx(context: c.Context): global plain_mention_regex, should_bridge_plaintext_highlights config = context.config dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)") diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 3e9992e4..70a13a55 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -33,8 +33,7 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, from mautrix_appservice import MatrixRequestError from mautrix_appservice.intent_api import IntentAPI -from .. import user as u, puppet as pu, portal as po -from ..context import Context +from .. import user as u, puppet as pu, portal as po, context as c from ..db import Message as DBMessage from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, trim_reply_fallback_text, unicode_to_html) @@ -321,6 +320,6 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool: return False -def init_tg(context: Context): +def init_tg(context: c.Context): global should_highlight_edits should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"] diff --git a/mautrix_telegram/util/__init__.py b/mautrix_telegram/util/__init__.py index 7d431396..99cdee2a 100644 --- a/mautrix_telegram/util/__init__.py +++ b/mautrix_telegram/util/__init__.py @@ -1,2 +1,3 @@ from .file_transfer import transfer_file_to_matrix, convert_image from .format_duration import format_duration +from .signed_token import sign_token, verify_token diff --git a/mautrix_telegram/util/signed_token.py b/mautrix_telegram/util/signed_token.py new file mode 100644 index 00000000..13281012 --- /dev/null +++ b/mautrix_telegram/util/signed_token.py @@ -0,0 +1,53 @@ +# -*- coding: future_fstrings -*- +# mautrix-telegram - A Matrix-Telegram puppeting bridge +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from typing import Optional +import json +import base64 +import hashlib + + +def _get_checksum(key: str, payload: bytes) -> str: + hasher = hashlib.sha256() + hasher.update(payload) + hasher.update(key.encode("utf-8")) + checksum = hasher.hexdigest() + return checksum + + +def sign_token(key: str, payload: dict) -> str: + payload = base64.urlsafe_b64encode(json.dumps(payload).encode("utf-8")) + checksum = _get_checksum(key, payload) + return f"{checksum}:{payload.decode('utf-8')}" + + +def verify_token(key: str, data: str) -> Optional[dict]: + if not data: + return None + + try: + checksum, payload = data.split(":", 1) + except ValueError: + return None + + if checksum != _get_checksum(key, payload.encode("utf-8")): + return None + + payload = base64.urlsafe_b64decode(payload).decode("utf-8") + try: + return json.loads(payload) + except json.JSONDecodeError: + return None diff --git a/mautrix_telegram/web/public/__init__.py b/mautrix_telegram/web/public/__init__.py index eab1f5fe..fb5f6de7 100644 --- a/mautrix_telegram/web/public/__init__.py +++ b/mautrix_telegram/web/public/__init__.py @@ -18,7 +18,11 @@ from aiohttp import web from mako.template import Template import pkg_resources import logging +import random +import string +import time +from ...util import sign_token, verify_token from ...user import User from ..common import AuthAPI @@ -28,6 +32,8 @@ class PublicBridgeWebsite(AuthAPI): def __init__(self, loop): super().__init__(loop) + self.secret_key = "".join( + random.choice(string.ascii_lowercase + string.digits) for _ in range(64)) self.login = Template( pkg_resources.resource_string("mautrix_telegram", "web/public/login.html.mako")) @@ -38,10 +44,24 @@ class PublicBridgeWebsite(AuthAPI): self.app.router.add_static("/", pkg_resources.resource_filename("mautrix_telegram", "web/public/")) - async def get_login(self, request): - state = "token" if request.rel_url.query.get("mode", "") == "bot" else "request" + def make_token(self, mxid, expires_in=900): + return sign_token(self.secret_key, { + "mxid": mxid, + "expiry": int(time.time()) + expires_in, + }) - mxid = request.rel_url.query.get("mxid", None) + def verify_token(self, token): + token = verify_token(self.secret_key, token) + if token and token.get("expiry", 0) > int(time.time()): + return token.get("mxid", None) + return None + + async def get_login(self, request): + state = "bot_token" if request.rel_url.query.get("mode", "") == "bot" else "request" + + mxid = self.verify_token(request.rel_url.query.get("token", None)) + if not mxid: + return self.get_login_response(status=401, state="invalid-token") user = User.get_by_mxid(mxid, create=False) if mxid else None if not user: @@ -62,11 +82,13 @@ class PublicBridgeWebsite(AuthAPI): message=message, mxid=mxid)) async def post_login(self, request): - data = await request.post() - if "mxid" not in data: - return self.get_login_response(error="Please enter your Matrix ID.", status=400) + mxid = self.verify_token(request.rel_url.query.get("token", None)) + if not mxid: + return self.get_login_response(status=401, state="invalid-token") - user = await User.get_by_mxid(data["mxid"]).ensure_started(even_if_no_session=True) + data = await request.post() + + user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True) if not user.puppet_whitelisted: return self.get_login_response(mxid=user.mxid, error="You are not whitelisted.", status=403) @@ -77,8 +99,8 @@ class PublicBridgeWebsite(AuthAPI): if "phone" in data: return await self.post_login_phone(user, data["phone"]) - elif "token" in data: - return await self.post_login_token(user, data["token"]) + elif "bot_token" in data: + return await self.post_login_token(user, data["bot_token"]) elif "code" in data: resp = await self.post_login_code(user, data["code"], password_in_data="password" in data) diff --git a/mautrix_telegram/web/public/login.html.mako b/mautrix_telegram/web/public/login.html.mako index 8c03cbdc..f00b6a69 100644 --- a/mautrix_telegram/web/public/login.html.mako +++ b/mautrix_telegram/web/public/login.html.mako @@ -76,6 +76,9 @@ along with this program. If not, see . management command first.

% endif + % elif state == "invalid-token": +

Invalid or expired token

+
Please ask the bridge bot for a new login link.
% else:

Log in to Telegram

% if error: @@ -87,8 +90,7 @@ along with this program. If not, see .
- + % if state == "request": @@ -96,9 +98,9 @@ along with this program. If not, see . - % elif state == "token": + % elif state == "bot_token": - + % elif state == "code": From 2b5426fda394877f80e08d3442f4f34337d9e8f4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jul 2018 18:57:46 +0300 Subject: [PATCH 21/65] Add portal info and user chat list endpoints --- mautrix_telegram/abstract_user.py | 4 +- mautrix_telegram/user.py | 15 +- mautrix_telegram/web/provisioning/__init__.py | 207 +++++++++++----- mautrix_telegram/web/provisioning/spec.yaml | 228 ++++++++++++++++-- 4 files changed, 362 insertions(+), 92 deletions(-) diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index d15e9e66..1095e993 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -96,9 +96,9 @@ class AbstractUser: except Exception: self.log.exception("Failed to handle Telegram update") - async def _get_dialogs(self, limit=None): + async def get_dialogs(self, limit=None) -> List[Union[Chat, Channel]]: if self.is_bot: - return + return [] dialogs = await self.client.get_dialogs(limit=limit) return [dialog.entity for dialog in dialogs if ( not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden)) diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index da62fad8..917cbf73 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -14,6 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Dict import logging import asyncio import re @@ -38,18 +39,18 @@ class User(AbstractUser): def __init__(self, mxid, tgid=None, username=None, db_contacts=None, saved_contacts=0, is_bot=False, db_portals=None, db_instance=None): super().__init__() - self.mxid = mxid - self.tgid = tgid - self.is_bot = is_bot - self.username = username + self.mxid = mxid # type: str + self.tgid = tgid # type: int + self.is_bot = is_bot # type: bool + self.username = username # type: str self.contacts = [] self.saved_contacts = saved_contacts self.db_contacts = db_contacts - self.portals = {} + self.portals = {} # type: Dict[str, po.Portal] self.db_portals = db_portals self._db_instance = db_instance - self.command_status = None + self.command_status = None # type: dict (self.relaybot_whitelisted, self.whitelisted, @@ -255,7 +256,7 @@ class User(AbstractUser): async def sync_dialogs(self, synchronous_create=False): creators = [] - for entity in await self._get_dialogs(limit=30): + for entity in await self.get_dialogs(limit=30): portal = po.Portal.get_by_entity(entity) self.portals[portal.tgid_full] = portal creators.append( diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index a792720b..becee3a2 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -15,10 +15,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from aiohttp import web +from typing import Tuple, Optional, Callable, Awaitable import logging import json +from telethon.utils import get_peer_id + from ...user import User +from ...portal import Portal from ..common import AuthAPI @@ -29,17 +33,117 @@ class ProvisioningAPI(AuthAPI): super().__init__(loop) self.secret = config["appservice.provisioning.shared_secret"] - self.app = web.Application(loop=loop) + self.app = web.Application(loop=loop, middlewares=[self.error_middleware]) - auth_prefix = "/auth/{mxid:@[^:]*:.+}" - self.app.router.add_route("GET", f"{auth_prefix}/get_me", self.get_me) - self.app.router.add_route("POST", f"{auth_prefix}/send_bot_token", self.send_bot_token) - self.app.router.add_route("POST", f"{auth_prefix}/request_code", self.request_code) - self.app.router.add_route("POST", f"{auth_prefix}/send_code", self.send_code) - self.app.router.add_route("POST", f"{auth_prefix}/send_password", self.send_password) + portal_prefix = "/portal/{mxid:![^/]+}" + self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal) + # self.app.router.add_route("POST", portal_prefix + "/connect/{chat_id:[0-9]+}", + # self.connect_chat) + # self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat) + # self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat) + + user_prefix = "/user/{mxid:@[^:]*:[^/]+}" + self.app.router.add_route("GET", f"{user_prefix}", self.get_me) + self.app.router.add_route("GET", f"{user_prefix}/chats", self.get_chats) + + self.app.router.add_route("POST", f"{user_prefix}/send_bot_token", self.send_bot_token) + self.app.router.add_route("POST", f"{user_prefix}/request_code", self.request_code) + self.app.router.add_route("POST", f"{user_prefix}/send_code", self.send_code) + self.app.router.add_route("POST", f"{user_prefix}/send_password", self.send_password) + + async def get_portal(self, request: web.Request) -> web.Response: + mxid = request.match_info["mxid"] + portal = Portal.get_by_mxid(mxid) + if not portal: + return self.get_error_response(404, "room_not_found", + "Portal with given Matrix ID not found.") + return web.json_response({ + "mxid": portal.mxid, + "chat_id": get_peer_id(portal.peer), + "peer_type": portal.peer_type, + "title": portal.title, + "about": portal.about, + "username": portal.username, + "megagroup": portal.megagroup, + }) + + async def get_me(self, request: web.Request) -> web.Response: + data, user, err = await self.get_user_request_info(request, require_logged_in=True) + if err is not None: + return err + + me = await user.client.get_me() + return web.json_response({ + "username": me.username, + "first_name": me.first_name, + "last_name": me.last_name, + "phone": me.phone, + "is_bot": me.bot, + }) + + async def get_chats(self, request: web.Request) -> web.Response: + data, user, err = await self.get_user_request_info(request, require_logged_in=True) + if err is not None: + return err + + if not user.is_bot: + chats = await user.get_dialogs() + return web.json_response([{ + "id": get_peer_id(chat), + "title": chat.title, + } for chat in chats]) + else: + return web.json_response([{ + "id": get_peer_id(chat.peer), + "title": chat.title, + } for chat in user.portals.values() if chat.tgid]) + + async def send_bot_token(self, request: web.Request) -> web.Response: + data, user, err = await self.get_user_request_info(request) + if err is not None: + return err + return await self.post_login_token(user, data.get("token", "")) + + async def request_code(self, request: web.Request) -> web.Response: + data, user, err = await self.get_user_request_info(request) + if err is not None: + return err + return await self.post_login_phone(user, data.get("phone", "")) + + async def send_code(self, request: web.Request) -> web.Response: + data, user, err = await self.get_user_request_info(request) + if err is not None: + return err + return await self.post_login_code(user, data.get("code", 0), password_in_data=False) + + async def send_password(self, request: web.Request) -> web.Response: + data, user, err = await self.get_user_request_info(request) + if err is not None: + return err + return await self.post_login_password(user, data.get("password", "")) + + @staticmethod + async def error_middleware(_, handler) -> Callable[[web.Request], Awaitable[web.Response]]: + async def middleware_handler(request: web.Request) -> web.Response: + try: + return await handler(request) + except web.HTTPException as ex: + return web.json_response({ + "error": f"Unhandled HTTP {ex.status}", + "errcode": f"unhandled_http_{ex.status}", + }, status=ex.status) + + return middleware_handler + + @staticmethod + def get_error_response(status=200, errcode="", error="") -> web.Response: + return web.json_response({ + "error": error, + "errcode": errcode, + }, status=status) def get_login_response(self, status=200, state="", username="", mxid="", message="", error="", - errcode=""): + errcode="") -> web.Response: if username: resp = { "state": "logged-in", @@ -58,7 +162,36 @@ class ProvisioningAPI(AuthAPI): } return web.json_response(resp, status=status) - async def get_request_info(self, request: web.Request, get_data=True, fail_on_logged_in=True): + def check_authorization(self, request: web.Request) -> bool: + return request.headers.get("Authorization", "") == f"Bearer {self.secret}" + + @staticmethod + async def get_data(request: web.Request) -> Optional[dict]: + try: + return await request.json() + except json.JSONDecodeError: + return None + + async def get_user(self, mxid: str, require_logged_in: bool = False + ) -> Tuple[Optional[User], Optional[web.Response]]: + user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True) + if not user.puppet_whitelisted: + return user, self.get_login_response(error="You are not whitelisted.", + errcode="mxid_not_whitelisted", status=403) + logged_in = await user.is_logged_in() + if not require_logged_in and logged_in: + return user, self.get_login_response(username=user.username, status=409, + error="You are already logged in.", + errcode="already_logged_in") + elif require_logged_in and not logged_in: + return user, self.get_login_response(status=403, error="You are not logged in.", + errcode="not_logged_in") + return user, None + + async def get_user_request_info(self, request: web.Request, require_logged_in: bool = False + ) -> (Tuple[Optional[dict], + Optional[User], + Optional[web.Response]]): auth = request.headers.get("Authorization", "") if auth != f"Bearer {self.secret}": return None, None, self.get_login_response(error="Shared secret is not valid.", @@ -66,61 +199,13 @@ class ProvisioningAPI(AuthAPI): status=401) data = None - if get_data: - try: - data = await request.json() - except json.JSONDecodeError: - pass + if request.method == "POST" or request.method == "PUT": + data = await self.get_data(request) if not data: return None, None, self.get_login_response(error="Invalid JSON.", errcode="json_invalid", status=400) mxid = request.match_info["mxid"] - user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True) - if not user.puppet_whitelisted: - return None, user, self.get_login_response(error="You are not whitelisted.", - errcode="mxid_not_whitelisted", status=403) - elif fail_on_logged_in and await user.is_logged_in(): - return None, user, self.get_login_response(username=user.username, status=409) - return data, user, None + user, err = await self.get_user(mxid, require_logged_in) - async def get_me(self, request: web.Request): - data, user, err = await self.get_request_info(request, get_data=False, - fail_on_logged_in=False) - if err is not None: - return err - if not await user.is_logged_in(): - return self.get_login_response(status=403, error="You are not logged in.", - errcode="not_logged_in") - me = await user.client.get_me() - return web.json_response({ - "username": me.username, - "first_name": me.first_name, - "last_name": me.last_name, - "phone": me.phone, - "is_bot": me.bot, - }) - - async def send_bot_token(self, request: web.Request): - data, user, err = await self.get_request_info(request) - if err is not None: - return err - return await self.post_login_token(user, data.get("token", "")) - - async def request_code(self, request: web.Request): - data, user, err = await self.get_request_info(request) - if err is not None: - return err - return await self.post_login_phone(user, data.get("phone", "")) - - async def send_code(self, request: web.Request): - data, user, err = await self.get_request_info(request) - if err is not None: - return err - return await self.post_login_code(user, data.get("code", 0), password_in_data=False) - - async def send_password(self, request: web.Request): - data, user, err = await self.get_request_info(request) - if err is not None: - return err - return await self.post_login_password(user, data.get("password", "")) + return data, user, err diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index c0b5e3a6..d3bd2ba8 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -4,16 +4,12 @@ info: title: mautrix-telegram provisioning version: 0.3.0 description: The provisioning API for mautrix-telegram. - contact: - name: Tulir Asokan - email: tulir@maunium.net - url: https://maunium.net license: name: AGPLv3 url: https://github.com/tulir/mautrix-telegram/blob/master/LICENSE externalDocs: - description: Provisioning API wiki page on GitHub. + description: Provisioning API wiki page on GitHub url: https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API basePath: /_matrix/provision @@ -23,14 +19,141 @@ consumes: [application/json] produces: [application/json] tags: +- name: User info - name: Authentication +- name: Bridging paths: - /auth/{mxid}/get_me: + /portal/{room_id}: + get: + operationId: get_portal + summary: Get the bridging status and info of the connected Telegram chat + tags: [Bridging] + responses: + 200: + description: Room is bridged + schema: + $ref: "#/definitions/PortalInfo" + 400: + $ref: "#/responses/MissingMXIDError" + 404: + description: Unknown portal + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - portal_not_found + error: + $ref: "#/definitions/HumanReadableError" + parameters: + - name: room_id + in: path + description: The Matrix ID of the room whose bridging status to get + required: true + type: string + /portal/{room_id}/connect/{chat_id}: + post: + operationId: connect_portal + summary: Connect an existing Telegram chat to the given room + tags: [Bridging] + parameters: + - name: room_id + in: path + description: The Matrix ID of the room to which the Telegram chat should be connected + required: true + type: string + - name: chat_id + in: path + description: The ID of the Telegram chat to connect + required: true + type: integer + format: int32 + responses: + 400: + $ref: "#/responses/MissingMXIDError" + 409: + description: Room is already bridged + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - room_already_bridged + error: + $ref: "#/definitions/HumanReadableError" + /portal/{room_id}/create: + post: + operationId: create_portal + summary: Create a new Telegram chat for the given room + tags: [Bridging] + responses: + 400: + $ref: "#/responses/MissingMXIDError" + 409: + description: Room is already bridged + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - room_already_bridged + error: + $ref: "#/definitions/HumanReadableError" + parameters: + - name: room_id + in: path + description: The Matrix ID of the room whose bridging status to get + required: true + type: string + /portal/{room_id}/disconnect: + post: + operationId: disconnect_portal + summary: Disconnect the Telegram chat from the room + tags: [Bridging] + responses: + 202: + description: Room unbridging initiated + 400: + $ref: "#/responses/MissingMXIDError" + 404: + description: Unknown portal + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - portal_not_found + error: + $ref: "#/definitions/HumanReadableError" + parameters: + - name: room_id + in: path + description: The Matrix ID of the room whose bridging status to get + required: true + type: string + + /user/{user_id}: get: operationId: get_me summary: Get the info of the Telegram user the given Matrix user is logged in as - tags: [Authentication] + tags: [User info] responses: 200: description: User is logged in @@ -56,22 +179,48 @@ paths: 500: $ref: "#/responses/UnknownError" parameters: - - name: mxid + - name: user_id in: path description: The Matrix ID of the user who to log in as required: true type: string - - name: body - in: body - required: true + /user/{user_id}/chats: + get: + operationId: get_chats + summary: Get the list of Telegram chats the given Matrix user has access to + tags: [User info] + responses: + 200: + description: User is logged in + schema: + $ref: "#/definitions/UserChats" + 400: + $ref: "#/responses/MissingMXIDError" + 403: + description: User is not logged in or not whitelisted schema: type: object + title: Error properties: - token: + errcode: type: string - description: The access token of the bot to log in as - example: "297900271:IXjeGEcAN61zHnjPgkWnYWyvVp9K4ulHBEv" - /auth/{mxid}/send_bot_token: + title: Error code + description: A machine-readable error code + enum: + - not_logged_in + - mxid_not_whitelisted + error: + $ref: "#/definitions/HumanReadableError" + 500: + $ref: "#/responses/UnknownError" + parameters: + - name: user_id + in: path + description: The Matrix ID of the user who to log in as + required: true + type: string + + /user/{user_id}/login/bot_token: post: operationId: post_bot_token summary: Log in with a bot token @@ -107,7 +256,7 @@ paths: 500: $ref: "#/responses/UnknownError" parameters: - - name: mxid + - name: user_id in: path description: The Matrix ID of the user who to log in as required: true @@ -122,7 +271,7 @@ paths: type: string description: The access token of the bot to log in as example: "297900271:IXjeGEcAN61zHnjPgkWnYWyvVp9K4ulHBEv" - /auth/{mxid}/request_code: + /user/{user_id}/login/request_code: post: operationId: post_login_phone summary: Request a phone code from Telegram @@ -214,7 +363,7 @@ paths: 500: $ref: "#/responses/UnknownError" parameters: - - name: mxid + - name: user_id in: path description: The Matrix ID of the user who to log in as required: true @@ -229,7 +378,7 @@ paths: type: string description: The phone number to log in as. example: "+123456789" - /auth/{mxid}/send_code: + /user/{user_id}/login/send_code: post: operationId: post_login_code summary: Send the login code @@ -281,7 +430,7 @@ paths: 500: $ref: "#/responses/UnknownError" parameters: - - name: mxid + - name: user_id in: path description: The Matrix ID of the user who to log in as required: true @@ -297,7 +446,7 @@ paths: description: The phone code from Telegram. format: int32 example: 123456 - /auth/{mxid}/send_password: + /user/{user_id}/login/send_password: post: operationId: post_login_password summary: Send the two-factor auth password @@ -346,7 +495,7 @@ paths: 500: $ref: "#/responses/UnknownError" parameters: - - name: mxid + - name: user_id in: path description: The Matrix ID of the user who to log in as required: true @@ -451,6 +600,41 @@ definitions: is_bot: type: boolean example: false + UserChats: + type: array + items: + type: object + properties: + id: + type: integer + example: -123456789 + description: A bot API style chat ID. + title: + type: string + + PortalInfo: + type: object + properties: + mxid: + type: string + example: "!foo:example.com" + chat_id: + type: integer + example: -100123456789 + peer_type: + type: string + enum: + - user + - chat + - channel + megagroup: + type: boolean + username: + type: string + title: + type: string + about: + type: string AuthSuccess: type: object From bbc7912a497b29f1aa8f14c67cd721046a85ae28 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jul 2018 19:24:05 +0300 Subject: [PATCH 22/65] Allow getting user info of unauthenticated users and add /portal/{chat_id} --- mautrix_telegram/config.py | 2 +- mautrix_telegram/user.py | 3 +- mautrix_telegram/web/provisioning/__init__.py | 86 ++++++++---- mautrix_telegram/web/provisioning/spec.yaml | 125 +++++++++++++----- 4 files changed, 157 insertions(+), 59 deletions(-) diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index facb4806..77cc6bfb 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -250,7 +250,7 @@ class Config(DictWithRecursion): puppeting = level == "full" or admin user = level == "user" or puppeting relaybot = level == "relaybot" or user - return relaybot, user, puppeting, admin + return relaybot, user, puppeting, admin, level def get_permissions(self, mxid): permissions = self["bridge.permissions"] or {} diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index 917cbf73..1163e503 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -55,7 +55,8 @@ class User(AbstractUser): (self.relaybot_whitelisted, self.whitelisted, self.puppet_whitelisted, - self.is_admin) = config.get_permissions(self.mxid) + self.is_admin, + self.permissions) = config.get_permissions(self.mxid) self.by_mxid[mxid] = self if tgid: diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index becee3a2..dc8feace 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -36,14 +36,15 @@ class ProvisioningAPI(AuthAPI): self.app = web.Application(loop=loop, middlewares=[self.error_middleware]) portal_prefix = "/portal/{mxid:![^/]+}" - self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal) + self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal_by_mxid) + self.app.router.add_route("GET", "/portal/{tgid:-[0-9]+}", self.get_portal_by_tgid) # self.app.router.add_route("POST", portal_prefix + "/connect/{chat_id:[0-9]+}", # self.connect_chat) # self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat) # self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat) user_prefix = "/user/{mxid:@[^:]*:[^/]+}" - self.app.router.add_route("GET", f"{user_prefix}", self.get_me) + self.app.router.add_route("GET", f"{user_prefix}", self.get_user_info) self.app.router.add_route("GET", f"{user_prefix}/chats", self.get_chats) self.app.router.add_route("POST", f"{user_prefix}/send_bot_token", self.send_bot_token) @@ -51,11 +52,11 @@ class ProvisioningAPI(AuthAPI): self.app.router.add_route("POST", f"{user_prefix}/send_code", self.send_code) self.app.router.add_route("POST", f"{user_prefix}/send_password", self.send_password) - async def get_portal(self, request: web.Request) -> web.Response: + async def get_portal_by_mxid(self, request: web.Request) -> web.Response: mxid = request.match_info["mxid"] portal = Portal.get_by_mxid(mxid) if not portal: - return self.get_error_response(404, "room_not_found", + return self.get_error_response(404, "portal_not_found", "Portal with given Matrix ID not found.") return web.json_response({ "mxid": portal.mxid, @@ -67,22 +68,53 @@ class ProvisioningAPI(AuthAPI): "megagroup": portal.megagroup, }) - async def get_me(self, request: web.Request) -> web.Response: - data, user, err = await self.get_user_request_info(request, require_logged_in=True) + async def get_portal_by_tgid(self, request: web.Request) -> web.Response: + try: + tgid = int(request.match_info["tgid"]) + except ValueError: + return self.get_error_response(400, "tgid_invalid", + "Given chat ID is not an integer.") + + portal = Portal.get_by_tgid(tgid) + if not portal: + return self.get_error_response(404, "portal_not_found", + "Portal to given Telegram chat not found.") + return web.json_response({ + "mxid": portal.mxid, + "chat_id": get_peer_id(portal.peer), + "peer_type": portal.peer_type, + "title": portal.title, + "about": portal.about, + "username": portal.username, + "megagroup": portal.megagroup, + }) + + async def get_user_info(self, request: web.Request) -> web.Response: + data, user, err = await self.get_user_request_info(request, expect_logged_in=None, + require_puppeting=False) if err is not None: return err - me = await user.client.get_me() + user_data = None + if await user.is_logged_in(): + me = await user.client.get_me() + await user.update_info(me) + user_data = { + "id": user.tgid, + "username": user.username, + "first_name": me.first_name, + "last_name": me.last_name, + "phone": me.phone, + "is_bot": user.is_bot, + } return web.json_response({ - "username": me.username, - "first_name": me.first_name, - "last_name": me.last_name, - "phone": me.phone, - "is_bot": me.bot, + "telegram": user_data, + "mxid": user.mxid, + "permissions": user.permissions, }) async def get_chats(self, request: web.Request) -> web.Response: - data, user, err = await self.get_user_request_info(request, require_logged_in=True) + data, user, err = await self.get_user_request_info(request, expect_logged_in=True) if err is not None: return err @@ -172,23 +204,27 @@ class ProvisioningAPI(AuthAPI): except json.JSONDecodeError: return None - async def get_user(self, mxid: str, require_logged_in: bool = False + async def get_user(self, mxid: str, expect_logged_in: Optional[bool] = False, + require_puppeting: bool = True, ) -> Tuple[Optional[User], Optional[web.Response]]: user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True) - if not user.puppet_whitelisted: + if require_puppeting and not user.puppet_whitelisted: return user, self.get_login_response(error="You are not whitelisted.", errcode="mxid_not_whitelisted", status=403) - logged_in = await user.is_logged_in() - if not require_logged_in and logged_in: - return user, self.get_login_response(username=user.username, status=409, - error="You are already logged in.", - errcode="already_logged_in") - elif require_logged_in and not logged_in: - return user, self.get_login_response(status=403, error="You are not logged in.", - errcode="not_logged_in") + if expect_logged_in is not None: + logged_in = await user.is_logged_in() + if not expect_logged_in and logged_in: + return user, self.get_login_response(username=user.username, status=409, + error="You are already logged in.", + errcode="already_logged_in") + elif expect_logged_in and not logged_in: + return user, self.get_login_response(status=403, error="You are not logged in.", + errcode="not_logged_in") return user, None - async def get_user_request_info(self, request: web.Request, require_logged_in: bool = False + async def get_user_request_info(self, request: web.Request, + expect_logged_in: Optional[bool] = False, + require_puppeting: bool = False, ) -> (Tuple[Optional[dict], Optional[User], Optional[web.Response]]): @@ -206,6 +242,6 @@ class ProvisioningAPI(AuthAPI): errcode="json_invalid", status=400) mxid = request.match_info["mxid"] - user, err = await self.get_user(mxid, require_logged_in) + user, err = await self.get_user(mxid, expect_logged_in, require_puppeting) return data, user, err diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index d3bd2ba8..775821b9 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -56,6 +56,52 @@ paths: description: The Matrix ID of the room whose bridging status to get required: true type: string + pattern: "![^/]+" + /portal/{chat_id}: + get: + operationId: get_portal_by_tgid + summary: Get the bridging status and info of the connected Telegram chat + tags: [Bridging] + responses: + 200: + description: Chat is bridged + schema: + $ref: "#/definitions/PortalInfo" + 400: + description: Invalid Telegram chat ID + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - tgid_invalid + error: + $ref: "#/definitions/HumanReadableError" + 404: + description: Unknown portal + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - portal_not_found + error: + $ref: "#/definitions/HumanReadableError" + parameters: + - name: chat_id + in: path + description: The Matrix ID of the room whose bridging status to get + required: true + type: integer + pattern: "-[0-9]+" /portal/{room_id}/connect/{chat_id}: post: operationId: connect_portal @@ -72,12 +118,20 @@ paths: description: The ID of the Telegram chat to connect required: true type: integer - format: int32 + pattern: "-[0-9]+" + - name: force + in: query + description: Set to force bridging by unbridging or deleting existing portal rooms. + required: false + type: string + enum: + - delete + - unbridge responses: 400: $ref: "#/responses/MissingMXIDError" 409: - description: Room is already bridged + description: Matrix room or Telegram chat is already bridged schema: type: object title: Error @@ -86,8 +140,10 @@ paths: type: string title: Error code description: A machine-readable error code + example: _already_bridged enum: - room_already_bridged + - chat_already_bridged error: $ref: "#/definitions/HumanReadableError" /portal/{room_id}/create: @@ -156,26 +212,13 @@ paths: tags: [User info] responses: 200: - description: User is logged in + description: User found schema: - $ref: "#/definitions/AuthInfo" + $ref: "#/definitions/UserInfo" 400: $ref: "#/responses/MissingMXIDError" 403: - description: User is not logged in or not whitelisted - schema: - type: object - title: Error - properties: - errcode: - type: string - title: Error code - description: A machine-readable error code - enum: - - not_logged_in - - mxid_not_whitelisted - error: - $ref: "#/definitions/HumanReadableError" + $ref: "#/responses/NotWhitelistedError" 500: $ref: "#/responses/UnknownError" parameters: @@ -582,24 +625,42 @@ definitions: type: string description: A human-readable description of the error example: A human-readable description of the error - AuthInfo: + UserInfo: type: object properties: - username: + mxid: type: string - example: username - first_name: + example: "@usern:example.com" + permissions: type: string - example: Usern - last_name: - type: string - example: A. - phone: - type: string - example: +123456789 - is_bot: - type: boolean - example: false + example: user + enum: + - none + - relaybot + - user + - full + - admin + telegram: + type: object + properties: + id: + type: integer + example: 123456789 + username: + type: string + example: username + first_name: + type: string + example: Usern + last_name: + type: string + example: A. + phone: + type: string + example: +123456789 + is_bot: + type: boolean + example: false UserChats: type: array items: From 34cc810d620dca72f0a9ea5e388f88ebdfab6f3a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jul 2018 19:33:55 +0300 Subject: [PATCH 23/65] Fix /portal/{chat_id} --- mautrix_telegram/web/provisioning/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index dc8feace..e3fa8347 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -19,7 +19,7 @@ from typing import Tuple, Optional, Callable, Awaitable import logging import json -from telethon.utils import get_peer_id +from telethon.utils import get_peer_id, resolve_id from ...user import User from ...portal import Portal @@ -70,11 +70,10 @@ class ProvisioningAPI(AuthAPI): async def get_portal_by_tgid(self, request: web.Request) -> web.Response: try: - tgid = int(request.match_info["tgid"]) + tgid, _ = resolve_id(int(request.match_info["tgid"])) except ValueError: return self.get_error_response(400, "tgid_invalid", - "Given chat ID is not an integer.") - + "Given chat ID is not valid.") portal = Portal.get_by_tgid(tgid) if not portal: return self.get_error_response(404, "portal_not_found", From 4cef2be0db00e7dc2df9bb924689707824bbb17f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jul 2018 23:14:04 +0300 Subject: [PATCH 24/65] Implement /portal/{mxid}/create --- mautrix_telegram/__main__.py | 2 +- mautrix_telegram/commands/portal.py | 26 ++-- mautrix_telegram/user.py | 3 + mautrix_telegram/web/common/auth_api.py | 10 +- mautrix_telegram/web/provisioning/__init__.py | 85 +++++++++++- mautrix_telegram/web/provisioning/spec.yaml | 126 ++++++++++++++---- 6 files changed, 205 insertions(+), 47 deletions(-) diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index 1969437b..bd31ab9d 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -93,7 +93,7 @@ if config["appservice.public.enabled"]: appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public_website.app) if config["appservice.provisioning.enabled"]: - provisioning_api = ProvisioningAPI(config, loop) + provisioning_api = ProvisioningAPI(config, appserv, loop) appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning", provisioning_api.app) diff --git a/mautrix_telegram/commands/portal.py b/mautrix_telegram/commands/portal.py index c87ade32..e9e29547 100644 --- a/mautrix_telegram/commands/portal.py +++ b/mautrix_telegram/commands/portal.py @@ -19,10 +19,11 @@ import asyncio from telethon.errors import * from telethon.tl.types import ChatForbidden, ChannelForbidden -from mautrix_appservice import MatrixRequestError +from mautrix_appservice import MatrixRequestError, IntentAPI from .. import portal as po, user as u -from . import command_handler, CommandEvent, SECTION_ADMIN, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT +from . import (command_handler, CommandEvent, + SECTION_ADMIN, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT) @command_handler(needs_admin=True, needs_auth=False, name="set-pl", @@ -65,7 +66,7 @@ async def invite_link(evt: CommandEvent): return await evt.reply("You don't have the permission to create an invite link.") -async def _has_access_to(room: str, intent, sender: u.User, event: str, default: int = 50): +async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50): if sender.is_admin: return True # Make sure the state store contains the power levels. @@ -87,7 +88,7 @@ async def _get_portal_and_check_permission(evt: CommandEvent, permission: str, that_this = "This" if room_id == evt.room_id else "That" return await evt.reply(f"{that_this} is not a portal room."), False - if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, permission): + if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission): action = action or f"{permission.replace('_', ' ')}s" return await evt.reply(f"You do not have the permissions to {action} that portal."), False return portal, True @@ -166,8 +167,8 @@ async def bridge(evt: CommandEvent): if portal: return await evt.reply(f"{that_this} room is already a portal room.") - if not await _has_access_to(room_id, evt.az.intent, evt.sender, "bridge"): - return await evt.reply("You do not have the permissions to bridge that room.") + if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"): + return await evt.reply(f"You do not have the permissions to bridge {that_this} room.") # The /id bot command provides the prefixed ID, so we assume tgid = evt.args[0] @@ -192,7 +193,7 @@ async def bridge(evt: CommandEvent): has_portal_message = ( "That Telegram chat already has a portal at " f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ") - if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, "unbridge"): + if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"): return await evt.reply(f"{has_portal_message}" "Additionally, you do not have the permissions to unbridge " "that room.") @@ -291,7 +292,7 @@ async def confirm_bridge(evt: CommandEvent): direct = False portal.mxid = bridge_to_mxid - portal.title, portal.about, levels = await _get_initial_state(evt) + portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id) portal.photo_id = "" portal.save() @@ -301,8 +302,8 @@ async def confirm_bridge(evt: CommandEvent): return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.") -async def _get_initial_state(evt: CommandEvent): - state = await evt.az.intent.get_room_state(evt.room_id) +async def get_initial_state(intent: IntentAPI, room_id: str): + state = await intent.get_room_state(room_id) title = None about = None levels = None @@ -336,7 +337,10 @@ async def create(evt: CommandEvent): if po.Portal.get_by_mxid(evt.room_id): return await evt.reply("This is already a portal room.") - title, about, levels = await _get_initial_state(evt) + if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"): + return await evt.reply("You do not have the permissions to bridge this room.") + + title, about, levels = await get_initial_state(evt.az.intent, evt.room_id) if not title: return await evt.reply("Please set a title before creating a Telegram chat.") diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index 1163e503..ea0b92e3 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -310,6 +310,9 @@ class User(AbstractUser): @classmethod def get_by_mxid(cls, mxid, create=True): + if not mxid: + raise ValueError("Matrix ID can't be empty") + try: return cls.by_mxid[mxid] except KeyError: diff --git a/mautrix_telegram/web/common/auth_api.py b/mautrix_telegram/web/common/auth_api.py index 2ea50bca..14d29963 100644 --- a/mautrix_telegram/web/common/auth_api.py +++ b/mautrix_telegram/web/common/auth_api.py @@ -21,8 +21,8 @@ import logging from telethon.errors import * -from mautrix_telegram.commands.auth import enter_password -from mautrix_telegram.util import format_duration +from ...commands.auth import enter_password +from ...util import format_duration class AuthAPI(abc.ABC): @@ -70,7 +70,7 @@ class AuthAPI(abc.ABC): except Exception: self.log.exception("Error requesting phone code") return self.get_login_response(mxid=user.mxid, state="request", status=500, - errcode="exception", + errcode="unknown_error", error="Internal server error while requesting code.") async def post_login_token(self, user, token): @@ -124,7 +124,7 @@ class AuthAPI(abc.ABC): except Exception: self.log.exception("Error sending phone code") return self.get_login_response(mxid=user.mxid, state="code", status=500, - errcode="exception", + errcode="unknown_error", error="Internal server error while sending code.") async def post_login_password(self, user, password): @@ -146,5 +146,5 @@ class AuthAPI(abc.ABC): except Exception: self.log.exception("Error sending password") return self.get_login_response(mxid=user.mxid, state="password", status=500, - errcode="exception", + errcode="unknown_error", error="Internal server error while sending password.") diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index e3fa8347..dc640c93 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -16,32 +16,37 @@ # along with this program. If not, see . from aiohttp import web from typing import Tuple, Optional, Callable, Awaitable +import asyncio import logging import json from telethon.utils import get_peer_id, resolve_id +from mautrix_appservice import AppService, MatrixRequestError, IntentError from ...user import User from ...portal import Portal +from ...commands.portal import user_has_power_level, get_initial_state +from ...config import Config from ..common import AuthAPI class ProvisioningAPI(AuthAPI): log = logging.getLogger("mau.web.provisioning") - def __init__(self, config, loop): + def __init__(self, config: Config, az: AppService, loop: asyncio.AbstractEventLoop): super().__init__(loop) self.secret = config["appservice.provisioning.shared_secret"] + self.az = az self.app = web.Application(loop=loop, middlewares=[self.error_middleware]) portal_prefix = "/portal/{mxid:![^/]+}" self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal_by_mxid) self.app.router.add_route("GET", "/portal/{tgid:-[0-9]+}", self.get_portal_by_tgid) - # self.app.router.add_route("POST", portal_prefix + "/connect/{chat_id:[0-9]+}", - # self.connect_chat) - # self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat) - # self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat) + self.app.router.add_route("POST", portal_prefix + "/connect/{chat_id:[0-9]+}", + self.connect_chat) + self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat) + self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat) user_prefix = "/user/{mxid:@[^:]*:[^/]+}" self.app.router.add_route("GET", f"{user_prefix}", self.get_user_info) @@ -88,6 +93,69 @@ class ProvisioningAPI(AuthAPI): "megagroup": portal.megagroup, }) + async def connect_chat(self, request: web.Request) -> web.Response: + return web.Response(status=501) + + async def create_chat(self, request: web.Request) -> web.Response: + data = await self.get_data(request) + if not data: + return self.get_error_response(400, "json_invalid", "Invalid JSON.") + + room_id = request.match_info["mxid"] + if Portal.get_by_mxid(room_id): + return self.get_error_response(409, "room_already_bridged", + "Room is already bridged to another Telegram chat.") + + user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None, + require_puppeting=False) + if err is not None: + return err + elif not await user.is_logged_in() or user.is_bot: + return self.get_error_response(403, "not_logged_in_real_account", + "You are not logged in with a real account.") + elif not await user_has_power_level(room_id, self.az.intent, user, "bridge"): + return self.get_error_response(403, "not_enough_permissions", + "You do not have the permissions to bridge that room.") + + try: + title, about, _ = await get_initial_state(self.az.intent, room_id) + except (MatrixRequestError, IntentError): + return self.get_error_response(403, "bot_not_in_room", + "The bridge bot is not in the given room.") + + about = data.get("about", about) + + title = data.get("title", title) + if len(title) == 0: + return self.get_error_response(400, "body_value_invalid", "Title can not be empty.") + + type = data.get("type", "") + if type not in ("group", "chat", "supergroup", "channel"): + return self.get_error_response(400, "body_value_invalid", + "Given chat type is not valid.") + + supergroup = type == "supergroup" + type = { + "supergroup": "channel", + "channel": "channel", + "chat": "chat", + "group": "chat", + }[type] + + portal = Portal(tgid=None, mxid=room_id, title=title, about=about, peer_type=type) + try: + await portal.create_telegram_chat(user, supergroup=supergroup) + except ValueError as e: + portal.delete() + return self.get_error_response(500, "unknown_error", e.args[0]) + + return web.json_response({ + "chat_id": portal.tgid, + }) + + async def disconnect_chat(self, request: web.Request) -> web.Response: + return web.Response(status=501) + async def get_user_info(self, request: web.Request) -> web.Response: data, user, err = await self.get_user_request_info(request, expect_logged_in=None, require_puppeting=False) @@ -187,10 +255,11 @@ class ProvisioningAPI(AuthAPI): } else: resp = { - "state": state, "error": error, "errcode": errcode, } + if state: + resp["state"] = state return web.json_response(resp, status=status) def check_authorization(self, request: web.Request) -> bool: @@ -206,6 +275,10 @@ class ProvisioningAPI(AuthAPI): async def get_user(self, mxid: str, expect_logged_in: Optional[bool] = False, require_puppeting: bool = True, ) -> Tuple[Optional[User], Optional[web.Response]]: + if not mxid: + return None, self.get_login_response(error="User ID not given.", + errcode="mxid_empty", status=400) + user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True) if require_puppeting and not user.puppet_whitelisted: return user, self.get_login_response(error="You are not whitelisted.", diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index 775821b9..b107d47c 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -35,7 +35,7 @@ paths: schema: $ref: "#/definitions/PortalInfo" 400: - $ref: "#/responses/MissingMXIDError" + $ref: "#/responses/BadRequest" 404: description: Unknown portal schema: @@ -127,9 +127,16 @@ paths: enum: - delete - unbridge + - name: user_id + in: query + description: Optional Matrix user ID to check if the user has permissions to do the bridging. + required: false + type: string responses: 400: - $ref: "#/responses/MissingMXIDError" + $ref: "#/responses/BadRequest" + 401: + $ref: "#/responses/PermissionError" 409: description: Matrix room or Telegram chat is already bridged schema: @@ -152,8 +159,33 @@ paths: summary: Create a new Telegram chat for the given room tags: [Bridging] responses: + 200: + description: Telegram chat created + schema: + type: object + properties: + chat_id: + type: integer 400: - $ref: "#/responses/MissingMXIDError" + $ref: "#/responses/BadRequest" + 401: + $ref: "#/responses/PermissionError" + 403: + description: "Given user isn't logged in with a real account or doesn't have permission to bridge the room, or the bridge bot is not in the room" + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - not_logged_in_real_account + - not_enough_permissions + - bot_not_in_room + error: + $ref: "#/definitions/HumanReadableError" 409: description: Room is already bridged schema: @@ -174,6 +206,34 @@ paths: description: The Matrix ID of the room whose bridging status to get required: true type: string + - name: body + in: body + required: true + schema: + type: object + required: [type] + properties: + type: + description: The type of chat to create + type: string + example: supergroup + enum: + - chat + - supergroup + - channel + title: + description: Title for the new chat + type: string + example: Mautrix-Telegram Bridge + about: + description: About text for the new chat + type: string + example: Discussion about mautrix-telegram + - name: user_id + in: query + description: Matrix user to create the chat as. + required: true + type: string /portal/{room_id}/disconnect: post: operationId: disconnect_portal @@ -183,7 +243,9 @@ paths: 202: description: Room unbridging initiated 400: - $ref: "#/responses/MissingMXIDError" + $ref: "#/responses/BadRequest" + 401: + $ref: "#/responses/PermissionError" 404: description: Unknown portal schema: @@ -204,6 +266,11 @@ paths: description: The Matrix ID of the room whose bridging status to get required: true type: string + - name: user_id + in: query + description: Optional Matrix user ID to check if the user has permissions to do the bridging. + required: false + type: string /user/{user_id}: get: @@ -216,7 +283,7 @@ paths: schema: $ref: "#/definitions/UserInfo" 400: - $ref: "#/responses/MissingMXIDError" + $ref: "#/responses/BadRequest" 403: $ref: "#/responses/NotWhitelistedError" 500: @@ -238,7 +305,7 @@ paths: schema: $ref: "#/definitions/UserChats" 400: - $ref: "#/responses/MissingMXIDError" + $ref: "#/responses/BadRequest" 403: description: User is not logged in or not whitelisted schema: @@ -274,7 +341,7 @@ paths: schema: $ref: "#/definitions/AuthSuccess" 400: - $ref: "#/responses/MissingMXIDError" + $ref: "#/responses/BadRequest" 401: description: Invalid or expired bot token or invalid shared secret schema: @@ -325,7 +392,7 @@ paths: schema: $ref: "#/definitions/AuthSuccess" 400: - description: Invalid phone number or JSON or missing Matrix ID + description: Invalid phone number or JSON schema: type: object title: Error @@ -337,7 +404,6 @@ paths: example: machine_readable_error enum: - phone_number_invalid - - mxid_empty - json_invalid error: $ref: "#/definitions/HumanReadableError" @@ -436,7 +502,7 @@ paths: schema: $ref: "#/definitions/AuthSuccess" 400: - $ref: "#/responses/MissingMXIDError" + $ref: "#/responses/BadRequest" 401: description: Invalid phone code or shared secret schema: @@ -500,7 +566,7 @@ paths: schema: $ref: "#/definitions/AuthSuccess" 400: - description: Missing password or Matrix ID or invalid JSON + description: Missing password or invalid JSON schema: type: object title: Error @@ -512,7 +578,6 @@ paths: example: _empty enum: - password_empty - - mxid_empty - json_invalid error: $ref: "#/definitions/HumanReadableError" @@ -582,8 +647,8 @@ responses: username: type: string description: The Telegram username the user is logged in as. - MissingMXIDError: - description: Missing Matrix ID or invalid JSON. + BadRequest: + description: Invalid JSON. schema: type: object title: Error @@ -593,8 +658,10 @@ responses: title: Error code description: A machine-readable error code enum: - - mxid_empty - json_invalid + - mxid_empty + - body_value_missing + - body_value_invalid error: $ref: "#/definitions/HumanReadableError" UnknownError: @@ -608,23 +675,30 @@ responses: title: Error code description: A machine-readable error code enum: - - exception + - unknown_error + - unhandled_error error: type: string title: Error description: A human-readable description of the error example: Internal server error while . + PermissionError: + description: The given Matrix user doesn't have the permissions to do that. + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + example: not_enough_permissions enum: - - Internal server error while requesting code. - - Internal server error while sending code. - - Internal server error while sending password. - - Internal server error while sending token. + - not_enough_permissions + error: + $ref: "#/definitions/HumanReadableError" definitions: - HumanReadableError: - type: string - description: A human-readable description of the error - example: A human-readable description of the error UserInfo: type: object properties: @@ -713,6 +787,10 @@ definitions: type: string description: The Telegram username the user is logged in as. Only applicable if state=logged-in + HumanReadableError: + type: string + description: A human-readable description of the error + example: A human-readable description of the error security: - Bearer: [] From cea52102902480534610fc277a12c13df8bdc523 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jul 2018 23:15:28 +0300 Subject: [PATCH 25/65] Add /v1 prefix to provisioning API by default --- example-config.yaml | 2 +- mautrix_telegram/web/provisioning/spec.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example-config.yaml b/example-config.yaml index eb6c57d8..69c832f3 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -39,7 +39,7 @@ appservice: # Whether or not the provisioning API should be enabled. enabled: true # The prefix to use in the provisioning API endpoints. - prefix: /_matrix/provision + prefix: /_matrix/provision/v1 # The shared secret to authorize users of the API. # If you leave the default token, a random token will be generated and saved at startup. shared_secret: "Very secret shared secret" diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index b107d47c..fdd5a5b9 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -12,7 +12,7 @@ externalDocs: description: Provisioning API wiki page on GitHub url: https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API -basePath: /_matrix/provision +basePath: /_matrix/provision/v1 schemes: [https] consumes: [application/json] From 6cef4d81c6385546819cdb0841fd8560f65b0b76 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jul 2018 23:22:36 +0300 Subject: [PATCH 26/65] Add .codeclimate.yml --- .codeclimate.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .codeclimate.yml diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 00000000..ad83e79a --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,6 @@ +engines: + sonar-python: + enabled: true + checks: + python:S107: + enabled: false From e16e53c261dc068b780960c6e4a8a9f8de8fb181 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jul 2018 23:31:11 +0300 Subject: [PATCH 27/65] Ignore alembic in code climate --- .codeclimate.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.codeclimate.yml b/.codeclimate.yml index ad83e79a..e2fdfb75 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -4,3 +4,5 @@ engines: checks: python:S107: enabled: false +exclude_patterns: +- "alembic/" From 2a65ccc6748318e5ba6da752140ee0d438ec7051 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 00:07:45 +0300 Subject: [PATCH 28/65] Cache RoomStates and UserProfiles --- mautrix_telegram/db.py | 6 +-- mautrix_telegram/sqlstatestore.py | 75 +++++++++++++++++++------------ 2 files changed, 49 insertions(+), 32 deletions(-) diff --git a/mautrix_telegram/db.py b/mautrix_telegram/db.py index 4709fbff..5393acad 100644 --- a/mautrix_telegram/db.py +++ b/mautrix_telegram/db.py @@ -90,9 +90,9 @@ class RoomState(Base): _power_levels_text = Column("power_levels", Text, nullable=True) _power_levels_json = None -# def __init__(self, *args, **kwargs): -# super().__init__(*args, **kwargs) -# self._power_levels_json = None + @property + def has_power_levels(self): + return bool(self._power_levels_text) @property def power_levels(self): diff --git a/mautrix_telegram/sqlstatestore.py b/mautrix_telegram/sqlstatestore.py index 1d9442e2..63b030d2 100644 --- a/mautrix_telegram/sqlstatestore.py +++ b/mautrix_telegram/sqlstatestore.py @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import json +from typing import Dict, Tuple from mautrix_appservice import StateStore @@ -26,6 +26,8 @@ class SQLStateStore(StateStore): def __init__(self, db): super().__init__() self.db = db + self.profile_cache = {} # type: Dict[Tuple[str, str], UserProfile] + self.room_state_cache = {} # type: Dict[str, RoomState] def is_registered(self, user: str) -> bool: puppet = pu.Puppet.get_by_mxid(user) @@ -44,42 +46,60 @@ class SQLStateStore(StateStore): elif event_type == "m.room.member": self.set_member(event["room_id"], event["state_key"], event["content"]) - def get_member(self, room: str, user: str) -> dict: - profile = UserProfile.query.get((room, user)) + def _get_user_profile(self, room_id: str, user_id: str, create: bool = True) -> UserProfile: + key = (room_id, user_id) + try: + return self.profile_cache[key] + except KeyError: + pass + + profile = UserProfile.query.get(key) if profile: - return profile.dict() - return {} + self.profile_cache[key] = profile + elif create: + profile = UserProfile(room_id=room_id, user_id=user_id) + self.db.add(profile) + self.db.commit() + self.profile_cache[key] = profile + return profile + + def get_member(self, room: str, user: str) -> dict: + return self._get_user_profile(room, user).dict() def set_member(self, room: str, user: str, member: dict): - profile = UserProfile(room_id=room, user_id=user, - membership=member.get("membership", "leave"), - displayname=member.get("displayname", None), - avatar_url=member.get("avatar_url", None)) - self.db.merge(profile) + profile = self._get_user_profile(room, user) + profile.membership = member.get("membership", profile.membership or "leave") + profile.displayname = member.get("displayname", profile.displayname) + profile.avatar_url = member.get("avatar_url", profile.avatar_url) self.db.commit() def set_membership(self, room: str, user: str, membership: str): - profile = UserProfile.query.get((room, user)) - if not profile: - profile = UserProfile(room_id=room, user_id=user, membership=membership) - self.db.add(profile) - else: - profile.membership = membership - self.db.commit() + self.set_member(room, user, { + "membership": membership, + }) + + def _get_room_state(self, room_id: str, create: bool = True) -> RoomState: + try: + return self.room_state_cache[room_id] + except KeyError: + pass + + room = RoomState.query.get(room_id) + if room: + self.room_state_cache[room_id] = room + elif create: + room = RoomState(room_id=room_id) + self.room_state_cache[room_id] = room + return room def has_power_levels(self, room: str) -> bool: - room = RoomState.query.get(room) - return room and room._power_levels_text + return self._get_room_state(room).has_power_levels def get_power_levels(self, room: str) -> dict: - return RoomState.query.get(room).power_levels + return self._get_room_state(room).power_levels def set_power_level(self, room: str, user: str, level: int): - room_state = RoomState.query.get(room) - if not room_state: - room_state = RoomState(room) - self.db.add(room_state) - + room_state = self._get_room_state(room) power_levels = room_state.power_levels if not power_levels: power_levels = { @@ -91,9 +111,6 @@ class SQLStateStore(StateStore): self.db.commit() def set_power_levels(self, room: str, content: dict): - state = RoomState.query.get(room) - if not state: - state = RoomState(room_id=room) - self.db.add(state) + state = self._get_room_state(room) state.power_levels = content self.db.commit() From 0f1ac98b9f8fc3ae751bebd9de9968c9b5fc606c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 00:14:43 +0300 Subject: [PATCH 29/65] Remove old things from gitignore --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index b7e3188b..2eca9ad6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,3 @@ __pycache__ config.yaml registration.yaml *.db -*.session -*.json From b4dd05ab041633e4b203c2cbbb5af7d42274613b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 00:49:24 +0300 Subject: [PATCH 30/65] Simplify docker setup --- .dockerignore | 4 ++++ Dockerfile | 18 +++++++-------- .../mautrix-telegram/run => docker-run.sh | 22 +++++++++---------- docker/root/etc/s6.d/.s6-svscan/finish | 1 - docker/root/etc/s6.d/mautrix-telegram/finish | 2 -- 5 files changed, 23 insertions(+), 24 deletions(-) create mode 100644 .dockerignore rename docker/root/etc/s6.d/mautrix-telegram/run => docker-run.sh (67%) delete mode 100755 docker/root/etc/s6.d/.s6-svscan/finish delete mode 100755 docker/root/etc/s6.d/mautrix-telegram/finish diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ec191c92 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.editorconfig +.codeclimate.yml +*.png +*.md diff --git a/Dockerfile b/Dockerfile index 6c6b5c28..5ac19659 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,14 @@ -FROM docker.io/alpine:3.7 +FROM docker.io/alpine:3.8 ENV UID=1337 \ - GID=1337 + GID=1337 \ + FFMPEG_BINARY=/usr/bin/ffmpeg -COPY . /opt/mautrixtelegram +COPY . /opt/mautrix-telegram +WORKDIR /opt/mautrix-telegram RUN apk add --no-cache \ python3-dev \ + build-base \ py3-virtualenv \ py3-pillow \ py3-aiohttp \ @@ -14,17 +17,12 @@ RUN apk add --no-cache \ py3-numpy \ py3-asn1crypto \ py3-sqlalchemy \ - build-base \ + py3-markdown \ ffmpeg \ - bash \ ca-certificates \ su-exec \ - s6 \ - && cd /opt/mautrixtelegram \ - && cp -r docker/root/* / \ - && rm docker -rf \ && pip3 install -r requirements.txt -r optional-requirements.txt VOLUME /data -CMD ["/bin/s6-svscan", "/etc/s6.d"] +CMD ["/opt/mautrix-telegram/run.sh"] diff --git a/docker/root/etc/s6.d/mautrix-telegram/run b/docker-run.sh similarity index 67% rename from docker/root/etc/s6.d/mautrix-telegram/run rename to docker-run.sh index 41eb26aa..228e9f2f 100755 --- a/docker/root/etc/s6.d/mautrix-telegram/run +++ b/docker-run.sh @@ -1,22 +1,22 @@ -#!/bin/bash +#!/bin/sh -# Define functions +# Define functions. function fixperms { - chown -R ${UID}:${GID} /data /opt/mautrixtelegram + chown -R $UID:$GID /data /opt/mautrix-telegram } - -# Go into env -cd /opt/mautrixtelegram -export FFMPEG_BINARY=/usr/bin/ffmpeg +cd /opt/mautrix-telegram # Replace database path in config. sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml +if [ -f /data/mx-state.json ]; then + ln -s /data/mx-state.json +fi # Check that database is in the right state alembic -x config=/data/config.yaml upgrade head -if [[ ! -f /data/config.yaml ]]; then +if [ ! -f /data/config.yaml ]; then cp example-config.yaml /data/config.yaml echo "Didn't find a config file." echo "Copied default config file to /data/config.yaml" @@ -26,14 +26,14 @@ if [[ ! -f /data/config.yaml ]]; then exit fi -if [[ ! -f /data/registration.yaml ]]; then +if [ ! -f /data/registration.yaml ]; then python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml echo "Didn't find a registration file." - echo "Generated ode for you." + echo "Generated one for you." echo "Copy that over to synapses app service directory." fixperms exit fi fixperms -exec su-exec ${UID}:${GID} python3 -m mautrix_telegram -c /data/config.yaml +exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml diff --git a/docker/root/etc/s6.d/.s6-svscan/finish b/docker/root/etc/s6.d/.s6-svscan/finish deleted file mode 100755 index 1a248525..00000000 --- a/docker/root/etc/s6.d/.s6-svscan/finish +++ /dev/null @@ -1 +0,0 @@ -#!/bin/sh diff --git a/docker/root/etc/s6.d/mautrix-telegram/finish b/docker/root/etc/s6.d/mautrix-telegram/finish deleted file mode 100755 index e90c4912..00000000 --- a/docker/root/etc/s6.d/mautrix-telegram/finish +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -s6-svscanctl -t /etc/s6.d From e71f7280b84dd31a40ae42023fe6c612bb442208 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 01:22:14 +0300 Subject: [PATCH 31/65] Fix command in dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5ac19659..b371bd44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,4 +25,4 @@ RUN apk add --no-cache \ VOLUME /data -CMD ["/opt/mautrix-telegram/run.sh"] +CMD ["/opt/mautrix-telegram/docker-run.sh"] From a46cc7a78895ced8b9519933c18f0f97f7ed1d3c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 12:38:24 +0300 Subject: [PATCH 32/65] Add logout endpoint --- mautrix_telegram/web/provisioning/__init__.py | 20 +++++++++---- mautrix_telegram/web/provisioning/spec.yaml | 30 +++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index dc640c93..f853d041 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -52,10 +52,11 @@ class ProvisioningAPI(AuthAPI): self.app.router.add_route("GET", f"{user_prefix}", self.get_user_info) self.app.router.add_route("GET", f"{user_prefix}/chats", self.get_chats) - self.app.router.add_route("POST", f"{user_prefix}/send_bot_token", self.send_bot_token) - self.app.router.add_route("POST", f"{user_prefix}/request_code", self.request_code) - self.app.router.add_route("POST", f"{user_prefix}/send_code", self.send_code) - self.app.router.add_route("POST", f"{user_prefix}/send_password", self.send_password) + self.app.router.add_route("POST", f"{user_prefix}/logout", self.logout) + self.app.router.add_route("POST", f"{user_prefix}/login/bot_token", self.send_bot_token) + self.app.router.add_route("POST", f"{user_prefix}/login/request_code", self.request_code) + self.app.router.add_route("POST", f"{user_prefix}/login/send_code", self.send_code) + self.app.router.add_route("POST", f"{user_prefix}/login/send_password", self.send_password) async def get_portal_by_mxid(self, request: web.Request) -> web.Response: mxid = request.match_info["mxid"] @@ -221,6 +222,14 @@ class ProvisioningAPI(AuthAPI): return err return await self.post_login_password(user, data.get("password", "")) + async def logout(self, request: web.Request) -> web.Response: + _, user, err = await self.get_user_request_info(request, expect_logged_in=True, + require_puppeting=False, + want_data=False) + if err is not None: + return err + await user.log_out() + @staticmethod async def error_middleware(_, handler) -> Callable[[web.Request], Awaitable[web.Response]]: async def middleware_handler(request: web.Request) -> web.Response: @@ -297,6 +306,7 @@ class ProvisioningAPI(AuthAPI): async def get_user_request_info(self, request: web.Request, expect_logged_in: Optional[bool] = False, require_puppeting: bool = False, + want_data: bool = True, ) -> (Tuple[Optional[dict], Optional[User], Optional[web.Response]]): @@ -307,7 +317,7 @@ class ProvisioningAPI(AuthAPI): status=401) data = None - if request.method == "POST" or request.method == "PUT": + if want_data and (request.method == "POST" or request.method == "PUT"): data = await self.get_data(request) if not data: return None, None, self.get_login_response(error="Invalid JSON.", diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index fdd5a5b9..94457ce4 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -619,6 +619,36 @@ paths: description: The two-factor auth password format: password example: hunter2 + /user/{user_id}/logout: + post: + operationId: logout + summary: Log out + tags: [Authentication] + responses: + 200: + description: Logout successful + 403: + description: User was not logged in + schema: + type: object + title: Error + properties: + errcode: + type: string + title: Error code + description: A machine-readable error code + enum: + - not_logged_in + error: + $ref: "#/definitions/HumanReadableError" + 500: + $ref: "#/responses/UnknownError" + parameters: + - name: user_id + in: path + description: The Matrix ID of the user who to log out as + required: true + type: string responses: NotWhitelistedError: From c2879408ccd49f77faf2e9a94cc64634ee27481c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 14:51:56 +0300 Subject: [PATCH 33/65] Make bridging permission checks consistent --- mautrix_telegram/commands/portal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix_telegram/commands/portal.py b/mautrix_telegram/commands/portal.py index e9e29547..38998224 100644 --- a/mautrix_telegram/commands/portal.py +++ b/mautrix_telegram/commands/portal.py @@ -117,7 +117,7 @@ def _get_portal_murder_function(action: str, room_id: str, function: Callable, c "Only works for group chats; to delete a private chat portal, simply " "leave the room.") async def delete_portal(evt: CommandEvent): - portal, ok = await _get_portal_and_check_permission(evt, "delete_portal") + portal, ok = await _get_portal_and_check_permission(evt, "unbridge") if not ok: return @@ -138,7 +138,7 @@ async def delete_portal(evt: CommandEvent): help_section=SECTION_PORTAL_MANAGEMENT, help_text="Remove puppets from the current portal room and forget the portal.") async def unbridge(evt: CommandEvent): - portal, ok = await _get_portal_and_check_permission(evt, "unbridge_room") + portal, ok = await _get_portal_and_check_permission(evt, "unbridge") if not ok: return From c55967c9f0ef98b75bc8e526dfbdd5b60f91f5fa Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 15:19:37 +0300 Subject: [PATCH 34/65] Implement disconnecting portals via provisioning API --- mautrix_telegram/web/common/auth_api.py | 2 +- mautrix_telegram/web/provisioning/__init__.py | 72 ++++++++++++++++--- mautrix_telegram/web/provisioning/spec.yaml | 12 ++++ 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/mautrix_telegram/web/common/auth_api.py b/mautrix_telegram/web/common/auth_api.py index 14d29963..70b66136 100644 --- a/mautrix_telegram/web/common/auth_api.py +++ b/mautrix_telegram/web/common/auth_api.py @@ -29,7 +29,7 @@ class AuthAPI(abc.ABC): log = logging.getLogger("mau.web.auth") def __init__(self, loop): - self.loop = loop + self.loop = loop # type: asyncio.AbstractEventLoop @abstractmethod def get_login_response(self, status=200, state="", username="", mxid="", message="", error="", diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index f853d041..63bb208d 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -59,6 +59,10 @@ class ProvisioningAPI(AuthAPI): self.app.router.add_route("POST", f"{user_prefix}/login/send_password", self.send_password) async def get_portal_by_mxid(self, request: web.Request) -> web.Response: + err = self.check_authorization(request) + if err is not None: + return err + mxid = request.match_info["mxid"] portal = Portal.get_by_mxid(mxid) if not portal: @@ -75,6 +79,10 @@ class ProvisioningAPI(AuthAPI): }) async def get_portal_by_tgid(self, request: web.Request) -> web.Response: + err = self.check_authorization(request) + if err is not None: + return err + try: tgid, _ = resolve_id(int(request.match_info["tgid"])) except ValueError: @@ -95,9 +103,19 @@ class ProvisioningAPI(AuthAPI): }) async def connect_chat(self, request: web.Request) -> web.Response: - return web.Response(status=501) + err = self.check_authorization(request) + if err is not None: + return err + + return self.get_error_response(501, "not_implemented", + "Connecting existing Matrix rooms to existing Telegram " + "chats via the provisioning API is not yet implemented.") async def create_chat(self, request: web.Request) -> web.Response: + err = self.check_authorization(request) + if err is not None: + return err + data = await self.get_data(request) if not data: return self.get_error_response(400, "json_invalid", "Invalid JSON.") @@ -155,7 +173,36 @@ class ProvisioningAPI(AuthAPI): }) async def disconnect_chat(self, request: web.Request) -> web.Response: - return web.Response(status=501) + err = self.check_authorization(request) + if err is not None: + return err + + portal = Portal.get_by_mxid(request.match_info["mxid"]) + if not portal or not portal.tgid: + return self.get_error_response(404, "portal_not_found", + "Room is not a portal.") + + user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None, + require_puppeting=False, require_user=False) + if err is not None: + return err + elif user and not await user_has_power_level(portal.mxid, self.az.intent, user, "unbridge"): + return self.get_error_response(403, "not_enough_permissions", + "You do not have the permissions to unbridge that room.") + + delete = request.query.get("delete", "").lower() in ("true", "t", "1", "yes", "y") + sync = request.query.get("delete", "").lower() in ("true", "t", "1", "yes", "y") + + coro = portal.cleanup_and_delete() if delete else portal.unbridge() + if sync: + try: + await coro + except Exception: + self.log.exception("Failed to disconnect chat") + return self.get_error_response(500, "exception", "Failed to disconnect chat") + else: + asyncio.ensure_future(coro, loop=self.loop) + return web.json_response({}, status=200 if sync else 202) async def get_user_info(self, request: web.Request) -> web.Response: data, user, err = await self.get_user_request_info(request, expect_logged_in=None, @@ -271,8 +318,13 @@ class ProvisioningAPI(AuthAPI): resp["state"] = state return web.json_response(resp, status=status) - def check_authorization(self, request: web.Request) -> bool: - return request.headers.get("Authorization", "") == f"Bearer {self.secret}" + def check_authorization(self, request: web.Request) -> Optional[web.Response]: + auth = request.headers.get("Authorization", "") + if auth != f"Bearer {self.secret}": + return self.get_error_response(error="Shared secret is not valid.", + errcode="shared_secret_invalid", + status=401) + return None @staticmethod async def get_data(request: web.Request) -> Optional[dict]: @@ -282,9 +334,11 @@ class ProvisioningAPI(AuthAPI): return None async def get_user(self, mxid: str, expect_logged_in: Optional[bool] = False, - require_puppeting: bool = True, + require_puppeting: bool = True, require_user: bool = True ) -> Tuple[Optional[User], Optional[web.Response]]: if not mxid: + if not require_user: + return None, None return None, self.get_login_response(error="User ID not given.", errcode="mxid_empty", status=400) @@ -310,11 +364,9 @@ class ProvisioningAPI(AuthAPI): ) -> (Tuple[Optional[dict], Optional[User], Optional[web.Response]]): - auth = request.headers.get("Authorization", "") - if auth != f"Bearer {self.secret}": - return None, None, self.get_login_response(error="Shared secret is not valid.", - errcode="shared_secret_invalid", - status=401) + err = self.check_authorization(request) + if err is not None: + return err data = None if want_data and (request.method == "POST" or request.method == "PUT"): diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index 94457ce4..c44becdc 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -271,6 +271,18 @@ paths: description: Optional Matrix user ID to check if the user has permissions to do the bridging. required: false type: string + - name: delete + in: query + description: Whether or not to delete the room completely (kick all users instead of just Telegram puppets) + required: false + type: boolean + default: false + - name: sync + in: query + description: Whether or not to wait for the unbridging to be completed before responding. **Could cause timeouts in large rooms** + required: false + type: boolean + default: false /user/{user_id}: get: From 64391ae20da8f5c691e56020ff63f399097a043f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 15:19:59 +0300 Subject: [PATCH 35/65] Ignore .log files instead of logs/ --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e3545879..d19a6d68 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,5 @@ __pycache__ config.yaml registration.yaml -logs/ +*.log *.db From 4d63cd75d42eff61eec3852d3192d133225bfd4a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 15:32:37 +0300 Subject: [PATCH 36/65] Update spec metadata --- mautrix_telegram/web/provisioning/spec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix_telegram/web/provisioning/spec.yaml b/mautrix_telegram/web/provisioning/spec.yaml index c44becdc..4c1c44c4 100644 --- a/mautrix_telegram/web/provisioning/spec.yaml +++ b/mautrix_telegram/web/provisioning/spec.yaml @@ -1,9 +1,9 @@ swagger: "2.0" info: - title: mautrix-telegram provisioning + title: Mautrix-Telegram provisioning version: 0.3.0 - description: The provisioning API for mautrix-telegram. + description: The provisioning API for Mautrix-Telegram, the Matrix-Telegram puppeting/relaybot bridge. license: name: AGPLv3 url: https://github.com/tulir/mautrix-telegram/blob/master/LICENSE From dfca56b292599fdad8e9751f5c5756a5d89d566b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 15:46:28 +0300 Subject: [PATCH 37/65] Fix cleaning up management rooms. Fixes #172 --- mautrix_telegram/commands/clean_rooms.py | 29 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/mautrix_telegram/commands/clean_rooms.py b/mautrix_telegram/commands/clean_rooms.py index 4c713f4a..e9031d3b 100644 --- a/mautrix_telegram/commands/clean_rooms.py +++ b/mautrix_telegram/commands/clean_rooms.py @@ -14,17 +14,24 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from mautrix_appservice import MatrixRequestError +from typing import Tuple, List + +from mautrix_appservice import MatrixRequestError, IntentAPI from . import command_handler, CommandEvent, SECTION_ADMIN from .. import puppet as pu, portal as po +ManagementRoomList = List[Tuple[str, str]] +RoomIDList = List[str] +PortalList = List[po.Portal] -async def _find_rooms(intent): - management_rooms = [] - unidentified_rooms = [] - portals = [] - empty_portals = [] + +async def _find_rooms(intent: IntentAPI) -> Tuple[ + ManagementRoomList, RoomIDList, PortalList, PortalList]: + management_rooms = [] # type: ManagementRoomList + unidentified_rooms = [] # type: RoomIDList + portals = [] # type: PortalList + empty_portals = [] # type: PortalList rooms = await intent.get_joined_rooms() for room in rooms: @@ -88,7 +95,7 @@ async def clean_rooms(evt: CommandEvent): "where `range` is the range (e.g. `5-21`) prefixed with the first letter of" "the group name."), "", - ("Please note that you will have to re-run `$cmdprefix+sp cleanrooms` " + ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` " "between each use of the commands above.")] evt.sender.command_status = { @@ -100,7 +107,9 @@ async def clean_rooms(evt: CommandEvent): return await evt.reply("\n".join(reply)) -async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals, empty_portals): +async def set_rooms_to_clean(evt, management_rooms: ManagementRoomList, + unidentified_rooms: RoomIDList, portals: PortalList, + empty_portals: PortalList): command = evt.args[0] rooms_to_clean = [] if command == "clean-recommended": @@ -110,7 +119,7 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals, return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]") groups_to_clean = evt.args[1] if "M" in groups_to_clean: - rooms_to_clean += management_rooms + rooms_to_clean += [room_id for (room_id, user_id) in management_rooms] if "A" in groups_to_clean: rooms_to_clean += portals if "U" in groups_to_clean: @@ -124,7 +133,7 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals, start, end = range.split("-") start, end = int(start), int(end) if group == "M": - group = management_rooms + group = [room_id for (room_id, user_id) in management_rooms] elif group == "A": group = portals elif group == "U": From 87dc1a44b27f7828d0915886d7374d4252ee4fb7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 16:03:46 +0300 Subject: [PATCH 38/65] Add bot_avatar config field --- example-config.yaml | 7 +++++-- mautrix_telegram/config.py | 3 ++- mautrix_telegram/context.py | 9 +++------ mautrix_telegram/matrix.py | 9 +++++++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/example-config.yaml b/example-config.yaml index 69c832f3..9fa15ad6 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -41,14 +41,17 @@ appservice: # The prefix to use in the provisioning API endpoints. prefix: /_matrix/provision/v1 # The shared secret to authorize users of the API. - # If you leave the default token, a random token will be generated and saved at startup. - shared_secret: "Very secret shared secret" + # Set to "generate" to generate and save a new token. + shared_secret: generate # The unique ID of this appservice. id: telegram # Username of the appservice bot. bot_username: telegrambot + # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty + # to leave display name/avatar as-is. bot_displayname: Telegram bridge bot + bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. as_token: "This value is generated when generating the registration" diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 77cc6bfb..1dad0fb4 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -162,12 +162,13 @@ class Config(DictWithRecursion): copy("appservice.provisioning.enabled") copy("appservice.provisioning.prefix") copy("appservice.provisioning.shared_secret") - if base["appservice.provisioning.shared_secret"] == "Very secret shared secret": + if base["appservice.provisioning.shared_secret"] == "generate": base["appservice.provisioning.shared_secret"] = self._new_token() copy("appservice.id") copy("appservice.bot_username") copy("appservice.bot_displayname") + copy("appservice.bot_avatar") copy("appservice.as_token") copy("appservice.hs_token") diff --git a/mautrix_telegram/context.py b/mautrix_telegram/context.py index ad48d7e4..5b0b6193 100644 --- a/mautrix_telegram/context.py +++ b/mautrix_telegram/context.py @@ -14,13 +14,13 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Tuple import asyncio from sqlalchemy.orm import scoped_session from alchemysession import AlchemySessionContainer from mautrix_appservice import AppService - class Context: def __init__(self, az, db, config, loop, bot, mx, session_container, public_website, provisioning_api): @@ -38,10 +38,7 @@ class Context: self.session_container = session_container # type: AlchemySessionContainer self.public_website = public_website # type: PublicBridgeWebsite self.provisioning_api = provisioning_api # type: ProvisioningAPI + self.t = (self.az, self.db, self.config, self.loop, self.bot) def __iter__(self): - yield self.az - yield self.db - yield self.config - yield self.loop - yield self.bot + return iter(self.t) diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index dc9e3f75..49dfa649 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -36,8 +36,13 @@ class MatrixHandler: self.az.matrix_event_handler(self.handle_event) async def init_as_bot(self): - await self.az.intent.set_display_name( - self.config.get("appservice.bot_displayname", "Telegram bridge bot")) + displayname = self.config["appservice.bot_displayname"] + if displayname: + await self.az.intent.set_display_name(displayname if displayname != "remove" else "") + + avatar = self.config["appservice.bot_avatar"] + if avatar: + await self.az.intent.set_avatar(avatar if avatar != "remove" else "") async def handle_puppet_invite(self, room, puppet, inviter): self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}") From 23232cf88cd16418965e80b9d106d75c3e66ac77 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 16:13:02 +0300 Subject: [PATCH 39/65] Don't crash on TimeoutError when initializing AS bot. Fixes #179 --- mautrix_telegram/matrix.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index 49dfa649..540fc39b 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -38,11 +38,17 @@ class MatrixHandler: async def init_as_bot(self): displayname = self.config["appservice.bot_displayname"] if displayname: - await self.az.intent.set_display_name(displayname if displayname != "remove" else "") + try: + await self.az.intent.set_display_name(displayname if displayname != "remove" else "") + except asyncio.TimeoutError: + self.log.exception("TimeoutError when trying to set displayname") avatar = self.config["appservice.bot_avatar"] if avatar: - await self.az.intent.set_avatar(avatar if avatar != "remove" else "") + try: + await self.az.intent.set_avatar(avatar if avatar != "remove" else "") + except asyncio.TimeoutError: + self.log.exception("TimeoutError when trying to set avatar") async def handle_puppet_invite(self, room, puppet, inviter): self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}") From 996e8ab4454c2f8f1e4a9855d7979f0006ad0c0c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 16:21:11 +0300 Subject: [PATCH 40/65] Update alembic version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 51ba9393..96fc29db 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setuptools.setup( "aiohttp>=3.0.1,<4", "mautrix-appservice>=0.3.0,<0.4.0", "SQLAlchemy>=1.2.3,<2", - "alembic>=0.9.8,<0.10", + "alembic>=1.0.0,<2", "Markdown>=2.6.11,<3", "ruamel.yaml>=0.15.35,<0.16", "future-fstrings>=0.4.2", From 122699593de088a9807d5d452b8a03c20ec5bfd3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 15 Jul 2018 22:39:03 -0600 Subject: [PATCH 41/65] Enable user-level access to bridge and unbridge commands --- mautrix_telegram/commands/portal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix_telegram/commands/portal.py b/mautrix_telegram/commands/portal.py index 38998224..0c88ca74 100644 --- a/mautrix_telegram/commands/portal.py +++ b/mautrix_telegram/commands/portal.py @@ -134,7 +134,7 @@ async def delete_portal(evt: CommandEvent): "bridge, use `$cmdprefix+sp unbridge` instead.") -@command_handler(needs_auth=False, +@command_handler(needs_auth=False, needs_puppeting=False, help_section=SECTION_PORTAL_MANAGEMENT, help_text="Remove puppets from the current portal room and forget the portal.") async def unbridge(evt: CommandEvent): @@ -150,7 +150,7 @@ async def unbridge(evt: CommandEvent): "by typing `$cmdprefix+sp confirm-unbridge`") -@command_handler(needs_auth=False, +@command_handler(needs_auth=False, needs_puppeting=False, help_section=SECTION_PORTAL_MANAGEMENT, help_args="[_id_]", help_text="Bridge the current Matrix room to the Telegram chat with the given " From 0625b2d661ee17d6ff397071cd058dc2443ed16b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Jul 2018 16:26:45 +0300 Subject: [PATCH 42/65] Handle FileNotFoundError when migrating state store --- .../6ca3d74d51e4_move_state_store_to_main_database.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py b/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py index 7e06de1f..6a66c02d 100644 --- a/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py +++ b/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py @@ -77,8 +77,11 @@ def upgrade(): session = orm.scoping.scoped_session(session) Puppet.query = session.query_property() - with open("mx-state.json") as file: - data = json.load(file) + try: + with open("mx-state.json") as file: + data = json.load(file) + except FileNotFoundError: + return if not data: return registrations = data.get("registrations", []) From 6926bce139b8afaf46e40574ddc2e69110d41b77 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 16 Jul 2018 23:21:14 +0300 Subject: [PATCH 43/65] Remove unnecessary __init__s and fix telematrix import script program name --- mautrix_telegram/scripts/__init__.py | 0 mautrix_telegram/scripts/telematrix_import/__init__.py | 0 mautrix_telegram/scripts/telematrix_import/__main__.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 mautrix_telegram/scripts/__init__.py delete mode 100644 mautrix_telegram/scripts/telematrix_import/__init__.py diff --git a/mautrix_telegram/scripts/__init__.py b/mautrix_telegram/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mautrix_telegram/scripts/telematrix_import/__init__.py b/mautrix_telegram/scripts/telematrix_import/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mautrix_telegram/scripts/telematrix_import/__main__.py b/mautrix_telegram/scripts/telematrix_import/__main__.py index 27d70735..8d009a6f 100644 --- a/mautrix_telegram/scripts/telematrix_import/__main__.py +++ b/mautrix_telegram/scripts/telematrix_import/__main__.py @@ -9,7 +9,7 @@ from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as parser = argparse.ArgumentParser( description="mautrix-telegram telematrix import script", - prog="python -m scripts/telematrix_import") + prog="python -m mautrix_telegram.scripts.telematrix_import") parser.add_argument("-c", "--config", type=str, default="config.yaml", metavar="", help="the path to your mautrix-telegram config file") parser.add_argument("-b", "--bot-id", type=int, required=True, From 983084270736b4c0510b540905f684f5ab563440 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 16 Jul 2018 23:21:22 +0300 Subject: [PATCH 44/65] Add db_migrate script. Fixes #178 --- .../scripts/db_migrate/__main__.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 mautrix_telegram/scripts/db_migrate/__main__.py diff --git a/mautrix_telegram/scripts/db_migrate/__main__.py b/mautrix_telegram/scripts/db_migrate/__main__.py new file mode 100644 index 00000000..73d243aa --- /dev/null +++ b/mautrix_telegram/scripts/db_migrate/__main__.py @@ -0,0 +1,59 @@ +import argparse +import sqlalchemy as sql +from sqlalchemy import orm +from sqlalchemy.ext.declarative import declarative_base + +from alchemysession import AlchemySessionContainer + +parser = argparse.ArgumentParser(description="mautrix-telegram database migration script", + prog="python -m mautrix_telegram.scripts.db_migrate") +parser.add_argument("-f", "--from-uri", type=str, required=True, metavar="", + help="the old database path") +parser.add_argument("-t", "--to-uri", type=str, required=True, metavar="", + help="the new database path") +args = parser.parse_args() + + +def connect(to): + import mautrix_telegram.base as base + base.Base = declarative_base() + from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile, + Contact, Puppet, BotChat, TelegramFile) + db_engine = sql.create_engine(to) + db_factory = orm.sessionmaker(bind=db_engine) + db_session = orm.scoped_session(db_factory) # type: orm.Session + base.Base.metadata.bind = db_engine + session_container = AlchemySessionContainer(engine=db_engine, session=db_session, + table_base=base.Base, table_prefix="telethon_", + manage_tables=False) + + return db_session, { + "Version": session_container.Version, + "Session": session_container.Session, + "Entity": session_container.Entity, + "SentFile": session_container.SentFile, + "UpdateState": session_container.UpdateState, + "Portal": Portal, + "Message": Message, + "Puppet": Puppet, + "User": User, + "UserPortal": UserPortal, + "RoomState": RoomState, + "UserProfile": UserProfile, + "Contact": Contact, + "BotChat": BotChat, + "TelegramFile": TelegramFile, + } + + +session, tables = connect(args.from_uri) + +data = {} +for name, table in tables.items(): + data[name] = session.query(table).all() + +session, tables = connect(args.to_uri) +for name, table in tables.items(): + for row in data[name]: + session.merge(row) +session.commit() From d4b725a508e32a1a59887b43dfe4685a9561291b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 16 Jul 2018 23:27:06 +0300 Subject: [PATCH 45/65] Add comment about supported DBMSes --- example-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/example-config.yaml b/example-config.yaml index 9fa15ad6..7668c236 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -18,7 +18,8 @@ appservice: hostname: 0.0.0.0 port: 8080 - # The full URI to the database. + # The full URI to the database. SQLite and Postgres are fully supported. + # Other DBMSes supported by SQLAlchemy may or may not work. database: sqlite:///mautrix-telegram.db # Public part of web server for out-of-Matrix interaction with the bridge. From e9348c9550ec70e2bc06cbbc1d346c9f3fcd39bd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 16 Jul 2018 23:30:02 +0300 Subject: [PATCH 46/65] Rename db_migrate script to dbms_migrate --- .../scripts/{db_migrate => dbms_migrate}/__main__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename mautrix_telegram/scripts/{db_migrate => dbms_migrate}/__main__.py (86%) diff --git a/mautrix_telegram/scripts/db_migrate/__main__.py b/mautrix_telegram/scripts/dbms_migrate/__main__.py similarity index 86% rename from mautrix_telegram/scripts/db_migrate/__main__.py rename to mautrix_telegram/scripts/dbms_migrate/__main__.py index 73d243aa..bc33bcc9 100644 --- a/mautrix_telegram/scripts/db_migrate/__main__.py +++ b/mautrix_telegram/scripts/dbms_migrate/__main__.py @@ -5,11 +5,11 @@ from sqlalchemy.ext.declarative import declarative_base from alchemysession import AlchemySessionContainer -parser = argparse.ArgumentParser(description="mautrix-telegram database migration script", - prog="python -m mautrix_telegram.scripts.db_migrate") -parser.add_argument("-f", "--from-uri", type=str, required=True, metavar="", +parser = argparse.ArgumentParser(description="mautrix-telegram dbms migration script", + prog="python -m mautrix_telegram.scripts.dbms_migrate") +parser.add_argument("-f", "--from-url", type=str, required=True, metavar="", help="the old database path") -parser.add_argument("-t", "--to-uri", type=str, required=True, metavar="", +parser.add_argument("-t", "--to-url", type=str, required=True, metavar="", help="the new database path") args = parser.parse_args() @@ -46,13 +46,13 @@ def connect(to): } -session, tables = connect(args.from_uri) +session, tables = connect(args.from_url) data = {} for name, table in tables.items(): data[name] = session.query(table).all() -session, tables = connect(args.to_uri) +session, tables = connect(args.to_url) for name, table in tables.items(): for row in data[name]: session.merge(row) From c8e44d4ab444fd9ca397ea6708f4dcbb35699482 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 16 Jul 2018 18:03:54 -0600 Subject: [PATCH 47/65] De-duplicate objects in the Telematrix import --- .../scripts/telematrix_import/__main__.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/mautrix_telegram/scripts/telematrix_import/__main__.py b/mautrix_telegram/scripts/telematrix_import/__main__.py index 8d009a6f..2de531c7 100644 --- a/mautrix_telegram/scripts/telematrix_import/__main__.py +++ b/mautrix_telegram/scripts/telematrix_import/__main__.py @@ -38,8 +38,14 @@ telematrix.close() telematrix_db_engine.dispose() portals = {} +chats = {} +messages = {} +puppets = {} for chat_link in chat_links: + if type(chat_link.tg_room) is str: + print("Expected tg_room to be a number, got a string. Ignoring %s" % chat_link.tg_room) + continue if chat_link.tg_room >= 0: print("Unexpected unprefixed telegram chat ID: %s, ignoring..." % chat_link.tg_room) continue @@ -55,11 +61,9 @@ for chat_link in chat_links: portal = Portal(tgid=tgid, tg_receiver=tgid, peer_type=peer_type, megagroup=megagroup, mxid=chat_link.matrix_room) - portals[chat_link.tg_room] = portal - mxtg.add(portal) - bot_chat = BotChat(id=tgid, type=peer_type) - mxtg.add(bot_chat) + portals[chat_link.tg_room] = portal + chats[tgid] = bot_chat for tm_msg in messages: try: @@ -70,8 +74,18 @@ for tm_msg in messages: tg_space = portal.tgid if portal.peer_type == "channel" else args.bot_id message = Message(mxid=tm_msg.matrix_event_id, mx_room=tm_msg.matrix_room_id, tgid=tm_msg.tg_message_id, tg_space=tg_space) - mxtg.add(message) + messages[tm_msg.matrix_event_id] = message + +for user in tg_users: + puppets[user.tg_id] = Puppet(id=user.tg_id, displayname=user.name, displayname_source=args.bot_id) + +for k, v in portals.items(): + mxtg.add(v) +for k, v in chats.items(): + mxtg.add(v) +for k, v in messages.items(): + mxtg.add(v) +for k, v in puppets.items(): + mxtg.add(v) -mxtg.add_all(Puppet(id=user.tg_id, displayname=user.name, displayname_source=args.bot_id) - for user in tg_users) mxtg.commit() From e4ce73cbba3318c865cf93c3a7e12ee2c8d9f65a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 17 Jul 2018 09:48:59 +0300 Subject: [PATCH 48/65] Revert Context iter changes in 87dc1a44b27f7828d0915886d7374d4252ee4fb7 and fix a f-string Closes #185 --- mautrix_telegram/context.py | 7 +++++-- mautrix_telegram/matrix.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mautrix_telegram/context.py b/mautrix_telegram/context.py index 5b0b6193..1324e5f1 100644 --- a/mautrix_telegram/context.py +++ b/mautrix_telegram/context.py @@ -38,7 +38,10 @@ class Context: self.session_container = session_container # type: AlchemySessionContainer self.public_website = public_website # type: PublicBridgeWebsite self.provisioning_api = provisioning_api # type: ProvisioningAPI - self.t = (self.az, self.db, self.config, self.loop, self.bot) def __iter__(self): - return iter(self.t) + yield self.az + yield self.db + yield self.config + yield self.loop + yield self.bot diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index 540fc39b..30604296 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -208,7 +208,7 @@ class MatrixHandler: self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:" " User is not whitelisted.") return - self.log.debug("Received Matrix event \"{message}\" from {sender} in {room}") + self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}") portal = Portal.get_by_mxid(room) if not is_command and portal and (await sender.is_logged_in() or portal.has_bot): From 64d726ec2b26b23991338ba79026f5ca4b99ab0b Mon Sep 17 00:00:00 2001 From: "Kai A. Hiller" Date: Fri, 20 Jul 2018 02:02:09 +0200 Subject: [PATCH 49/65] Fix install of web resources --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 96fc29db..1ed8106d 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ setuptools.setup( mautrix-telegram=mautrix_telegram.__main__:main """, package_data={"mautrix_telegram": [ - "public/*.mako", "public/*.png", "public/*.css", + "web/public/*.mako", "web/public/*.png", "web/public/*.css", ]}, data_files=[ (".", ["example-config.yaml", "alembic.ini"]), From ad7b7f5c0669523c8adb88d40adf1f6250b4e347 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 20 Jul 2018 10:05:12 -0400 Subject: [PATCH 50/65] Stop using f-strings in Alembic migrations. Fixes #189 --- .../versions/6ca3d74d51e4_move_state_store_to_main_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py b/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py index 6a66c02d..b5cfe450 100644 --- a/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py +++ b/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py @@ -93,7 +93,7 @@ def upgrade(): username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}") hs_domain = mxtg_config["homeserver.domain"] localpart = username_template.format(userid="(.+)") - mxid_regex = re.compile(f"@{localpart}:{hs_domain}") + mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain)) for user in registrations: match = mxid_regex.match(user) if not match: From 2b92483c50ed0bd12da2a1eb726ab33bbd5623f9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 20 Jul 2018 12:35:22 -0400 Subject: [PATCH 51/65] Initial option to replace Matrix puppet of own Telegram account --- ...dd_access_token_and_custom_mxid_fields_.py | 26 +++++++++ mautrix_telegram/abstract_user.py | 4 +- mautrix_telegram/commands/auth.py | 20 +++++++ mautrix_telegram/db.py | 2 + mautrix_telegram/matrix.py | 30 +++++----- mautrix_telegram/puppet.py | 55 ++++++++++++++----- mautrix_telegram/user.py | 4 +- 7 files changed, 108 insertions(+), 33 deletions(-) create mode 100644 alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py diff --git a/alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py b/alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py new file mode 100644 index 00000000..5c5a940a --- /dev/null +++ b/alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py @@ -0,0 +1,26 @@ +"""Add access_token and custom_mxid fields for puppets + +Revision ID: d5f7b8b4b456 +Revises: 6ca3d74d51e4 +Create Date: 2018-07-20 12:09:30.277960 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "d5f7b8b4b456" +down_revision = "6ca3d74d51e4" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True)) + op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True)) + + +def downgrade(): + with op.batch_alter_table("puppet") as batch_op: + batch_op.drop_column("custom_mxid") + batch_op.drop_column("access_token") diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index 1095e993..a4ae028c 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -229,9 +229,9 @@ class AbstractUser: async def update_status(self, update): puppet = pu.Puppet.get(update.user_id) if isinstance(update.status, UserStatusOnline): - await puppet.intent.set_presence("online") + await puppet.default_mxid_intent.set_presence("online") elif isinstance(update.status, UserStatusOffline): - await puppet.intent.set_presence("offline") + await puppet.default_mxid_intent.set_presence("offline") else: self.log.warning("Unexpected user status update: %s", update) return diff --git a/mautrix_telegram/commands/auth.py b/mautrix_telegram/commands/auth.py index 67c7ab64..38bcdf52 100644 --- a/mautrix_telegram/commands/auth.py +++ b/mautrix_telegram/commands/auth.py @@ -49,6 +49,26 @@ async def ping_bot(evt: CommandEvent): "To use the bot, simply invite it to a portal room.") +@command_handler(needs_auth=True, management_only=True, + help_section=SECTION_AUTH, + help_args="<_token_>", + help_text="Replace your Telegram account's Matrix puppet with your own Matrix " + "account") +async def login_matrix(evt: CommandEvent): + puppet = pu.Puppet.get(evt.sender.tgid) + prev_info = puppet.custom_mxid, puppet.access_token + puppet.custom_mxid = evt.sender.mxid + puppet.access_token = " ".join(evt.args) + puppet.refresh_intents() + if not await puppet.get_profile(): + puppet.custom_mxid, puppet.access_token = prev_info + puppet.refresh_intents() + return await evt.reply("Failed to verify access token.") + puppet.save() + return await evt.reply( + f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.") + + @command_handler(needs_auth=False, management_only=True, help_section=SECTION_AUTH, help_args="<_phone_> <_full name_>", diff --git a/mautrix_telegram/db.py b/mautrix_telegram/db.py index 5393acad..32099908 100644 --- a/mautrix_telegram/db.py +++ b/mautrix_telegram/db.py @@ -137,6 +137,8 @@ class Puppet(Base): __tablename__ = "puppet" id = Column(Integer, primary_key=True) + custom_mxid = Column(String, nullable=True) + access_token = Column(String, nullable=True) displayname = Column(String, nullable=True) displayname_source = Column(Integer, nullable=True) username = Column(String, nullable=True) diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index 30604296..6da5a9f5 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -39,7 +39,8 @@ class MatrixHandler: displayname = self.config["appservice.bot_displayname"] if displayname: try: - await self.az.intent.set_display_name(displayname if displayname != "remove" else "") + await self.az.intent.set_display_name( + displayname if displayname != "remove" else "") except asyncio.TimeoutError: self.log.exception("TimeoutError when trying to set displayname") @@ -51,19 +52,20 @@ class MatrixHandler: self.log.exception("TimeoutError when trying to set avatar") async def handle_puppet_invite(self, room, puppet, inviter): + intent = puppet.default_mxid_intent self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}") if not await inviter.is_logged_in(): - await puppet.intent.error_and_leave( + await intent.error_and_leave( room, text="Please log in before inviting Telegram puppets.") return portal = Portal.get_by_mxid(room) if portal: if portal.peer_type == "user": - await puppet.intent.error_and_leave( + await intent.error_and_leave( room, text="You can not invite additional users to private chats.") return await portal.invite_telegram(inviter, puppet) - await puppet.intent.join_room(room) + await intent.join_room(room) return try: members = await self.az.intent.get_room_members(room) @@ -71,34 +73,34 @@ class MatrixHandler: members = [] if self.az.bot_mxid not in members: if len(members) > 1: - await puppet.intent.error_and_leave(room, text=None, html=( + await intent.error_and_leave(room, text=None, html=( f"Please invite " f"the bridge bot " f"first if you want to create a Telegram chat.")) return - await puppet.intent.join_room(room) + await intent.join_room(room) portal = Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user") if portal.mxid: try: - await puppet.intent.invite(portal.mxid, inviter.mxid) - await puppet.intent.send_notice(room, text=None, html=( + await intent.invite(portal.mxid, inviter.mxid) + await intent.send_notice(room, text=None, html=( "You already have a private chat with me: " f"" "Link to room" "")) - await puppet.intent.leave_room(room) + await intent.leave_room(room) return except MatrixRequestError: pass portal.mxid = room portal.save() inviter.register_portal(portal) - await puppet.intent.send_notice(room, "Portal to private chat created.") + await intent.send_notice(room, "Portal to private chat created.") else: - await puppet.intent.join_room(room) - await puppet.intent.send_notice(room, "This puppet will remain inactive until a " - "Telegram chat is created for this room.") + await intent.join_room(room) + await intent.send_notice(room, "This puppet will remain inactive until a " + "Telegram chat is created for this room.") async def accept_bot_invite(self, room, inviter): tries = 0 @@ -215,7 +217,7 @@ class MatrixHandler: await portal.handle_matrix_message(sender, message, event_id) return - if not sender.whitelisted or message["msgtype"] != "m.text": + if not sender.whitelisted or message.get("msgtype", "m.unknown") != "m.text": return try: diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index b0977036..1403e64d 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -15,10 +15,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from difflib import SequenceMatcher +from typing import Optional import re import logging from telethon.tl.types import UserProfilePhoto +from mautrix_appservice import MatrixError, IntentAPI from .db import Puppet as DBPuppet from . import util @@ -35,10 +37,15 @@ class Puppet: hs_domain = None cache = {} - def __init__(self, id=None, username=None, displayname=None, displayname_source=None, - photo_id=None, is_bot=None, is_registered=False, db_instance=None): + def __init__(self, id=None, access_token=None, custom_mxid=None, username=None, + displayname=None, displayname_source=None, photo_id=None, is_bot=None, + is_registered=False, db_instance=None): self.id = id - self.mxid = self.get_mxid_from_id(self.id) + self.access_token = access_token + self.custom_mxid = custom_mxid + self.is_real_user = self.custom_mxid and self.access_token + self.default_mxid = self.get_mxid_from_id(self.id) + self.mxid = self.custom_mxid or self.default_mxid self.username = username self.displayname = displayname @@ -48,10 +55,23 @@ class Puppet: self.is_registered = is_registered self._db_instance = db_instance - self.intent = self.az.intent.user(self.mxid) + self.default_mxid_intent = self.az.intent.user(self.default_mxid) + self.intent = None # type: IntentAPI + self.refresh_intents() self.cache[id] = self + def refresh_intents(self): + self.is_real_user = self.custom_mxid and self.access_token + self.intent = (self.az.intent.user(self.custom_mxid, self.access_token) + if self.is_real_user else self.default_mxid_intent) + + async def get_profile(self): + try: + return await self.intent.get_profile(self.custom_mxid) + except MatrixError: + return None + @property def tgid(self): return self.id @@ -66,17 +86,21 @@ class Puppet: return self._db_instance def new_db_instance(self): - return DBPuppet(id=self.id, username=self.username, displayname=self.displayname, + return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid, + username=self.username, displayname=self.displayname, displayname_source=self.displayname_source, photo_id=self.photo_id, is_bot=self.is_bot, matrix_registered=self.is_registered) @classmethod def from_db(cls, db_puppet): - return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname, - db_puppet.displayname_source, db_puppet.photo_id, db_puppet.is_bot, - db_puppet.matrix_registered, db_instance=db_puppet) + return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid, + db_puppet.username, db_puppet.displayname, db_puppet.displayname_source, + db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered, + db_instance=db_puppet) def save(self): + self.db_instance.access_token = self.access_token + self.db_instance.custom_mxid = self.custom_mxid self.db_instance.username = self.username self.db_instance.displayname = self.displayname self.db_instance.displayname_source = self.displayname_source @@ -145,7 +169,7 @@ class Puppet: displayname = self.get_displayname(info) if displayname != self.displayname: - await self.intent.set_display_name(displayname) + await self.default_mxid_intent.set_display_name(displayname) self.displayname = displayname self.displayname_source = source.tgid return True @@ -156,15 +180,16 @@ class Puppet: async def update_avatar(self, source, photo): photo_id = f"{photo.volume_id}-{photo.local_id}" if self.photo_id != photo_id: - file = await util.transfer_file_to_matrix(self.db, source.client, self.intent, photo) + file = await util.transfer_file_to_matrix(self.db, source.client, + self.default_mxid_intent, photo) if file: - await self.intent.set_avatar(file.mxc) + await self.default_mxid_intent.set_avatar(file.mxc) self.photo_id = photo_id return True return False @classmethod - def get(cls, id, create=True): + def get(cls, id, create=True) -> "Optional[Puppet]": try: return cls.cache[id] except KeyError: @@ -183,7 +208,7 @@ class Puppet: return None @classmethod - def get_by_mxid(cls, mxid, create=True): + def get_by_mxid(cls, mxid, create=True) -> "Optional[Puppet]": tgid = cls.get_id_from_mxid(mxid) return cls.get(tgid, create) if tgid else None @@ -199,7 +224,7 @@ class Puppet: return f"@{cls.username_template.format(userid=id)}:{cls.hs_domain}" @classmethod - def find_by_username(cls, username): + def find_by_username(cls, username) -> "Optional[Puppet]": if not username: return None @@ -214,7 +239,7 @@ class Puppet: return None @classmethod - def find_by_displayname(cls, displayname): + def find_by_displayname(cls, displayname) -> "Optional[Puppet]": if not displayname: return None diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index ea0b92e3..43ad5dec 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -111,14 +111,14 @@ class User(AbstractUser): def new_db_instance(self): return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username, - contacts=self.db_contacts, saved_contacts=self.saved_contacts, + contacts=self.db_contacts, saved_contacts=self.saved_contacts or 0, portals=self.db_portals) def save(self): self.db_instance.tgid = self.tgid self.db_instance.username = self.username self.db_instance.contacts = self.db_contacts - self.db_instance.saved_contacts = self.saved_contacts + self.db_instance.saved_contacts = self.saved_contacts or 0 self.db_instance.portals = self.db_portals self.db.commit() From ecdca21e32ecee9e0cc578422cf2bdfe12d27a50 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 20 Jul 2018 14:13:13 -0400 Subject: [PATCH 52/65] Stop handling events from custom puppets --- mautrix_telegram/__main__.py | 6 ++- mautrix_telegram/abstract_user.py | 2 +- mautrix_telegram/commands/auth.py | 12 ++---- mautrix_telegram/db.py | 23 +++++------ mautrix_telegram/portal.py | 7 +++- mautrix_telegram/puppet.py | 67 ++++++++++++++++++++++++++++--- mautrix_telegram/user.py | 11 +++-- 7 files changed, 94 insertions(+), 34 deletions(-) diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index 430696eb..fff2d868 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -110,8 +110,10 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st context.mx = MatrixHandler(context) init_formatter(context) init_portal(context) - init_puppet(context) - startup_actions = init_user(context) + [start, context.mx.init_as_bot()] + startup_actions = (init_puppet(context) + + init_user(context) + + [start, + context.mx.init_as_bot()]) if context.bot: startup_actions.append(context.bot.start()) diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index a4ae028c..dd17234f 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -124,7 +124,7 @@ class AbstractUser: self.log.debug("%s connected: %s", self.mxid, self.connected) return self - async def ensure_started(self, even_if_no_session=False): + async def ensure_started(self, even_if_no_session=False) -> "AbstractUser": if not self.puppet_whitelisted: return self self.log.debug("ensure_started(%s, connected=%s, even_if_no_session=%s, session_count=%s)", diff --git a/mautrix_telegram/commands/auth.py b/mautrix_telegram/commands/auth.py index 38bcdf52..33c88cb7 100644 --- a/mautrix_telegram/commands/auth.py +++ b/mautrix_telegram/commands/auth.py @@ -56,15 +56,11 @@ async def ping_bot(evt: CommandEvent): "account") async def login_matrix(evt: CommandEvent): puppet = pu.Puppet.get(evt.sender.tgid) - prev_info = puppet.custom_mxid, puppet.access_token - puppet.custom_mxid = evt.sender.mxid - puppet.access_token = " ".join(evt.args) - puppet.refresh_intents() - if not await puppet.get_profile(): - puppet.custom_mxid, puppet.access_token = prev_info - puppet.refresh_intents() + resp = puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid) + if resp == 2: + return await evt.reply("You can only log in as your own Matrix user.") + elif resp == 1: return await evt.reply("Failed to verify access token.") - puppet.save() return await evt.reply( f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.") diff --git a/mautrix_telegram/db.py b/mautrix_telegram/db.py index 32099908..81bc0598 100644 --- a/mautrix_telegram/db.py +++ b/mautrix_telegram/db.py @@ -17,14 +17,14 @@ from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer, BigInteger, String, Boolean, Text) from sqlalchemy.sql import expression -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, Query import json from .base import Base class Portal(Base): - query = None + query = None # type: Query __tablename__ = "portal" # Telegram chat information @@ -42,9 +42,8 @@ class Portal(Base): about = Column(String, nullable=True) photo_id = Column(String, nullable=True) - class Message(Base): - query = None + query = None # type: Query __tablename__ = "message" mxid = Column(String) @@ -56,7 +55,7 @@ class Message(Base): class UserPortal(Base): - query = None + query = None # type: Query __tablename__ = "user_portal" user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"), @@ -70,7 +69,7 @@ class UserPortal(Base): class User(Base): - query = None + query = None # type: Query __tablename__ = "user" mxid = Column(String, primary_key=True) @@ -83,7 +82,7 @@ class User(Base): class RoomState(Base): - query = None + query = None # type: Query __tablename__ = "mx_room_state" room_id = Column(String, primary_key=True) @@ -107,7 +106,7 @@ class RoomState(Base): class UserProfile(Base): - query = None + query = None # type: Query __tablename__ = "mx_user_profile" room_id = Column(String, primary_key=True) @@ -125,7 +124,7 @@ class UserProfile(Base): class Contact(Base): - query = None + query = None # type: Query __tablename__ = "contact" user = Column(Integer, ForeignKey("user.tgid"), primary_key=True) @@ -133,7 +132,7 @@ class Contact(Base): class Puppet(Base): - query = None + query = None # type: Query __tablename__ = "puppet" id = Column(Integer, primary_key=True) @@ -149,14 +148,14 @@ class Puppet(Base): # Fucking Telegram not telling bots what chats they are in 3:< class BotChat(Base): - query = None + query = None # type: Query __tablename__ = "bot_chat" id = Column(Integer, primary_key=True) type = Column(String, nullable=False) class TelegramFile(Base): - query = None + query = None # type: Query __tablename__ = "telegram_file" id = Column(String, primary_key=True) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 35872438..c78869d2 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -824,7 +824,12 @@ class Portal: mxid=event_id)) self.db.commit() - async def handle_matrix_message(self, sender, message, event_id): + async def handle_matrix_message(self, sender: u.User, message: dict, event_id: str): + puppet = p.Puppet.get_by_custom_mxid(sender.mxid) + if puppet and message.get("net.maunium.telegram.puppet", False): + self.log.debug("Ignoring puppet-sent message by confirmed puppet user %s", sender.mxid) + return + logged_in = not await sender.needs_relaybot(self) client = sender.client if logged_in else self.bot.client sender_id = sender.tgid if logged_in else self.bot.tgid diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index 1403e64d..f708196e 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -36,6 +36,7 @@ class Puppet: username_template = None hs_domain = None cache = {} + by_custom_mxid = {} def __init__(self, id=None, access_token=None, custom_mxid=None, username=None, displayname=None, displayname_source=None, photo_id=None, is_bot=None, @@ -60,22 +61,51 @@ class Puppet: self.refresh_intents() self.cache[id] = self + if self.custom_mxid: + self.by_custom_mxid[self.custom_mxid] = self def refresh_intents(self): self.is_real_user = self.custom_mxid and self.access_token self.intent = (self.az.intent.user(self.custom_mxid, self.access_token) if self.is_real_user else self.default_mxid_intent) - async def get_profile(self): - try: - return await self.intent.get_profile(self.custom_mxid) - except MatrixError: - return None - @property def tgid(self): return self.id + async def switch_mxid(self, access_token, mxid): + prev_mxid = self.custom_mxid + self.custom_mxid = mxid + self.access_token = access_token + self.refresh_intents() + + err = await self.test_custom_mxid() + if err != 0: + return err + + try: + del self.by_custom_mxid[prev_mxid] + except KeyError: + pass + self.mxid = self.custom_mxid or self.default_mxid + self.by_custom_mxid[self.mxid] = self + self.save() + return 0 + + async def test_custom_mxid(self): + if not self.is_real_user: + return 0 + + mxid = await self.intent.whoami() + if not mxid or mxid != self.custom_mxid: + self.custom_mxid = None + self.access_token = None + self.refresh_intents() + if mxid != self.custom_mxid: + return 2 + return 1 + return 0 + async def is_logged_in(self): return True @@ -212,6 +242,30 @@ class Puppet: tgid = cls.get_id_from_mxid(mxid) return cls.get(tgid, create) if tgid else None + @classmethod + def get_by_custom_mxid(cls, mxid): + if not mxid: + raise ValueError("Matrix ID can't be empty") + + try: + return cls.by_custom_mxid[mxid] + except KeyError: + pass + + puppet = DBPuppet.query.filter(DBPuppet.custom_mxid == mxid).one_or_none() + if puppet: + puppet = cls.from_db(puppet) + return puppet + + return None + + @classmethod + def get_all_with_custom_mxid(cls): + return [cls.by_custom_mxid[puppet.mxid] + if puppet.custom_mxid in cls.by_custom_mxid + else cls.from_db(puppet) + for puppet in DBPuppet.query.filter(DBPuppet.custom_mxid is not None).all()] + @classmethod def get_id_from_mxid(cls, mxid): match = cls.mxid_regex.match(mxid) @@ -261,3 +315,4 @@ def init(context): Puppet.hs_domain = config["homeserver"]["domain"] localpart = Puppet.username_template.format(userid="(.+)") Puppet.mxid_regex = re.compile(f"@{localpart}:{Puppet.hs_domain}") + return [puppet.test_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()] diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index 43ad5dec..110c9f8e 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict +from typing import Dict, Awaitable, Optional import logging import asyncio import re @@ -185,6 +185,9 @@ class User(AbstractUser): # endregion # region Telegram actions that need custom methods + def ensure_started(self, even_if_no_session=False) -> "Awaitable[User]": + return super().ensure_started(even_if_no_session) + async def update_info(self, info: User = None): info = info or await self.client.get_me() changed = False @@ -309,7 +312,7 @@ class User(AbstractUser): # region Class instance lookup @classmethod - def get_by_mxid(cls, mxid, create=True): + def get_by_mxid(cls, mxid, create=True) -> "Optional[User]": if not mxid: raise ValueError("Matrix ID can't be empty") @@ -332,7 +335,7 @@ class User(AbstractUser): return None @classmethod - def get_by_tgid(cls, tgid): + def get_by_tgid(cls, tgid) -> "Optional[User]": try: return cls.by_tgid[tgid] except KeyError: @@ -346,7 +349,7 @@ class User(AbstractUser): return None @classmethod - def find_by_username(cls, username): + def find_by_username(cls, username) -> "Optional[User]": if not username: return None From 54287c344fa55c441df52dfad4b7a66e47f50376 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 21 Jul 2018 10:45:29 -0400 Subject: [PATCH 53/65] Implement syncing with custom puppets --- mautrix_telegram/matrix.py | 21 ++++++--- mautrix_telegram/puppet.py | 96 ++++++++++++++++++++++++++++++++++---- 2 files changed, 103 insertions(+), 14 deletions(-) diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index 6da5a9f5..350ee029 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -289,17 +289,26 @@ class MatrixHandler: await portal.name_change_matrix(user, displayname, prev_displayname, event_id) def filter_matrix_event(self, event): - return (event["sender"] == self.az.bot_mxid - or Puppet.get_id_from_mxid(event["sender"]) is not None) + sender = event.get("sender", None) + if not sender: + return False + return (sender == self.az.bot_mxid + or Puppet.get_id_from_mxid(sender) is not None) + + async def try_handle_event(self, evt): + try: + await self.handle_event(evt) + except Exception: + self.log.exception("Error handling manually received Matrix event") async def handle_event(self, evt): if self.filter_matrix_event(evt): return self.log.debug("Received event: %s", evt) - type = evt["type"] - room_id = evt["room_id"] - event_id = evt["event_id"] - sender = evt["sender"] + type = evt.get("type", "m.unknown") + room_id = evt.get("room_id", None) + event_id = evt.get("event_id", None) + sender = evt.get("sender", None) content = evt.get("content", {}) if type == "m.room.member": state_key = evt["state_key"] diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index f708196e..1fbe6f76 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -15,15 +15,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from difflib import SequenceMatcher -from typing import Optional +from typing import Optional, Awaitable import re import logging +import asyncio from telethon.tl.types import UserProfilePhoto -from mautrix_appservice import MatrixError, IntentAPI +from mautrix_appservice import AppService, IntentAPI, MatrixRequestError from .db import Puppet as DBPuppet -from . import util +from . import util, matrix config = None @@ -31,7 +32,9 @@ config = None class Puppet: log = logging.getLogger("mau.puppet") db = None - az = None + az = None # type: AppService + mx = None # type: matrix.MatrixHandler + loop = None # type: asyncio.AbstractEventLoop mxid_regex = None username_template = None hs_domain = None @@ -79,7 +82,7 @@ class Puppet: self.access_token = access_token self.refresh_intents() - err = await self.test_custom_mxid() + err = await self.init_custom_mxid() if err != 0: return err @@ -92,7 +95,7 @@ class Puppet: self.save() return 0 - async def test_custom_mxid(self): + async def init_custom_mxid(self): if not self.is_real_user: return 0 @@ -104,8 +107,84 @@ class Puppet: if mxid != self.custom_mxid: return 2 return 1 + asyncio.ensure_future(self.sync(), loop=self.loop) return 0 + def create_sync_filter(self) -> Awaitable[str]: + return self.intent.client.create_filter(self.custom_mxid, { + "room": { + "include_leave": False, + "state": { + "types": [] + }, + "timeline": { + "types": [], + }, + "ephemeral": { + "types": ["m.typing", "m.receipt"] + }, + "account_data": { + "types": [] + } + }, + "account_data": { + "types": [], + }, + "presence": { + "types": ["m.presence"] + }, + }) + + def handle_sync(self, presence, ephemeral): + presence = [self.mx.try_handle_event(event) for event in presence] + + for room_id, events in ephemeral.items(): + for event in events: + event["room_id"] = room_id + + ephemeral = [self.mx.try_handle_event(event) + for events in ephemeral.values() + for event in events] + + events = ephemeral + presence + coro = asyncio.gather(*events, loop=self.loop) + asyncio.ensure_future(coro, loop=self.loop) + + async def sync(self): + try: + await self._sync() + except Exception: + self.log.exception("Fatal error syncing") + + async def _sync(self): + if not self.is_real_user: + self.log.warning("Called sync() for non-custom puppet.") + return + custom_mxid = self.custom_mxid + access_token_at_start = self.access_token + errors = 0 + next_batch = None + filter_id = await self.create_sync_filter() + self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.") + while access_token_at_start == self.access_token: + try: + sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch) + errors = 0 + if next_batch is not None: + presence = sync_resp.get("presence", {}).get("events", []) + ephemeral = {room: data.get("ephemeral", {}).get("events", []) + for room, data + in sync_resp.get("rooms", {}).get("join", {}).items()} + self.handle_sync(presence, ephemeral) + next_batch = sync_resp.get("next_batch", None) + except MatrixRequestError as e: + wait = min(errors, 11) ** 2 + self.log.warning(f"Syncer for {custom_mxid} errored: {e}. " + f"Waiting for {wait} seconds...") + errors += 1 + await asyncio.sleep(wait) + self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.") + async def is_logged_in(self): return True @@ -310,9 +389,10 @@ class Puppet: def init(context): global config - Puppet.az, Puppet.db, config, _, _ = context + Puppet.az, Puppet.db, config, Puppet.loop, _ = context + Puppet.mx = context.mx Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}") Puppet.hs_domain = config["homeserver"]["domain"] localpart = Puppet.username_template.format(userid="(.+)") Puppet.mxid_regex = re.compile(f"@{localpart}:{Puppet.hs_domain}") - return [puppet.test_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()] + return [puppet.init_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()] From 54d7ac5542019cf9c52ed1e206d5ba5142405db6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 21 Jul 2018 11:19:19 -0400 Subject: [PATCH 54/65] Implement Matrix->Telegram typing notifications --- mautrix_telegram/abstract_user.py | 18 +++++++------- mautrix_telegram/matrix.py | 39 +++++++++++++++++++++++++++++++ mautrix_telegram/user.py | 5 ++++ 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index dd17234f..0de32c91 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -36,15 +36,15 @@ class AbstractUser: az = None def __init__(self): - self.puppet_whitelisted = False - self.whitelisted = False - self.relaybot_whitelisted = False - self.is_admin = False - self.client = None - self.tgid = None - self.mxid = None - self.is_relaybot = False - self.is_bot = False + self.puppet_whitelisted = False # type: bool + self.whitelisted = False # type: bool + self.relaybot_whitelisted = False # type: bool + self.is_admin = False # type: bool + self.client = None # type: MautrixTelegramClient + self.tgid = None # type: int + self.mxid = None # type: str + self.is_relaybot = False # type: bool + self.is_bot = False # type: bool @property def connected(self): diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index 350ee029..e78e8a49 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -14,6 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import List import logging import asyncio import re @@ -32,6 +33,7 @@ class MatrixHandler: def __init__(self, context): self.az, self.db, self.config, _, self.tgbot = context self.commands = CommandProcessor(context) + self.previously_typing = [] self.az.matrix_event_handler(self.handle_event) @@ -288,6 +290,37 @@ class MatrixHandler: if await user.needs_relaybot(portal): await portal.name_change_matrix(user, displayname, prev_displayname, event_id) + @staticmethod + def parse_read_receipts(content: dict) -> dict: + return {user_id: event_id + for event_id, receipts in content.items() + for user_id in receipts.get("m.read", {})} + + async def handle_read_receipts(self, room_id: str, receipts: dict): + pass + + async def handle_presence(self, user: str, presence: str): + pass + + async def handle_typing(self, room_id: str, now_typing: List[str]): + portal = Portal.get_by_mxid(room_id) + if not portal: + return + + for user_id in set(self.previously_typing + now_typing): + is_typing = user_id in now_typing + was_typing = user_id in self.previously_typing + if is_typing and was_typing: + continue + + user = await User.get_by_mxid(user_id).ensure_started() + if not user.tgid: + continue + + await user.set_typing(portal.peer, is_typing) + + self.previously_typing = now_typing + def filter_matrix_event(self, event): sender = event.get("sender", None) if not sender: @@ -346,3 +379,9 @@ class MatrixHandler: except KeyError: old_events = set() await self.handle_room_pin(room_id, sender, new_events, old_events) + elif type == "m.receipt": + await self.handle_read_receipts(room_id, self.parse_read_receipts(content)) + elif type == "m.presence": + await self.handle_presence(sender, content.get("presence", "offline")) + elif type == "m.typing": + await self.handle_typing(room_id, content.get("user_ids", [])) diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index 110c9f8e..fb9e426f 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -22,6 +22,7 @@ import re from telethon.tl.types import * from telethon.tl.types.contacts import ContactsNotModified from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest +from telethon.tl.functions.messages import SetTypingRequest from mautrix_appservice import MatrixRequestError from .db import User as DBUser, Contact as DBContact @@ -203,6 +204,10 @@ class User(AbstractUser): if changed: self.save() + def set_typing(self, peer, typing=True, action=SendMessageTypingAction): + return self.client( + SetTypingRequest(peer, action() if typing else SendMessageCancelAction())) + async def log_out(self): for _, portal in self.portals.items(): if portal.has_bot: From e4e100a184e7b97aae5233459bfd45dcccb50414 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 21 Jul 2018 11:23:34 -0400 Subject: [PATCH 55/65] Add option to disable /syncing with custom puppets --- example-config.yaml | 3 +++ mautrix_telegram/config.py | 1 + mautrix_telegram/puppet.py | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/example-config.yaml b/example-config.yaml index 7668c236..4f0aa3b1 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -125,6 +125,9 @@ bridge: # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down. # WARNING: Probably buggy, might get stuck in infinite loop. catch_up: false + # Whether or not to use /sync to get presence, read receipts and typing notifications when using + # your own Matrix account as the Matrix puppet for your Telegram account. + sync_with_custom_puppets: true # The formats to use when sending messages to Telegram via the relay bot. # diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 1dad0fb4..b7766f47 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -191,6 +191,7 @@ class Config(DictWithRecursion): copy("bridge.public_portals") copy("bridge.native_stickers") copy("bridge.catch_up") + copy("bridge.sync_with_custom_puppets") if "bridge.message_formats.m_text" in self: del self["bridge.message_formats"] diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index 1fbe6f76..f3da0c37 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -107,7 +107,8 @@ class Puppet: if mxid != self.custom_mxid: return 2 return 1 - asyncio.ensure_future(self.sync(), loop=self.loop) + if config["bridge.sync_with_custom_puppets"]: + asyncio.ensure_future(self.sync(), loop=self.loop) return 0 def create_sync_filter(self) -> Awaitable[str]: From af46aee191c29b79f52d5a52cd0c4c42d8ad81d8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 22 Jul 2018 17:27:56 -0400 Subject: [PATCH 56/65] Implement Matrix->Telegram read receipts --- mautrix_telegram/matrix.py | 20 ++++++++++++++------ mautrix_telegram/portal.py | 19 +++++++++++++++++++ mautrix_telegram/user.py | 5 ----- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index e78e8a49..81660a9b 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import List +from typing import List, Dict import logging import asyncio import re @@ -291,13 +291,21 @@ class MatrixHandler: await portal.name_change_matrix(user, displayname, prev_displayname, event_id) @staticmethod - def parse_read_receipts(content: dict) -> dict: + def parse_read_receipts(content: dict) -> Dict[str, str]: return {user_id: event_id for event_id, receipts in content.items() for user_id in receipts.get("m.read", {})} - async def handle_read_receipts(self, room_id: str, receipts: dict): - pass + async def handle_read_receipts(self, room_id: str, receipts: Dict[str, str]): + portal = Portal.get_by_mxid(room_id) + if not portal: + return + + for user_id, event_id in receipts.items(): + user = await User.get_by_mxid(user_id).ensure_started() + if not await user.is_logged_in(): + continue + await portal.mark_read(user, event_id) async def handle_presence(self, user: str, presence: str): pass @@ -314,10 +322,10 @@ class MatrixHandler: continue user = await User.get_by_mxid(user_id).ensure_started() - if not user.tgid: + if not await user.is_logged_in(): continue - await user.set_typing(portal.peer, is_typing) + await portal.set_typing(user, is_typing) self.previously_typing = now_typing diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index c78869d2..687d137a 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -31,6 +31,8 @@ from sqlalchemy.orm.exc import FlushError from telethon.tl.functions.messages import * from telethon.tl.functions.channels import * +from telethon.tl.functions.messages import ReadHistoryRequest +from telethon.tl.functions.channels import ReadHistoryRequest as ReadChannelHistoryRequest from telethon.errors import * from telethon.tl.types import * from mautrix_appservice import MatrixRequestError, IntentError @@ -652,6 +654,23 @@ class Portal: return (await self.main_intent.get_displayname(self.mxid, user.mxid) or user.mxid_localpart) + def set_typing(self, user, typing=True, action=SendMessageTypingAction): + return user.client( + SetTypingRequest(self.peer, action() if typing else SendMessageCancelAction())) + + async def mark_read(self, user, event_id): + space = self.tgid if self.peer_type == "channel" else user.tgid + message = DBMessage.query.filter(DBMessage.mxid == event_id, + DBMessage.mx_room == self.mxid, + DBMessage.tg_space == space).one_or_none() + if not message: + return + if self.peer_type == "channel": + await user.client(ReadChannelHistoryRequest( + channel=await self.get_input_entity(user), max_id=message.tgid)) + else: + await user.client(ReadHistoryRequest(peer=self.peer, max_id=message.tgid)) + async def leave_matrix(self, user, source, event_id): if await user.needs_relaybot(self): async with self.require_send_lock(self.bot.tgid): diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index fb9e426f..110c9f8e 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -22,7 +22,6 @@ import re from telethon.tl.types import * from telethon.tl.types.contacts import ContactsNotModified from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest -from telethon.tl.functions.messages import SetTypingRequest from mautrix_appservice import MatrixRequestError from .db import User as DBUser, Contact as DBContact @@ -204,10 +203,6 @@ class User(AbstractUser): if changed: self.save() - def set_typing(self, peer, typing=True, action=SendMessageTypingAction): - return self.client( - SetTypingRequest(peer, action() if typing else SendMessageCancelAction())) - async def log_out(self): for _, portal in self.portals.items(): if portal.has_bot: From 76410ee7cbfa7c4357dc5f9b18b13a4b82a0b67f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 22 Jul 2018 17:42:29 -0400 Subject: [PATCH 57/65] Implement Matrix->Telegram presence --- mautrix_telegram/matrix.py | 5 ++++- mautrix_telegram/user.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index 81660a9b..28cf6796 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -308,7 +308,10 @@ class MatrixHandler: await portal.mark_read(user, event_id) async def handle_presence(self, user: str, presence: str): - pass + user = await User.get_by_mxid(user).ensure_started() + if not await user.is_logged_in(): + return + await user.set_presence(presence == "online") async def handle_typing(self, room_id: str, now_typing: List[str]): portal = Portal.get_by_mxid(room_id) diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index 110c9f8e..9c42089a 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -22,6 +22,7 @@ import re from telethon.tl.types import * from telethon.tl.types.contacts import ContactsNotModified from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest +from telethon.tl.functions.account import UpdateStatusRequest from mautrix_appservice import MatrixRequestError from .db import User as DBUser, Contact as DBContact @@ -188,6 +189,9 @@ class User(AbstractUser): def ensure_started(self, even_if_no_session=False) -> "Awaitable[User]": return super().ensure_started(even_if_no_session) + def set_presence(self, online: bool = True): + return self.client(UpdateStatusRequest(offline=not online)) + async def update_info(self, info: User = None): info = info or await self.client.get_me() changed = False From ab098879fdeb48a8b9ea0594141ae90161ca2948 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 22 Jul 2018 18:08:18 -0400 Subject: [PATCH 58/65] Don't set presence when /syncing custom puppets --- mautrix_telegram/puppet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index f3da0c37..e128561f 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -169,7 +169,8 @@ class Puppet: self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.") while access_token_at_start == self.access_token: try: - sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch) + sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch, + set_presence="offline") errors = 0 if next_batch is not None: presence = sync_resp.get("presence", {}).get("events", []) From f3e1c755ebf7b148bb8150b84a0031ed21c33e93 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 22 Jul 2018 18:22:13 -0400 Subject: [PATCH 59/65] Bump mautrix-appservice version requirement --- mautrix_telegram/__main__.py | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index fff2d868..ad4cb4ef 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -86,7 +86,8 @@ state_store = SQLStateStore(db_session) appserv = AppService(config["homeserver.address"], config["homeserver.domain"], config["appservice.as_token"], config["appservice.hs_token"], config["appservice.bot_username"], log="mau.as", loop=loop, - verify_ssl=config["homeserver.verify_ssl"], state_store=state_store) + verify_ssl=config["homeserver.verify_ssl"], state_store=state_store, + real_user_content_key="net.maunium.telegram.puppet") public_website = None provisioning_api = None diff --git a/setup.py b/setup.py index 1ed8106d..c2d72e38 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setuptools.setup( install_requires=[ "aiohttp>=3.0.1,<4", - "mautrix-appservice>=0.3.0,<0.4.0", + "mautrix-appservice>=0.3.1,<0.4.0", "SQLAlchemy>=1.2.3,<2", "alembic>=1.0.0,<2", "Markdown>=2.6.11,<3", From 473668645450a8389e22ba6f3def015f60a7abe4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 23 Jul 2018 11:49:42 -0400 Subject: [PATCH 60/65] Implement Matrix login with web interface --- mautrix_telegram/commands/auth.py | 58 ++++++++++++-- mautrix_telegram/puppet.py | 36 +++++++-- mautrix_telegram/web/common/auth_api.py | 28 +++++++ mautrix_telegram/web/provisioning/__init__.py | 4 + mautrix_telegram/web/public/__init__.py | 69 ++++++++++++++-- mautrix_telegram/web/public/login.css | 53 ++++++++++++- mautrix_telegram/web/public/login.html.mako | 10 +-- .../web/public/matrix-login.html.mako | 78 +++++++++++++++++++ 8 files changed, 311 insertions(+), 25 deletions(-) create mode 100644 mautrix_telegram/web/public/matrix-login.html.mako diff --git a/mautrix_telegram/commands/auth.py b/mautrix_telegram/commands/auth.py index 33c88cb7..9d52ab0d 100644 --- a/mautrix_telegram/commands/auth.py +++ b/mautrix_telegram/commands/auth.py @@ -49,14 +49,62 @@ async def ping_bot(evt: CommandEvent): "To use the bot, simply invite it to a portal room.") +@command_handler(needs_auth=True, + help_section=SECTION_AUTH, + help_text="Revert your Telegram account's Matrix puppet to use the default Matrix " + "account.") +async def logout_matrix(evt: CommandEvent): + puppet = pu.Puppet.get(evt.sender.tgid) + if not puppet.is_real_user: + return await evt.reply("You are not logged in with your Matrix account.") + await puppet.switch_mxid(None, None) + await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.") + + @command_handler(needs_auth=True, management_only=True, help_section=SECTION_AUTH, - help_args="<_token_>", help_text="Replace your Telegram account's Matrix puppet with your own Matrix " "account") async def login_matrix(evt: CommandEvent): puppet = pu.Puppet.get(evt.sender.tgid) - resp = puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid) + if puppet.is_real_user: + return await evt.reply("You have already logged in with your Matrix account. " + "Log out with `$cmdprefix+sp logout-matrix` first.") + allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True) + if allow_matrix_login: + evt.sender.command_status = { + "next": enter_matrix_token, + "action": "Matrix login", + } + if evt.config["appservice.public.enabled"]: + prefix = evt.config["appservice.public.external"] + url = f"{prefix}/matrix-login?token={evt.public_website.make_token(evt.sender.mxid, '/matrix-login')}" + if allow_matrix_login: + return await evt.reply( + "This bridge instance allows you to log in inside or outside Matrix.\n\n" + "If you would like to log in within Matrix, please send your Matrix access token " + "here.\n" + f"If you would like to log in outside of Matrix, [click here]({url}).\n\n" + "Logging in outside of Matrix is recommended, because in-Matrix login would save " + "your access token in the message history.") + return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n" + f"Please visit [the login page]({url}) to log in.") + elif allow_matrix_login: + return await evt.reply( + "This bridge instance does not allow you to log in outside of Matrix.\n\n" + "Please send your Matrix access token here to log in.") + return await evt.reply("This bridge instance has been configured to not allow logging in.") + + +async def enter_matrix_token(evt: CommandEvent): + evt.sender.command_status = None + + puppet = pu.Puppet.get(evt.sender.tgid) + if puppet.is_real_user: + return await evt.reply("You have already logged in with your Matrix account. " + "Log out with `$cmdprefix+sp logout-matrix` first.") + + resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid) if resp == 2: return await evt.reply("You can only log in as your own Matrix user.") elif resp == 1: @@ -130,8 +178,8 @@ async def login(evt: CommandEvent): if evt.config["appservice.public.enabled"]: prefix = evt.config["appservice.public.external"] - url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid)}" - if evt.config.get("bridge.allow_matrix_login", True): + url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}" + if allow_matrix_login: return await evt.reply( "This bridge instance allows you to log in inside or outside Matrix.\n\n" "If you would like to log in within Matrix, please send your phone number or bot " @@ -144,7 +192,7 @@ async def login(evt: CommandEvent): elif allow_matrix_login: return await evt.reply( "This bridge instance does not allow you to log in outside of Matrix.\n\n" - "Please send your phone number or bot aut token here to start the login process.") + "Please send your phone number or bot auth token here to start the login process.") return await evt.reply("This bridge instance has been configured to not allow logging in.") diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index e128561f..222fd972 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -21,7 +21,7 @@ import logging import asyncio from telethon.tl.types import UserProfilePhoto -from mautrix_appservice import AppService, IntentAPI, MatrixRequestError +from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError from .db import Puppet as DBPuppet from . import util, matrix @@ -67,15 +67,19 @@ class Puppet: if self.custom_mxid: self.by_custom_mxid[self.custom_mxid] = self + @property + def tgid(self): + return self.id + + async def is_logged_in(self): + return True + + # region Custom puppet management def refresh_intents(self): self.is_real_user = self.custom_mxid and self.access_token self.intent = (self.az.intent.user(self.custom_mxid, self.access_token) if self.is_real_user else self.default_mxid_intent) - @property - def tgid(self): - return self.id - async def switch_mxid(self, access_token, mxid): prev_mxid = self.custom_mxid self.custom_mxid = mxid @@ -91,7 +95,9 @@ class Puppet: except KeyError: pass self.mxid = self.custom_mxid or self.default_mxid - self.by_custom_mxid[self.mxid] = self + if self.mxid != self.default_mxid: + self.by_custom_mxid[self.mxid] = self + await self.leave_rooms_with_default_user() self.save() return 0 @@ -111,6 +117,14 @@ class Puppet: asyncio.ensure_future(self.sync(), loop=self.loop) return 0 + async def leave_rooms_with_default_user(self): + for room_id in await self.default_mxid_intent.get_joined_rooms(): + try: + await self.default_mxid_intent.leave_room(room_id) + await self.intent.ensure_joined(room_id) + except (IntentError, MatrixRequestError): + pass + def create_sync_filter(self) -> Awaitable[str]: return self.intent.client.create_filter(self.custom_mxid, { "room": { @@ -187,8 +201,8 @@ class Puppet: await asyncio.sleep(wait) self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.") - async def is_logged_in(self): - return True + # endregion + # region DB conversion @property def db_instance(self): @@ -220,6 +234,8 @@ class Puppet: self.db_instance.matrix_registered = self.is_registered self.db.commit() + # endregion + # region Info updating def similarity(self, query): username_similarity = (SequenceMatcher(None, self.username, query).ratio() if self.username else 0) @@ -299,6 +315,9 @@ class Puppet: return True return False + # endregion + # region Getters + @classmethod def get(cls, id, create=True) -> "Optional[Puppet]": try: @@ -387,6 +406,7 @@ class Puppet: return cls.from_db(puppet) return None + # endregion def init(context): diff --git a/mautrix_telegram/web/common/auth_api.py b/mautrix_telegram/web/common/auth_api.py index 70b66136..24fa74e9 100644 --- a/mautrix_telegram/web/common/auth_api.py +++ b/mautrix_telegram/web/common/auth_api.py @@ -23,6 +23,8 @@ from telethon.errors import * from ...commands.auth import enter_password from ...util import format_duration +from ...puppet import Puppet +from ...user import User class AuthAPI(abc.ABC): @@ -36,6 +38,32 @@ class AuthAPI(abc.ABC): errcode=""): raise NotImplementedError() + @abstractmethod + def get_mx_login_response(self, status=200, state="", username="", mxid="", message="", + error="", errcode=""): + raise NotImplementedError() + + async def post_matrix_token(self, user: User, token): + puppet = Puppet.get(user.tgid) + if puppet.is_real_user: + return self.get_mx_login_response(state="already-logged-in", status=409, + error="You have already logged in with your Matrix " + "account.", errcode="already-logged-in") + + resp = await puppet.switch_mxid(token, user.mxid) + if resp == 2: + return self.get_mx_login_response(status=403, errcode="only-login-self", + error="You can only log in as your own Matrix user.") + elif resp == 1: + return self.get_mx_login_response(status=401, errcode="invalid-access-token", + error="Failed to verify access token.") + + return self.get_mx_login_response(mxid=user.mxid, status=200, state="logged-in") + + async def post_matrix_password(self, user, password): + return self.get_mx_login_response(mxid=user.mxid, status=501, error="Not yet implemented", + errcode="not-yet-implemented") + async def post_login_phone(self, user, phone): try: await user.client.sign_in(phone or "+123") diff --git a/mautrix_telegram/web/provisioning/__init__.py b/mautrix_telegram/web/provisioning/__init__.py index 63bb208d..731a91ff 100644 --- a/mautrix_telegram/web/provisioning/__init__.py +++ b/mautrix_telegram/web/provisioning/__init__.py @@ -297,6 +297,10 @@ class ProvisioningAPI(AuthAPI): "errcode": errcode, }, status=status) + def get_mx_login_response(self, status=200, state="", username="", mxid="", message="", + error="", errcode=""): + raise NotImplementedError() + def get_login_response(self, status=200, state="", username="", mxid="", message="", error="", errcode="") -> web.Response: if username: diff --git a/mautrix_telegram/web/public/__init__.py b/mautrix_telegram/web/public/__init__.py index fb5f6de7..be6bb3b0 100644 --- a/mautrix_telegram/web/public/__init__.py +++ b/mautrix_telegram/web/public/__init__.py @@ -24,6 +24,7 @@ import time from ...util import sign_token, verify_token from ...user import User +from ...puppet import Puppet from ..common import AuthAPI @@ -38,28 +39,35 @@ class PublicBridgeWebsite(AuthAPI): self.login = Template( pkg_resources.resource_string("mautrix_telegram", "web/public/login.html.mako")) + self.mx_login = Template( + pkg_resources.resource_string("mautrix_telegram", "web/public/matrix-login.html.mako")) + self.app = web.Application(loop=loop) self.app.router.add_route("GET", "/login", self.get_login) self.app.router.add_route("POST", "/login", self.post_login) + self.app.router.add_route("GET", "/matrix-login", self.get_matrix_login) + self.app.router.add_route("POST", "/matrix-login", self.post_matrix_login) self.app.router.add_static("/", pkg_resources.resource_filename("mautrix_telegram", "web/public/")) - def make_token(self, mxid, expires_in=900): + def make_token(self, mxid, endpoint="/login", expires_in=900): return sign_token(self.secret_key, { "mxid": mxid, + "endpoint": endpoint, "expiry": int(time.time()) + expires_in, }) - def verify_token(self, token): + def verify_token(self, token, endpoint="/login"): token = verify_token(self.secret_key, token) - if token and token.get("expiry", 0) > int(time.time()): + if token and (token.get("expiry", 0) > int(time.time()) and + token.get("endpoint", None) == endpoint): return token.get("mxid", None) return None async def get_login(self, request): state = "bot_token" if request.rel_url.query.get("mode", "") == "bot" else "request" - mxid = self.verify_token(request.rel_url.query.get("token", None)) + mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login") if not mxid: return self.get_login_response(status=401, state="invalid-token") user = User.get_by_mxid(mxid, create=False) if mxid else None @@ -75,14 +83,65 @@ class PublicBridgeWebsite(AuthAPI): return self.get_login_response(mxid=user.mxid, username=user.username) + async def get_matrix_login(self, request): + mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login") + if not mxid: + return self.get_mx_login_response(status=401, state="invalid-token") + user = User.get_by_mxid(mxid, create=False) if mxid else None + + if not user: + return self.get_mx_login_response(mxid=mxid) + elif not user.puppet_whitelisted: + return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.", + status=403) + await user.ensure_started() + if not await user.is_logged_in(): + return self.get_mx_login_response(mxid=user.mxid, status=403, + error="You are not logged in to Telegram.") + + puppet = Puppet.get(user.tgid) + if puppet.is_real_user: + return self.get_mx_login_response(state="already-logged-in", status=409) + + return self.get_mx_login_response(mxid=user.mxid) + def get_login_response(self, status=200, state="", username="", mxid="", message="", error="", errcode=""): return web.Response(status=status, content_type="text/html", text=self.login.render(username=username, state=state, error=error, message=message, mxid=mxid)) + def get_mx_login_response(self, status=200, state="", username="", mxid="", message="", + error="", errcode=""): + return web.Response(status=status, content_type="text/html", + text=self.mx_login.render(username=username, state=state, error=error, + message=message, mxid=mxid)) + + async def post_matrix_login(self, request): + mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login") + if not mxid: + return self.get_mx_login_response(status=401, state="invalid-token") + + data = await request.post() + + user = await User.get_by_mxid(mxid).ensure_started() + if not user.puppet_whitelisted: + return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.", + status=403) + elif not await user.is_logged_in(): + return self.get_mx_login_response(mxid=user.mxid, status=403, + error="You are not logged in to Telegram.") + mode = data.get("mode", "access_token") + if mode == "password": + return await self.post_matrix_password(user, data["value"]) + elif mode == "access_token": + return await self.post_matrix_token(user, data["value"]) + return self.get_mx_login_response(mxid=user.mxid, status=400, + error="You must provide an access token or " + "password.") + async def post_login(self, request): - mxid = self.verify_token(request.rel_url.query.get("token", None)) + mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login") if not mxid: return self.get_login_response(status=401, state="invalid-token") diff --git a/mautrix_telegram/web/public/login.css b/mautrix_telegram/web/public/login.css index c7ade95b..7b035792 100644 --- a/mautrix_telegram/web/public/login.css +++ b/mautrix_telegram/web/public/login.css @@ -19,8 +19,8 @@ form > div { display: none; } -form[data-status="request"] > div.status-request, -form[data-status="code"] > div.status-code, +form[data-status="request"] > div.status-request, +form[data-status="code"] > div.status-code, form[data-status="password"] > div.status-password { display: initial; } @@ -48,3 +48,52 @@ form[data-status="password"] > div.status-password { background-color: #d4edda; color: #155724; } + +[type="checkbox"], [type="radio"] { + position: absolute; + opacity: 0; +} + +[type="checkbox"] + label, [type="radio"] + label { + position: relative; + padding-left: 2.5rem; + cursor: pointer; + display: inline-block; +} + +[type="checkbox"] + label:before, [type="radio"] + label:before { + content: ''; + position: absolute; + left: 0; + top: 0.4rem; + width: 1.8rem; + height: 1.8rem; + border: 0.1rem solid #d1d1d1; +} + +[type="radio"] + label:before, [type="radio"] + label:after { + border-radius: 50%; +} + +[type="checkbox"]:checked + label:after, +[type="radio"]:checked + label:after { + content: ''; + width: 0.8rem; + height: 0.8rem; + background: #9b4dca; + position: absolute; + top: 0.9rem; + left: 0.5rem; +} + +[type="radio"]:disabled + label:before, [type="checkbox"]:disabled + label:before { + background-color: #d1d1d1; +} + +[type="radio"]:disabled + label, [type="checkbox"]:disabled + label { + color: #d1d1d1; +} + +[type="radio"]:disabled:checked + label:after, [type="checkbox"]:disabled:checked + label:after { + background: #606c76; +} diff --git a/mautrix_telegram/web/public/login.html.mako b/mautrix_telegram/web/public/login.html.mako index f00b6a69..96db7bac 100644 --- a/mautrix_telegram/web/public/login.html.mako +++ b/mautrix_telegram/web/public/login.html.mako @@ -18,9 +18,9 @@ along with this program. If not, see . - Mautrix-Telegram bridge + Login - Mautrix-Telegram bridge - + @@ -40,10 +40,10 @@ along with this program. If not, see . function goBack() { let params = new URLSearchParams(location.search.slice(1)) - const mxid = params.get("mxid") + const token = params.get("token") params = new URLSearchParams() - if (mxid) { - params.set("mxid", mxid) + if (token) { + params.set("token", token) } location.replace(location.href.split("?")[0] + "?" + params.toString()) } diff --git a/mautrix_telegram/web/public/matrix-login.html.mako b/mautrix_telegram/web/public/matrix-login.html.mako new file mode 100644 index 00000000..6135cb0d --- /dev/null +++ b/mautrix_telegram/web/public/matrix-login.html.mako @@ -0,0 +1,78 @@ + + + + + Matrix login - Mautrix-Telegram bridge + + + + + + + + + + + +
+ % if state == "logged-in": +

Logged in successfully!

+

+ Logged in as ${mxid}. + You can now close this page. +

+ % elif state == "already-logged-in": +

You're already logged in!

+

+ If you want to log in with another account, log out using the + logout-matrix management command first. +

+ % elif state == "invalid-token": +

Invalid or expired token

+
Please ask the bridge bot for a new login link.
+ % else: +

Log in to Matrix

+ % if error: +
${error}
+ % endif + % if message: +
${message}
+ % endif + +
+ + + + +
+ +
+ + + + + +
+ + % endif +
+ + From c5bec3740139da94103af9f561f20df3e927dd0a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 23 Jul 2018 13:50:49 -0400 Subject: [PATCH 61/65] Disable unimplemented password login checkbox in Matrix web login --- mautrix_telegram/web/public/matrix-login.html.mako | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix_telegram/web/public/matrix-login.html.mako b/mautrix_telegram/web/public/matrix-login.html.mako index 6135cb0d..2c8516fe 100644 --- a/mautrix_telegram/web/public/matrix-login.html.mako +++ b/mautrix_telegram/web/public/matrix-login.html.mako @@ -62,7 +62,7 @@ along with this program. If not, see .
- +
From a22b83de444fb5065c1435c9ec5da6479b916fd1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 24 Jul 2018 12:46:54 -0400 Subject: [PATCH 62/65] Disable presence and read receipt bridging for bots. Fixes #194 --- mautrix_telegram/portal.py | 2 ++ mautrix_telegram/user.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 687d137a..9331c9bc 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -659,6 +659,8 @@ class Portal: SetTypingRequest(self.peer, action() if typing else SendMessageCancelAction())) async def mark_read(self, user, event_id): + if user.is_bot: + return space = self.tgid if self.peer_type == "channel" else user.tgid message = DBMessage.query.filter(DBMessage.mxid == event_id, DBMessage.mx_room == self.mxid, diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index 9c42089a..8e229f94 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -190,6 +190,8 @@ class User(AbstractUser): return super().ensure_started(even_if_no_session) def set_presence(self, online: bool = True): + if self.is_bot: + return return self.client(UpdateStatusRequest(offline=not online)) async def update_info(self, info: User = None): From 55b6773b5e883060c6d0d37ca20c3c5ca278c12d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 24 Jul 2018 12:47:27 -0400 Subject: [PATCH 63/65] Limit custom puppet syncing to own EDUs to prevent echoing/duplicates --- mautrix_telegram/puppet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index 222fd972..c6b1b22b 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -136,7 +136,8 @@ class Puppet: "types": [], }, "ephemeral": { - "types": ["m.typing", "m.receipt"] + "types": ["m.typing", "m.receipt"], + "senders": [self.custom_mxid], }, "account_data": { "types": [] @@ -146,7 +147,8 @@ class Puppet: "types": [], }, "presence": { - "types": ["m.presence"] + "types": ["m.presence"], + "senders": [self.custom_mxid], }, }) From ae334b9a04198ba6309b1b3cd5a003d91cdc7172 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 24 Jul 2018 14:37:51 -0400 Subject: [PATCH 64/65] Add hacky local filtering for ephemeral events --- mautrix_telegram/puppet.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index c6b1b22b..20b6af8a 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -137,7 +137,6 @@ class Puppet: }, "ephemeral": { "types": ["m.typing", "m.receipt"], - "senders": [self.custom_mxid], }, "account_data": { "types": [] @@ -152,6 +151,33 @@ class Puppet: }, }) + def filter_events(self, events): + new_events = [] + for event in events: + type = event.get("type", None) + event.setdefault("content", {}) + if type == "m.typing": + is_typing = self.custom_mxid in event["content"].get("user_ids", []) + event["content"]["user_ids"] = [self.custom_mxid] if is_typing else [] + elif type == "m.receipt": + val = None + evt = None + for event_id in event["content"]: + try: + val = event["content"][event_id]["m.read"][self.custom_mxid] + evt = event_id + break + except KeyError: + pass + if val and evt: + event["content"] = {evt: {"m.read": { + self.custom_mxid: val + }}} + else: + continue + new_events.append(event) + return new_events + def handle_sync(self, presence, ephemeral): presence = [self.mx.try_handle_event(event) for event in presence] @@ -161,7 +187,7 @@ class Puppet: ephemeral = [self.mx.try_handle_event(event) for events in ephemeral.values() - for event in events] + for event in self.filter_events(events)] events = ephemeral + presence coro = asyncio.gather(*events, loop=self.loop) From dbfb980bdea668c8b70f7bce5747f54018778ff7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 25 Jul 2018 10:40:31 -0400 Subject: [PATCH 65/65] Add more type hints --- mautrix_telegram/__main__.py | 39 +- mautrix_telegram/abstract_user.py | 125 ++++-- mautrix_telegram/base.py | 2 +- mautrix_telegram/bot.py | 40 +- mautrix_telegram/commands/clean_rooms.py | 13 +- mautrix_telegram/commands/portal.py | 2 +- mautrix_telegram/config.py | 43 +- mautrix_telegram/context.py | 30 +- mautrix_telegram/db.py | 1 + mautrix_telegram/formatter/from_matrix.py | 51 +-- mautrix_telegram/formatter/from_telegram.py | 44 +- mautrix_telegram/formatter/util.py | 4 +- mautrix_telegram/matrix.py | 222 +++++----- mautrix_telegram/portal.py | 445 +++++++++++--------- mautrix_telegram/puppet.py | 60 +-- mautrix_telegram/sqlstatestore.py | 10 +- mautrix_telegram/tgclient.py | 11 +- mautrix_telegram/user.py | 112 ++--- mautrix_telegram/util/file_transfer.py | 84 ++-- mautrix_telegram/util/format_duration.py | 8 +- 20 files changed, 751 insertions(+), 595 deletions(-) diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index ad4cb4ef..ed566c6d 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -14,34 +14,33 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional import argparse -import sys -import logging -import logging.config import asyncio +import logging.config +import sys -import sqlalchemy as sql from sqlalchemy import orm +import sqlalchemy as sql -from alchemysession import AlchemySessionContainer from mautrix_appservice import AppService +from alchemysession import AlchemySessionContainer -from .base import Base -from .config import Config -from .matrix import MatrixHandler - -from . import __version__ -from .db import init as init_db +from .web.provisioning import ProvisioningAPI +from .web.public import PublicBridgeWebsite from .abstract_user import init as init_abstract_user -from .user import init as init_user, User +from .base import Base from .bot import init as init_bot +from .config import Config +from .context import Context +from .db import init as init_db +from .formatter import init as init_formatter +from .matrix import MatrixHandler from .portal import init as init_portal from .puppet import init as init_puppet -from .formatter import init as init_formatter -from .web.public import PublicBridgeWebsite -from .web.provisioning import ProvisioningAPI -from .context import Context from .sqlstatestore import SQLStateStore +from .user import User, init as init_user +from . import __version__ parser = argparse.ArgumentParser( description="A Matrix-Telegram puppeting bridge.", @@ -68,7 +67,7 @@ if args.generate_registration: sys.exit(0) logging.config.dictConfig(config["logging"]) -log = logging.getLogger("mau.init") +log = logging.getLogger("mau.init") # type: logging.Logger log.debug(f"Initializing mautrix-telegram {__version__}") db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db") @@ -80,7 +79,7 @@ session_container = AlchemySessionContainer(engine=db_engine, session=db_session table_base=Base, table_prefix="telethon_", manage_tables=False) -loop = asyncio.get_event_loop() +loop = asyncio.get_event_loop() # type: asyncio.AbstractEventLoop state_store = SQLStateStore(db_session) appserv = AppService(config["homeserver.address"], config["homeserver.domain"], @@ -89,8 +88,8 @@ appserv = AppService(config["homeserver.address"], config["homeserver.domain"], verify_ssl=config["homeserver.verify_ssl"], state_store=state_store, real_user_content_key="net.maunium.telegram.puppet") -public_website = None -provisioning_api = None +public_website = None # type: Optional[PublicBridgeWebsite] +provisioning_api = None # type: Optional[ProvisioningAPI] if config["appservice.public.enabled"]: public_website = PublicBridgeWebsite(loop) diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py index 0de32c91..49632378 100644 --- a/mautrix_telegram/abstract_user.py +++ b/mautrix_telegram/abstract_user.py @@ -14,26 +14,48 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Tuple, Optional, List, Union, TYPE_CHECKING +from abc import ABC, abstractmethod +import asyncio +import logging import platform -from telethon.tl.types import * -from mautrix_appservice import MatrixRequestError +from sqlalchemy import orm +from telethon.tl.types import Channel, ChannelForbidden, Chat, ChatForbidden, Message, \ + MessageActionChannelMigrateFrom, MessageService, PeerUser, TypeUpdate, \ + UpdateChannelPinnedMessage, UpdateChatAdmins, UpdateChatParticipantAdmin, \ + UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, \ + UpdateDeleteMessages, UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, \ + UpdateNewMessage, UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, \ + UpdateUserName, UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, \ + UserStatusOnline + +from mautrix_appservice import MatrixRequestError, AppService +from alchemysession import AlchemySessionContainer -from .tgclient import MautrixTelegramClient -from .db import Message as DBMessage from . import portal as po, puppet as pu, __version__ +from .db import Message as DBMessage +from .tgclient import MautrixTelegramClient -config = None +if TYPE_CHECKING: + from .context import Context + from .config import Config + +config = None # type: Config # Value updated from config in init() -MAX_DELETIONS = 10 +MAX_DELETIONS = 10 # type: int + +UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage, + UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage] +UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService] -class AbstractUser: - session_container = None - loop = None - log = None - db = None - az = None +class AbstractUser(ABC): + session_container = None # type: AlchemySessionContainer + loop = None # type: asyncio.AbstractEventLoop + log = None # type: logging.Logger + db = None # type: orm.Session + az = None # type: AppService def __init__(self): self.puppet_whitelisted = False # type: bool @@ -47,22 +69,22 @@ class AbstractUser: self.is_bot = False # type: bool @property - def connected(self): + def connected(self) -> bool: return self.client and self.client.is_connected() @property - def _proxy_settings(self): - type = config["telegram.proxy.type"].lower() - if type == "disabled": + def _proxy_settings(self) -> Optional[Tuple[int, str, str, str, str, str]]: + proxy_type = config["telegram.proxy.type"].lower() + if proxy_type == "disabled": return None - elif type == "socks4": - type = 1 - elif type == "socks5": - type = 2 - elif type == "http": - type = 3 + elif proxy_type == "socks4": + proxy_type = 1 + elif proxy_type == "socks5": + proxy_type = 2 + elif proxy_type == "http": + proxy_type = 3 - return (type, + return (proxy_type, config["telegram.proxy.address"], config["telegram.proxy.port"], config["telegram.proxy.rdns"], config["telegram.proxy.username"], config["telegram.proxy.password"]) @@ -83,20 +105,30 @@ class AbstractUser: proxy=self._proxy_settings) self.client.add_event_handler(self._update_catch) - async def update(self, update): + @abstractmethod + async def update(self, update: TypeUpdate) -> bool: return False + @abstractmethod async def post_login(self): raise NotImplementedError() - async def _update_catch(self, update): + @abstractmethod + def register_portal(self, portal: po.Portal): + raise NotImplementedError() + + @abstractmethod + def unregister_portal(self, portal: po.Portal): + raise NotImplementedError() + + async def _update_catch(self, update: TypeUpdate): try: if not await self.update(update): await self._update(update) except Exception: self.log.exception("Failed to handle Telegram update") - async def get_dialogs(self, limit=None) -> List[Union[Chat, Channel]]: + async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]: if self.is_bot: return [] dialogs = await self.client.get_dialogs(limit=limit) @@ -106,18 +138,19 @@ class AbstractUser: and (dialog.entity.deactivated or dialog.entity.left)))] @property - def name(self): + @abstractmethod + def name(self) -> str: raise NotImplementedError() - async def is_logged_in(self): + async def is_logged_in(self) -> bool: return self.client and await self.client.is_user_authorized() - async def has_full_access(self, allow_bot=False): + async def has_full_access(self, allow_bot: bool = False) -> bool: return (self.puppet_whitelisted and (not self.is_bot or allow_bot) and await self.is_logged_in()) - async def start(self, delete_unless_authenticated=False): + async def start(self, delete_unless_authenticated: bool = False) -> "AbstractUser": if not self.client: self._init_client() await self.client.connect() @@ -144,7 +177,7 @@ class AbstractUser: # region Telegram update handling - async def _update(self, update): + async def _update(self, update: TypeUpdate): if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage, UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)): await self.update_message(update) @@ -169,17 +202,19 @@ class AbstractUser: else: self.log.debug("Unhandled update: %s", update) - async def update_pinned_messages(self, update): + @staticmethod + async def update_pinned_messages(update: UpdateChannelPinnedMessage): portal = po.Portal.get_by_tgid(update.channel_id) if portal and portal.mxid: await portal.receive_telegram_pin_id(update.id) - async def update_participants(self, update): + @staticmethod + async def update_participants(update: UpdateChatParticipants): portal = po.Portal.get_by_tgid(update.participants.chat_id) if portal and portal.mxid: await portal.update_telegram_participants(update.participants.participants) - async def update_read_receipt(self, update): + async def update_read_receipt(self, update: UpdateReadHistoryOutbox): if not isinstance(update.peer, PeerUser): self.log.debug("Unexpected read receipt peer: %s", update.peer) return @@ -196,7 +231,7 @@ class AbstractUser: puppet = pu.Puppet.get(update.peer.user_id) await puppet.intent.mark_read(portal.mxid, message.mxid) - async def update_admin(self, update): + async def update_admin(self, update: Union[UpdateChatAdmins, UpdateChatParticipantAdmin]): # TODO duplication not checked portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat") if isinstance(update, UpdateChatAdmins): @@ -206,7 +241,7 @@ class AbstractUser: else: self.log.warning("Unexpected admin status update: %s", update) - async def update_typing(self, update): + async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]): if isinstance(update, UpdateUserTyping): portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user") else: @@ -214,7 +249,7 @@ class AbstractUser: sender = pu.Puppet.get(update.user_id) await portal.handle_telegram_typing(sender, update) - async def update_others_info(self, update): + async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]): # TODO duplication not checked puppet = pu.Puppet.get(update.user_id) if isinstance(update, UpdateUserName): @@ -226,7 +261,7 @@ class AbstractUser: else: self.log.warning("Unexpected other user info update: %s", update) - async def update_status(self, update): + async def update_status(self, update: UpdateUserStatus): puppet = pu.Puppet.get(update.user_id) if isinstance(update.status, UserStatusOnline): await puppet.default_mxid_intent.set_presence("online") @@ -236,7 +271,9 @@ class AbstractUser: self.log.warning("Unexpected user status update: %s", update) return - def get_message_details(self, update): + def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent, + Optional[pu.Puppet], + Optional[po.Portal]]: if isinstance(update, UpdateShortChatMessage): portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat") sender = pu.Puppet.get(update.from_id) @@ -259,7 +296,7 @@ class AbstractUser: return update, sender, portal @staticmethod - async def _try_redact(portal, message): + async def _try_redact(portal: po.Portal, message: DBMessage): if not portal: return try: @@ -267,7 +304,7 @@ class AbstractUser: except MatrixRequestError: pass - async def delete_message(self, update): + async def delete_message(self, update: UpdateDeleteMessages): if len(update.messages) > MAX_DELETIONS: return @@ -283,7 +320,7 @@ class AbstractUser: await self._try_redact(portal, message) self.db.commit() - async def delete_channel_message(self, update): + async def delete_channel_message(self, update: UpdateDeleteChannelMessages): if len(update.messages) > MAX_DELETIONS: return @@ -299,7 +336,7 @@ class AbstractUser: await self._try_redact(portal, message) self.db.commit() - async def update_message(self, original_update): + async def update_message(self, original_update: UpdateMessage): update, sender, portal = self.get_message_details(original_update) if isinstance(update, MessageService): @@ -325,7 +362,7 @@ class AbstractUser: # endregion -def init(context): +def init(context: "Context"): global config, MAX_DELETIONS AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, _ = context AbstractUser.session_container = context.session_container diff --git a/mautrix_telegram/base.py b/mautrix_telegram/base.py index c64447da..0b62d886 100644 --- a/mautrix_telegram/base.py +++ b/mautrix_telegram/base.py @@ -1,2 +1,2 @@ from sqlalchemy.ext.declarative import declarative_base -Base = declarative_base() +Base = declarative_base() # type: declarative_base diff --git a/mautrix_telegram/bot.py b/mautrix_telegram/bot.py index c05a62aa..51a6a110 100644 --- a/mautrix_telegram/bot.py +++ b/mautrix_telegram/bot.py @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Awaitable, Callable +from typing import Awaitable, Callable, Pattern, Dict, TYPE_CHECKING import logging import re @@ -27,27 +27,31 @@ from .abstract_user import AbstractUser from .db import BotChat from . import puppet as pu, portal as po, user as u -config = None +if TYPE_CHECKING: + from .config import Config + +config = None # type: Config ReplyFunc = Callable[[str], Awaitable[Message]] class Bot(AbstractUser): - log = logging.getLogger("mau.bot") - mxid_regex = re.compile("@.+:.+") + log = logging.getLogger("mau.bot") # type: logging.Logger + mxid_regex = re.compile("@.+:.+") # type: Pattern def __init__(self, token: str): super().__init__() - self.token = token - self.puppet_whitelisted = True - self.whitelisted = True - self.relaybot_whitelisted = True - self.username = None - self.is_relaybot = True - self.is_bot = True - self.chats = {chat.id: chat.type for chat in BotChat.query.all()} - self.tg_whitelist = [] - self.whitelist_group_admins = config["bridge.relaybot.whitelist_group_admins"] or False + self.token = token # type: str + self.puppet_whitelisted = True # type: bool + self.whitelisted = True # type: bool + self.relaybot_whitelisted = True # type: bool + self.username = None # type: str + self.is_relaybot = True # type: bool + self.is_bot = True # type: bool + self.chats = {chat.id: chat.type for chat in BotChat.query.all()} # type: Dict[int, str] + self.tg_whitelist = [] # type: List[int] + self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"] + or False) # type: bool async def init_permissions(self): whitelist = config["bridge.relaybot.whitelist"] or [] @@ -61,7 +65,7 @@ class Bot(AbstractUser): if isinstance(id, int): self.tg_whitelist.append(id) - async def start(self, delete_unless_authenticated=False): + async def start(self, delete_unless_authenticated: bool = False) -> "Bot": await super().start(delete_unless_authenticated) if not await self.is_logged_in(): await self.client.sign_in(bot_token=self.token) @@ -118,7 +122,7 @@ class Bot(AbstractUser): self.db.delete(existing_chat) self.db.commit() - async def _can_use_commands(self, chat, tgid): + async def _can_use_commands(self, chat: TypePeer, tgid: int) -> bool: if tgid in self.tg_whitelist: return True @@ -138,7 +142,7 @@ class Bot(AbstractUser): if p.user_id == tgid: return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin)) - async def check_can_use_commands(self, event: Message, reply: ReplyFunc): + async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool: if not await self._can_use_commands(event.to_id, event.from_id): await reply("You do not have the permission to use that command.") return False @@ -262,7 +266,7 @@ class Bot(AbstractUser): return "bot" -def init(context): +def init(context) -> Optional[Bot]: global config config = context.config token = config["telegram.bot_token"] diff --git a/mautrix_telegram/commands/clean_rooms.py b/mautrix_telegram/commands/clean_rooms.py index e9031d3b..aac5a54d 100644 --- a/mautrix_telegram/commands/clean_rooms.py +++ b/mautrix_telegram/commands/clean_rooms.py @@ -23,15 +23,14 @@ from .. import puppet as pu, portal as po ManagementRoomList = List[Tuple[str, str]] RoomIDList = List[str] -PortalList = List[po.Portal] -async def _find_rooms(intent: IntentAPI) -> Tuple[ - ManagementRoomList, RoomIDList, PortalList, PortalList]: +async def _find_rooms(intent: IntentAPI) -> Tuple[ManagementRoomList, RoomIDList, + List["po.Portal"], List["po.Portal"]]: management_rooms = [] # type: ManagementRoomList unidentified_rooms = [] # type: RoomIDList - portals = [] # type: PortalList - empty_portals = [] # type: PortalList + portals = [] # type: List[po.Portal] + empty_portals = [] # type: List[po.Portal] rooms = await intent.get_joined_rooms() for room in rooms: @@ -108,8 +107,8 @@ async def clean_rooms(evt: CommandEvent): async def set_rooms_to_clean(evt, management_rooms: ManagementRoomList, - unidentified_rooms: RoomIDList, portals: PortalList, - empty_portals: PortalList): + unidentified_rooms: RoomIDList, portals: List["po.Portal"], + empty_portals: List["po.Portal"]): command = evt.args[0] rooms_to_clean = [] if command == "clean-recommended": diff --git a/mautrix_telegram/commands/portal.py b/mautrix_telegram/commands/portal.py index 0c88ca74..c2ff2347 100644 --- a/mautrix_telegram/commands/portal.py +++ b/mautrix_telegram/commands/portal.py @@ -222,7 +222,7 @@ async def bridge(evt: CommandEvent): "chat to this room, use `$cmdprefix+sp continue`") -async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: po.Portal): +async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"): if not portal.mxid: await evt.reply("The portal seems to have lost its Matrix room between you" "calling `$cmdprefix+sp bridge` and this command.\n\n" diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index b7766f47..72e61f27 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -14,6 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Tuple, Any, Optional from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap import random @@ -24,28 +25,28 @@ yaml.indent(4) class DictWithRecursion: - def __init__(self, data=None): - self._data = data or CommentedMap() + def __init__(self, data: CommentedMap = None): + self._data = data or CommentedMap() # type: CommentedMap - def _recursive_get(self, data, key, default_value): + def _recursive_get(self, data: CommentedMap, key: str, default_value: Any) -> Any: if '.' in key: key, next_key = key.split('.', 1) next_data = data.get(key, CommentedMap()) return self._recursive_get(next_data, next_key, default_value) return data.get(key, default_value) - def get(self, key, default_value, allow_recursion=True): + def get(self, key: str, default_value: Any, allow_recursion: bool = True) -> Any: if allow_recursion and '.' in key: return self._recursive_get(self._data, key, default_value) return self._data.get(key, default_value) - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return self.get(key, None) - def __contains__(self, key): + def __contains__(self, key: str) -> bool: return self[key] is not None - def _recursive_set(self, data, key, value): + def _recursive_set(self, data: CommentedMap, key: str, value: Any): if '.' in key: key, next_key = key.split('.', 1) if key not in data: @@ -55,16 +56,16 @@ class DictWithRecursion: return data[key] = value - def set(self, key, value, allow_recursion=True): + def set(self, key: str, value: Any, allow_recursion: bool = True): if allow_recursion and '.' in key: self._recursive_set(self._data, key, value) return self._data[key] = value - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any): self.set(key, value) - def _recursive_del(self, data, key): + def _recursive_del(self, data: CommentedMap, key: str): if '.' in key: key, next_key = key.split('.', 1) if key not in data: @@ -78,7 +79,7 @@ class DictWithRecursion: except KeyError: pass - def delete(self, key, allow_recursion=True): + def delete(self, key: str, allow_recursion: bool = True): if allow_recursion and '.' in key: self._recursive_del(self._data, key) return @@ -88,23 +89,23 @@ class DictWithRecursion: except KeyError: pass - def __delitem__(self, key): + def __delitem__(self, key: str): self.delete(key) class Config(DictWithRecursion): - def __init__(self, path, registration_path, base_path): + def __init__(self, path: str, registration_path: str, base_path: str): super().__init__() - self.path = path - self.registration_path = registration_path - self.base_path = base_path - self._registration = None + self.path = path # type: str + self.registration_path = registration_path # type: str + self.base_path = base_path # type: str + self._registration = None # type: dict def load(self): with open(self.path, 'r') as stream: self._data = yaml.load(stream) - def load_base(self): + def load_base(self) -> Optional[DictWithRecursion]: try: with open(self.base_path, 'r') as stream: return DictWithRecursion(yaml.load(stream)) @@ -120,7 +121,7 @@ class Config(DictWithRecursion): yaml.dump(self._registration, stream) @staticmethod - def _new_token(): + def _new_token() -> str: return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64)) def update(self): @@ -246,7 +247,7 @@ class Config(DictWithRecursion): self._data = base._data self.save() - def _get_permissions(self, key): + def _get_permissions(self, key: str) -> Tuple[bool, bool, bool, bool, bool]: level = self["bridge.permissions"].get(key, "") admin = level == "admin" puppeting = level == "full" or admin @@ -254,7 +255,7 @@ class Config(DictWithRecursion): relaybot = level == "relaybot" or user return relaybot, user, puppeting, admin, level - def get_permissions(self, mxid): + def get_permissions(self, mxid: str) -> Tuple[bool, bool, bool, bool, bool]: permissions = self["bridge.permissions"] or {} if mxid in permissions: return self._get_permissions(mxid) diff --git a/mautrix_telegram/context.py b/mautrix_telegram/context.py index 1324e5f1..76f75ded 100644 --- a/mautrix_telegram/context.py +++ b/mautrix_telegram/context.py @@ -14,21 +14,27 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Tuple -import asyncio +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import asyncio + + from sqlalchemy.orm import scoped_session + + from alchemysession import AlchemySessionContainer + from mautrix_appservice import AppService + + from .web import PublicBridgeWebsite, ProvisioningAPI + from .config import Config + from .bot import Bot + from .matrix import MatrixHandler -from sqlalchemy.orm import scoped_session -from alchemysession import AlchemySessionContainer -from mautrix_appservice import AppService class Context: - def __init__(self, az, db, config, loop, bot, mx, session_container, public_website, - provisioning_api): - from .web import PublicBridgeWebsite, ProvisioningAPI - from .config import Config - from .bot import Bot - from .matrix import MatrixHandler - + def __init__(self, az: "AppService", db: "scoped_session", config: "Config", + loop: "asyncio.AbstractEventLoop", bot: "Bot", mx: "MatrixHandler", + session_container: "AlchemySessionContainer", + public_website: "PublicBridgeWebsite", provisioning_api: "ProvisioningAPI"): self.az = az # type: AppService self.db = db # type: scoped_session self.config = config # type: Config diff --git a/mautrix_telegram/db.py b/mautrix_telegram/db.py index 81bc0598..5a0baf70 100644 --- a/mautrix_telegram/db.py +++ b/mautrix_telegram/db.py @@ -42,6 +42,7 @@ class Portal(Base): about = Column(String, nullable=True) photo_id = Column(String, nullable=True) + class Message(Base): query = None # type: Query __tablename__ = "message" diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py index f98d3ad5..6619ef02 100644 --- a/mautrix_telegram/formatter/from_matrix.py +++ b/mautrix_telegram/formatter/from_matrix.py @@ -14,10 +14,10 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import (Optional, List, Tuple, Type, Callable, Dict, Any, Pattern, Deque, Match, TYPE_CHECKING) from html import unescape from html.parser import HTMLParser from collections import deque -from typing import Optional, List, Tuple, Type, Callable, Dict, Any import math import re import logging @@ -27,37 +27,40 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, M MessageEntityItalic, MessageEntityCode, MessageEntityPre, MessageEntityBotCommand, TypeMessageEntity) -from .. import user as u, puppet as pu, portal as po, context as c +from .. import user as u, puppet as pu, portal as po from ..db import Message as DBMessage from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, trim_reply_fallback_text, html_to_unicode) -log = logging.getLogger("mau.fmt.mx") -should_bridge_plaintext_highlights = False +if TYPE_CHECKING: + from ..context import Context + +log = logging.getLogger("mau.fmt.mx") # type: logging.Logger +should_bridge_plaintext_highlights = False # type: bool class MatrixParser(HTMLParser): - mention_regex = re.compile("https://matrix.to/#/(@.+:.+)") - room_regex = re.compile("https://matrix.to/#/(#.+:.+)") + mention_regex = re.compile("https://matrix.to/#/(@.+:.+)") # type: Pattern + room_regex = re.compile("https://matrix.to/#/(#.+:.+)") # type: Pattern block_tags = ("br", "p", "pre", "blockquote", "ol", "ul", "li", "h1", "h2", "h3", "h4", "h5", "h6", - "div", "hr", "table") + "div", "hr", "table") # type: Tuple[str, ...] def __init__(self): super().__init__() - self.text = "" - self.entities = [] - self._building_entities = {} - self._list_counter = 0 - self._open_tags = deque() - self._open_tags_meta = deque() - self._line_is_new = True - self._list_entry_is_new = False + self.text = "" # type: str + self.entities = [] # type: List[TypeMessageEntity] + self._building_entities = {} # type: Dict[str, TypeMessageEntity] + self._list_counter = 0 # type: int + self._open_tags = deque() # type: Deque[str] + self._open_tags_meta = deque() # type: Deque[Any] + self._line_is_new = True # type: bool + self._list_entry_is_new = False # type: bool def _parse_url(self, url: str, args: Dict[str, Any] ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]: - mention = self.mention_regex.match(url) + mention = self.mention_regex.match(url) # type: Match if mention: mxid = mention.group(1) user = (pu.Puppet.get_by_mxid(mxid) @@ -72,7 +75,7 @@ class MatrixParser(HTMLParser): else: return None, None - room = self.room_regex.match(url) + room = self.room_regex.match(url) # type: Match if room: username = po.Portal.get_username_from_mx_alias(room.group(1)) portal = po.Portal.find_by_username(username) @@ -92,8 +95,8 @@ class MatrixParser(HTMLParser): self._open_tags_meta.appendleft(0) attrs = dict(attrs) - entity_type = None - args = {} + entity_type = None # type: type(TypeMessageEntity) + args = {} # type: Dict[str, Any] if tag in ("strong", "b"): entity_type = MessageEntityBold elif tag in ("em", "i"): @@ -243,12 +246,12 @@ class MatrixParser(HTMLParser): self._newline(allow_multi=tag == "br") -command_regex = re.compile(r"^!([A-Za-z0-9@]+)") -not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)") -plain_mention_regex = None +command_regex = re.compile(r"^!([A-Za-z0-9@]+)") # type: Pattern +not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)") # type: Pattern +plain_mention_regex = None # type: Pattern -def plain_mention_to_html(match): +def plain_mention_to_html(match: Match) -> str: puppet = pu.Puppet.find_by_displayname(match.group(2)) if puppet: return (f"{match.group(1)}" @@ -351,7 +354,7 @@ def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], st return entities, replacer -def init_mx(context: c.Context): +def init_mx(context: "Context"): global plain_mention_regex, should_bridge_plaintext_highlights config = context.config dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)") diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py index 70a13a55..33f8a335 100644 --- a/mautrix_telegram/formatter/from_telegram.py +++ b/mautrix_telegram/formatter/from_telegram.py @@ -14,13 +14,8 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional, List, Tuple, TYPE_CHECKING from html import escape -from typing import Optional, List, Tuple - -try: - from lxml.html.diff import htmldiff -except ImportError: - htmldiff = None # type: function import logging import re @@ -33,16 +28,26 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, from mautrix_appservice import MatrixRequestError from mautrix_appservice.intent_api import IntentAPI -from .. import user as u, puppet as pu, portal as po, context as c +from .. import user as u, puppet as pu, portal as po from ..db import Message as DBMessage from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html, trim_reply_fallback_text, unicode_to_html) -log = logging.getLogger("mau.fmt.tg") -should_highlight_edits = False +if TYPE_CHECKING: + from ..abstract_user import AbstractUser + from ..context import Context + +try: + from lxml.html.diff import htmldiff +except ImportError: + htmldiff = None # type: function -def telegram_reply_to_matrix(evt: Message, source: u.User) -> dict: +log = logging.getLogger("mau.fmt.tg") # type: logging.Logger +should_highlight_edits = False # type: bool + + +def telegram_reply_to_matrix(evt: Message, source: "AbstractUser") -> dict: if evt.reply_to_msg_id: space = (evt.to_id.channel_id if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel) @@ -78,7 +83,7 @@ async def _add_forward_header(source, text: str, html: Optional[str], if not fwd_from_text: user = await source.client.get_entity(PeerUser(fwd_from.from_id)) if user: - fwd_from_text = pu.Puppet.get_displayname(user, format=False) + fwd_from_text = pu.Puppet.get_displayname(user, False) fwd_from_html = f"{fwd_from_text}" if not fwd_from_text: @@ -110,8 +115,9 @@ def highlight_edits(new_html: str, old_html: str) -> str: return new_html -async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, relates_to: dict, - main_intent: IntentAPI, is_edit: bool) -> Tuple[str, str]: +async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message, + relates_to: dict, main_intent: IntentAPI, is_edit: bool + ) -> Tuple[str, str]: space = (evt.to_id.channel_id if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel) else source.tgid) @@ -142,7 +148,7 @@ async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, if is_edit and should_highlight_edits: html = highlight_edits(html or escape(text), r_html_body) - except (ValueError, KeyError, MatrixRequestError) as e: + except (ValueError, KeyError, MatrixRequestError): r_sender_link = "unknown user" r_displayname = "unknown user" r_text_body = "Failed to fetch message" @@ -154,8 +160,9 @@ async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, r_keyword = "In reply to" if not is_edit else "Edit to" r_msg_link = f"{r_keyword}" - html = (f"
{r_msg_link} {r_sender_link}\n{r_html_body}
" - + (html or escape(text))) + html = ( + f"
{r_msg_link} {r_sender_link}\n{r_html_body}
" + + (html or escape(text))) lines = r_text_body.strip().split("\n") text_with_quote = f"> <{r_displayname}> {lines.pop(0)}" @@ -167,7 +174,8 @@ async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, return text_with_quote, html -async def telegram_to_matrix(evt: Message, source: u.User, main_intent: Optional[IntentAPI] = None, +async def telegram_to_matrix(evt: Message, source: "AbstractUser", + main_intent: Optional[IntentAPI] = None, is_edit: bool = False, prefix_text: Optional[str] = None, prefix_html: Optional[str] = None) -> Tuple[str, str, dict]: text = add_surrogates(evt.message) @@ -320,6 +328,6 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool: return False -def init_tg(context: c.Context): +def init_tg(context: "Context"): global should_highlight_edits should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"] diff --git a/mautrix_telegram/formatter/util.py b/mautrix_telegram/formatter/util.py index f464ffe5..2a296614 100644 --- a/mautrix_telegram/formatter/util.py +++ b/mautrix_telegram/formatter/util.py @@ -14,8 +14,8 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional, Pattern from html import escape -from typing import Optional import struct import re @@ -47,7 +47,7 @@ def trim_reply_fallback_text(text: str) -> str: html_reply_fallback_regex = re.compile("^" r"[\s\S]+?" - "") + "") # type: Pattern def trim_reply_fallback_html(html: str) -> str: diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index 28cf6796..8feed9f4 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -14,26 +14,23 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import List, Dict +from typing import List, Dict, Tuple, Set, Match import logging import asyncio import re from mautrix_appservice import MatrixRequestError, IntentError -from .user import User -from .portal import Portal -from .puppet import Puppet -from .commands import CommandProcessor +from . import user as u, portal as po, puppet as pu, commands as com class MatrixHandler: - log = logging.getLogger("mau.mx") + log = logging.getLogger("mau.mx") # type: logging.Logger def __init__(self, context): self.az, self.db, self.config, _, self.tgbot = context - self.commands = CommandProcessor(context) - self.previously_typing = [] + self.commands = com.CommandProcessor(context) # type: com.CommandProcessor + self.previously_typing = [] # type: List[str] self.az.matrix_event_handler(self.handle_event) @@ -53,68 +50,68 @@ class MatrixHandler: except asyncio.TimeoutError: self.log.exception("TimeoutError when trying to set avatar") - async def handle_puppet_invite(self, room, puppet, inviter): + async def handle_puppet_invite(self, room_id, puppet: pu.Puppet, inviter: u.User): intent = puppet.default_mxid_intent - self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}") + self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}") if not await inviter.is_logged_in(): await intent.error_and_leave( - room, text="Please log in before inviting Telegram puppets.") + room_id, text="Please log in before inviting Telegram puppets.") return - portal = Portal.get_by_mxid(room) + portal = po.Portal.get_by_mxid(room_id) if portal: if portal.peer_type == "user": await intent.error_and_leave( - room, text="You can not invite additional users to private chats.") + room_id, text="You can not invite additional users to private chats.") return await portal.invite_telegram(inviter, puppet) - await intent.join_room(room) + await intent.join_room(room_id) return try: - members = await self.az.intent.get_room_members(room) + members = await self.az.intent.get_room_members(room_id) except MatrixRequestError: members = [] if self.az.bot_mxid not in members: if len(members) > 1: - await intent.error_and_leave(room, text=None, html=( + await intent.error_and_leave(room_id, text=None, html=( f"Please invite " f"the bridge bot " f"first if you want to create a Telegram chat.")) return - await intent.join_room(room) - portal = Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user") + await intent.join_room(room_id) + portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user") if portal.mxid: try: await intent.invite(portal.mxid, inviter.mxid) - await intent.send_notice(room, text=None, html=( + await intent.send_notice(room_id, text=None, html=( "You already have a private chat with me: " f"" "Link to room" "")) - await intent.leave_room(room) + await intent.leave_room(room_id) return except MatrixRequestError: pass - portal.mxid = room + portal.mxid = room_id portal.save() inviter.register_portal(portal) - await intent.send_notice(room, "Portal to private chat created.") + await intent.send_notice(room_id, "po.Portal to private chat created.") else: - await intent.join_room(room) - await intent.send_notice(room, "This puppet will remain inactive until a " - "Telegram chat is created for this room.") + await intent.join_room(room_id) + await intent.send_notice(room_id, "This puppet will remain inactive until a " + "Telegram chat is created for this room.") - async def accept_bot_invite(self, room, inviter): + async def accept_bot_invite(self, room_id: str, inviter: u.User): tries = 0 while tries < 5: try: - await self.az.intent.join_room(room) + await self.az.intent.join_room(room_id) break - except (IntentError, MatrixRequestError) as e: + except (IntentError, MatrixRequestError): tries += 1 wait_for_seconds = (tries + 1) * 10 if tries < 5: - self.log.exception(f"Failed to join room {room} with bridge bot, " + self.log.exception(f"Failed to join room {room_id} with bridge bot, " f"retrying in {wait_for_seconds} seconds...") await asyncio.sleep(wait_for_seconds) else: @@ -123,81 +120,81 @@ class MatrixHandler: if not inviter.whitelisted: await self.az.intent.send_notice( - room, text=None, + room_id, text=None, html="You are not whitelisted to use this bridge.

" "If you are the owner of this bridge, see the " "bridge.permissions section in your config file.") - await self.az.intent.leave_room(room) + await self.az.intent.leave_room(room_id) - async def handle_invite(self, room, user, inviter): - self.log.debug(f"{inviter} invited {user} to {room}") - inviter = await User.get_by_mxid(inviter).ensure_started() - if user == self.az.bot_mxid: - return await self.accept_bot_invite(room, inviter) + async def handle_invite(self, room_id: str, user_id: str, inviter_mxid: str): + self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}") + inviter = await u.User.get_by_mxid(inviter_mxid).ensure_started() + if user_id == self.az.bot_mxid: + return await self.accept_bot_invite(room_id, inviter) elif not inviter.whitelisted: return - puppet = Puppet.get_by_mxid(user) + puppet = pu.Puppet.get_by_mxid(user_id) if puppet: - await self.handle_puppet_invite(room, puppet, inviter) + await self.handle_puppet_invite(room_id, puppet, inviter) return - user = User.get_by_mxid(user, create=False) + user = u.User.get_by_mxid(user_id, create=False) if not user: return await user.ensure_started() - portal = Portal.get_by_mxid(room) + portal = po.Portal.get_by_mxid(room_id) if user and await user.has_full_access(allow_bot=True) and portal: await portal.invite_telegram(inviter, user) return # The rest can probably be ignored - async def handle_join(self, room, user, event_id): - user = await User.get_by_mxid(user).ensure_started() + async def handle_join(self, room_id: str, user_id: str, event_id: str): + user = await u.User.get_by_mxid(user_id).ensure_started() - portal = Portal.get_by_mxid(room) + portal = po.Portal.get_by_mxid(room_id) if not portal: return if not user.relaybot_whitelisted: - await portal.main_intent.kick(room, user.mxid, + await portal.main_intent.kick(room_id, user.mxid, "You are not whitelisted on this Telegram bridge.") return elif not await user.is_logged_in() and not portal.has_bot: - await portal.main_intent.kick(room, user.mxid, + await portal.main_intent.kick(room_id, user.mxid, "This chat does not have a bot relaying " "messages for unauthenticated users.") return - self.log.debug(f"{user} joined {room}") + self.log.debug(f"{user} joined {room_id}") if await user.is_logged_in() or portal.has_bot: await portal.join_matrix(user, event_id) - async def handle_part(self, room, user, sender, event_id): - self.log.debug(f"{user} left {room}") + async def handle_part(self, room_id: str, user_id, sender_mxid: str, event_id: str): + self.log.debug(f"{user_id} left {room_id}") - sender = User.get_by_mxid(sender, create=False) + sender = u.User.get_by_mxid(sender_mxid, create=False) if not sender: return await sender.ensure_started() - portal = Portal.get_by_mxid(room) + portal = po.Portal.get_by_mxid(room_id) if not portal: return - puppet = Puppet.get_by_mxid(user) + puppet = pu.Puppet.get_by_mxid(user_id) if sender and puppet: await portal.leave_matrix(puppet, sender, event_id) - user = User.get_by_mxid(user, create=False) + user = u.User.get_by_mxid(user_id, create=False) if not user: return await user.ensure_started() if await user.is_logged_in() or portal.has_bot: await portal.leave_matrix(user, sender, event_id) - def is_command(self, message): + def is_command(self, message: dict) -> Tuple[bool, str]: text = message.get("body", "") prefix = self.config["bridge.command_prefix"] is_command = text.startswith(prefix) @@ -207,14 +204,14 @@ class MatrixHandler: async def handle_message(self, room, sender, message, event_id): is_command, text = self.is_command(message) - sender = await User.get_by_mxid(sender).ensure_started() + sender = await u.User.get_by_mxid(sender).ensure_started() if not sender.relaybot_whitelisted: self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:" - " User is not whitelisted.") + " u.User is not whitelisted.") return self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}") - portal = Portal.get_by_mxid(room) + portal = po.Portal.get_by_mxid(room) if not is_command and portal and (await sender.is_logged_in() or portal.has_bot): await portal.handle_matrix_message(sender, message, event_id) return @@ -239,39 +236,44 @@ class MatrixHandler: await self.commands.handle(room, sender, command, args, is_management, is_portal=portal is not None) - async def handle_redaction(self, room, sender, event_id): - sender = await User.get_by_mxid(sender).ensure_started() + @staticmethod + async def handle_redaction(room_id: str, sender_mxid: str, event_id: str): + sender = await u.User.get_by_mxid(sender_mxid).ensure_started() if not sender.relaybot_whitelisted: return - portal = Portal.get_by_mxid(room) + portal = po.Portal.get_by_mxid(room_id) if not portal: return await portal.handle_matrix_deletion(sender, event_id) - async def handle_power_levels(self, room, sender, new, old): - portal = Portal.get_by_mxid(room) - sender = await User.get_by_mxid(sender).ensure_started() + @staticmethod + async def handle_power_levels(room_id: str, sender_mxid: str, new: dict, old: dict): + portal = po.Portal.get_by_mxid(room_id) + sender = await u.User.get_by_mxid(sender_mxid).ensure_started() if await sender.has_full_access(allow_bot=True) and portal: await portal.handle_matrix_power_levels(sender, new["users"], old["users"]) - async def handle_room_meta(self, type, room, sender, content): - portal = Portal.get_by_mxid(room) - sender = await User.get_by_mxid(sender).ensure_started() + @staticmethod + async def handle_room_meta(evt_type: str, room_id: str, sender_mxid: str, content: dict): + portal = po.Portal.get_by_mxid(room_id) + sender = await u.User.get_by_mxid(sender_mxid).ensure_started() if await sender.has_full_access(allow_bot=True) and portal: handler, content_key = { "m.room.name": (portal.handle_matrix_title, "name"), "m.room.topic": (portal.handle_matrix_about, "topic"), "m.room.avatar": (portal.handle_matrix_avatar, "url"), - }[type] + }[evt_type] if content_key not in content: return await handler(sender, content[content_key]) - async def handle_room_pin(self, room, sender, new_events, old_events): - portal = Portal.get_by_mxid(room) - sender = await User.get_by_mxid(sender).ensure_started() + @staticmethod + async def handle_room_pin(room_id: str, sender_mxid: str, new_events: Set[str], + old_events: Set[str]): + portal = po.Portal.get_by_mxid(room_id) + sender = await u.User.get_by_mxid(sender_mxid).ensure_started() if await sender.has_full_access(allow_bot=True) and portal: events = new_events - old_events if len(events) > 0: @@ -281,12 +283,14 @@ class MatrixHandler: # All pinned events removed, remove pinned event in Telegram. await portal.handle_matrix_pin(sender, None) - async def handle_name_change(self, room, user, displayname, prev_displayname, event_id): - portal = Portal.get_by_mxid(room) + @staticmethod + async def handle_name_change(room_id: str, user_id: str, displayname: str, + prev_displayname: str, event_id: str): + portal = po.Portal.get_by_mxid(room_id) if not portal or not portal.has_bot: return - user = await User.get_by_mxid(user).ensure_started() + user = await u.User.get_by_mxid(user_id).ensure_started() if await user.needs_relaybot(portal): await portal.name_change_matrix(user, displayname, prev_displayname, event_id) @@ -296,25 +300,27 @@ class MatrixHandler: for event_id, receipts in content.items() for user_id in receipts.get("m.read", {})} - async def handle_read_receipts(self, room_id: str, receipts: Dict[str, str]): - portal = Portal.get_by_mxid(room_id) + @staticmethod + async def handle_read_receipts(room_id: str, receipts: Dict[str, str]): + portal = po.Portal.get_by_mxid(room_id) if not portal: return for user_id, event_id in receipts.items(): - user = await User.get_by_mxid(user_id).ensure_started() + user = await u.User.get_by_mxid(user_id).ensure_started() if not await user.is_logged_in(): continue await portal.mark_read(user, event_id) - async def handle_presence(self, user: str, presence: str): - user = await User.get_by_mxid(user).ensure_started() + @staticmethod + async def handle_presence(user_id: str, presence: str): + user = await u.User.get_by_mxid(user_id).ensure_started() if not await user.is_logged_in(): return await user.set_presence(presence == "online") async def handle_typing(self, room_id: str, now_typing: List[str]): - portal = Portal.get_by_mxid(room_id) + portal = po.Portal.get_by_mxid(room_id) if not portal: return @@ -324,7 +330,7 @@ class MatrixHandler: if is_typing and was_typing: continue - user = await User.get_by_mxid(user_id).ensure_started() + user = await u.User.get_by_mxid(user_id).ensure_started() if not await user.is_logged_in(): continue @@ -332,38 +338,38 @@ class MatrixHandler: self.previously_typing = now_typing - def filter_matrix_event(self, event): + def filter_matrix_event(self, event: dict): sender = event.get("sender", None) if not sender: return False return (sender == self.az.bot_mxid - or Puppet.get_id_from_mxid(sender) is not None) + or pu.Puppet.get_id_from_mxid(sender) is not None) - async def try_handle_event(self, evt): + async def try_handle_event(self, evt: dict): try: await self.handle_event(evt) except Exception: self.log.exception("Error handling manually received Matrix event") - async def handle_event(self, evt): + async def handle_event(self, evt: dict): if self.filter_matrix_event(evt): return self.log.debug("Received event: %s", evt) - type = evt.get("type", "m.unknown") - room_id = evt.get("room_id", None) - event_id = evt.get("event_id", None) - sender = evt.get("sender", None) - content = evt.get("content", {}) - if type == "m.room.member": - state_key = evt["state_key"] - prev_content = evt.get("unsigned", {}).get("prev_content", {}) - membership = content.get("membership", "") - prev_membership = prev_content.get("membership", "leave") + evt_type = evt.get("type", "m.unknown") # type: str + room_id = evt.get("room_id", None) # type: str + event_id = evt.get("event_id", None) # type: str + sender = evt.get("sender", None) # type: str + content = evt.get("content", {}) # type: dict + if evt_type == "m.room.member": + state_key = evt["state_key"] # type: str + prev_content = evt.get("unsigned", {}).get("prev_content", {}) # type: dict + membership = content.get("membership", "") # type: str + prev_membership = prev_content.get("membership", "leave") # type: str if membership == prev_membership: - match = re.compile("@(.+):(.+)").match(state_key) - localpart = match.group(1) - displayname = content.get("displayname", localpart) - prev_displayname = prev_content.get("displayname", localpart) + match = re.compile("@(.+):(.+)").match(state_key) # type: Match + localpart = match.group(1) # type: str + displayname = content.get("displayname", localpart) # type: str + prev_displayname = prev_content.get("displayname", localpart) # type: str if displayname != prev_displayname: await self.handle_name_change(room_id, state_key, displayname, prev_displayname, event_id) @@ -373,26 +379,26 @@ class MatrixHandler: await self.handle_part(room_id, state_key, sender, event_id) elif membership == "join": await self.handle_join(room_id, state_key, event_id) - elif type in ("m.room.message", "m.sticker"): - if type != "m.room.message": - content["msgtype"] = type + elif evt_type in ("m.room.message", "m.sticker"): + if evt_type != "m.room.message": + content["msgtype"] = evt_type await self.handle_message(room_id, sender, content, event_id) - elif type == "m.room.redaction": + elif evt_type == "m.room.redaction": await self.handle_redaction(room_id, sender, evt["redacts"]) - elif type == "m.room.power_levels": + elif evt_type == "m.room.power_levels": await self.handle_power_levels(room_id, sender, evt["content"], evt["prev_content"]) - elif type in ("m.room.name", "m.room.avatar", "m.room.topic"): - await self.handle_room_meta(type, room_id, sender, evt["content"]) - elif type == "m.room.pinned_events": + elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"): + await self.handle_room_meta(evt_type, room_id, sender, evt["content"]) + elif evt_type == "m.room.pinned_events": new_events = set(evt["content"]["pinned"]) try: old_events = set(evt["unsigned"]["prev_content"]["pinned"]) except KeyError: old_events = set() await self.handle_room_pin(room_id, sender, new_events, old_events) - elif type == "m.receipt": + elif evt_type == "m.receipt": await self.handle_read_receipts(room_id, self.parse_read_receipts(content)) - elif type == "m.presence": + elif evt_type == "m.presence": await self.handle_presence(sender, content.get("presence", "offline")) - elif type == "m.typing": + elif evt_type == "m.typing": await self.handle_typing(room_id, content.get("user_ids", [])) diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py index 9331c9bc..a4f80776 100644 --- a/mautrix_telegram/portal.py +++ b/mautrix_telegram/portal.py @@ -14,6 +14,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Pattern, Dict, Tuple, Awaitable, TYPE_CHECKING from collections import deque from datetime import datetime from string import Template @@ -24,65 +25,82 @@ import mimetypes import unicodedata import hashlib import logging +import re import magic +from sqlalchemy import orm from sqlalchemy.exc import IntegrityError, InvalidRequestError from sqlalchemy.orm.exc import FlushError from telethon.tl.functions.messages import * from telethon.tl.functions.channels import * -from telethon.tl.functions.messages import ReadHistoryRequest +from telethon.tl.functions.messages import ReadHistoryRequest as ReadMessageHistoryRequest from telethon.tl.functions.channels import ReadHistoryRequest as ReadChannelHistoryRequest -from telethon.errors import * +from telethon.errors import ChatAdminRequiredError, ChatNotModifiedError from telethon.tl.types import * -from mautrix_appservice import MatrixRequestError, IntentError +from mautrix_appservice import MatrixRequestError, IntentError, AppService, IntentAPI -from .db import Portal as DBPortal, Message as DBMessage +from .context import Context +from .db import Portal as DBPortal, Message as DBMessage, TelegramFile as DBTelegramFile from . import puppet as p, user as u, formatter, util +if TYPE_CHECKING: + from .bot import Bot + from .abstract_user import AbstractUser + from .config import Config + from .tgclient import MautrixTelegramClient + mimetypes.init() -config = None +config = None # type: Config + +TypeMessage = Union[Message, MessageService] +TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant] +DedupMXID = Tuple[str, int] +InviteList = Union[str, List[str]] class Portal: - log = logging.getLogger("mau.portal") - db = None - az = None - bot = None - loop = None - filter_mode = None - filter_list = None - bridge_notices = False - alias_template = None - mx_alias_regex = None - hs_domain = None - by_mxid = {} - by_tgid = {} + log = logging.getLogger("mau.portal") # type: logging.Logger + db = None # type: orm.Session + az = None # type: AppService + bot = None # type: Bot + loop = None # type: asyncio.AbstractEventLoop + filter_mode = None # type: str + filter_list = None # type: List[str] + bridge_notices = False # type: bool + alias_template = None # type: str + mx_alias_regex = None # type: Pattern + hs_domain = None # type: str + by_mxid = {} # type: Dict[str, Portal] + by_tgid = {} # type: Dict[Tuple[int, int], Portal] - def __init__(self, tgid, peer_type, tg_receiver=None, mxid=None, username=None, - megagroup=False, title=None, about=None, photo_id=None, db_instance=None): - self.mxid = mxid - self.tgid = tgid - self.tg_receiver = tg_receiver or tgid - self.peer_type = peer_type - self.username = username - self.megagroup = megagroup - self.title = title - self.about = about - self.photo_id = photo_id - self._db_instance = db_instance + def __init__(self, tgid: int, peer_type: str, tg_receiver: Optional[int] = None, + mxid: Optional[str] = None, username: Optional[str] = None, + megagroup: Optional[bool] = False, title: Optional[str] = None, + about: Optional[str] = None, photo_id: Optional[str] = None, + db_instance: DBPortal = None): + self.mxid = mxid # type: str + self.tgid = tgid # type: int + self.tg_receiver = tg_receiver or tgid # type: int + self.peer_type = peer_type # type: str + self.username = username # type: str + self.megagroup = megagroup # type: bool + self.title = title # type: str + self.about = about # type: str + self.photo_id = photo_id # type: str + self._db_instance = db_instance # type: DBPortal - self._main_intent = None - self._room_create_lock = asyncio.Lock() - self._temp_pinned_message_id = None - self._temp_pinned_message_sender = None + self._main_intent = None # type: IntentAPI + self._room_create_lock = asyncio.Lock() # type: asyncio.Lock + self._temp_pinned_message_id = None # type: Optional[int] + self._temp_pinned_message_sender = None # type: Optional[p.Puppet] - self._dedup = deque() - self._dedup_mxid = {} - self._dedup_action = deque() + self._dedup = deque() # type: deque + self._dedup_mxid = {} # type: Dict[str, DedupMXID] + self._dedup_action = deque() # type: deque - self._send_locks = {} + self._send_locks = {} # type: Dict[int, asyncio.Lock] if tgid: self.by_tgid[self.tgid_full] = self @@ -92,17 +110,17 @@ class Portal: # region Propegrties @property - def tgid_full(self): + def tgid_full(self) -> Tuple[int, int]: return self.tgid, self.tg_receiver @property - def tgid_log(self): + def tgid_log(self) -> str: if self.tgid == self.tg_receiver: - return self.tgid + return str(self.tgid) return f"{self.tg_receiver}<->{self.tgid}" @property - def peer(self): + def peer(self) -> TypePeer: if self.peer_type == "user": return PeerUser(user_id=self.tgid) elif self.peer_type == "chat": @@ -111,11 +129,11 @@ class Portal: return PeerChannel(channel_id=self.tgid) @property - def has_bot(self): + def has_bot(self) -> bool: return self.bot and self.bot.is_in_chat(self.tgid) @property - def main_intent(self): + def main_intent(self) -> IntentAPI: if not self._main_intent: direct = self.peer_type == "user" puppet = p.Puppet.get(self.tgid) if direct else None @@ -125,7 +143,7 @@ class Portal: # endregion # region Filtering - def allow_bridging(self, tgid=None): + def allow_bridging(self, tgid: Optional[int] = None) -> bool: tgid = tgid or self.tgid if self.peer_type == "user": return True @@ -139,7 +157,7 @@ class Portal: # region Deduplication @staticmethod - def _hash_event(event): + def _hash_event(event: TypeMessage) -> str: # Non-channel messages are unique per-user (wtf telegram), so we have no other choice than # to deduplicate based on a hash of the message content. @@ -165,48 +183,54 @@ class Portal: .encode("utf-8") ).hexdigest() - def is_duplicate_action(self, event): - hash = self._hash_event(event) if self.peer_type != "channel" else event.id - if hash in self._dedup_action: + def is_duplicate_action(self, event: TypeMessage) -> bool: + evt_hash = self._hash_event(event) if self.peer_type != "channel" else event.id + if evt_hash in self._dedup_action: return True - self._dedup_action.append(hash) + self._dedup_action.append(evt_hash) if len(self._dedup_action) > 20: self._dedup_action.popleft() return False - def update_duplicate(self, event, mxid=None, expected_mxid=None, force_hash=False): - hash = self._hash_event(event) if self.peer_type != "channel" or force_hash else event.id + def update_duplicate(self, event: TypeMessage, mxid: DedupMXID = None, + expected_mxid: Optional[DedupMXID] = None, force_hash: bool = False + ) -> Optional[DedupMXID]: + evt_hash = self._hash_event( + event) if self.peer_type != "channel" or force_hash else event.id try: - found_mxid = self._dedup_mxid[hash] + found_mxid = self._dedup_mxid[evt_hash] except KeyError: - return 0, "None" + return "None", 0 if found_mxid != expected_mxid: return found_mxid - self._dedup_mxid[hash] = mxid + self._dedup_mxid[evt_hash] = mxid return None - def is_duplicate(self, event, mxid=None, force_hash=False): - hash = self._hash_event(event) if self.peer_type != "channel" or force_hash else event.id - if hash in self._dedup: - return self._dedup_mxid[hash] + def is_duplicate(self, event: TypeMessage, mxid: DedupMXID = None, force_hash: bool = False + ) -> Optional[DedupMXID]: + evt_hash = (self._hash_event(event) + if self.peer_type != "channel" or force_hash + else event.id) + if evt_hash in self._dedup: + return self._dedup_mxid[evt_hash] - self._dedup_mxid[hash] = mxid - self._dedup.append(hash) + self._dedup_mxid[evt_hash] = mxid + self._dedup.append(evt_hash) if len(self._dedup) > 20: del self._dedup_mxid[self._dedup.popleft()] return None - def get_input_entity(self, user): + def get_input_entity(self, user: u.User) -> Awaitable[TypeInputPeer]: return user.client.get_input_entity(self.peer) # endregion # region Matrix room info updating - async def invite_to_matrix(self, users): + async def invite_to_matrix(self, users: InviteList): if isinstance(users, str): await self.main_intent.invite(self.mxid, users, check_cache=True) elif isinstance(users, list): @@ -215,8 +239,10 @@ class Portal: else: raise ValueError("Invalid invite identifier given to invite_matrix()") - async def update_matrix_room(self, user, entity, direct, puppet=None, - levels=None, users=None, participants=None): + async def update_matrix_room(self, user: "AbstractUser", entity: TypeChat, direct: bool, + puppet: p.Puppet = None, levels: dict = None, + users: List[User] = None, + participants: List[TypeParticipant] = None): if not direct: await self.update_info(user, entity) if not users or not participants: @@ -229,8 +255,9 @@ class Portal: await puppet.update_info(user, entity) await puppet.intent.join_room(self.mxid) - async def create_matrix_room(self, user, entity=None, invites=None, update_if_exists=True, - synchronous=False): + async def create_matrix_room(self, user: "AbstractUser", entity: TypeChat = None, + invites: InviteList = None, update_if_exists: bool = True, + synchronous: bool = False) -> Optional[str]: if self.mxid: if update_if_exists: if not entity: @@ -245,7 +272,8 @@ class Portal: async with self._room_create_lock: return await self._create_matrix_room(user, entity, invites) - async def _create_matrix_room(self, user, entity, invites): + async def _create_matrix_room(self, user: "AbstractUser", entity: TypeChat, invites: InviteList + ) -> Optional[str]: direct = self.peer_type == "user" if self.mxid: @@ -310,7 +338,7 @@ class Portal: participants=participants), loop=self.loop) - def _get_base_power_levels(self, levels=None, entity=None): + def _get_base_power_levels(self, levels: dict = None, entity: TypeChat = None) -> dict: levels = levels or {} power_level_requirement = (0 if self.peer_type == "chat" and not entity.admins_enabled else 50) @@ -336,27 +364,27 @@ class Portal: return levels @property - def alias(self): + def alias(self) -> Optional[str]: if not self.username: return None return f"#{self._get_alias_localpart()}:{self.hs_domain}" - def _get_alias_localpart(self, username=None): + def _get_alias_localpart(self, username: Optional[str] = None) -> Optional[str]: username = username or self.username if not username: return None return self.alias_template.format(groupname=username) - def add_bot_chat(self, entity): - if self.bot and entity.id == self.bot.tgid: + def add_bot_chat(self, bot: User): + if self.bot and bot.id == self.bot.tgid: self.bot.add_chat(self.tgid, self.peer_type) return - user = u.User.get_by_tgid(entity.id) + user = u.User.get_by_tgid(bot.id) if user and user.is_bot: user.register_portal(self) - async def sync_telegram_users(self, source, users): + async def sync_telegram_users(self, source: "AbstractUser", users: List[User]): allowed_tgids = set() for entity in users: puppet = p.Puppet.get(entity.id) @@ -398,7 +426,7 @@ class Portal: "You had left this Telegram chat.") continue - async def add_telegram_user(self, user_id, source=None): + async def add_telegram_user(self, user_id: int, source: Optional["AbstractUser"] = None): puppet = p.Puppet.get(user_id) if source: entity = await source.client.get_entity(PeerUser(user_id)) @@ -410,7 +438,7 @@ class Portal: user.register_portal(self) await self.invite_to_matrix(user.mxid) - async def delete_telegram_user(self, user_id, sender): + async def delete_telegram_user(self, user_id: int, sender: p.Puppet): puppet = p.Puppet.get(user_id) user = u.User.get_by_tgid(user_id) kick_message = (f"Kicked by {sender.displayname}" @@ -424,7 +452,7 @@ class Portal: user.unregister_portal(self) await self.main_intent.kick(self.mxid, user.mxid, kick_message) - async def update_info(self, user, entity=None): + async def update_info(self, user: "AbstractUser", entity: TypeChat = None): if self.peer_type == "user": self.log.warning(f"Called update_info() for direct chat portal {self.tgid_log}") return @@ -448,7 +476,7 @@ class Portal: if changed: self.save() - async def update_username(self, username, save=False): + async def update_username(self, username: str, save: bool = False) -> bool: if self.username != username: if self.username: await self.main_intent.remove_room_alias(self._get_alias_localpart()) @@ -465,7 +493,7 @@ class Portal: return True return False - async def update_about(self, about, save=False): + async def update_about(self, about: str, save: bool = False) -> bool: if self.about != about: self.about = about await self.main_intent.set_room_topic(self.mxid, self.about) @@ -474,7 +502,7 @@ class Portal: return True return False - async def update_title(self, title, save=False): + async def update_title(self, title: str, save: bool = False) -> bool: if self.title != title: self.title = title await self.main_intent.set_room_name(self.mxid, self.title) @@ -484,17 +512,18 @@ class Portal: return False @staticmethod - def _get_largest_photo_size(photo): + def _get_largest_photo_size(photo: Photo) -> TypePhotoSize: return max(photo.sizes, key=(lambda photo2: ( len(photo2.bytes) if isinstance(photo2, PhotoCachedSize) else photo2.size))) - async def remove_avatar(self, user, save=False): + async def remove_avatar(self, _: "AbstractUser", save: bool = False): await self.main_intent.set_room_avatar(self.mxid, None) self.photo_id = None if save: self.save() - async def update_avatar(self, user, photo, save=False): + async def update_avatar(self, user: "AbstractUser", photo: FileLocation, + save: bool = False) -> bool: photo_id = f"{photo.volume_id}-{photo.local_id}" if self.photo_id != photo_id: file = await util.transfer_file_to_matrix(self.db, user.client, self.main_intent, @@ -507,7 +536,9 @@ class Portal: return True return False - async def _get_users(self, user, entity): + async def _get_users(self, user: "AbstractUser", entity: Union[TypeInputPeer, InputUser, + TypeChat, TypeUser] + ) -> Tuple[List[TypeUser], List[TypeParticipant]]: if self.peer_type == "chat": chat = await user.client(GetFullChatRequest(chat_id=self.tgid)) return chat.users, chat.full_chat.participants.participants @@ -544,7 +575,7 @@ class Portal: elif self.peer_type == "user": return [entity], [] - async def get_invite_link(self, user): + async def get_invite_link(self, user: u.User) -> str: if self.peer_type == "user": raise ValueError("You can't invite users to private chats.") elif self.peer_type == "chat": @@ -562,7 +593,7 @@ class Portal: return link.link - async def get_authenticated_matrix_users(self): + async def get_authenticated_matrix_users(self) -> List[u.User]: try: members = await self.main_intent.get_room_members(self.mxid) except MatrixRequestError: @@ -573,13 +604,14 @@ class Portal: if p.Puppet.get_id_from_mxid(member) or member == self.main_intent.mxid: continue user = await u.User.get_by_mxid(member).ensure_started() - if (has_bot and user.relaybot_whitelisted) or await user.has_full_access( - allow_bot=True): + authenticated_through_bot = has_bot and user.relaybot_whitelisted + if authenticated_through_bot or await user.has_full_access(allow_bot=True): authenticated.append(user) return authenticated @staticmethod - async def cleanup_room(intent, room_id, message="Portal deleted", puppets_only=False): + async def cleanup_room(intent: IntentAPI, room_id: str, message: str = "Portal deleted", + puppets_only: bool = False): try: members = await intent.get_room_members(room_id) except MatrixRequestError: @@ -608,7 +640,7 @@ class Portal: # region Matrix event handling @staticmethod - def _get_file_meta(body, mime): + def _get_file_meta(body: str, mime: str) -> str: try: current_extension = body[body.rindex("."):] if mimetypes.types_map[current_extension] == mime: @@ -620,7 +652,8 @@ class Portal: else: return "" - async def _get_state_change_message(self, event, user, arguments=None): + async def _get_state_change_message(self, event: str, user: u.User, + arguments: Optional[dict] = None) -> Optional[dict]: tpl = config[f"bridge.state_event_formats.{event}"] if len(tpl) == 0: # Empty format means they don't want the message @@ -637,7 +670,8 @@ class Portal: "formatted_body": message, } - async def name_change_matrix(self, user, displayname, prev_displayname, event_id): + async def name_change_matrix(self, user: u.User, displayname: str, prev_displayname: str, + event_id: str): async with self.require_send_lock(self.bot.tgid): message = await self._get_state_change_message( "name_change", user, @@ -650,15 +684,15 @@ class Portal: space = self.tgid if self.peer_type == "channel" else self.bot.tgid self.is_duplicate(response, (event_id, space)) - async def get_displayname(self, user): + async def get_displayname(self, user: u.User) -> str: return (await self.main_intent.get_displayname(self.mxid, user.mxid) or user.mxid_localpart) - def set_typing(self, user, typing=True, action=SendMessageTypingAction): - return user.client( - SetTypingRequest(self.peer, action() if typing else SendMessageCancelAction())) + def set_typing(self, user: u.User, typing: bool = True, action=SendMessageTypingAction): + return user.client(SetTypingRequest( + self.peer, action() if typing else SendMessageCancelAction())) - async def mark_read(self, user, event_id): + async def mark_read(self, user: u.User, event_id: str): if user.is_bot: return space = self.tgid if self.peer_type == "channel" else user.tgid @@ -671,9 +705,9 @@ class Portal: await user.client(ReadChannelHistoryRequest( channel=await self.get_input_entity(user), max_id=message.tgid)) else: - await user.client(ReadHistoryRequest(peer=self.peer, max_id=message.tgid)) + await user.client(ReadMessageHistoryRequest(peer=self.peer, max_id=message.tgid)) - async def leave_matrix(self, user, source, event_id): + async def leave_matrix(self, user: u.User, source: u.User, event_id: str): if await user.needs_relaybot(self): async with self.require_send_lock(self.bot.tgid): message = await self._get_state_change_message("leave", user) @@ -709,7 +743,7 @@ class Portal: channel = await self.get_input_entity(user) await user.client(LeaveChannelRequest(channel=channel)) - async def join_matrix(self, user, event_id): + async def join_matrix(self, user: u.User, event_id: str): if await user.needs_relaybot(self): async with self.require_send_lock(self.bot.tgid): message = await self._get_state_change_message("join", user) @@ -728,7 +762,7 @@ class Portal: # We'll just assume the user is already in the chat. pass - async def _apply_msg_format(self, sender, msgtype, message): + async def _apply_msg_format(self, sender: u.User, msgtype: str, message: dict): if "formatted_body" not in message: message["format"] = "org.matrix.custom.html" message["formatted_body"] = escape_html(message.get("body", "")) @@ -743,7 +777,7 @@ class Portal: message=body) message["formatted_body"] = Template(tpl).safe_substitute(tpl_args) - async def _preprocess_matrix_message(self, sender, use_relaybot, message): + async def _pre_process_matrix_message(self, sender: u.User, use_relaybot: bool, message: dict): msgtype = message.get("msgtype", "m.text") if msgtype == "m.emote": await self._apply_msg_format(sender, msgtype, message) @@ -751,7 +785,8 @@ class Portal: elif use_relaybot: await self._apply_msg_format(sender, msgtype, message) - def _matrix_event_to_entities(self, event): + @staticmethod + def _matrix_event_to_entities(event: dict) -> Tuple[str, Optional[List[TypeMessageEntity]]]: try: if event.get("format", None) == "org.matrix.custom.html": message, entities = formatter.matrix_to_telegram(event["formatted_body"]) @@ -761,32 +796,33 @@ class Portal: message, entities = None, None return message, entities - def require_send_lock(self, id): - if id is None: - return None + def require_send_lock(self, user_id: int) -> asyncio.Lock: + if user_id is None: + raise ValueError("Required send lock for none id") try: - return self._send_locks[id] + return self._send_locks[user_id] except KeyError: - self._send_locks[id] = asyncio.Lock() - return self._send_locks[id] + self._send_locks[user_id] = asyncio.Lock() + return self._send_locks[user_id] - def optional_send_lock(self, id): - if id is None: + def optional_send_lock(self, user_id: int) -> Optional[asyncio.Lock]: + if user_id is None: return None try: - return self._send_locks[id] + return self._send_locks[user_id] except KeyError: return None - async def _handle_matrix_text(self, sender_id, event_id, space, client, message, reply_to): + async def _handle_matrix_text(self, sender_id: int, event_id: str, space: int, + client: "MautrixTelegramClient", message: dict, reply_to: int): lock = self.require_send_lock(sender_id) async with lock: response = await client.send_message(self.peer, message, reply_to=reply_to, parse_mode=self._matrix_event_to_entities) self._add_telegram_message_to_db(event_id, space, response) - async def _handle_matrix_file(self, type, sender_id, event_id, space, client, message, - reply_to): + async def _handle_matrix_file(self, msgtype: str, sender_id: int, event_id: str, space: int, + client: "MautrixTelegramClient", message: dict, reply_to: int): file = await self.main_intent.download_file(message["url"]) info = message.get("info", {}) @@ -794,7 +830,7 @@ class Portal: w, h = None, None - if type == "m.sticker": + if msgtype == "m.sticker": if mime != "image/gif": mime, file, w, h = util.convert_image(file, source_mime=mime, target_type="webp") else: @@ -812,14 +848,16 @@ class Portal: caption = message["body"] if message["body"] != file_name else None - media = await client.upload_file(file, mime, attributes, file_name) + media = await client.upload_file_direct(file, mime, attributes, file_name) lock = self.require_send_lock(sender_id) async with lock: response = await client.send_media(self.peer, media, reply_to=reply_to, caption=caption) self._add_telegram_message_to_db(event_id, space, response) - async def _handle_matrix_location(self, sender_id, event_id, space, client, message, reply_to): + async def _handle_matrix_location(self, sender_id: int, event_id: str, space: int, + client: "MautrixTelegramClient", message: dict, + reply_to: int): try: lat, long = message["geo_uri"][len("geo:"):].split(",") lat, long = float(lat), float(long) @@ -827,7 +865,7 @@ class Portal: self.log.exception("Failed to parse location") return None message, entities = self._matrix_event_to_entities(message) - media = MessageMediaGeo(geo=GeoPoint(lat, long)) + media = MessageMediaGeo(geo=GeoPoint(lat, long, access_hash=0)) lock = self.require_send_lock(sender_id) async with lock: @@ -835,7 +873,7 @@ class Portal: caption=message, entities=entities) self._add_telegram_message_to_db(event_id, space, response) - def _add_telegram_message_to_db(self, event_id, space, response): + def _add_telegram_message_to_db(self, event_id: str, space: int, response: TypeMessage): self.log.debug("Handled Matrix message: %s", response) self.is_duplicate(response, (event_id, space)) self.db.add(DBMessage( @@ -859,21 +897,21 @@ class Portal: reply_to = formatter.matrix_reply_to_telegram(message, space, room_id=self.mxid) message["mxtg_filename"] = message["body"] - await self._preprocess_matrix_message(sender, not logged_in, message) - type = message["msgtype"] + await self._pre_process_matrix_message(sender, not logged_in, message) + msgtype = message["msgtype"] - if type == "m.text" or (self.bridge_notices and type == "m.notice"): + if msgtype == "m.text" or (self.bridge_notices and msgtype == "m.notice"): await self._handle_matrix_text(sender_id, event_id, space, client, message, reply_to) - elif type == "m.location": + elif msgtype == "m.location": await self._handle_matrix_location(sender_id, event_id, space, client, message, reply_to) - elif type in ("m.sticker", "m.image", "m.file", "m.audio", "m.video"): - await self._handle_matrix_file(type, sender_id, event_id, space, client, message, + elif msgtype in ("m.sticker", "m.image", "m.file", "m.audio", "m.video"): + await self._handle_matrix_file(msgtype, sender_id, event_id, space, client, message, reply_to) else: self.log.debug(f"Unhandled Matrix event: {message}") - async def handle_matrix_pin(self, sender, pinned_message): + async def handle_matrix_pin(self, sender: u.User, pinned_message: Optional[str]): if self.peer_type != "channel": return try: @@ -887,7 +925,7 @@ class Portal: except ChatNotModifiedError: pass - async def handle_matrix_deletion(self, deleter, event_id): + async def handle_matrix_deletion(self, deleter: u.User, event_id: str): deleter = deleter if not await deleter.needs_relaybot(self) else self.bot space = self.tgid if self.peer_type == "channel" else deleter.tgid message = DBMessage.query.filter(DBMessage.mxid == event_id, @@ -897,7 +935,7 @@ class Portal: return await deleter.client.delete_messages(self.peer, [message.tgid]) - async def _update_telegram_power_level(self, sender, user_id, level): + async def _update_telegram_power_level(self, sender: u.User, user_id: int, level: int): if self.peer_type == "chat": await sender.client(EditChatAdminRequest( chat_id=self.tgid, user_id=user_id, is_admin=level >= 50)) @@ -913,7 +951,8 @@ class Portal: EditAdminRequest(channel=await self.get_input_entity(sender), user_id=user_id, admin_rights=rights)) - async def handle_matrix_power_levels(self, sender, new_users, old_users): + async def handle_matrix_power_levels(self, sender: u.User, new_users: Dict[str, int], + old_users: Dict[str, int]): # TODO handle all power level changes and bridge exact admin rights to supergroups/channels for user, level in new_users.items(): if not user or user == self.main_intent.mxid or user == sender.mxid: @@ -929,7 +968,7 @@ class Portal: if user not in old_users or level != old_users[user]: await self._update_telegram_power_level(sender, user_id, level) - async def handle_matrix_about(self, sender, about): + async def handle_matrix_about(self, sender: u.User, about: str): if self.peer_type not in {"channel"}: return channel = await self.get_input_entity(sender) @@ -937,7 +976,7 @@ class Portal: self.about = about self.save() - async def handle_matrix_title(self, sender, title): + async def handle_matrix_title(self, sender: u.User, title: str): if self.peer_type not in {"chat", "channel"}: return @@ -950,7 +989,7 @@ class Portal: self.title = title self.save() - async def handle_matrix_avatar(self, sender, url): + async def handle_matrix_avatar(self, sender: u.User, url: str): if self.peer_type not in {"chat", "channel"}: # Invalid peer type return @@ -958,7 +997,7 @@ class Portal: file = await self.main_intent.download_file(url) mime = magic.from_buffer(file, mime=True) ext = mimetypes.guess_extension(mime) - uploaded = await sender.client.upload_file(file, file_name=f"avatar{ext}") + uploaded = await sender.client.upload_file_direct(file, file_name=f"avatar{ext}") photo = InputChatUploadedPhoto(file=uploaded) if self.peer_type == "chat": @@ -977,7 +1016,7 @@ class Portal: self.save() break - def _register_outgoing_actions_for_dedup(self, response): + def _register_outgoing_actions_for_dedup(self, response: TypeUpdates): for update in response.updates: check_dedup = (isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)) and isinstance(update.message, MessageService)) @@ -987,7 +1026,7 @@ class Portal: # endregion # region Telegram chat info updating - async def _get_telegram_users_in_matrix_room(self): + async def _get_telegram_users_in_matrix_room(self) -> List[int]: user_tgids = set() user_mxids = await self.main_intent.get_room_members(self.mxid, ("join", "invite")) for user in user_mxids: @@ -1001,13 +1040,13 @@ class Portal: user_tgids.add(puppet_id) return list(user_tgids) - async def upgrade_telegram_chat(self, source): + async def upgrade_telegram_chat(self, source: u.User): if self.peer_type != "chat": raise ValueError("Only normal group chats are upgradable to supergroups.") - updates = await source.client(MigrateChatRequest(chat_id=self.tgid)) + response = await source.client(MigrateChatRequest(chat_id=self.tgid)) entity = None - for chat in updates.chats: + for chat in response.chats: if isinstance(chat, Channel): entity = chat break @@ -1017,7 +1056,7 @@ class Portal: self.migrate_and_save(entity.id) await self.update_info(source, entity) - async def set_telegram_username(self, source, username): + async def set_telegram_username(self, source: u.User, username: str): if self.peer_type != "channel": raise ValueError("Only channels and supergroups have usernames.") await source.client( @@ -1025,7 +1064,7 @@ class Portal: if await self.update_username(username): self.save() - async def create_telegram_chat(self, source, supergroup=False): + async def create_telegram_chat(self, source: u.User, supergroup: bool = False): if not self.mxid: raise ValueError("Can't create Telegram chat for portal without Matrix room.") elif self.tgid: @@ -1036,13 +1075,13 @@ class Portal: raise ValueError("Not enough Telegram users to create a chat") if self.peer_type == "chat": - updates = await source.client(CreateChatRequest(title=self.title, users=invites)) - entity = updates.chats[0] + response = await source.client(CreateChatRequest(title=self.title, users=invites)) + entity = response.chats[0] elif self.peer_type == "channel": - updates = await source.client(CreateChannelRequest(title=self.title, - about=self.about or "", - megagroup=supergroup)) - entity = updates.chats[0] + response = await source.client(CreateChannelRequest(title=self.title, + about=self.about or "", + megagroup=supergroup)) + entity = response.chats[0] await source.client(InviteToChannelRequest( channel=await source.client.get_input_entity(entity), users=invites)) @@ -1066,7 +1105,7 @@ class Portal: await self.main_intent.set_power_levels(self.mxid, levels) await self.handle_matrix_power_levels(source, levels["users"], {}) - async def invite_telegram(self, source, puppet): + async def invite_telegram(self, source: u.User, puppet: Union[p.Puppet, "AbstractUser"]): if self.peer_type == "chat": await source.client( AddChatUserRequest(chat_id=self.tgid, user_id=puppet.tgid, fwd_limit=0)) @@ -1078,16 +1117,18 @@ class Portal: # endregion # region Telegram event handling - async def handle_telegram_typing(self, user, event): + async def handle_telegram_typing(self, user: p.Puppet, + _: Union[UpdateUserTyping, UpdateChatUserTyping]): if self.mxid: await user.intent.set_typing(self.mxid, is_typing=True) - def get_external_url(self, evt: Message): + def get_external_url(self, evt: Message) -> Optional[str]: if self.peer_type == "channel" and self.username is not None: return f"https://t.me/{self.username}/{evt.id}" return None - async def handle_telegram_photo(self, source: u.User, intent, evt: Message, relates_to=None): + async def handle_telegram_photo(self, source: "AbstractUser", intent: IntentAPI, evt: Message, + relates_to=None): largest_size = self._get_largest_photo_size(evt.media.photo) file = await util.transfer_file_to_matrix(self.db, source.client, intent, largest_size.location) @@ -1117,7 +1158,7 @@ class Portal: external_url=self.get_external_url(evt)) @staticmethod - def _parse_telegram_document_attributes(attributes): + def _parse_telegram_document_attributes(attributes: List[TypeDocumentAttribute]) -> dict: attrs = { "name": None, "mime_type": None, @@ -1138,7 +1179,8 @@ class Portal: return attrs @staticmethod - def _parse_telegram_document_meta(evt, file, attrs): + def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: dict + ) -> Tuple[dict, str]: document = evt.media.document name = evt.message or attrs["name"] if attrs["is_sticker"]: @@ -1170,7 +1212,9 @@ class Portal: return info, name - async def handle_telegram_document(self, source, intent, evt: Message, relates_to=None): + async def handle_telegram_document(self, source: "AbstractUser", intent: IntentAPI, + evt: Message, + relates_to: dict = None) -> Optional[dict]: document = evt.media.document attrs = self._parse_telegram_document_attributes(document.attributes) @@ -1207,7 +1251,8 @@ class Portal: kwargs["file_type"] = "m.file" return await intent.send_file(**kwargs) - def handle_telegram_location(self, source, intent, evt, relates_to=None): + def handle_telegram_location(self, _: "AbstractUser", intent: IntentAPI, evt: Message, + relates_to: dict = None) -> Awaitable[dict]: location = evt.media.geo long = location.long lat = location.lat @@ -1234,7 +1279,8 @@ class Portal: "m.relates_to": relates_to or None, }, timestamp=evt.date, external_url=self.get_external_url(evt)) - async def handle_telegram_text(self, source, intent, is_bot, evt): + async def handle_telegram_text(self, source: "AbstractUser", intent: IntentAPI, is_bot: bool, + evt: Message) -> dict: self.log.debug(f"Sending {evt.message} to {self.mxid} by {intent.mxid}") text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent) await intent.set_typing(self.mxid, is_typing=False) @@ -1243,7 +1289,7 @@ class Portal: msgtype=msgtype, timestamp=evt.date, external_url=self.get_external_url(evt)) - async def handle_telegram_edit(self, source, sender, evt): + async def handle_telegram_edit(self, source: "AbstractUser", sender: p.Puppet, evt: Message): if not self.mxid: return elif not config["bridge.edits_as_replies"]: @@ -1290,7 +1336,7 @@ class Portal: .update({"mxid": mxid}) self.db.commit() - async def handle_telegram_message(self, source, sender, evt): + async def handle_telegram_message(self, source: "AbstractUser", sender: p.Puppet, evt: Message): if not self.mxid: await self.create_matrix_room(source, invites=[source.mxid], update_if_exists=False) @@ -1373,19 +1419,21 @@ class Portal: self.db.rollback() await intent.redact(self.mxid, mxid) - async def _create_room_on_action(self, source, action): + async def _create_room_on_action(self, source: "AbstractUser", + action: TypeMessageAction) -> bool: if source.is_relaybot: return False create_and_exit = (MessageActionChatCreate, MessageActionChannelCreate) create_and_continue = (MessageActionChatAddUser, MessageActionChatJoinedByLink) - if isinstance(action, create_and_exit + create_and_continue): + if isinstance(action, create_and_exit) or isinstance(action, create_and_continue): await self.create_matrix_room(source, invites=[source.mxid], update_if_exists=isinstance(action, create_and_exit)) if not isinstance(action, create_and_continue): return False return True - async def handle_telegram_action(self, source, sender, update): + async def handle_telegram_action(self, source: "AbstractUser", sender: p.Puppet, + update: MessageService): action = update.action should_ignore = ((not self.mxid and not await self._create_room_on_action(source, action)) or self.is_duplicate_action(update)) @@ -1415,7 +1463,7 @@ class Portal: else: self.log.debug("Unhandled Telegram action in %s: %s", self.title, action) - async def set_telegram_admin(self, user_id): + async def set_telegram_admin(self, user_id: int): puppet = p.Puppet.get(user_id) user = await u.User.get_by_tgid(user_id) @@ -1426,7 +1474,7 @@ class Portal: levels["users"][puppet.mxid] = 50 await self.main_intent.set_power_levels(self.mxid, levels) - async def receive_telegram_pin_sender(self, sender): + async def receive_telegram_pin_sender(self, sender: p.Puppet): self._temp_pinned_message_sender = sender if self._temp_pinned_message_id: await self.update_telegram_pin() @@ -1434,25 +1482,25 @@ class Portal: async def update_telegram_pin(self): intent = (self._temp_pinned_message_sender.intent if self._temp_pinned_message_sender else self.main_intent) - id = self._temp_pinned_message_id + msg_id = self._temp_pinned_message_id self._temp_pinned_message_id = None self._temp_pinned_message_sender = None - message = DBMessage.query.get((id, self.tgid)) + message = DBMessage.query.get((msg_id, self.tgid)) if message: await intent.set_pinned_messages(self.mxid, [message.mxid]) else: await intent.set_pinned_messages(self.mxid, []) - async def receive_telegram_pin_id(self, id): - if id == 0: + async def receive_telegram_pin_id(self, msg_id: int): + if msg_id == 0: return await self.update_telegram_pin() - self._temp_pinned_message_id = id + self._temp_pinned_message_id = msg_id if self._temp_pinned_message_sender: await self.update_telegram_pin() @staticmethod - def _get_level_from_participant(participant, _): + def _get_level_from_participant(participant: TypeParticipant, _) -> int: # TODO use the power level requirements to get better precision in channels if isinstance(participant, (ChatParticipantAdmin, ChannelParticipantAdmin)): return 50 @@ -1461,7 +1509,8 @@ class Portal: return 0 @staticmethod - def _participant_to_power_levels(levels, user, new_level, bot_level): + def _participant_to_power_levels(levels: dict, user: Union[u.User, p.Puppet], new_level: int, + bot_level: int) -> bool: new_level = min(new_level, bot_level) default_level = levels["users_default"] if "users_default" in levels else 0 try: @@ -1473,7 +1522,7 @@ class Portal: return True return False - def _get_bot_level(self, levels): + def _get_bot_level(self, levels: dict) -> int: try: return levels["users"][self.main_intent.mxid] except KeyError: @@ -1483,7 +1532,7 @@ class Portal: return 0 @staticmethod - def _get_powerlevel_level(levels): + def _get_powerlevel_level(levels: dict) -> int: try: return levels["events"]["m.room.power_levels"] except KeyError: @@ -1492,7 +1541,8 @@ class Portal: except KeyError: return 50 - def _participants_to_power_levels(self, participants, levels): + def _participants_to_power_levels(self, participants: List[TypeParticipant], levels: dict + ) -> bool: bot_level = self._get_bot_level(levels) if bot_level < self._get_powerlevel_level(levels): return False @@ -1517,13 +1567,14 @@ class Portal: bot_level) or changed return changed - async def update_telegram_participants(self, participants, levels=None): + async def update_telegram_participants(self, participants: List[TypeParticipant], + levels: dict = None): if not levels: levels = await self.main_intent.get_power_levels(self.mxid) if self._participants_to_power_levels(participants, levels): await self.main_intent.set_power_levels(self.mxid, levels) - async def set_telegram_admins_enabled(self, enabled): + async def set_telegram_admins_enabled(self, enabled: bool): level = 50 if enabled else 10 levels = await self.main_intent.get_power_levels(self.mxid) levels["invite"] = level @@ -1535,17 +1586,17 @@ class Portal: # region Database conversion @property - def db_instance(self): + def db_instance(self) -> DBPortal: if not self._db_instance: self._db_instance = self.new_db_instance() return self._db_instance - def new_db_instance(self): + def new_db_instance(self) -> DBPortal: return DBPortal(tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type, mxid=self.mxid, username=self.username, megagroup=self.megagroup, title=self.title, about=self.about, photo_id=self.photo_id) - def migrate_and_save(self, new_id): + def migrate_and_save(self, new_id: int): existing = DBPortal.query.get(self.tgid_full) if existing: self.db.delete(existing) @@ -1580,7 +1631,7 @@ class Portal: self.db.commit() @classmethod - def from_db(cls, db_portal): + def from_db(cls, db_portal: DBPortal) -> "Portal": return Portal(tgid=db_portal.tgid, tg_receiver=db_portal.tg_receiver, peer_type=db_portal.peer_type, mxid=db_portal.mxid, username=db_portal.username, megagroup=db_portal.megagroup, @@ -1591,7 +1642,7 @@ class Portal: # region Class instance lookup @classmethod - def get_by_mxid(cls, mxid): + def get_by_mxid(cls, mxid: str) -> Optional["Portal"]: try: return cls.by_mxid[mxid] except KeyError: @@ -1604,14 +1655,14 @@ class Portal: return None @classmethod - def get_username_from_mx_alias(cls, alias): + def get_username_from_mx_alias(cls, alias: str) -> Optional[str]: match = cls.mx_alias_regex.match(alias) if match: return match.group(1) return None @classmethod - def find_by_username(cls, username): + def find_by_username(cls, username: str) -> Optional["Portal"]: if not username: return None @@ -1626,7 +1677,8 @@ class Portal: return None @classmethod - def get_by_tgid(cls, tgid, tg_receiver=None, peer_type=None): + def get_by_tgid(cls, tgid: int, tg_receiver: int = None, peer_type: str = None + ) -> Optional["Portal"]: tg_receiver = tg_receiver or tgid tgid_full = (tgid, tg_receiver) try: @@ -1647,36 +1699,37 @@ class Portal: return None @classmethod - def get_by_entity(cls, entity, receiver_id=None, create=True): + def get_by_entity(cls, entity: Union[TypeChat, TypePeer, TypeUser, TypeUserFull, TypeInputPeer], + receiver_id: int = None, create: bool = True) -> Optional["Portal"]: entity_type = type(entity) if entity_type in {Chat, ChatFull}: type_name = "chat" - id = entity.id + entity_id = entity.id elif entity_type in {PeerChat, InputPeerChat}: type_name = "chat" - id = entity.chat_id + entity_id = entity.chat_id elif entity_type in {Channel, ChannelFull}: type_name = "channel" - id = entity.id + entity_id = entity.id elif entity_type in {PeerChannel, InputPeerChannel, InputChannel}: type_name = "channel" - id = entity.channel_id + entity_id = entity.channel_id elif entity_type in {User, UserFull}: type_name = "user" - id = entity.id + entity_id = entity.id elif entity_type in {PeerUser, InputPeerUser, InputUser}: type_name = "user" - id = entity.user_id + entity_id = entity.user_id else: raise ValueError(f"Unknown entity type {entity_type.__name__}") - return cls.get_by_tgid(id, - receiver_id if type_name == "user" else id, + return cls.get_by_tgid(entity_id, + receiver_id if type_name == "user" else entity_id, type_name if create else None) # endregion -def init(context): +def init(context: Context): global config Portal.az, Portal.db, config, Portal.loop, Portal.bot = context Portal.bridge_notices = config["bridge.bridge_notices"] @@ -1684,5 +1737,5 @@ def init(context): Portal.filter_list = config["bridge.filter.list"] Portal.alias_template = config.get("bridge.alias_template", "telegram_{groupname}") Portal.hs_domain = config["homeserver.domain"] - localpart = Portal.alias_template.format(groupname="(.+)") - Portal.mx_alias_regex = re.compile(f"#{localpart}:{Portal.hs_domain}") + Portal.mx_alias_regex = re.compile( + f"#{Portal.alias_template.format(groupname='(.+)')}:{Portal.hs_domain}") diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index 20b6af8a..f5642bc2 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -14,32 +14,39 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional, Awaitable, Pattern, Dict, List, TYPE_CHECKING from difflib import SequenceMatcher -from typing import Optional, Awaitable import re import logging import asyncio +from sqlalchemy import orm + from telethon.tl.types import UserProfilePhoto from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError from .db import Puppet as DBPuppet -from . import util, matrix +from . import util -config = None +if TYPE_CHECKING: + from .matrix import MatrixHandler + from .config import Config + from .context import Context + +config = None # type: Config class Puppet: - log = logging.getLogger("mau.puppet") - db = None + log = logging.getLogger("mau.puppet") # type: logging.Logger + db = None # type: orm.Session az = None # type: AppService - mx = None # type: matrix.MatrixHandler + mx = None # type: MatrixHandler loop = None # type: asyncio.AbstractEventLoop - mxid_regex = None - username_template = None - hs_domain = None - cache = {} - by_custom_mxid = {} + mxid_regex = None # type: Pattern + username_template = None # type: str + hs_domain = None # type: str + cache = {} # type: Dict[str, Puppet] + by_custom_mxid = {} # type: Dict[str, Puppet] def __init__(self, id=None, access_token=None, custom_mxid=None, username=None, displayname=None, displayname_source=None, photo_id=None, is_bot=None, @@ -71,7 +78,8 @@ class Puppet: def tgid(self): return self.id - async def is_logged_in(self): + @staticmethod + async def is_logged_in(): return True # region Custom puppet management @@ -154,12 +162,12 @@ class Puppet: def filter_events(self, events): new_events = [] for event in events: - type = event.get("type", None) + evt_type = event.get("type", None) event.setdefault("content", {}) - if type == "m.typing": + if evt_type == "m.typing": is_typing = self.custom_mxid in event["content"].get("user_ids", []) event["content"]["user_ids"] = [self.custom_mxid] if is_typing else [] - elif type == "m.receipt": + elif evt_type == "m.receipt": val = None evt = None for event_id in event["content"]: @@ -273,7 +281,7 @@ class Puppet: return round(similarity * 1000) / 10 @staticmethod - def get_displayname(info, format=True): + def get_displayname(info, enable_format=True): data = { "phone number": info.phone if hasattr(info, "phone") else None, "username": info.username, @@ -295,7 +303,7 @@ class Puppet: elif not name: name = info.id - if not format: + if not enable_format: return name return config.get("bridge.displayname_template", "{displayname} (Telegram)").format( displayname=name) @@ -347,18 +355,18 @@ class Puppet: # region Getters @classmethod - def get(cls, id, create=True) -> "Optional[Puppet]": + def get(cls, tgid, create=True) -> "Optional[Puppet]": try: - return cls.cache[id] + return cls.cache[tgid] except KeyError: pass - puppet = DBPuppet.query.get(id) + puppet = DBPuppet.query.get(tgid) if puppet: return cls.from_db(puppet) if create: - puppet = cls(id) + puppet = cls(tgid) cls.db.add(puppet.db_instance) cls.db.commit() return puppet @@ -402,8 +410,8 @@ class Puppet: return None @classmethod - def get_mxid_from_id(cls, id): - return f"@{cls.username_template.format(userid=id)}:{cls.hs_domain}" + def get_mxid_from_id(cls, tgid): + return f"@{cls.username_template.format(userid=tgid)}:{cls.hs_domain}" @classmethod def find_by_username(cls, username) -> "Optional[Puppet]": @@ -437,12 +445,12 @@ class Puppet: # endregion -def init(context): +def init(context: "Context") -> List[Awaitable[int]]: global config Puppet.az, Puppet.db, config, Puppet.loop, _ = context Puppet.mx = context.mx Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}") Puppet.hs_domain = config["homeserver"]["domain"] - localpart = Puppet.username_template.format(userid="(.+)") - Puppet.mxid_regex = re.compile(f"@{localpart}:{Puppet.hs_domain}") + Puppet.mxid_regex = re.compile( + f"@{Puppet.username_template.format(userid='(.+)')}:{Puppet.hs_domain}") return [puppet.init_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()] diff --git a/mautrix_telegram/sqlstatestore.py b/mautrix_telegram/sqlstatestore.py index 63b030d2..68e9fd9d 100644 --- a/mautrix_telegram/sqlstatestore.py +++ b/mautrix_telegram/sqlstatestore.py @@ -16,6 +16,8 @@ # along with this program. If not, see . from typing import Dict, Tuple +from sqlalchemy import orm + from mautrix_appservice import StateStore from . import puppet as pu @@ -25,15 +27,17 @@ from .db import RoomState, UserProfile class SQLStateStore(StateStore): def __init__(self, db): super().__init__() - self.db = db + self.db = db # type: orm.Session self.profile_cache = {} # type: Dict[Tuple[str, str], UserProfile] self.room_state_cache = {} # type: Dict[str, RoomState] - def is_registered(self, user: str) -> bool: + @staticmethod + def is_registered(user: str) -> bool: puppet = pu.Puppet.get_by_mxid(user) return puppet.is_registered if puppet else False - def registered(self, user: str): + @staticmethod + def registered(user: str): puppet = pu.Puppet.get_by_mxid(user) if puppet: puppet.is_registered = True diff --git a/mautrix_telegram/tgclient.py b/mautrix_telegram/tgclient.py index 302515d8..4534524e 100644 --- a/mautrix_telegram/tgclient.py +++ b/mautrix_telegram/tgclient.py @@ -17,10 +17,14 @@ from telethon import TelegramClient, utils from telethon.tl.functions.messages import SendMediaRequest from telethon.tl.types import * +from telethon.tl import custom class MautrixTelegramClient(TelegramClient): - async def upload_file(self, file, mime_type=None, attributes=None, file_name=None): + async def upload_file_direct(self, file: bytes, mime_type: str = None, + attributes: List[TypeDocumentAttribute] = None, + file_name: str = None + ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]: file_handle = await super().upload_file(file, file_name=file_name, use_cache=False) if mime_type == "image/png" or mime_type == "image/jpeg": @@ -34,7 +38,10 @@ class MautrixTelegramClient(TelegramClient): mime_type=mime_type or "application/octet-stream", attributes=list(attr_dict.values())) - async def send_media(self, entity, media, caption=None, entities=None, reply_to=None): + async def send_media(self, entity: Union[TypeInputPeer, TypePeer], + media: Union[TypeInputMedia, TypeMessageMedia], + caption: str = None, entities: List[TypeMessageEntity] = None, + reply_to: int = None) -> Optional[custom.Message]: entity = await self.get_input_entity(entity) reply_to = utils.get_message_id(reply_to) request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [], diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py index 8e229f94..c2bdf780 100644 --- a/mautrix_telegram/user.py +++ b/mautrix_telegram/user.py @@ -14,42 +14,51 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, Awaitable, Optional +from typing import Dict, Awaitable, Optional, Match, Tuple, TYPE_CHECKING import logging import asyncio import re from telethon.tl.types import * +from telethon.tl.types import User as TLUser from telethon.tl.types.contacts import ContactsNotModified from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest from telethon.tl.functions.account import UpdateStatusRequest from mautrix_appservice import MatrixRequestError -from .db import User as DBUser, Contact as DBContact +from .db import User as DBUser, Contact as DBContact, Portal as DBPortal from .abstract_user import AbstractUser from . import portal as po, puppet as pu -config = None +if TYPE_CHECKING: + from .config import Config + from .context import Context + +config = None # type: Config + +SearchResults = List[Tuple["pu.Puppet", int]] class User(AbstractUser): - log = logging.getLogger("mau.user") - by_mxid = {} - by_tgid = {} + log = logging.getLogger("mau.user") # type: logging.Logger + by_mxid = {} # type: Dict[str, User] + by_tgid = {} # type: Dict[int, User] - def __init__(self, mxid, tgid=None, username=None, db_contacts=None, saved_contacts=0, - is_bot=False, db_portals=None, db_instance=None): + def __init__(self, mxid: str, tgid: Optional[int] = None, username: Optional[str] = None, + db_contacts: Optional[List[DBContact]] = None, saved_contacts: int = 0, + is_bot: bool = False, db_portals: Optional[List[DBPortal]] = None, + db_instance: Optional[DBUser] = None): super().__init__() self.mxid = mxid # type: str self.tgid = tgid # type: int self.is_bot = is_bot # type: bool self.username = username # type: str - self.contacts = [] - self.saved_contacts = saved_contacts - self.db_contacts = db_contacts - self.portals = {} # type: Dict[str, po.Portal] - self.db_portals = db_portals - self._db_instance = db_instance + self.contacts = [] # type: List[pu.Puppet] + self.saved_contacts = saved_contacts # type: int + self.db_contacts = db_contacts # type: List[DBContact] + self.portals = {} # type: Dict[Tuple[int, int], po.Portal] + self.db_portals = db_portals # type: List[DBPortal] + self._db_instance = db_instance # type: DBUser self.command_status = None # type: dict @@ -64,53 +73,47 @@ class User(AbstractUser): self.by_tgid[tgid] = self @property - def name(self): + def name(self) -> str: return self.mxid @property - def mxid_localpart(self): - match = re.compile("@(.+):(.+)").match(self.mxid) + def mxid_localpart(self) -> str: + match = re.compile("@(.+):(.+)").match(self.mxid) # type: Match return match.group(1) # TODO replace with proper displayname getting everywhere @property - def displayname(self): + def displayname(self) -> str: return self.mxid_localpart @property - def db_contacts(self): + def db_contacts(self) -> List[DBContact]: return [self.db.merge(DBContact(user=self.tgid, contact=puppet.id)) for puppet in self.contacts] @db_contacts.setter - def db_contacts(self, contacts): - if contacts: - self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts] - else: - self.contacts = [] + def db_contacts(self, contacts: List[DBContact]): + self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts] if contacts else [] @property - def db_portals(self): + def db_portals(self) -> List[DBPortal]: return [portal.db_instance for portal in self.portals.values()] @db_portals.setter - def db_portals(self, portals): - if portals: - self.portals = {(portal.tgid, portal.tg_receiver): - po.Portal.get_by_tgid(portal.tgid, portal.tg_receiver) - for portal in portals} - else: - self.portals = {} + def db_portals(self, portals: List[DBPortal]): + self.portals = {(portal.tgid, portal.tg_receiver): + po.Portal.get_by_tgid(portal.tgid, portal.tg_receiver) + for portal in portals} if portals else {} # region Database conversion @property - def db_instance(self): + def db_instance(self) -> DBUser: if not self._db_instance: self._db_instance = self.new_db_instance() return self._db_instance - def new_db_instance(self): + def new_db_instance(self) -> DBUser: return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username, contacts=self.db_contacts, saved_contacts=self.saved_contacts or 0, portals=self.db_portals) @@ -134,14 +137,14 @@ class User(AbstractUser): self.db.commit() @classmethod - def from_db(cls, db_user): + def from_db(cls, db_user: DBUser) -> "User": return User(db_user.mxid, db_user.tgid, db_user.tg_username, db_user.contacts, False, db_user.saved_contacts, db_user.portals, db_instance=db_user) # endregion # region Telegram connection management - async def start(self, delete_unless_authenticated=False): + async def start(self, delete_unless_authenticated: bool = False) -> "User": await super().start() if await self.is_logged_in(): self.log.debug(f"Ensuring post_login() for {self.name}") @@ -152,7 +155,7 @@ class User(AbstractUser): self.client.session.delete() return self - async def post_login(self, info=None): + async def post_login(self, info: TLUser = None): try: await self.update_info(info) if not self.is_bot: @@ -163,7 +166,7 @@ class User(AbstractUser): except Exception: self.log.exception("Failed to run post-login functions for %s", self.mxid) - async def update(self, update): + async def update(self, update: TypeUpdate): if not self.is_bot: return @@ -186,7 +189,7 @@ class User(AbstractUser): # endregion # region Telegram actions that need custom methods - def ensure_started(self, even_if_no_session=False) -> "Awaitable[User]": + def ensure_started(self, even_if_no_session: bool = False) -> "Awaitable[User]": return super().ensure_started(even_if_no_session) def set_presence(self, online: bool = True): @@ -194,7 +197,7 @@ class User(AbstractUser): return return self.client(UpdateStatusRequest(offline=not online)) - async def update_info(self, info: User = None): + async def update_info(self, info: TLUser = None): info = info or await self.client.get_me() changed = False if self.is_bot != info.bot: @@ -233,8 +236,9 @@ class User(AbstractUser): self.delete() return True - def _search_local(self, query, max_results=5, min_similarity=45): - results = [] + def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45 + ) -> SearchResults: + results = [] # type: SearchResults for contact in self.contacts: similarity = contact.similarity(query) if similarity >= min_similarity: @@ -242,11 +246,11 @@ class User(AbstractUser): results.sort(key=lambda tup: tup[1], reverse=True) return results[0:max_results] - async def _search_remote(self, query, max_results=5): + async def _search_remote(self, query: str, max_results: int = 5) -> SearchResults: if len(query) < 5: return [] server_results = await self.client(SearchRequest(q=query, limit=max_results)) - results = [] + results = [] # type: SearchResults for user in server_results.users: puppet = pu.Puppet.get(user.id) await puppet.update_info(self, user) @@ -254,7 +258,7 @@ class User(AbstractUser): results.sort(key=lambda tup: tup[1], reverse=True) return results[0:max_results] - async def search(self, query, force_remote=False): + async def search(self, query: str, force_remote: bool = False) -> Tuple[SearchResults, bool]: if force_remote: return await self._search_remote(query), True @@ -264,7 +268,7 @@ class User(AbstractUser): return await self._search_remote(query), True - async def sync_dialogs(self, synchronous_create=False): + async def sync_dialogs(self, synchronous_create: bool = False): creators = [] for entity in await self.get_dialogs(limit=30): portal = po.Portal.get_by_entity(entity) @@ -275,7 +279,7 @@ class User(AbstractUser): self.save() await asyncio.gather(*creators, loop=self.loop) - def register_portal(self, portal): + def register_portal(self, portal: po.Portal): try: if self.portals[portal.tgid_full] == portal: return @@ -284,18 +288,18 @@ class User(AbstractUser): self.portals[portal.tgid_full] = portal self.save() - def unregister_portal(self, portal): + def unregister_portal(self, portal: po.Portal): try: del self.portals[portal.tgid_full] self.save() except KeyError: pass - async def needs_relaybot(self, portal): + async def needs_relaybot(self, portal: po.Portal) -> bool: return not await self.is_logged_in() or ( self.is_bot and portal.tgid_full not in self.portals) - def _hash_contacts(self): + def _hash_contacts(self) -> int: acc = 0 for id in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]): acc = (acc * 20261 + id) & 0xffffffff @@ -318,7 +322,7 @@ class User(AbstractUser): # region Class instance lookup @classmethod - def get_by_mxid(cls, mxid, create=True) -> "Optional[User]": + def get_by_mxid(cls, mxid: str, create: bool=True) -> "Optional[User]": if not mxid: raise ValueError("Matrix ID can't be empty") @@ -341,7 +345,7 @@ class User(AbstractUser): return None @classmethod - def get_by_tgid(cls, tgid) -> "Optional[User]": + def get_by_tgid(cls, tgid: int) -> "Optional[User]": try: return cls.by_tgid[tgid] except KeyError: @@ -355,7 +359,7 @@ class User(AbstractUser): return None @classmethod - def find_by_username(cls, username) -> "Optional[User]": + def find_by_username(cls, username: str) -> "Optional[User]": if not username: return None @@ -371,7 +375,7 @@ class User(AbstractUser): # endregion -def init(context): +def init(context: "Context") -> List[Awaitable[User]]: global config config = context.config diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py index e927cd77..d950b2a0 100644 --- a/mautrix_telegram/util/file_transfer.py +++ b/mautrix_telegram/util/file_transfer.py @@ -14,15 +14,25 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Optional, Tuple, Union, Dict from io import BytesIO import time import logging import asyncio import magic +from sqlalchemy import orm from sqlalchemy.exc import IntegrityError, InvalidRequestError from sqlalchemy.orm.exc import FlushError +from telethon.tl.types import (Document, FileLocation, InputFileLocation, + InputDocumentFileLocation, PhotoSize, PhotoCachedSize) +from telethon.errors import * +from mautrix_appservice import IntentAPI + +from ..tgclient import MautrixTelegramClient +from ..db import TelegramFile as DBTelegramFile + try: from PIL import Image except ImportError: @@ -36,20 +46,18 @@ try: except ImportError: VideoFileClip = random = string = os = mimetypes = None -from telethon.tl.types import (Document, FileLocation, InputFileLocation, - InputDocumentFileLocation, PhotoSize, PhotoCachedSize) -from telethon.errors import * +log = logging.getLogger("mau.util") # type: logging.Logger -from ..db import TelegramFile as DBTelegramFile - -log = logging.getLogger("mau.util") +TypeLocation = Union[Document, InputDocumentFileLocation, FileLocation, InputFileLocation] -def convert_image(file, source_mime="image/webp", target_type="png", thumbnail_to=None): +def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png", + thumbnail_to: Optional[Tuple[int, int]] = None + ) -> Tuple[str, bytes, Optional[int], Optional[int]]: if not Image: return source_mime, file, None, None try: - image = Image.open(BytesIO(file)).convert("RGBA") + image = Image.open(BytesIO(file)).convert("RGBA") # type: Image.Image if thumbnail_to: image.thumbnail(thumbnail_to, Image.ANTIALIAS) new_file = BytesIO() @@ -61,13 +69,14 @@ def convert_image(file, source_mime="image/webp", target_type="png", thumbnail_t return source_mime, file, None, None -def _temp_file_name(ext): +def _temp_file_name(ext: str) -> str: return ("/tmp/mxtg-video-" + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + ext) -def _read_video_thumbnail(data, video_ext="mp4", frame_ext="png", max_size=(1024, 720)): +def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str = "png", + max_size: Tuple[int, int] = (1024, 720)) -> Tuple[bytes, int, int]: # We don't have any way to read the video from memory, so save it to disk. temp_file = _temp_file_name(video_ext) with open(temp_file, "wb") as file: @@ -90,21 +99,21 @@ def _read_video_thumbnail(data, video_ext="mp4", frame_ext="png", max_size=(1024 return thumbnail_file.getvalue(), w, h -def _location_to_id(location): +def _location_to_id(location: TypeLocation) -> str: if isinstance(location, (Document, InputDocumentFileLocation)): return f"{location.id}-{location.version}" elif isinstance(location, (FileLocation, InputFileLocation)): return f"{location.volume_id}-{location.local_id}" - else: - return None -async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mime): +async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI, + thumbnail_loc: TypeLocation, video: bytes, + mime: str) -> Optional[DBTelegramFile]: if not Image or not VideoFileClip: return None - id = _location_to_id(thumbnail_loc) - if not id: + loc_id = _location_to_id(thumbnail_loc) + if not loc_id: return None video_ext = mimetypes.guess_extension(mime) @@ -121,36 +130,40 @@ async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mim content_uri = await intent.upload_file(file, mime_type) - return DBTelegramFile(id=id, mxc=content_uri, mime_type=mime_type, + return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type, was_converted=False, timestamp=int(time.time()), size=len(file), width=width, height=height) -transfer_locks = {} -transfer_locks_lock = asyncio.Lock() +transfer_locks = {} # type: Dict[str, asyncio.Lock] -async def transfer_file_to_matrix(db, client, intent, location, thumbnail=None, is_sticker=False): - id = _location_to_id(location) - if not id: +async def transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient, intent: IntentAPI, + location: TypeLocation, thumbnail: Optional[TypeLocation] = None, + is_sticker: bool = False) -> Optional[DBTelegramFile]: + location_id = _location_to_id(location) + if not location_id: return None - db_file = DBTelegramFile.query.get(id) + db_file = DBTelegramFile.query.get(location_id) if db_file: return db_file - async with transfer_locks_lock: - try: - lock = transfer_locks[id] - except KeyError: - lock = asyncio.Lock() - transfer_locks[id] = lock + try: + lock = transfer_locks[location_id] + except KeyError: + lock = asyncio.Lock() + transfer_locks[location_id] = lock async with lock: - return await _unlocked_transfer_file_to_matrix(db, client, intent, id, location, thumbnail, is_sticker) + return await _unlocked_transfer_file_to_matrix(db, client, intent, location_id, location, + thumbnail, is_sticker) -async def _unlocked_transfer_file_to_matrix(db, client, intent, id, location, thumbnail, is_sticker): - db_file = DBTelegramFile.query.get(id) +async def _unlocked_transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient, + intent: IntentAPI, loc_id: str, location: TypeLocation, + thumbnail: Optional[TypeLocation], + is_sticker: bool) -> Optional[DBTelegramFile]: + db_file = DBTelegramFile.query.get(loc_id) if db_file: return db_file @@ -167,15 +180,16 @@ async def _unlocked_transfer_file_to_matrix(db, client, intent, id, location, th image_converted = False if mime_type == "image/webp": - new_mime_type, file, width, height = convert_image(file, source_mime="image/webp", target_type="png", thumbnail_to=( - 256, 256) if is_sticker else None) + new_mime_type, file, width, height = convert_image( + file, source_mime="image/webp", target_type="png", + thumbnail_to=(256, 256) if is_sticker else None) image_converted = new_mime_type != mime_type mime_type = new_mime_type thumbnail = None content_uri = await intent.upload_file(file, mime_type) - db_file = DBTelegramFile(id=id, mxc=content_uri, + db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type, was_converted=image_converted, timestamp=int(time.time()), size=len(file), width=width, height=height) diff --git a/mautrix_telegram/util/format_duration.py b/mautrix_telegram/util/format_duration.py index c873e9e5..9402b83e 100644 --- a/mautrix_telegram/util/format_duration.py +++ b/mautrix_telegram/util/format_duration.py @@ -16,10 +16,12 @@ # along with this program. If not, see . -def format_duration(seconds): - def pluralize(count, singular): return singular if count == 1 else singular + "s" +def format_duration(seconds: int) -> str: + def pluralize(count, singular): + return singular if count == 1 else singular + "s" - def include(count, word): return f"{count} {pluralize(count, word)}" if count > 0 else "" + def include(count, word): + return f"{count} {pluralize(count, word)}" if count > 0 else "" minutes, seconds = divmod(seconds, 60) hours, minutes = divmod(minutes, 60)