diff --git a/.codeclimate.yml b/.codeclimate.yml index ad83e79a..e2fdfb75 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -4,3 +4,5 @@ engines: checks: python:S107: enabled: false +exclude_patterns: +- "alembic/" diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ec191c92 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.editorconfig +.codeclimate.yml +*.png +*.md diff --git a/.gitignore b/.gitignore index 7d5457da..e3545879 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,3 @@ config.yaml registration.yaml logs/ *.db -*.session -*.json diff --git a/Dockerfile b/Dockerfile index 6c6b5c28..b371bd44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,14 @@ -FROM docker.io/alpine:3.7 +FROM docker.io/alpine:3.8 ENV UID=1337 \ - GID=1337 + GID=1337 \ + FFMPEG_BINARY=/usr/bin/ffmpeg -COPY . /opt/mautrixtelegram +COPY . /opt/mautrix-telegram +WORKDIR /opt/mautrix-telegram RUN apk add --no-cache \ python3-dev \ + build-base \ py3-virtualenv \ py3-pillow \ py3-aiohttp \ @@ -14,17 +17,12 @@ RUN apk add --no-cache \ py3-numpy \ py3-asn1crypto \ py3-sqlalchemy \ - build-base \ + py3-markdown \ ffmpeg \ - bash \ ca-certificates \ su-exec \ - s6 \ - && cd /opt/mautrixtelegram \ - && cp -r docker/root/* / \ - && rm docker -rf \ && pip3 install -r requirements.txt -r optional-requirements.txt VOLUME /data -CMD ["/bin/s6-svscan", "/etc/s6.d"] +CMD ["/opt/mautrix-telegram/docker-run.sh"] diff --git a/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py b/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py new file mode 100644 index 00000000..7e06de1f --- /dev/null +++ b/alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py @@ -0,0 +1,126 @@ +"""Move state store to main database + +Revision ID: 6ca3d74d51e4 +Revises: 2228d49c383f +Create Date: 2018-06-26 21:31:26.911307 + +""" +from alembic import context, op +import sqlalchemy.orm as orm +import sqlalchemy as sa +import json +import re + +from mautrix_telegram.config import Config +from mautrix_telegram.base import Base + +# revision identifiers, used by Alembic. +revision = "6ca3d74d51e4" +down_revision = "2228d49c383f" +branch_labels = None +depends_on = None + + +class RoomState(Base): + query = None + __tablename__ = "mx_room_state" + __table_args__ = {"extend_existing": True} + + room_id = sa.Column(sa.String, primary_key=True) + power_levels = sa.Column("power_levels", sa.Text, nullable=True) + + +class UserProfile(Base): + query = None + __tablename__ = "mx_user_profile" + __table_args__ = {"extend_existing": True} + + room_id = sa.Column(sa.String, primary_key=True) + user_id = sa.Column(sa.String, primary_key=True) + membership = sa.Column(sa.String, nullable=False, default="leave") + displayname = sa.Column(sa.String, nullable=True) + avatar_url = sa.Column(sa.String, nullable=True) + + +class Puppet(Base): + query = None + __tablename__ = "puppet" + __table_args__ = {"extend_existing": True} + + id = sa.Column(sa.Integer, primary_key=True) + displayname = sa.Column(sa.String, nullable=True) + displayname_source = sa.Column(sa.Integer, nullable=True) + username = sa.Column(sa.String, nullable=True) + photo_id = sa.Column(sa.String, nullable=True) + is_bot = sa.Column(sa.Boolean, nullable=True) + matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False) + + +def upgrade(): + op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False, + server_default=sa.sql.expression.false())) + op.create_table("mx_room_state", + sa.Column("room_id", sa.String(), nullable=False), + sa.Column("power_levels", sa.Text(), nullable=True), + sa.PrimaryKeyConstraint("room_id")) + op.create_table("mx_user_profile", + sa.Column("room_id", sa.String(), nullable=False), + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("membership", sa.String(), nullable=False, + default="leave"), + sa.Column("displayname", sa.String(), nullable=True), + sa.Column("avatar_url", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("room_id", "user_id")) + + conn = op.get_bind() + session = orm.sessionmaker(bind=conn) + session = orm.scoping.scoped_session(session) + Puppet.query = session.query_property() + + with open("mx-state.json") as file: + data = json.load(file) + if not data: + return + registrations = data.get("registrations", []) + + mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml") + mxtg_config = Config(mxtg_config_path, None, None) + mxtg_config.load() + + username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}") + hs_domain = mxtg_config["homeserver.domain"] + localpart = username_template.format(userid="(.+)") + mxid_regex = re.compile(f"@{localpart}:{hs_domain}") + for user in registrations: + match = mxid_regex.match(user) + if not match: + continue + + puppet = Puppet.query.get(match.group(1)) + if not puppet: + continue + + puppet.matrix_registered = True + session.merge(puppet) + session.commit() + + user_profiles = [UserProfile(room_id=room, user_id=user, + membership=member.get("membership", "leave"), + displayname=member.get("displayname", None), + avatar_url=member.get("avatar_url", None)) + for room, members in data.get("members", {}).items() + for user, member in members.items()] + session.add_all(user_profiles) + session.commit() + + room_state = [RoomState(room_id=room, power_levels=json.dumps(levels)) + for room, levels in data.get("power_levels", {}).items()] + session.add_all(room_state) + session.commit() + + +def downgrade(): + op.drop_table("mx_user_profile") + op.drop_table("mx_room_state") + with op.batch_alter_table("puppet") as batch_op: + batch_op.drop_column("matrix_registered") diff --git a/docker/root/etc/s6.d/mautrix-telegram/run b/docker-run.sh similarity index 67% rename from docker/root/etc/s6.d/mautrix-telegram/run rename to docker-run.sh index 41eb26aa..228e9f2f 100755 --- a/docker/root/etc/s6.d/mautrix-telegram/run +++ b/docker-run.sh @@ -1,22 +1,22 @@ -#!/bin/bash +#!/bin/sh -# Define functions +# Define functions. function fixperms { - chown -R ${UID}:${GID} /data /opt/mautrixtelegram + chown -R $UID:$GID /data /opt/mautrix-telegram } - -# Go into env -cd /opt/mautrixtelegram -export FFMPEG_BINARY=/usr/bin/ffmpeg +cd /opt/mautrix-telegram # Replace database path in config. sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml +if [ -f /data/mx-state.json ]; then + ln -s /data/mx-state.json +fi # Check that database is in the right state alembic -x config=/data/config.yaml upgrade head -if [[ ! -f /data/config.yaml ]]; then +if [ ! -f /data/config.yaml ]; then cp example-config.yaml /data/config.yaml echo "Didn't find a config file." echo "Copied default config file to /data/config.yaml" @@ -26,14 +26,14 @@ if [[ ! -f /data/config.yaml ]]; then exit fi -if [[ ! -f /data/registration.yaml ]]; then +if [ ! -f /data/registration.yaml ]; then python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml echo "Didn't find a registration file." - echo "Generated ode for you." + echo "Generated one for you." echo "Copy that over to synapses app service directory." fixperms exit fi fixperms -exec su-exec ${UID}:${GID} python3 -m mautrix_telegram -c /data/config.yaml +exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml diff --git a/docker/root/etc/s6.d/.s6-svscan/finish b/docker/root/etc/s6.d/.s6-svscan/finish deleted file mode 100755 index 1a248525..00000000 --- a/docker/root/etc/s6.d/.s6-svscan/finish +++ /dev/null @@ -1 +0,0 @@ -#!/bin/sh diff --git a/docker/root/etc/s6.d/mautrix-telegram/finish b/docker/root/etc/s6.d/mautrix-telegram/finish deleted file mode 100755 index e90c4912..00000000 --- a/docker/root/etc/s6.d/mautrix-telegram/finish +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -s6-svscanctl -t /etc/s6.d diff --git a/mautrix_telegram/__main__.py b/mautrix_telegram/__main__.py index bd31ab9d..430696eb 100644 --- a/mautrix_telegram/__main__.py +++ b/mautrix_telegram/__main__.py @@ -41,6 +41,7 @@ from .formatter import init as init_formatter from .web.public import PublicBridgeWebsite from .web.provisioning import ProvisioningAPI from .context import Context +from .sqlstatestore import SQLStateStore parser = argparse.ArgumentParser( description="A Matrix-Telegram puppeting bridge.", @@ -81,10 +82,12 @@ session_container = AlchemySessionContainer(engine=db_engine, session=db_session loop = asyncio.get_event_loop() +state_store = SQLStateStore(db_session) appserv = AppService(config["homeserver.address"], config["homeserver.domain"], config["appservice.as_token"], config["appservice.hs_token"], config["appservice.bot_username"], log="mau.as", loop=loop, - verify_ssl=config["homeserver.verify_ssl"]) + verify_ssl=config["homeserver.verify_ssl"], state_store=state_store) + public_website = None provisioning_api = None diff --git a/mautrix_telegram/db.py b/mautrix_telegram/db.py index 2ef9525e..5393acad 100644 --- a/mautrix_telegram/db.py +++ b/mautrix_telegram/db.py @@ -15,8 +15,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer, - BigInteger, String, Boolean) + BigInteger, String, Boolean, Text) +from sqlalchemy.sql import expression from sqlalchemy.orm import relationship +import json from .base import Base @@ -80,6 +82,48 @@ class User(Base): portals = relationship("Portal", secondary="user_portal") +class RoomState(Base): + query = None + __tablename__ = "mx_room_state" + + room_id = Column(String, primary_key=True) + _power_levels_text = Column("power_levels", Text, nullable=True) + _power_levels_json = None + + @property + def has_power_levels(self): + return bool(self._power_levels_text) + + @property + def power_levels(self): + if not self._power_levels_json and self._power_levels_text: + self._power_levels_json = json.loads(self._power_levels_text) + return self._power_levels_json or {} + + @power_levels.setter + def power_levels(self, val): + self._power_levels_json = val + self._power_levels_text = json.dumps(val) + + +class UserProfile(Base): + query = None + __tablename__ = "mx_user_profile" + + room_id = Column(String, primary_key=True) + user_id = Column(String, primary_key=True) + membership = Column(String, nullable=False, default="leave") + displayname = Column(String, nullable=True) + avatar_url = Column(String, nullable=True) + + def dict(self): + return { + "membership": self.membership, + "displayname": self.displayname, + "avatar_url": self.avatar_url, + } + + class Contact(Base): query = None __tablename__ = "contact" @@ -98,6 +142,7 @@ class Puppet(Base): username = Column(String, nullable=True) photo_id = Column(String, nullable=True) is_bot = Column(Boolean, nullable=True) + matrix_registered = Column(Boolean, nullable=False, server_default=expression.false()) # Fucking Telegram not telling bots what chats they are in 3:< @@ -132,3 +177,5 @@ def init(db_session): Puppet.query = db_session.query_property() BotChat.query = db_session.query_property() TelegramFile.query = db_session.query_property() + UserProfile.query = db_session.query_property() + RoomState.query = db_session.query_property() diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py index ad0648bc..b0977036 100644 --- a/mautrix_telegram/puppet.py +++ b/mautrix_telegram/puppet.py @@ -36,7 +36,7 @@ class Puppet: cache = {} def __init__(self, id=None, username=None, displayname=None, displayname_source=None, - photo_id=None, is_bot=None, db_instance=None): + photo_id=None, is_bot=None, is_registered=False, db_instance=None): self.id = id self.mxid = self.get_mxid_from_id(self.id) @@ -45,6 +45,7 @@ class Puppet: self.displayname_source = displayname_source self.photo_id = photo_id self.is_bot = is_bot + self.is_registered = is_registered self._db_instance = db_instance self.intent = self.az.intent.user(self.mxid) @@ -67,13 +68,13 @@ class Puppet: def new_db_instance(self): return DBPuppet(id=self.id, username=self.username, displayname=self.displayname, displayname_source=self.displayname_source, photo_id=self.photo_id, - is_bot=self.is_bot) + is_bot=self.is_bot, matrix_registered=self.is_registered) @classmethod def from_db(cls, db_puppet): return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname, db_puppet.displayname_source, db_puppet.photo_id, db_puppet.is_bot, - db_instance=db_puppet) + db_puppet.matrix_registered, db_instance=db_puppet) def save(self): self.db_instance.username = self.username @@ -81,6 +82,7 @@ class Puppet: self.db_instance.displayname_source = self.displayname_source self.db_instance.photo_id = self.photo_id self.db_instance.is_bot = self.is_bot + self.db_instance.matrix_registered = self.is_registered self.db.commit() def similarity(self, query): diff --git a/mautrix_telegram/sqlstatestore.py b/mautrix_telegram/sqlstatestore.py new file mode 100644 index 00000000..63b030d2 --- /dev/null +++ b/mautrix_telegram/sqlstatestore.py @@ -0,0 +1,116 @@ +# -*- coding: future_fstrings -*- +# mautrix-telegram - A Matrix-Telegram puppeting bridge +# Copyright (C) 2018 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from typing import Dict, Tuple + +from mautrix_appservice import StateStore + +from . import puppet as pu +from .db import RoomState, UserProfile + + +class SQLStateStore(StateStore): + def __init__(self, db): + super().__init__() + self.db = db + self.profile_cache = {} # type: Dict[Tuple[str, str], UserProfile] + self.room_state_cache = {} # type: Dict[str, RoomState] + + def is_registered(self, user: str) -> bool: + puppet = pu.Puppet.get_by_mxid(user) + return puppet.is_registered if puppet else False + + def registered(self, user: str): + puppet = pu.Puppet.get_by_mxid(user) + if puppet: + puppet.is_registered = True + puppet.save() + + def update_state(self, event: dict): + event_type = event["type"] + if event_type == "m.room.power_levels": + self.set_power_levels(event["room_id"], event["content"]) + elif event_type == "m.room.member": + self.set_member(event["room_id"], event["state_key"], event["content"]) + + def _get_user_profile(self, room_id: str, user_id: str, create: bool = True) -> UserProfile: + key = (room_id, user_id) + try: + return self.profile_cache[key] + except KeyError: + pass + + profile = UserProfile.query.get(key) + if profile: + self.profile_cache[key] = profile + elif create: + profile = UserProfile(room_id=room_id, user_id=user_id) + self.db.add(profile) + self.db.commit() + self.profile_cache[key] = profile + return profile + + def get_member(self, room: str, user: str) -> dict: + return self._get_user_profile(room, user).dict() + + def set_member(self, room: str, user: str, member: dict): + profile = self._get_user_profile(room, user) + profile.membership = member.get("membership", profile.membership or "leave") + profile.displayname = member.get("displayname", profile.displayname) + profile.avatar_url = member.get("avatar_url", profile.avatar_url) + self.db.commit() + + def set_membership(self, room: str, user: str, membership: str): + self.set_member(room, user, { + "membership": membership, + }) + + def _get_room_state(self, room_id: str, create: bool = True) -> RoomState: + try: + return self.room_state_cache[room_id] + except KeyError: + pass + + room = RoomState.query.get(room_id) + if room: + self.room_state_cache[room_id] = room + elif create: + room = RoomState(room_id=room_id) + self.room_state_cache[room_id] = room + return room + + def has_power_levels(self, room: str) -> bool: + return self._get_room_state(room).has_power_levels + + def get_power_levels(self, room: str) -> dict: + return self._get_room_state(room).power_levels + + def set_power_level(self, room: str, user: str, level: int): + room_state = self._get_room_state(room) + power_levels = room_state.power_levels + if not power_levels: + power_levels = { + "users": {}, + "events": {}, + } + power_levels[room]["users"][user] = level + room_state.power_levels = power_levels + self.db.commit() + + def set_power_levels(self, room: str, content: dict): + state = self._get_room_state(room) + state.power_levels = content + self.db.commit()