diff --git a/Dockerfile b/Dockerfile index 28d739a5..7ac870c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,22 +20,22 @@ RUN apk add --no-cache libpng libpng-dev zlib zlib-dev \ FROM docker.io/alpine:3.11 +RUN echo "@edge_main http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories +RUN echo "@edge_testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +RUN echo "@edge_community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories + RUN apk add --no-cache \ py3-virtualenv \ py3-pillow \ py3-aiohttp \ py3-magic \ py3-sqlalchemy \ + py3-alembic@edge_testing \ py3-psycopg2 \ py3-ruamel.yaml \ + py3-commonmark@edge_testing \ # Indirect dependencies py3-idna \ - #commonmark - py3-future \ - #alembic - py3-mako \ - py3-dateutil \ - py3-markupsafe \ #moviepy py3-decorator \ py3-tqdm \ @@ -44,6 +44,8 @@ RUN apk add --no-cache \ py3-numpy \ #telethon py3-rsa \ + # Optional for socks proxies + py3-pysocks \ # cryptg py3-cffi \ py3-brotli \ @@ -53,7 +55,22 @@ RUN apk add --no-cache \ su-exec \ netcat-openbsd \ # lottieconverter - zlib libpng + zlib libpng \ + # olm + olm-dev@edge_community \ + # matrix-nio? + py3-future \ + py3-atomicwrites \ + py3-pycryptodome@edge_main \ + py3-peewee@edge_community \ + py3-pyrsistent@edge_community \ + py3-jsonschema \ + py3-aiofiles \ + py3-cachetools@edge_community \ + py3-prometheus-client@edge_community \ + py3-unpaddedbase64 \ + py3-pyaes@edge_testing \ + py3-logbook@edge_testing COPY requirements.txt /opt/mautrix-telegram/requirements.txt COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt @@ -67,7 +84,7 @@ RUN apk add --virtual .build-deps \ && apk del .build-deps COPY . /opt/mautrix-telegram -RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics] && apk del git +RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics,e2be] && apk del git COPY --from=lottieconverter /usr/lib/librlottie* /usr/lib/ COPY --from=lottieconverter /build/LottieConverter/dist/Debug/GNU-Linux/lottieconverter /usr/local/bin/lottieconverter diff --git a/alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py b/alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py new file mode 100644 index 00000000..8cd1ba59 --- /dev/null +++ b/alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py @@ -0,0 +1,27 @@ +"""Add encrypted field for portals + +Revision ID: 24f31fc8a72b +Revises: a7c04a56041b +Create Date: 2020-03-28 20:14:29.046699 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "24f31fc8a72b" +down_revision = "a7c04a56041b" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("portal") as batch_op: + batch_op.add_column(sa.Column("encrypted", sa.Boolean(), nullable=False, + server_default=sa.sql.expression.false())) + + +def downgrade(): + with op.batch_alter_table("portal") as batch_op: + batch_op.drop_column("encrypted") diff --git a/alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py b/alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py new file mode 100644 index 00000000..1469f2a8 --- /dev/null +++ b/alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py @@ -0,0 +1,26 @@ +"""Add decryption info field for reuploaded telegram files + +Revision ID: d3c922a6acd2 +Revises: 24f31fc8a72b +Create Date: 2020-03-30 20:07:17.340346 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd3c922a6acd2' +down_revision = '24f31fc8a72b' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("telegram_file") as batch_op: + batch_op.add_column(sa.Column("decryption_info", sa.Text(), nullable=True)) + + +def downgrade(): + with op.batch_alter_table("telegram_file") as batch_op: + batch_op.drop_column("decryption_info") diff --git a/alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py b/alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py new file mode 100644 index 00000000..6a16f2dc --- /dev/null +++ b/alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py @@ -0,0 +1,71 @@ +"""Add matrix-nio state store to main db + +Revision ID: dff56c93da8d +Revises: d3c922a6acd2 +Create Date: 2020-03-31 22:04:04.014048 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'dff56c93da8d' +down_revision = 'd3c922a6acd2' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('nio_account', + sa.Column('user_id', sa.String(length=255), nullable=False), + sa.Column('device_id', sa.String(length=255), nullable=False), + sa.Column('shared', sa.Boolean(), nullable=False), + sa.Column('sync_token', sa.Text(), nullable=False), + sa.Column('account', sa.LargeBinary(), nullable=False), + sa.PrimaryKeyConstraint('user_id', 'device_id') + ) + op.create_table('nio_device_key', + sa.Column('user_id', sa.String(length=255), nullable=False), + sa.Column('device_id', sa.String(length=255), nullable=False), + sa.Column('display_name', sa.String(length=255), nullable=False), + sa.Column('deleted', sa.Boolean(), nullable=False), + sa.Column('keys', sa.PickleType(), nullable=False), + sa.PrimaryKeyConstraint('user_id', 'device_id') + ) + op.create_table('nio_megolm_inbound_session', + sa.Column('session_id', sa.String(length=255), nullable=False), + sa.Column('sender_key', sa.String(length=255), nullable=False), + sa.Column('fp_key', sa.String(length=255), nullable=False), + sa.Column('room_id', sa.String(length=255), nullable=False), + sa.Column('session', sa.LargeBinary(), nullable=False), + sa.Column('forwarded_chains', sa.PickleType(), nullable=False), + sa.PrimaryKeyConstraint('session_id') + ) + op.create_table('nio_olm_session', + sa.Column('session_id', sa.String(length=255), nullable=False), + sa.Column('sender_key', sa.String(length=255), nullable=False), + sa.Column('session', sa.LargeBinary(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('last_used', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('session_id') + ) + op.create_table('nio_outgoing_key_request', + sa.Column('request_id', sa.String(length=255), nullable=False), + sa.Column('session_id', sa.String(length=255), nullable=False), + sa.Column('room_id', sa.String(length=255), nullable=False), + sa.Column('algorithm', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('request_id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('nio_outgoing_key_request') + op.drop_table('nio_olm_session') + op.drop_table('nio_megolm_inbound_session') + op.drop_table('nio_device_key') + op.drop_table('nio_account') + # ### end Alembic commands ### diff --git a/docker-run.sh b/docker-run.sh index 228e9f2f..bf904d45 100755 --- a/docker-run.sh +++ b/docker-run.sh @@ -17,7 +17,7 @@ fi alembic -x config=/data/config.yaml upgrade head if [ ! -f /data/config.yaml ]; then - cp example-config.yaml /data/config.yaml + cp mautrix_telegram/example-config.yaml /data/config.yaml echo "Didn't find a config file." echo "Copied default config file to /data/config.yaml" echo "Modify that config file to your liking." diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index a1ca0182..904f3a43 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -118,6 +118,8 @@ class Config(BaseBridgeConfig): copy("bridge.federate_rooms") copy("bridge.animated_sticker.target") copy("bridge.animated_sticker.args") + copy("bridge.encryption.allow") + copy("bridge.encryption.default") copy("bridge.initial_power_level_overrides.group") copy("bridge.initial_power_level_overrides.user") diff --git a/mautrix_telegram/db/__init__.py b/mautrix_telegram/db/__init__.py index 92a824f0..0725d6ae 100644 --- a/mautrix_telegram/db/__init__.py +++ b/mautrix_telegram/db/__init__.py @@ -24,6 +24,11 @@ from .puppet import Puppet from .telegram_file import TelegramFile from .user import User, UserPortal, Contact +try: + from mautrix.bridge.db.nio_state_store import init as init_nio_db +except ImportError: + init_nio_db = None + def init(db_engine: Engine) -> None: for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile, @@ -32,3 +37,5 @@ def init(db_engine: Engine) -> None: table.t = table.__table__ table.c = table.t.c table.column_names = table.c.keys() + if init_nio_db: + init_nio_db(db_engine) diff --git a/mautrix_telegram/db/portal.py b/mautrix_telegram/db/portal.py index bae8c76a..8112e9ad 100644 --- a/mautrix_telegram/db/portal.py +++ b/mautrix_telegram/db/portal.py @@ -15,7 +15,7 @@ # along with this program. If not, see . from typing import Optional -from sqlalchemy import Column, Integer, String, Boolean, Text, func +from sqlalchemy import Column, Integer, String, Boolean, Text, func, sql from mautrix.types import RoomID from mautrix.util.db import Base @@ -34,6 +34,7 @@ class Portal(Base): # Matrix portal information mxid: RoomID = Column(String, unique=True, nullable=True) + encrypted: bool = Column(Boolean, nullable=False, server_default=sql.expression.false()) config: str = Column(Text, nullable=True) diff --git a/mautrix_telegram/db/telegram_file.py b/mautrix_telegram/db/telegram_file.py index 8cc0c9c0..82f1e17c 100644 --- a/mautrix_telegram/db/telegram_file.py +++ b/mautrix_telegram/db/telegram_file.py @@ -13,15 +13,37 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Optional +from typing import Optional, cast, Dict, Any -from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean +from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text, + TypeDecorator) from sqlalchemy.engine.result import RowProxy -from mautrix.types import ContentURI +from mautrix.types import ContentURI, EncryptedFile from mautrix.util.db import Base +class DBEncryptedFile(TypeDecorator): + impl = Text + + @property + def python_type(self): + return EncryptedFile + + def process_bind_param(self, value: EncryptedFile, dialect) -> Optional[str]: + if value is not None: + return value.json() + return None + + def process_result_value(self, value: str, dialect) -> Optional[EncryptedFile]: + if value is not None: + return EncryptedFile.parse_json(value) + return None + + def process_literal_param(self, value, dialect): + return value + + class TelegramFile(Base): __tablename__ = "telegram_file" @@ -33,12 +55,13 @@ class TelegramFile(Base): size: Optional[int] = Column(Integer, nullable=True) width: Optional[int] = Column(Integer, nullable=True) height: Optional[int] = Column(Integer, nullable=True) + decryption_info: Optional[Dict[str, Any]] = Column(DBEncryptedFile, nullable=True) thumbnail_id: str = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True) thumbnail: Optional['TelegramFile'] = None @classmethod def scan(cls, row: RowProxy) -> 'TelegramFile': - telegram_file: TelegramFile = super().scan(row) + telegram_file = cast(TelegramFile, super().scan(row)) if isinstance(telegram_file.thumbnail, str): telegram_file.thumbnail = cls.get(telegram_file.thumbnail) return telegram_file @@ -52,5 +75,5 @@ class TelegramFile(Base): conn.execute(self.t.insert().values( id=self.id, mxc=self.mxc, mime_type=self.mime_type, was_converted=self.was_converted, timestamp=self.timestamp, size=self.size, - width=self.width, height=self.height, + width=self.width, height=self.height, decryption_info=self.decryption_info, thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id)) diff --git a/mautrix_telegram/example-config.yaml b/mautrix_telegram/example-config.yaml index f468124e..794f6d6e 100644 --- a/mautrix_telegram/example-config.yaml +++ b/mautrix_telegram/example-config.yaml @@ -193,6 +193,17 @@ bridge: height: 256 background: "020202" # only for gif fps: 30 # only for webm + # End-to-bridge encryption support options. These require matrix-nio to be installed with pip + # and login_shared_secret to be configured in order to get a device for the bridge bot. + # + # Additionally, https://github.com/matrix-org/synapse/pull/5758 is required if using a normal + # application service. + encryption: + # Allow encryption, work in group chat rooms with e2ee enabled + allow: false + # Default to encryption, force-enable encryption in all portals the bridge creates + # This will cause the bridge bot to be in private chats for the encryption to work properly. + default: false # Overrides for base power levels. initial_power_level_overrides: diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py index 27d6f164..6d600669 100644 --- a/mautrix_telegram/matrix.py +++ b/mautrix_telegram/matrix.py @@ -13,14 +13,15 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Dict, Set, Tuple, Union, Iterable, TYPE_CHECKING +from typing import Dict, Set, Tuple, Union, Iterable, List, TYPE_CHECKING from mautrix.bridge import BaseMatrixHandler from mautrix.types import (Event, EventType, RoomID, UserID, EventID, ReceiptEvent, ReceiptType, ReceiptEventContent, PresenceEvent, PresenceState, TypingEvent, MessageEvent, StateEvent, RedactionEvent, RoomNameStateEventContent, RoomAvatarStateEventContent, RoomTopicStateEventContent, - MemberStateEventContent) + MemberStateEventContent, EncryptedEvent, TextMessageEventContent, + MessageType) from mautrix.errors import MatrixError from . import user as u, portal as po, puppet as pu, commands as com @@ -47,8 +48,15 @@ class MatrixHandler(BaseMatrixHandler): previously_typing: Dict[RoomID, Set[UserID]] def __init__(self, context: 'Context') -> None: + prefix, suffix = context.config["bridge.username_template"].format(userid=":").split(":") + homeserver = context.config["homeserver.domain"] + self.user_id_prefix = f"@{prefix}" + self.user_id_suffix = f"{suffix}:{homeserver}" + super(MatrixHandler, self).__init__(context.az, context.config, loop=context.loop, - command_processor=com.CommandProcessor(context)) + command_processor=com.CommandProcessor(context), + bridge=context.bridge) + self.bot = context.bot self.previously_typing = {} @@ -104,14 +112,38 @@ class MatrixHandler(BaseMatrixHandler): except MatrixError: pass portal.mxid = room_id + e2be_ok = None + if self.config["bridge.encryption.default"] and self.e2ee: + e2be_ok = await self.enable_dm_encryption(portal, members=members) portal.save() inviter.register_portal(portal) - await intent.send_notice(room_id, "Portal to private chat created.") + if e2be_ok is True: + evt_type, content = await self.e2ee.encrypt( + room_id, EventType.ROOM_MESSAGE, + TextMessageEventContent(msgtype=MessageType.NOTICE, + body="Portal to private chat created and end-to-bridge" + " encryption enabled.")) + await intent.send_message_event(room_id, evt_type, content) + else: + message = "Portal to private chat created." + if e2be_ok is False: + message += "\n\nWarning: Failed to enable end-to-bridge encryption" + await intent.send_notice(room_id, message) else: await intent.join_room(room_id) await intent.send_notice(room_id, "This puppet will remain inactive until a " "Telegram chat is created for this room.") + async def enable_dm_encryption(self, portal: po.Portal, members: List[UserID]) -> bool: + ok = await super().enable_dm_encryption(portal, members) + if ok: + try: + puppet = pu.Puppet.get(portal.tgid) + await portal.main_intent.set_room_name(portal.mxid, puppet.displayname) + except Exception: + self.log.warning(f"Failed to set room name for {portal.mxid}", exc_info=True) + return ok + async def send_welcome_message(self, room_id: RoomID, inviter: 'u.User') -> None: try: is_management = len(await self.az.intent.get_room_members(room_id)) == 2 @@ -156,7 +188,7 @@ class MatrixHandler(BaseMatrixHandler): "messages for unauthenticated users.") return - self.log.debug(f"{user} joined {room_id}") + self.log.debug(f"{user.mxid} joined {room_id}") if await user.is_logged_in() or portal.has_bot: await portal.join_matrix(user, event_id) @@ -355,7 +387,7 @@ class MatrixHandler(BaseMatrixHandler): self.previously_typing[room_id] = now_typing def filter_matrix_event(self, evt: Event) -> bool: - if not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent)): + if not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent, EncryptedEvent)): return True return evt.sender and (evt.sender == self.az.bot_mxid or pu.Puppet.get_id_from_mxid(evt.sender) is not None) @@ -387,6 +419,11 @@ class MatrixHandler(BaseMatrixHandler): await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events) elif evt.type == EventType.ROOM_TOMBSTONE: await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room) + elif evt.type == EventType.ROOM_ENCRYPTION: + portal = po.Portal.get_by_mxid(evt.room_id) + if portal: + portal.encrypted = True + portal.save() async def log_event_handle_duration(self, evt: Event, duration: float) -> None: if EVENT_TIME: diff --git a/mautrix_telegram/portal/base.py b/mautrix_telegram/portal/base.py index c1332b0e..1d2d6d71 100644 --- a/mautrix_telegram/portal/base.py +++ b/mautrix_telegram/portal/base.py @@ -44,6 +44,7 @@ if TYPE_CHECKING: from ..bot import Bot from ..abstract_user import AbstractUser from ..config import Config + from ..matrix import MatrixHandler from . import Portal TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant] @@ -58,6 +59,7 @@ class BasePortal(ABC): az: AppService = None bot: 'Bot' = None loop: asyncio.AbstractEventLoop = None + matrix: 'MatrixHandler' = None # Config cache filter_mode: str = None @@ -85,6 +87,7 @@ class BasePortal(ABC): about: Optional[str] photo_id: Optional[str] local_config: Dict[str, Any] + encrypted: bool deleted: bool backfilling: bool backfill_leave: Optional[Set[IntentAPI]] @@ -102,7 +105,8 @@ class BasePortal(ABC): mxid: Optional[RoomID] = None, username: Optional[str] = None, megagroup: Optional[bool] = False, title: Optional[str] = None, about: Optional[str] = None, photo_id: Optional[str] = None, - local_config: Optional[str] = None, db_instance: DBPortal = None) -> None: + local_config: Optional[str] = None, encrypted: Optional[bool] = False, + db_instance: DBPortal = None) -> None: self.mxid = mxid self.tgid = tgid self.tg_receiver = tg_receiver or tgid @@ -113,6 +117,7 @@ class BasePortal(ABC): self.about = about self.photo_id = photo_id self.local_config = json.loads(local_config or "{}") + self.encrypted = encrypted self._db_instance = db_instance self._main_intent = None self.deleted = False @@ -328,12 +333,12 @@ class BasePortal(ABC): return DBPortal(tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type, mxid=self.mxid, username=self.username, megagroup=self.megagroup, title=self.title, about=self.about, photo_id=self.photo_id, - config=json.dumps(self.local_config)) + config=json.dumps(self.local_config), encrypted=self.encrypted) def save(self) -> None: self.db_instance.edit(mxid=self.mxid, username=self.username, title=self.title, about=self.about, photo_id=self.photo_id, megagroup=self.megagroup, - config=json.dumps(self.local_config)) + config=json.dumps(self.local_config), encrypted=self.encrypted) def delete(self) -> None: try: @@ -352,10 +357,10 @@ class BasePortal(ABC): @classmethod def from_db(cls, db_portal: DBPortal) -> 'Portal': return cls(tgid=db_portal.tgid, tg_receiver=db_portal.tg_receiver, - peer_type=db_portal.peer_type, mxid=db_portal.mxid, - username=db_portal.username, megagroup=db_portal.megagroup, - title=db_portal.title, about=db_portal.about, photo_id=db_portal.photo_id, - local_config=db_portal.config, db_instance=db_portal) + peer_type=db_portal.peer_type, mxid=db_portal.mxid, username=db_portal.username, + megagroup=db_portal.megagroup, title=db_portal.title, about=db_portal.about, + photo_id=db_portal.photo_id, local_config=db_portal.config, + encrypted=db_portal.encrypted, db_instance=db_portal) # endregion # region Class instance lookup @@ -506,6 +511,7 @@ class BasePortal(ABC): def init(context: Context) -> None: global config BasePortal.az, config, BasePortal.loop, BasePortal.bot = context.core + BasePortal.matrix = context.mx BasePortal.max_initial_member_sync = config["bridge.max_initial_member_sync"] BasePortal.sync_channel_members = config["bridge.sync_channel_members"] BasePortal.sync_matrix_state = config["bridge.sync_matrix_state"] diff --git a/mautrix_telegram/portal/matrix.py b/mautrix_telegram/portal/matrix.py index e0ffb58e..afd25b6c 100644 --- a/mautrix_telegram/portal/matrix.py +++ b/mautrix_telegram/portal/matrix.py @@ -50,6 +50,11 @@ if TYPE_CHECKING: from ..tgclient import MautrixTelegramClient from ..config import Config +try: + from nio.crypto import decrypt_attachment +except ImportError: + decrypt_attachment = None + TypeMessage = Union[Message, MessageService] config: Optional['Config'] = None @@ -250,11 +255,20 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC): file_name = content["net.maunium.telegram.internal.filename"] max_image_size = config["bridge.image_as_file_size"] * 1000 ** 2 - if config["bridge.parallel_file_transfer"]: + if config["bridge.parallel_file_transfer"] and content.url: file_handle, file_size = await parallel_transfer_to_telegram(client, self.main_intent, content.url, sender_id) else: - file = await self.main_intent.download_media(content.url) + if content.file: + if not decrypt_attachment: + self.log.warning(f"Can't bridge encrypted media event {event_id}:" + " matrix-nio not installed") + return + file = await self.main_intent.download_media(content.file.url) + file = decrypt_attachment(file, content.file.key.key, + content.file.hashes.get("sha256"), content.file.iv) + else: + file = await self.main_intent.download_media(content.url) if content.msgtype == MessageType.STICKER: if mime != "image/gif": diff --git a/mautrix_telegram/portal/metadata.py b/mautrix_telegram/portal/metadata.py index 7c6ef837..c0cec1be 100644 --- a/mautrix_telegram/portal/metadata.py +++ b/mautrix_telegram/portal/metadata.py @@ -30,7 +30,7 @@ from telethon.tl.types import ( from mautrix.errors import MForbidden from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership, Member, - PowerLevelStateEventContent, RoomAlias) + PowerLevelStateEventContent) from mautrix.appservice import IntentAPI from ..types import TelegramID @@ -308,6 +308,17 @@ class PortalMetadata(BasePortal, ABC): "type": EventType.ROOM_POWER_LEVELS.serialize(), "content": power_levels.serialize(), }] + if config["bridge.encryption.default"] and self.matrix.e2ee: + self.encrypted = True + initial_state.append({ + "type": "m.room.encryption", + "content": {"algorithm": "m.megolm.v1.aes-sha2"}, + }) + if direct: + invites.append(self.az.bot_mxid) + # The bridge bot needs to join for e2ee, but that messes up the default name + # generation. If/when canonical DMs happen, this might not be necessary anymore. + self.title = puppet.displayname if config["appservice.community_id"]: initial_state.append({ "type": "m.room.related_groups", @@ -325,6 +336,16 @@ class PortalMetadata(BasePortal, ABC): if not room_id: raise Exception(f"Failed to create room") + if self.encrypted and self.matrix.e2ee: + members = [self.main_intent.mxid] + if direct: + try: + await self.az.intent.join_room_by_id(room_id) + members += [self.az.intent.mxid] + except Exception: + self.log.warning(f"Failed to add bridge bot to new private chat {room_id}") + await self.matrix.e2ee.add_room(room_id, members=members, encrypted=True) + self.mxid = RoomID(room_id) self.by_mxid[self.mxid] = self self.save() @@ -362,7 +383,7 @@ class PortalMetadata(BasePortal, ABC): levels.kick = overrides.get("kick", 50) levels.redact = overrides.get("redact", 50) levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0) - levels.events[EventType.ROOM_ENCRYPTED] = 99 + levels.events[EventType.ROOM_ENCRYPTION] = 99 levels.events[EventType.ROOM_TOMBSTONE] = 99 levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0 levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0 diff --git a/mautrix_telegram/portal/telegram.py b/mautrix_telegram/portal/telegram.py index 35c992bb..bd782797 100644 --- a/mautrix_telegram/portal/telegram.py +++ b/mautrix_telegram/portal/telegram.py @@ -38,7 +38,7 @@ from telethon.tl.types import ( from mautrix.appservice import IntentAPI from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType, EventType, MediaMessageEventContent, TextMessageEventContent, - LocationMessageEventContent, Format) + LocationMessageEventContent, Format, MessageEventContent) from ..types import TelegramID from ..db import Message as DBMessage, TelegramFile as DBTelegramFile @@ -71,10 +71,17 @@ class PortalTelegram(BasePortal, ABC): return f"https://t.me/c/{self.tgid}/{evt.id}" return None + async def _send_message(self, intent: IntentAPI, content: MessageEventContent, + event_type: EventType = EventType.ROOM_MESSAGE, **kwargs) -> EventID: + if self.encrypted and self.matrix.e2ee: + event_type, content = await self.matrix.e2ee.encrypt(self.mxid, event_type, content) + return await intent.send_message_event(self.mxid, event_type, content, **kwargs) + async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message, relates_to: Dict = None) -> Optional[EventID]: loc, largest_size = self._get_largest_photo_size(evt.media.photo) - file = await util.transfer_file_to_matrix(source.client, intent, loc) + file = await util.transfer_file_to_matrix(source.client, intent, loc, + encrypt=self.encrypted) if not file: return None if self.get_config("inline_images") and (evt.message @@ -85,22 +92,26 @@ class PortalTelegram(BasePortal, ABC): prefix_text="Inline image: ") content.external_url = self._get_external_url(evt) await intent.set_typing(self.mxid, is_typing=False) - return await intent.send_message(self.mxid, content, timestamp=evt.date) + return await self._send_message(intent, content, timestamp=evt.date) info = ImageInfo( height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type, size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize)) else largest_size.size)) name = f"image{sane_mimetypes.guess_extension(file.mime_type)}" await intent.set_typing(self.mxid, is_typing=False) - content = MediaMessageEventContent(url=file.mxc, msgtype=MessageType.IMAGE, info=info, + content = MediaMessageEventContent(msgtype=MessageType.IMAGE, info=info, body=name, relates_to=relates_to, external_url=self._get_external_url(evt)) - result = await intent.send_message(self.mxid, content, timestamp=evt.date) + if file.decryption_info: + content.file = file.decryption_info + else: + content.url = file.mxc + result = await self._send_message(intent, content, timestamp=evt.date) if evt.message: caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent, no_reply_fallback=True) caption_content.external_url = content.external_url - result = await intent.send_message(self.mxid, caption_content, timestamp=evt.date) + result = await self._send_message(intent, caption_content, timestamp=evt.date) return result @staticmethod @@ -147,13 +158,20 @@ class PortalTelegram(BasePortal, ABC): info.width, info.height = attrs.width, attrs.height if file.thumbnail: - info.thumbnail_url = file.thumbnail.mxc + if file.thumbnail.decryption_info: + info.thumbnail_file = file.thumbnail.decryption_info + else: + info.thumbnail_url = file.thumbnail.mxc info.thumbnail_info = ThumbnailInfo(mimetype=file.thumbnail.mime_type, height=file.thumbnail.height or thumb_size.h, width=file.thumbnail.width or thumb_size.w, size=file.thumbnail.size) else: - info.thumbnail_url = file.mxc + # This is a hack for bad clients like Riot iOS that require a thumbnail + if file.decryption_info: + info.thumbnail_file = file.decryption_info + else: + info.thumbnail_url = file.mxc info.thumbnail_info = ImageInfo.deserialize(info.serialize()) return info, name @@ -168,6 +186,7 @@ class PortalTelegram(BasePortal, ABC): if document.size > config["bridge.max_document_size"] * 1000 ** 2: name = attrs.name or "" caption = f"\n{evt.message}" if evt.message else "" + # TODO encrypt return await intent.send_notice(self.mxid, f"Too large file {name}{caption}") thumb_loc, thumb_size = self._get_largest_photo_size(document) @@ -179,7 +198,8 @@ class PortalTelegram(BasePortal, ABC): file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc, is_sticker=attrs.is_sticker, tgs_convert=config["bridge.animated_sticker"], - filename=attrs.name, parallel_id=parallel_id) + filename=attrs.name, parallel_id=parallel_id, + encrypt=self.encrypted) if not file: return None @@ -192,14 +212,18 @@ class PortalTelegram(BasePortal, ABC): if attrs.is_sticker and file.mime_type.startswith("image/"): event_type = EventType.STICKER content = MediaMessageEventContent( - body=name or "unnamed file", info=info, url=file.mxc, relates_to=relates_to, + body=name or "unnamed file", info=info, relates_to=relates_to, external_url=self._get_external_url(evt), msgtype={ "video/": MessageType.VIDEO, "audio/": MessageType.AUDIO, "image/": MessageType.IMAGE, }.get(info.mimetype[:6], MessageType.FILE)) - return await intent.send_message_event(self.mxid, event_type, content, timestamp=evt.date) + if file.decryption_info: + content.file = file.decryption_info + else: + content.url = file.mxc + return await self._send_message(intent, content, event_type=event_type, timestamp=evt.date) def handle_telegram_location(self, _: 'AbstractUser', intent: IntentAPI, evt: Message, relates_to: dict = None) -> Awaitable[EventID]: @@ -218,7 +242,7 @@ class PortalTelegram(BasePortal, ABC): content["format"] = str(Format.HTML) content["formatted_body"] = f"Location: {body}" - return intent.send_message(self.mxid, content, timestamp=evt.date) + return self._send_message(intent, content, timestamp=evt.date) async def handle_telegram_text(self, source: 'AbstractUser', intent: IntentAPI, is_bot: bool, evt: Message) -> EventID: @@ -228,7 +252,7 @@ class PortalTelegram(BasePortal, ABC): if is_bot and self.get_config("bot_messages_as_notices"): content.msgtype = MessageType.NOTICE await intent.set_typing(self.mxid, is_typing=False) - return await intent.send_message(self.mxid, content, timestamp=evt.date) + return await self._send_message(intent, content, timestamp=evt.date) async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI, evt: Message, relates_to: dict = None) -> EventID: @@ -241,7 +265,7 @@ class PortalTelegram(BasePortal, ABC): content.external_url = self._get_external_url(evt) content["net.maunium.telegram.unsupported"] = True await intent.set_typing(self.mxid, is_typing=False) - return await intent.send_message(self.mxid, content, timestamp=evt.date) + return await self._send_message(intent, content, timestamp=evt.date) async def handle_telegram_poll(self, source: 'AbstractUser', intent: IntentAPI, evt: Message, relates_to: RelatesTo) -> EventID: @@ -267,7 +291,7 @@ class PortalTelegram(BasePortal, ABC): relates_to=relates_to, external_url=self._get_external_url(evt)) await intent.set_typing(self.mxid, is_typing=False) - return await intent.send_message(self.mxid, content, timestamp=evt.date) + return await self._send_message(intent, content, timestamp=evt.date) @staticmethod def _int_to_bytes(i: int) -> bytes: @@ -309,7 +333,7 @@ class PortalTelegram(BasePortal, ABC): content["net.maunium.telegram.game"] = play_id await intent.set_typing(self.mxid, is_typing=False) - return await intent.send_message(self.mxid, content, timestamp=evt.date) + return await self._send_message(intent, content, timestamp=evt.date) async def handle_telegram_edit(self, source: 'AbstractUser', sender: p.Puppet, evt: Message ) -> None: @@ -353,7 +377,7 @@ class PortalTelegram(BasePortal, ABC): intent = sender.intent_for(self) if sender else self.main_intent await intent.set_typing(self.mxid, is_typing=False) - event_id = await intent.send_message(self.mxid, content) + event_id = await self._send_message(intent, content) prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg DBMessage(mxid=event_id, mx_room=self.mxid, tg_space=tg_space, tgid=TelegramID(evt.id), @@ -522,6 +546,7 @@ class PortalTelegram(BasePortal, ABC): elif isinstance(action, MessageActionChatMigrateTo): self.peer_type = "channel" self._migrate_and_save_telegram(TelegramID(action.channel_id)) + # TODO encrypt await sender.intent_for(self).send_emote(self.mxid, "upgraded this group to a supergroup.") elif isinstance(action, MessageActionGameScore): diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py index f9df31e5..36ae6a0c 100644 --- a/mautrix_telegram/util/file_transfer.py +++ b/mautrix_telegram/util/file_transfer.py @@ -29,11 +29,13 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio SecurityError, FileIdInvalidError) from mautrix.appservice import IntentAPI +from mautrix.types import EncryptedFile from ..tgclient import MautrixTelegramClient from ..db import TelegramFile as DBTelegramFile from ..util import sane_mimetypes from .parallel_file_transfer import parallel_transfer_to_matrix +from .tgs_converter import convert_tgs_to try: from PIL import Image @@ -49,7 +51,10 @@ try: except ImportError: VideoFileClip = random = string = os = mimetypes = None -from .tgs_converter import convert_tgs_to +try: + from nio.crypto import encrypt_attachment +except ImportError: + encrypt_attachment = None log: logging.Logger = logging.getLogger("mau.util") @@ -115,8 +120,8 @@ def _location_to_id(location: TypeLocation) -> str: async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI, - thumbnail_loc: TypeLocation, video: bytes, - mime: str) -> Optional[DBTelegramFile]: + thumbnail_loc: TypeLocation, video: bytes, mime: str, + encrypt: bool) -> Optional[DBTelegramFile]: if not Image or not VideoFileClip: return None @@ -140,11 +145,19 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In width, height = None, None mime_type = magic.from_buffer(file, mime=True) - content_uri = await intent.upload_media(file, mime_type) + decryption_info = None + upload_mime_type = mime_type + if encrypt: + file, decryption_info_dict = encrypt_attachment(file) + decryption_info = EncryptedFile.deserialize(decryption_info_dict) + upload_mime_type = "application/octet-stream" + content_uri = await intent.upload_media(file, upload_mime_type) + if decryption_info: + decryption_info.url = content_uri db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type, was_converted=False, timestamp=int(time.time()), size=len(file), - width=width, height=height) + width=width, height=height, decryption_info=decryption_info) try: db_file.insert() except (IntegrityError, InvalidRequestError) as e: @@ -160,10 +173,10 @@ TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]] async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI, - location: TypeLocation, thumbnail: TypeThumbnail = None, + location: TypeLocation, thumbnail: TypeThumbnail = None, *, is_sticker: bool = False, tgs_convert: Optional[dict] = None, - filename: Optional[str] = None, parallel_id: Optional[int] = None - ) -> Optional[DBTelegramFile]: + filename: Optional[str] = None, encrypt: bool = False, + parallel_id: Optional[int] = None) -> Optional[DBTelegramFile]: location_id = _location_to_id(location) if not location_id: return None @@ -180,14 +193,14 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA async with lock: return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location, thumbnail, is_sticker, tgs_convert, - filename, parallel_id) + filename, encrypt, parallel_id) async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI, loc_id: str, location: TypeLocation, thumbnail: TypeThumbnail, is_sticker: bool, tgs_convert: Optional[dict], filename: Optional[str], - parallel_id: Optional[int] + encrypt: bool, parallel_id: Optional[int] ) -> Optional[DBTelegramFile]: db_file = DBTelegramFile.get(loc_id) if db_file: @@ -195,7 +208,7 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten if parallel_id and isinstance(location, Document) and (not is_sticker or not tgs_convert): db_file = await parallel_transfer_to_matrix(client, intent, loc_id, location, filename, - parallel_id) + encrypt, parallel_id) mime_type = location.mime_type file = None else: @@ -228,9 +241,17 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten mime_type = new_mime_type thumbnail = None - content_uri = await intent.upload_media(file, mime_type) + decryption_info = None + upload_mime_type = mime_type + if encrypt and encrypt_attachment: + file, decryption_info_dict = encrypt_attachment(file) + decryption_info = EncryptedFile.deserialize(decryption_info_dict) + upload_mime_type = "application/octet-stream" + content_uri = await intent.upload_media(file, upload_mime_type) + if decryption_info: + decryption_info.url = content_uri - db_file = DBTelegramFile(id=loc_id, mxc=content_uri, + db_file = DBTelegramFile(id=loc_id, mxc=content_uri, decryption_info=decryption_info, mime_type=mime_type, was_converted=image_converted, timestamp=int(time.time()), size=len(file), width=width, height=height) @@ -239,7 +260,7 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten thumbnail = thumbnail.location try: db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file, - mime_type) + mime_type, encrypt) except FileIdInvalidError: log.warning(f"Failed to transfer thumbnail for {thumbnail!s}", exc_info=True) diff --git a/mautrix_telegram/util/parallel_file_transfer.py b/mautrix_telegram/util/parallel_file_transfer.py index 507646aa..5d316e48 100644 --- a/mautrix_telegram/util/parallel_file_transfer.py +++ b/mautrix_telegram/util/parallel_file_transfer.py @@ -34,11 +34,16 @@ from telethon.crypto import AuthKey from telethon import utils, helpers from mautrix.appservice import IntentAPI -from mautrix.types import ContentURI +from mautrix.types import ContentURI, EncryptedFile from ..tgclient import MautrixTelegramClient from ..db import TelegramFile as DBTelegramFile +try: + from nio.crypto import async_encrypt_attachment +except ImportError: + async_encrypt_attachment = None + log: logging.Logger = logging.getLogger("mau.util") TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation, @@ -242,18 +247,34 @@ parallel_transfer_locks: DefaultDict[int, asyncio.Lock] = defaultdict(lambda: as async def parallel_transfer_to_matrix(client: MautrixTelegramClient, intent: IntentAPI, loc_id: str, location: TypeLocation, filename: str, - parallel_id: int) -> DBTelegramFile: + encrypt: bool, parallel_id: int) -> DBTelegramFile: size = location.size mime_type = location.mime_type dc_id, location = utils.get_input_location(location) # We lock the transfers because telegram has connection count limits async with parallel_transfer_locks[parallel_id]: downloader = ParallelTransferrer(client, dc_id) - content_uri = await intent.upload_media(downloader.download(location, size), - mime_type=mime_type, filename=filename, size=size) + data = downloader.download(location, size) + decryption_info = None + up_mime_type = mime_type + if encrypt and async_encrypt_attachment: + async def encrypted(stream): + nonlocal decryption_info + async for chunk in async_encrypt_attachment(stream): + if isinstance(chunk, dict): + decryption_info = EncryptedFile.deserialize(chunk) + else: + yield chunk + + data = encrypted(data) + up_mime_type = "application/octet-stream" + content_uri = await intent.upload_media(data, mime_type=up_mime_type, filename=filename, + size=size if not encrypt else None) + if decryption_info: + decryption_info.url = content_uri return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type, was_converted=False, timestamp=int(time.time()), size=size, - width=None, height=None) + width=None, height=None, decryption_info=decryption_info) async def _internal_transfer_to_telegram(client: MautrixTelegramClient, response: ClientResponse diff --git a/optional-requirements.txt b/optional-requirements.txt index 0c404596..c30daef8 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -18,3 +18,6 @@ prometheus_client>=0.6,<0.8 #/postgres psycopg2-binary>=2,<3 + +#/e2be +matrix-nio[e2e]>=0.9,<0.11 diff --git a/requirements.txt b/requirements.txt index 4941f95a..feaf4bb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ ruamel.yaml>=0.15.35,<0.17 python-magic>=0.4,<0.5 commonmark>=0.8,<0.10 aiohttp>=3,<4 -mautrix==0.5.0.beta4 +mautrix==0.5.0.beta10 telethon>=1.10,<1.12 telethon-session-sqlalchemy>=0.2.14,<0.3