iartamonov/mautrix-telegram
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'e2be' into master

Browse Source
This commit is contained in:
Tulir Asokan
2020-04-04 20:38:55 +03:00
parent 7a6de144ce c957989abb
commit dc5fe62e3a
19 changed files with 402 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+25 -8
View File
@@ -20,22 +20,22 @@ RUN apk add --no-cache libpng libpng-dev zlib zlib-dev \
FROM docker.io/alpine:3.11
RUN echo "@edge_main http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories
RUN echo "@edge_testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
RUN echo "@edge_community http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories
RUN apk add --no-cache \
py3-virtualenv \
py3-pillow \
py3-aiohttp \
py3-magic \
py3-sqlalchemy \
py3-alembic@edge_testing \
py3-psycopg2 \
py3-ruamel.yaml \
py3-commonmark@edge_testing \
# Indirect dependencies
py3-idna \
#commonmark
py3-future \
#alembic
py3-mako \
py3-dateutil \
py3-markupsafe \
#moviepy
py3-decorator \
py3-tqdm \
@@ -44,6 +44,8 @@ RUN apk add --no-cache \
py3-numpy \
#telethon
py3-rsa \
# Optional for socks proxies
py3-pysocks \
# cryptg
py3-cffi \
py3-brotli \
@@ -53,7 +55,22 @@ RUN apk add --no-cache \
su-exec \
netcat-openbsd \
# lottieconverter
zlib libpng
zlib libpng \
# olm
olm-dev@edge_community \
# matrix-nio?
py3-future \
py3-atomicwrites \
py3-pycryptodome@edge_main \
py3-peewee@edge_community \
py3-pyrsistent@edge_community \
py3-jsonschema \
py3-aiofiles \
py3-cachetools@edge_community \
py3-prometheus-client@edge_community \
py3-unpaddedbase64 \
py3-pyaes@edge_testing \
py3-logbook@edge_testing
COPY requirements.txt /opt/mautrix-telegram/requirements.txt
COPY optional-requirements.txt /opt/mautrix-telegram/optional-requirements.txt
@@ -67,7 +84,7 @@ RUN apk add --virtual .build-deps \
&& apk del .build-deps
COPY . /opt/mautrix-telegram
RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics] && apk del git
RUN apk add git && pip3 install .[speedups,hq_thumbnails,metrics,e2be] && apk del git
COPY --from=lottieconverter /usr/lib/librlottie* /usr/lib/
COPY --from=lottieconverter /build/LottieConverter/dist/Debug/GNU-Linux/lottieconverter /usr/local/bin/lottieconverter
alembic/versions/24f31fc8a72b_add_encrypted_field_for_portals.py
+27
View File
@@ -0,0 +1,27 @@
"""Add encrypted field for portals
Revision ID: 24f31fc8a72b
Revises: a7c04a56041b
Create Date: 2020-03-28 20:14:29.046699
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = "24f31fc8a72b"
down_revision = "a7c04a56041b"
branch_labels = None
depends_on = None
def upgrade():
with op.batch_alter_table("portal") as batch_op:
batch_op.add_column(sa.Column("encrypted", sa.Boolean(), nullable=False,
server_default=sa.sql.expression.false()))
def downgrade():
with op.batch_alter_table("portal") as batch_op:
batch_op.drop_column("encrypted")
alembic/versions/d3c922a6acd2_add_decryption_info_field_for_.py
+26
View File
@@ -0,0 +1,26 @@
"""Add decryption info field for reuploaded telegram files
Revision ID: d3c922a6acd2
Revises: 24f31fc8a72b
Create Date: 2020-03-30 20:07:17.340346
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'd3c922a6acd2'
down_revision = '24f31fc8a72b'
branch_labels = None
depends_on = None
def upgrade():
with op.batch_alter_table("telegram_file") as batch_op:
batch_op.add_column(sa.Column("decryption_info", sa.Text(), nullable=True))
def downgrade():
with op.batch_alter_table("telegram_file") as batch_op:
batch_op.drop_column("decryption_info")
alembic/versions/dff56c93da8d_add_matrix_nio_state_store_to_main_db.py
+71
View File
@@ -0,0 +1,71 @@
"""Add matrix-nio state store to main db
Revision ID: dff56c93da8d
Revises: d3c922a6acd2
Create Date: 2020-03-31 22:04:04.014048
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'dff56c93da8d'
down_revision = 'd3c922a6acd2'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('nio_account',
sa.Column('user_id', sa.String(length=255), nullable=False),
sa.Column('device_id', sa.String(length=255), nullable=False),
sa.Column('shared', sa.Boolean(), nullable=False),
sa.Column('sync_token', sa.Text(), nullable=False),
sa.Column('account', sa.LargeBinary(), nullable=False),
sa.PrimaryKeyConstraint('user_id', 'device_id')
)
op.create_table('nio_device_key',
sa.Column('user_id', sa.String(length=255), nullable=False),
sa.Column('device_id', sa.String(length=255), nullable=False),
sa.Column('display_name', sa.String(length=255), nullable=False),
sa.Column('deleted', sa.Boolean(), nullable=False),
sa.Column('keys', sa.PickleType(), nullable=False),
sa.PrimaryKeyConstraint('user_id', 'device_id')
)
op.create_table('nio_megolm_inbound_session',
sa.Column('session_id', sa.String(length=255), nullable=False),
sa.Column('sender_key', sa.String(length=255), nullable=False),
sa.Column('fp_key', sa.String(length=255), nullable=False),
sa.Column('room_id', sa.String(length=255), nullable=False),
sa.Column('session', sa.LargeBinary(), nullable=False),
sa.Column('forwarded_chains', sa.PickleType(), nullable=False),
sa.PrimaryKeyConstraint('session_id')
)
op.create_table('nio_olm_session',
sa.Column('session_id', sa.String(length=255), nullable=False),
sa.Column('sender_key', sa.String(length=255), nullable=False),
sa.Column('session', sa.LargeBinary(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=False),
sa.Column('last_used', sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint('session_id')
)
op.create_table('nio_outgoing_key_request',
sa.Column('request_id', sa.String(length=255), nullable=False),
sa.Column('session_id', sa.String(length=255), nullable=False),
sa.Column('room_id', sa.String(length=255), nullable=False),
sa.Column('algorithm', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('request_id')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('nio_outgoing_key_request')
op.drop_table('nio_olm_session')
op.drop_table('nio_megolm_inbound_session')
op.drop_table('nio_device_key')
op.drop_table('nio_account')
# ### end Alembic commands ###
docker-run.sh
+1 -1
View File
@@ -17,7 +17,7 @@ fi
alembic -x config=/data/config.yaml upgrade head
if [ ! -f /data/config.yaml ]; then
cp example-config.yaml /data/config.yaml
cp mautrix_telegram/example-config.yaml /data/config.yaml
echo "Didn't find a config file."
echo "Copied default config file to /data/config.yaml"
echo "Modify that config file to your liking."
mautrix_telegram/config.py
+2
View File
@@ -118,6 +118,8 @@ class Config(BaseBridgeConfig):
copy("bridge.federate_rooms")
copy("bridge.animated_sticker.target")
copy("bridge.animated_sticker.args")
copy("bridge.encryption.allow")
copy("bridge.encryption.default")
copy("bridge.initial_power_level_overrides.group")
copy("bridge.initial_power_level_overrides.user")
mautrix_telegram/db/__init__.py
+7
View File
@@ -24,6 +24,11 @@ from .puppet import Puppet
from .telegram_file import TelegramFile
from .user import User, UserPortal, Contact
try:
from mautrix.bridge.db.nio_state_store import init as init_nio_db
except ImportError:
init_nio_db = None
def init(db_engine: Engine) -> None:
for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
@@ -32,3 +37,5 @@ def init(db_engine: Engine) -> None:
table.t = table.__table__
table.c = table.t.c
table.column_names = table.c.keys()
if init_nio_db:
init_nio_db(db_engine)
mautrix_telegram/db/portal.py
+2 -1
View File
@@ -15,7 +15,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Optional
from sqlalchemy import Column, Integer, String, Boolean, Text, func
from sqlalchemy import Column, Integer, String, Boolean, Text, func, sql
from mautrix.types import RoomID
from mautrix.util.db import Base
@@ -34,6 +34,7 @@ class Portal(Base):
# Matrix portal information
mxid: RoomID = Column(String, unique=True, nullable=True)
encrypted: bool = Column(Boolean, nullable=False, server_default=sql.expression.false())
config: str = Column(Text, nullable=True)
mautrix_telegram/db/telegram_file.py
+28 -5
View File
@@ -13,15 +13,37 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Optional
from typing import Optional, cast, Dict, Any
from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
from sqlalchemy import (Column, ForeignKey, Integer, BigInteger, String, Boolean, Text,
TypeDecorator)
from sqlalchemy.engine.result import RowProxy
from mautrix.types import ContentURI
from mautrix.types import ContentURI, EncryptedFile
from mautrix.util.db import Base
class DBEncryptedFile(TypeDecorator):
impl = Text
@property
def python_type(self):
return EncryptedFile
def process_bind_param(self, value: EncryptedFile, dialect) -> Optional[str]:
if value is not None:
return value.json()
return None
def process_result_value(self, value: str, dialect) -> Optional[EncryptedFile]:
if value is not None:
return EncryptedFile.parse_json(value)
return None
def process_literal_param(self, value, dialect):
return value
class TelegramFile(Base):
__tablename__ = "telegram_file"
@@ -33,12 +55,13 @@ class TelegramFile(Base):
size: Optional[int] = Column(Integer, nullable=True)
width: Optional[int] = Column(Integer, nullable=True)
height: Optional[int] = Column(Integer, nullable=True)
decryption_info: Optional[Dict[str, Any]] = Column(DBEncryptedFile, nullable=True)
thumbnail_id: str = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
thumbnail: Optional['TelegramFile'] = None
@classmethod
def scan(cls, row: RowProxy) -> 'TelegramFile':
telegram_file: TelegramFile = super().scan(row)
telegram_file = cast(TelegramFile, super().scan(row))
if isinstance(telegram_file.thumbnail, str):
telegram_file.thumbnail = cls.get(telegram_file.thumbnail)
return telegram_file
@@ -52,5 +75,5 @@ class TelegramFile(Base):
conn.execute(self.t.insert().values(
id=self.id, mxc=self.mxc, mime_type=self.mime_type,
was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
width=self.width, height=self.height,
width=self.width, height=self.height, decryption_info=self.decryption_info,
thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))
mautrix_telegram/example-config.yaml
+11
View File
@@ -193,6 +193,17 @@ bridge:
height: 256
background: "020202" # only for gif
fps: 30 # only for webm
# End-to-bridge encryption support options. These require matrix-nio to be installed with pip
# and login_shared_secret to be configured in order to get a device for the bridge bot.
#
# Additionally, https://github.com/matrix-org/synapse/pull/5758 is required if using a normal
# application service.
encryption:
# Allow encryption, work in group chat rooms with e2ee enabled
allow: false
# Default to encryption, force-enable encryption in all portals the bridge creates
# This will cause the bridge bot to be in private chats for the encryption to work properly.
default: false
# Overrides for base power levels.
initial_power_level_overrides:
mautrix_telegram/matrix.py
+43 -6
View File
@@ -13,14 +13,15 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Dict, Set, Tuple, Union, Iterable, TYPE_CHECKING
from typing import Dict, Set, Tuple, Union, Iterable, List, TYPE_CHECKING
from mautrix.bridge import BaseMatrixHandler
from mautrix.types import (Event, EventType, RoomID, UserID, EventID, ReceiptEvent, ReceiptType,
ReceiptEventContent, PresenceEvent, PresenceState, TypingEvent,
MessageEvent, StateEvent, RedactionEvent, RoomNameStateEventContent,
RoomAvatarStateEventContent, RoomTopicStateEventContent,
MemberStateEventContent)
MemberStateEventContent, EncryptedEvent, TextMessageEventContent,
MessageType)
from mautrix.errors import MatrixError
from . import user as u, portal as po, puppet as pu, commands as com
@@ -47,8 +48,15 @@ class MatrixHandler(BaseMatrixHandler):
previously_typing: Dict[RoomID, Set[UserID]]
def __init__(self, context: 'Context') -> None:
prefix, suffix = context.config["bridge.username_template"].format(userid=":").split(":")
homeserver = context.config["homeserver.domain"]
self.user_id_prefix = f"@{prefix}"
self.user_id_suffix = f"{suffix}:{homeserver}"
super(MatrixHandler, self).__init__(context.az, context.config, loop=context.loop,
command_processor=com.CommandProcessor(context))
command_processor=com.CommandProcessor(context),
bridge=context.bridge)
self.bot = context.bot
self.previously_typing = {}
@@ -104,14 +112,38 @@ class MatrixHandler(BaseMatrixHandler):
except MatrixError:
pass
portal.mxid = room_id
e2be_ok = None
if self.config["bridge.encryption.default"] and self.e2ee:
e2be_ok = await self.enable_dm_encryption(portal, members=members)
portal.save()
inviter.register_portal(portal)
await intent.send_notice(room_id, "Portal to private chat created.")
if e2be_ok is True:
evt_type, content = await self.e2ee.encrypt(
room_id, EventType.ROOM_MESSAGE,
TextMessageEventContent(msgtype=MessageType.NOTICE,
body="Portal to private chat created and end-to-bridge"
" encryption enabled."))
await intent.send_message_event(room_id, evt_type, content)
else:
message = "Portal to private chat created."
if e2be_ok is False:
message += "\n\nWarning: Failed to enable end-to-bridge encryption"
await intent.send_notice(room_id, message)
else:
await intent.join_room(room_id)
await intent.send_notice(room_id, "This puppet will remain inactive until a "
"Telegram chat is created for this room.")
async def enable_dm_encryption(self, portal: po.Portal, members: List[UserID]) -> bool:
ok = await super().enable_dm_encryption(portal, members)
if ok:
try:
puppet = pu.Puppet.get(portal.tgid)
await portal.main_intent.set_room_name(portal.mxid, puppet.displayname)
except Exception:
self.log.warning(f"Failed to set room name for {portal.mxid}", exc_info=True)
return ok
async def send_welcome_message(self, room_id: RoomID, inviter: 'u.User') -> None:
try:
is_management = len(await self.az.intent.get_room_members(room_id)) == 2
@@ -156,7 +188,7 @@ class MatrixHandler(BaseMatrixHandler):
"messages for unauthenticated users.")
return
self.log.debug(f"{user} joined {room_id}")
self.log.debug(f"{user.mxid} joined {room_id}")
if await user.is_logged_in() or portal.has_bot:
await portal.join_matrix(user, event_id)
@@ -355,7 +387,7 @@ class MatrixHandler(BaseMatrixHandler):
self.previously_typing[room_id] = now_typing
def filter_matrix_event(self, evt: Event) -> bool:
if not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent)):
if not isinstance(evt, (RedactionEvent, MessageEvent, StateEvent, EncryptedEvent)):
return True
return evt.sender and (evt.sender == self.az.bot_mxid
or pu.Puppet.get_id_from_mxid(evt.sender) is not None)
@@ -387,6 +419,11 @@ class MatrixHandler(BaseMatrixHandler):
await self.handle_room_pin(evt.room_id, evt.sender, new_events, old_events)
elif evt.type == EventType.ROOM_TOMBSTONE:
await self.handle_room_upgrade(evt.room_id, evt.sender, evt.content.replacement_room)
elif evt.type == EventType.ROOM_ENCRYPTION:
portal = po.Portal.get_by_mxid(evt.room_id)
if portal:
portal.encrypted = True
portal.save()
async def log_event_handle_duration(self, evt: Event, duration: float) -> None:
if EVENT_TIME:
mautrix_telegram/portal/base.py
+13 -7
View File
@@ -44,6 +44,7 @@ if TYPE_CHECKING:
from ..bot import Bot
from ..abstract_user import AbstractUser
from ..config import Config
from ..matrix import MatrixHandler
from . import Portal
TypeParticipant = Union[TypeChatParticipant, TypeChannelParticipant]
@@ -58,6 +59,7 @@ class BasePortal(ABC):
az: AppService = None
bot: 'Bot' = None
loop: asyncio.AbstractEventLoop = None
matrix: 'MatrixHandler' = None
# Config cache
filter_mode: str = None
@@ -85,6 +87,7 @@ class BasePortal(ABC):
about: Optional[str]
photo_id: Optional[str]
local_config: Dict[str, Any]
encrypted: bool
deleted: bool
backfilling: bool
backfill_leave: Optional[Set[IntentAPI]]
@@ -102,7 +105,8 @@ class BasePortal(ABC):
mxid: Optional[RoomID] = None, username: Optional[str] = None,
megagroup: Optional[bool] = False, title: Optional[str] = None,
about: Optional[str] = None, photo_id: Optional[str] = None,
local_config: Optional[str] = None, db_instance: DBPortal = None) -> None:
local_config: Optional[str] = None, encrypted: Optional[bool] = False,
db_instance: DBPortal = None) -> None:
self.mxid = mxid
self.tgid = tgid
self.tg_receiver = tg_receiver or tgid
@@ -113,6 +117,7 @@ class BasePortal(ABC):
self.about = about
self.photo_id = photo_id
self.local_config = json.loads(local_config or "{}")
self.encrypted = encrypted
self._db_instance = db_instance
self._main_intent = None
self.deleted = False
@@ -328,12 +333,12 @@ class BasePortal(ABC):
return DBPortal(tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
mxid=self.mxid, username=self.username, megagroup=self.megagroup,
title=self.title, about=self.about, photo_id=self.photo_id,
config=json.dumps(self.local_config))
config=json.dumps(self.local_config), encrypted=self.encrypted)
def save(self) -> None:
self.db_instance.edit(mxid=self.mxid, username=self.username, title=self.title,
about=self.about, photo_id=self.photo_id, megagroup=self.megagroup,
config=json.dumps(self.local_config))
config=json.dumps(self.local_config), encrypted=self.encrypted)
def delete(self) -> None:
try:
@@ -352,10 +357,10 @@ class BasePortal(ABC):
@classmethod
def from_db(cls, db_portal: DBPortal) -> 'Portal':
return cls(tgid=db_portal.tgid, tg_receiver=db_portal.tg_receiver,
peer_type=db_portal.peer_type, mxid=db_portal.mxid,
username=db_portal.username, megagroup=db_portal.megagroup,
title=db_portal.title, about=db_portal.about, photo_id=db_portal.photo_id,
local_config=db_portal.config, db_instance=db_portal)
peer_type=db_portal.peer_type, mxid=db_portal.mxid, username=db_portal.username,
megagroup=db_portal.megagroup, title=db_portal.title, about=db_portal.about,
photo_id=db_portal.photo_id, local_config=db_portal.config,
encrypted=db_portal.encrypted, db_instance=db_portal)
# endregion
# region Class instance lookup
@@ -506,6 +511,7 @@ class BasePortal(ABC):
def init(context: Context) -> None:
global config
BasePortal.az, config, BasePortal.loop, BasePortal.bot = context.core
BasePortal.matrix = context.mx
BasePortal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
BasePortal.sync_channel_members = config["bridge.sync_channel_members"]
BasePortal.sync_matrix_state = config["bridge.sync_matrix_state"]
mautrix_telegram/portal/matrix.py
+16 -2
View File
@@ -50,6 +50,11 @@ if TYPE_CHECKING:
from ..tgclient import MautrixTelegramClient
from ..config import Config
try:
from nio.crypto import decrypt_attachment
except ImportError:
decrypt_attachment = None
TypeMessage = Union[Message, MessageService]
config: Optional['Config'] = None
@@ -250,11 +255,20 @@ class PortalMatrix(BasePortal, MautrixBasePortal, ABC):
file_name = content["net.maunium.telegram.internal.filename"]
max_image_size = config["bridge.image_as_file_size"] * 1000 ** 2
if config["bridge.parallel_file_transfer"]:
if config["bridge.parallel_file_transfer"] and content.url:
file_handle, file_size = await parallel_transfer_to_telegram(client, self.main_intent,
content.url, sender_id)
else:
file = await self.main_intent.download_media(content.url)
if content.file:
if not decrypt_attachment:
self.log.warning(f"Can't bridge encrypted media event {event_id}:"
" matrix-nio not installed")
return
file = await self.main_intent.download_media(content.file.url)
file = decrypt_attachment(file, content.file.key.key,
content.file.hashes.get("sha256"), content.file.iv)
else:
file = await self.main_intent.download_media(content.url)
if content.msgtype == MessageType.STICKER:
if mime != "image/gif":
mautrix_telegram/portal/metadata.py
+23 -2
View File
@@ -30,7 +30,7 @@ from telethon.tl.types import (
from mautrix.errors import MForbidden
from mautrix.types import (RoomID, UserID, RoomCreatePreset, EventType, Membership, Member,
PowerLevelStateEventContent, RoomAlias)
PowerLevelStateEventContent)
from mautrix.appservice import IntentAPI
from ..types import TelegramID
@@ -308,6 +308,17 @@ class PortalMetadata(BasePortal, ABC):
"type": EventType.ROOM_POWER_LEVELS.serialize(),
"content": power_levels.serialize(),
}]
if config["bridge.encryption.default"] and self.matrix.e2ee:
self.encrypted = True
initial_state.append({
"type": "m.room.encryption",
"content": {"algorithm": "m.megolm.v1.aes-sha2"},
})
if direct:
invites.append(self.az.bot_mxid)
# The bridge bot needs to join for e2ee, but that messes up the default name
# generation. If/when canonical DMs happen, this might not be necessary anymore.
self.title = puppet.displayname
if config["appservice.community_id"]:
initial_state.append({
"type": "m.room.related_groups",
@@ -325,6 +336,16 @@ class PortalMetadata(BasePortal, ABC):
if not room_id:
raise Exception(f"Failed to create room")
if self.encrypted and self.matrix.e2ee:
members = [self.main_intent.mxid]
if direct:
try:
await self.az.intent.join_room_by_id(room_id)
members += [self.az.intent.mxid]
except Exception:
self.log.warning(f"Failed to add bridge bot to new private chat {room_id}")
await self.matrix.e2ee.add_room(room_id, members=members, encrypted=True)
self.mxid = RoomID(room_id)
self.by_mxid[self.mxid] = self
self.save()
@@ -362,7 +383,7 @@ class PortalMetadata(BasePortal, ABC):
levels.kick = overrides.get("kick", 50)
levels.redact = overrides.get("redact", 50)
levels.invite = overrides.get("invite", 50 if dbr.invite_users else 0)
levels.events[EventType.ROOM_ENCRYPTED] = 99
levels.events[EventType.ROOM_ENCRYPTION] = 99
levels.events[EventType.ROOM_TOMBSTONE] = 99
levels.events[EventType.ROOM_NAME] = 50 if dbr.change_info else 0
levels.events[EventType.ROOM_AVATAR] = 50 if dbr.change_info else 0
mautrix_telegram/portal/telegram.py
+42 -17
View File
@@ -38,7 +38,7 @@ from telethon.tl.types import (
from mautrix.appservice import IntentAPI
from mautrix.types import (EventID, UserID, ImageInfo, ThumbnailInfo, RelatesTo, MessageType,
EventType, MediaMessageEventContent, TextMessageEventContent,
LocationMessageEventContent, Format)
LocationMessageEventContent, Format, MessageEventContent)
from ..types import TelegramID
from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
@@ -71,10 +71,17 @@ class PortalTelegram(BasePortal, ABC):
return f"https://t.me/c/{self.tgid}/{evt.id}"
return None
async def _send_message(self, intent: IntentAPI, content: MessageEventContent,
event_type: EventType = EventType.ROOM_MESSAGE, **kwargs) -> EventID:
if self.encrypted and self.matrix.e2ee:
event_type, content = await self.matrix.e2ee.encrypt(self.mxid, event_type, content)
return await intent.send_message_event(self.mxid, event_type, content, **kwargs)
async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
relates_to: Dict = None) -> Optional[EventID]:
loc, largest_size = self._get_largest_photo_size(evt.media.photo)
file = await util.transfer_file_to_matrix(source.client, intent, loc)
file = await util.transfer_file_to_matrix(source.client, intent, loc,
encrypt=self.encrypted)
if not file:
return None
if self.get_config("inline_images") and (evt.message
@@ -85,22 +92,26 @@ class PortalTelegram(BasePortal, ABC):
prefix_text="Inline image: ")
content.external_url = self._get_external_url(evt)
await intent.set_typing(self.mxid, is_typing=False)
return await intent.send_message(self.mxid, content, timestamp=evt.date)
return await self._send_message(intent, content, timestamp=evt.date)
info = ImageInfo(
height=largest_size.h, width=largest_size.w, orientation=0, mimetype=file.mime_type,
size=(len(largest_size.bytes) if (isinstance(largest_size, PhotoCachedSize))
else largest_size.size))
name = f"image{sane_mimetypes.guess_extension(file.mime_type)}"
await intent.set_typing(self.mxid, is_typing=False)
content = MediaMessageEventContent(url=file.mxc, msgtype=MessageType.IMAGE, info=info,
content = MediaMessageEventContent(msgtype=MessageType.IMAGE, info=info,
body=name, relates_to=relates_to,
external_url=self._get_external_url(evt))
result = await intent.send_message(self.mxid, content, timestamp=evt.date)
if file.decryption_info:
content.file = file.decryption_info
else:
content.url = file.mxc
result = await self._send_message(intent, content, timestamp=evt.date)
if evt.message:
caption_content = await formatter.telegram_to_matrix(evt, source, self.main_intent,
no_reply_fallback=True)
caption_content.external_url = content.external_url
result = await intent.send_message(self.mxid, caption_content, timestamp=evt.date)
result = await self._send_message(intent, caption_content, timestamp=evt.date)
return result
@staticmethod
@@ -147,13 +158,20 @@ class PortalTelegram(BasePortal, ABC):
info.width, info.height = attrs.width, attrs.height
if file.thumbnail:
info.thumbnail_url = file.thumbnail.mxc
if file.thumbnail.decryption_info:
info.thumbnail_file = file.thumbnail.decryption_info
else:
info.thumbnail_url = file.thumbnail.mxc
info.thumbnail_info = ThumbnailInfo(mimetype=file.thumbnail.mime_type,
height=file.thumbnail.height or thumb_size.h,
width=file.thumbnail.width or thumb_size.w,
size=file.thumbnail.size)
else:
info.thumbnail_url = file.mxc
# This is a hack for bad clients like Riot iOS that require a thumbnail
if file.decryption_info:
info.thumbnail_file = file.decryption_info
else:
info.thumbnail_url = file.mxc
info.thumbnail_info = ImageInfo.deserialize(info.serialize())
return info, name
@@ -168,6 +186,7 @@ class PortalTelegram(BasePortal, ABC):
if document.size > config["bridge.max_document_size"] * 1000 ** 2:
name = attrs.name or ""
caption = f"\n{evt.message}" if evt.message else ""
# TODO encrypt
return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
thumb_loc, thumb_size = self._get_largest_photo_size(document)
@@ -179,7 +198,8 @@ class PortalTelegram(BasePortal, ABC):
file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
is_sticker=attrs.is_sticker,
tgs_convert=config["bridge.animated_sticker"],
filename=attrs.name, parallel_id=parallel_id)
filename=attrs.name, parallel_id=parallel_id,
encrypt=self.encrypted)
if not file:
return None
@@ -192,14 +212,18 @@ class PortalTelegram(BasePortal, ABC):
if attrs.is_sticker and file.mime_type.startswith("image/"):
event_type = EventType.STICKER
content = MediaMessageEventContent(
body=name or "unnamed file", info=info, url=file.mxc, relates_to=relates_to,
body=name or "unnamed file", info=info, relates_to=relates_to,
external_url=self._get_external_url(evt),
msgtype={
"video/": MessageType.VIDEO,
"audio/": MessageType.AUDIO,
"image/": MessageType.IMAGE,
}.get(info.mimetype[:6], MessageType.FILE))
return await intent.send_message_event(self.mxid, event_type, content, timestamp=evt.date)
if file.decryption_info:
content.file = file.decryption_info
else:
content.url = file.mxc
return await self._send_message(intent, content, event_type=event_type, timestamp=evt.date)
def handle_telegram_location(self, _: 'AbstractUser', intent: IntentAPI, evt: Message,
relates_to: dict = None) -> Awaitable[EventID]:
@@ -218,7 +242,7 @@ class PortalTelegram(BasePortal, ABC):
content["format"] = str(Format.HTML)
content["formatted_body"] = f"Location: <a href='{url}'>{body}</a>"
return intent.send_message(self.mxid, content, timestamp=evt.date)
return self._send_message(intent, content, timestamp=evt.date)
async def handle_telegram_text(self, source: 'AbstractUser', intent: IntentAPI, is_bot: bool,
evt: Message) -> EventID:
@@ -228,7 +252,7 @@ class PortalTelegram(BasePortal, ABC):
if is_bot and self.get_config("bot_messages_as_notices"):
content.msgtype = MessageType.NOTICE
await intent.set_typing(self.mxid, is_typing=False)
return await intent.send_message(self.mxid, content, timestamp=evt.date)
return await self._send_message(intent, content, timestamp=evt.date)
async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
evt: Message, relates_to: dict = None) -> EventID:
@@ -241,7 +265,7 @@ class PortalTelegram(BasePortal, ABC):
content.external_url = self._get_external_url(evt)
content["net.maunium.telegram.unsupported"] = True
await intent.set_typing(self.mxid, is_typing=False)
return await intent.send_message(self.mxid, content, timestamp=evt.date)
return await self._send_message(intent, content, timestamp=evt.date)
async def handle_telegram_poll(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
relates_to: RelatesTo) -> EventID:
@@ -267,7 +291,7 @@ class PortalTelegram(BasePortal, ABC):
relates_to=relates_to, external_url=self._get_external_url(evt))
await intent.set_typing(self.mxid, is_typing=False)
return await intent.send_message(self.mxid, content, timestamp=evt.date)
return await self._send_message(intent, content, timestamp=evt.date)
@staticmethod
def _int_to_bytes(i: int) -> bytes:
@@ -309,7 +333,7 @@ class PortalTelegram(BasePortal, ABC):
content["net.maunium.telegram.game"] = play_id
await intent.set_typing(self.mxid, is_typing=False)
return await intent.send_message(self.mxid, content, timestamp=evt.date)
return await self._send_message(intent, content, timestamp=evt.date)
async def handle_telegram_edit(self, source: 'AbstractUser', sender: p.Puppet, evt: Message
) -> None:
@@ -353,7 +377,7 @@ class PortalTelegram(BasePortal, ABC):
intent = sender.intent_for(self) if sender else self.main_intent
await intent.set_typing(self.mxid, is_typing=False)
event_id = await intent.send_message(self.mxid, content)
event_id = await self._send_message(intent, content)
prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
DBMessage(mxid=event_id, mx_room=self.mxid, tg_space=tg_space, tgid=TelegramID(evt.id),
@@ -522,6 +546,7 @@ class PortalTelegram(BasePortal, ABC):
elif isinstance(action, MessageActionChatMigrateTo):
self.peer_type = "channel"
self._migrate_and_save_telegram(TelegramID(action.channel_id))
# TODO encrypt
await sender.intent_for(self).send_emote(self.mxid,
"upgraded this group to a supergroup.")
elif isinstance(action, MessageActionGameScore):
mautrix_telegram/util/file_transfer.py
+35 -14
View File
@@ -29,11 +29,13 @@ from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, Locatio
SecurityError, FileIdInvalidError)
from mautrix.appservice import IntentAPI
from mautrix.types import EncryptedFile
from ..tgclient import MautrixTelegramClient
from ..db import TelegramFile as DBTelegramFile
from ..util import sane_mimetypes
from .parallel_file_transfer import parallel_transfer_to_matrix
from .tgs_converter import convert_tgs_to
try:
from PIL import Image
@@ -49,7 +51,10 @@ try:
except ImportError:
VideoFileClip = random = string = os = mimetypes = None
from .tgs_converter import convert_tgs_to
try:
from nio.crypto import encrypt_attachment
except ImportError:
encrypt_attachment = None
log: logging.Logger = logging.getLogger("mau.util")
@@ -115,8 +120,8 @@ def _location_to_id(location: TypeLocation) -> str:
async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
thumbnail_loc: TypeLocation, video: bytes,
mime: str) -> Optional[DBTelegramFile]:
thumbnail_loc: TypeLocation, video: bytes, mime: str,
encrypt: bool) -> Optional[DBTelegramFile]:
if not Image or not VideoFileClip:
return None
@@ -140,11 +145,19 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
width, height = None, None
mime_type = magic.from_buffer(file, mime=True)
content_uri = await intent.upload_media(file, mime_type)
decryption_info = None
upload_mime_type = mime_type
if encrypt:
file, decryption_info_dict = encrypt_attachment(file)
decryption_info = EncryptedFile.deserialize(decryption_info_dict)
upload_mime_type = "application/octet-stream"
content_uri = await intent.upload_media(file, upload_mime_type)
if decryption_info:
decryption_info.url = content_uri
db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
was_converted=False, timestamp=int(time.time()), size=len(file),
width=width, height=height)
width=width, height=height, decryption_info=decryption_info)
try:
db_file.insert()
except (IntegrityError, InvalidRequestError) as e:
@@ -160,10 +173,10 @@ TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
location: TypeLocation, thumbnail: TypeThumbnail = None,
location: TypeLocation, thumbnail: TypeThumbnail = None, *,
is_sticker: bool = False, tgs_convert: Optional[dict] = None,
filename: Optional[str] = None, parallel_id: Optional[int] = None
) -> Optional[DBTelegramFile]:
filename: Optional[str] = None, encrypt: bool = False,
parallel_id: Optional[int] = None) -> Optional[DBTelegramFile]:
location_id = _location_to_id(location)
if not location_id:
return None
@@ -180,14 +193,14 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA
async with lock:
return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
thumbnail, is_sticker, tgs_convert,
filename, parallel_id)
filename, encrypt, parallel_id)
async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
loc_id: str, location: TypeLocation,
thumbnail: TypeThumbnail, is_sticker: bool,
tgs_convert: Optional[dict], filename: Optional[str],
parallel_id: Optional[int]
encrypt: bool, parallel_id: Optional[int]
) -> Optional[DBTelegramFile]:
db_file = DBTelegramFile.get(loc_id)
if db_file:
@@ -195,7 +208,7 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
if parallel_id and isinstance(location, Document) and (not is_sticker or not tgs_convert):
db_file = await parallel_transfer_to_matrix(client, intent, loc_id, location, filename,
parallel_id)
encrypt, parallel_id)
mime_type = location.mime_type
file = None
else:
@@ -228,9 +241,17 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
mime_type = new_mime_type
thumbnail = None
content_uri = await intent.upload_media(file, mime_type)
decryption_info = None
upload_mime_type = mime_type
if encrypt and encrypt_attachment:
file, decryption_info_dict = encrypt_attachment(file)
decryption_info = EncryptedFile.deserialize(decryption_info_dict)
upload_mime_type = "application/octet-stream"
content_uri = await intent.upload_media(file, upload_mime_type)
if decryption_info:
decryption_info.url = content_uri
db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
db_file = DBTelegramFile(id=loc_id, mxc=content_uri, decryption_info=decryption_info,
mime_type=mime_type, was_converted=image_converted,
timestamp=int(time.time()), size=len(file),
width=width, height=height)
@@ -239,7 +260,7 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten
thumbnail = thumbnail.location
try:
db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
mime_type)
mime_type, encrypt)
except FileIdInvalidError:
log.warning(f"Failed to transfer thumbnail for {thumbnail!s}", exc_info=True)
mautrix_telegram/util/parallel_file_transfer.py
+26 -5
View File
@@ -34,11 +34,16 @@ from telethon.crypto import AuthKey
from telethon import utils, helpers
from mautrix.appservice import IntentAPI
from mautrix.types import ContentURI
from mautrix.types import ContentURI, EncryptedFile
from ..tgclient import MautrixTelegramClient
from ..db import TelegramFile as DBTelegramFile
try:
from nio.crypto import async_encrypt_attachment
except ImportError:
async_encrypt_attachment = None
log: logging.Logger = logging.getLogger("mau.util")
TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
@@ -242,18 +247,34 @@ parallel_transfer_locks: DefaultDict[int, asyncio.Lock] = defaultdict(lambda: as
async def parallel_transfer_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
loc_id: str, location: TypeLocation, filename: str,
parallel_id: int) -> DBTelegramFile:
encrypt: bool, parallel_id: int) -> DBTelegramFile:
size = location.size
mime_type = location.mime_type
dc_id, location = utils.get_input_location(location)
# We lock the transfers because telegram has connection count limits
async with parallel_transfer_locks[parallel_id]:
downloader = ParallelTransferrer(client, dc_id)
content_uri = await intent.upload_media(downloader.download(location, size),
mime_type=mime_type, filename=filename, size=size)
data = downloader.download(location, size)
decryption_info = None
up_mime_type = mime_type
if encrypt and async_encrypt_attachment:
async def encrypted(stream):
nonlocal decryption_info
async for chunk in async_encrypt_attachment(stream):
if isinstance(chunk, dict):
decryption_info = EncryptedFile.deserialize(chunk)
else:
yield chunk
data = encrypted(data)
up_mime_type = "application/octet-stream"
content_uri = await intent.upload_media(data, mime_type=up_mime_type, filename=filename,
size=size if not encrypt else None)
if decryption_info:
decryption_info.url = content_uri
return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
was_converted=False, timestamp=int(time.time()), size=size,
width=None, height=None)
width=None, height=None, decryption_info=decryption_info)
async def _internal_transfer_to_telegram(client: MautrixTelegramClient, response: ClientResponse
optional-requirements.txt
+3
View File
@@ -18,3 +18,6 @@ prometheus_client>=0.6,<0.8
#/postgres
psycopg2-binary>=2,<3
#/e2be
matrix-nio[e2e]>=0.9,<0.11
requirements.txt
+1 -1
View File
@@ -4,6 +4,6 @@ ruamel.yaml>=0.15.35,<0.17
python-magic>=0.4,<0.5
commonmark>=0.8,<0.10
aiohttp>=3,<4
mautrix==0.5.0.beta4
mautrix==0.5.0.beta10
telethon>=1.10,<1.12
telethon-session-sqlalchemy>=0.2.14,<0.3