iartamonov/mautrix-telegram
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Properly use takeout client for backfilling

Browse Source
This commit is contained in:
Tulir Asokan
2022-10-11 17:53:41 +03:00
parent 8ae34223c5
commit 27b23a96b6
5 changed files with 98 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mautrix_telegram/formatter/from_telegram.py
+13 -7
View File
@@ -53,6 +53,7 @@ from mautrix.types import Format, MessageType, TextMessageEventContent
from .. import abstract_user as au, portal as po, puppet as pu, user as u
from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
from ..tgclient import MautrixTelegramClient
from ..types import TelegramID
from ..util.file_transfer import UnicodeCustomEmoji, transfer_custom_emojis_to_matrix
@@ -60,7 +61,7 @@ log: logging.Logger = logging.getLogger("mau.fmt.tg")
async def _add_forward_header(
source: au.AbstractUser, content: TextMessageEventContent, fwd_from: MessageFwdHeader
client: MautrixTelegramClient, content: TextMessageEventContent, fwd_from: MessageFwdHeader
) -> None:
fwd_from_html, fwd_from_text = None, None
if isinstance(fwd_from.from_id, PeerUser):
@@ -81,7 +82,7 @@ async def _add_forward_header(
if not fwd_from_text:
try:
user = await source.client.get_entity(fwd_from.from_id)
user = await client.get_entity(fwd_from.from_id)
if user:
fwd_from_text, _ = pu.Puppet.get_displayname(user, False)
fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
@@ -104,7 +105,7 @@ async def _add_forward_header(
fwd_from_html = f"channel <b>{escape(fwd_from_text)}</b>"
else:
try:
channel = await source.client.get_entity(fwd_from.from_id)
channel = await client.get_entity(fwd_from.from_id)
if channel:
fwd_from_text = f"channel {channel.title}"
fwd_from_html = f"channel <b>{escape(channel.title)}</b>"
@@ -135,12 +136,14 @@ class ReuploadedCustomEmoji(MessageEntityCustomEmoji):
async def _convert_custom_emoji(
source: au.AbstractUser, entities: list[TypeMessageEntity]
source: au.AbstractUser,
entities: list[TypeMessageEntity],
client: MautrixTelegramClient | None = None,
) -> None:
emoji_ids = [
entity.document_id for entity in entities if isinstance(entity, MessageEntityCustomEmoji)
]
custom_emojis = await transfer_custom_emojis_to_matrix(source, emoji_ids)
custom_emojis = await transfer_custom_emojis_to_matrix(source, emoji_ids, client=client)
if len(custom_emojis) > 0:
for i, entity in enumerate(entities):
if isinstance(entity, MessageEntityCustomEmoji):
@@ -150,17 +153,20 @@ async def _convert_custom_emoji(
async def telegram_to_matrix(
evt: Message | SponsoredMessage,
source: au.AbstractUser,
client: MautrixTelegramClient | None = None,
override_text: str = None,
override_entities: list[TypeMessageEntity] = None,
require_html: bool = False,
) -> TextMessageEventContent:
if not client:
client = source.client
content = TextMessageEventContent(
msgtype=MessageType.TEXT,
body=override_text or evt.message,
)
entities = override_entities or evt.entities
if entities:
await _convert_custom_emoji(source, entities)
await _convert_custom_emoji(source, entities, client=client)
content.format = Format.HTML
html = await _telegram_entities_to_matrix_catch(add_surrogate(content.body), entities)
content.formatted_body = del_surrogate(html)
@@ -169,7 +175,7 @@ async def telegram_to_matrix(
content.ensure_has_html()
if getattr(evt, "fwd_from", None):
await _add_forward_header(source, content, evt.fwd_from)
await _add_forward_header(client, content, evt.fwd_from)
if isinstance(evt, Message) and evt.post and evt.post_author:
content.ensure_has_html()
mautrix_telegram/portal.py
+5 -4
View File
@@ -2785,6 +2785,7 @@ class Portal(DBPortal, BasePortal):
async def _convert_batch_msg(
self,
source: u.User,
client: MautrixTelegramClient,
msg: Message,
add_member: Callable[[IntentAPI, str, ContentURI], Awaitable[None]],
) -> tuple[putil.ConvertedMessage, IntentAPI]:
@@ -2800,8 +2801,8 @@ class Portal(DBPortal, BasePortal):
if sender:
intent = sender.intent_for(self)
if not sender.displayname:
entity = await source.client.get_entity(sender.peer)
await sender.update_info(source, entity)
entity = await client.get_entity(sender.peer)
await sender.update_info(source, entity, client_override=client)
else:
intent = self.main_intent
if intent.api.is_real_user and not self._can_double_puppet_backfill(intent.mxid):
@@ -2809,7 +2810,7 @@ class Portal(DBPortal, BasePortal):
if sender:
await add_member(intent, sender.displayname, sender.avatar_url)
is_bot = sender.is_bot if sender else False
converted = await self._msg_conv.convert(source, intent, is_bot, msg)
converted = await self._msg_conv.convert(source, intent, is_bot, msg, client=client)
return converted, intent
async def _wrap_batch_msg(
@@ -2902,7 +2903,7 @@ class Portal(DBPortal, BasePortal):
first_id = msg.id
before_first_msg_timestamp = int(msg.date.timestamp() * 1000) - 1
converted, intent = await self._convert_batch_msg(source, msg, add_member)
converted, intent = await self._convert_batch_msg(source, client, msg, add_member)
if converted is None:
continue
events.append(await self._wrap_batch_msg(intent, msg, converted))
mautrix_telegram/portal_util/message_convert.py
+46 -17
View File
@@ -78,6 +78,7 @@ from mautrix.util.logging import TraceLogger
from .. import abstract_user as au, formatter, matrix as m, portal as po, puppet as pu, util
from ..config import Config
from ..db import Message as DBMessage, TelegramFile as DBTelegramFile
from ..tgclient import MautrixTelegramClient
from ..types import TelegramID
from ..util import sane_mimetypes
@@ -149,12 +150,15 @@ class TelegramMessageConverter:
is_bot: bool,
evt: Message,
no_reply_fallback: bool = False,
client: MautrixTelegramClient | None = None,
) -> ConvertedMessage | None:
if not client:
client = source.client
if hasattr(evt, "media") and isinstance(evt.media, self._allowed_media):
convert_media = self._media_converters[type(evt.media)]
converted = await convert_media(source=source, intent=intent, evt=evt)
converted = await convert_media(source=source, intent=intent, evt=evt, client=client)
elif evt.message:
converted = await self._convert_text(source, intent, is_bot, evt)
converted = await self._convert_text(source, intent, is_bot, evt, client)
else:
self.log.debug("Unhandled Telegram message %d", evt.id)
return
@@ -326,9 +330,14 @@ class TelegramMessageConverter:
return beeper_link_preview
async def _convert_text(
self, source: au.AbstractUser, intent: IntentAPI, is_bot: bool, evt: Message
self,
source: au.AbstractUser,
intent: IntentAPI,
is_bot: bool,
evt: Message,
client: MautrixTelegramClient,
) -> ConvertedMessage:
content = await formatter.telegram_to_matrix(evt, source)
content = await formatter.telegram_to_matrix(evt, source, client)
if is_bot and self.portal.get_config("bot_messages_as_notices"):
content.msgtype = MessageType.NOTICE
@@ -344,7 +353,11 @@ class TelegramMessageConverter:
return ConvertedMessage(content=content)
async def _convert_photo(
self, source: au.AbstractUser, intent: IntentAPI, evt: Message
self,
source: au.AbstractUser,
intent: IntentAPI,
evt: Message,
client: MautrixTelegramClient,
) -> ConvertedMessage | None:
media: MessageMediaPhoto = evt.media
if media.photo is None and media.ttl_seconds:
@@ -362,7 +375,7 @@ class TelegramMessageConverter:
)
)
file = await util.transfer_file_to_matrix(
source.client,
client,
intent,
loc,
encrypt=self.portal.encrypted,
@@ -388,7 +401,9 @@ class TelegramMessageConverter:
content.file = file.decryption_info
else:
content.url = file.mxc
caption_content = await formatter.telegram_to_matrix(evt, source) if evt.message else None
caption_content = (
await formatter.telegram_to_matrix(evt, source, client) if evt.message else None
)
return ConvertedMessage(
content=content,
caption=caption_content,
@@ -396,7 +411,11 @@ class TelegramMessageConverter:
)
async def _convert_document(
self, source: au.AbstractUser, intent: IntentAPI, evt: Message
self,
source: au.AbstractUser,
intent: IntentAPI,
evt: Message,
client: MautrixTelegramClient,
) -> ConvertedMessage | None:
document = evt.media.document
@@ -419,7 +438,7 @@ class TelegramMessageConverter:
parallel_id = source.tgid if self.config["bridge.parallel_file_transfer"] else None
tgs_convert = self.config["bridge.animated_sticker"]
file = await util.transfer_file_to_matrix(
source.client,
client,
intent,
document,
thumb_loc,
@@ -486,7 +505,9 @@ class TelegramMessageConverter:
else:
content.url = file.mxc
caption_content = await formatter.telegram_to_matrix(evt, source) if evt.message else None
caption_content = (
await formatter.telegram_to_matrix(evt, source, client) if evt.message else None
)
return ConvertedMessage(
type=event_type,
@@ -527,13 +548,17 @@ class TelegramMessageConverter:
return ConvertedMessage(content=content)
@staticmethod
async def _convert_unsupported(source: au.AbstractUser, evt: Message, **_) -> ConvertedMessage:
async def _convert_unsupported(
source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
) -> ConvertedMessage:
override_text = (
"This message is not supported on your version of Mautrix-Telegram. "
"Please check https://github.com/mautrix/telegram or ask your "
"bridge administrator about possible updates."
)
content = await formatter.telegram_to_matrix(evt, source, override_text=override_text)
content = await formatter.telegram_to_matrix(
evt, source, client, override_text=override_text
)
content.msgtype = MessageType.NOTICE
content["fi.mau.telegram.unsupported"] = True
return ConvertedMessage(content=content)
@@ -589,7 +614,9 @@ class TelegramMessageConverter:
content["fi.mau.telegram.dice"] = {"emoticon": roll.emoticon, "value": roll.value}
return ConvertedMessage(content=content)
async def _convert_game(self, source: au.AbstractUser, evt: Message, **_) -> ConvertedMessage:
async def _convert_game(
self, source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
) -> ConvertedMessage:
game: Game = evt.media.game
play_id = self._encode_msgid(source, evt)
command = f"{self.command_prefix} play {play_id}"
@@ -599,7 +626,7 @@ class TelegramMessageConverter:
]
content = await formatter.telegram_to_matrix(
evt, source, override_text=override_text, override_entities=override_entities
evt, source, client, override_text=override_text, override_entities=override_entities
)
content.msgtype = MessageType.NOTICE
content["fi.mau.telegram.game"] = play_id
@@ -607,7 +634,9 @@ class TelegramMessageConverter:
return ConvertedMessage(content=content)
@staticmethod
async def _convert_contact(source: au.AbstractUser, evt: Message, **_) -> ConvertedMessage:
async def _convert_contact(
source: au.AbstractUser, evt: Message, client: MautrixTelegramClient, **_
) -> ConvertedMessage:
contact: MessageMediaContact = evt.media
name = " ".join(x for x in [contact.first_name, contact.last_name] if x)
formatted_phone = f"+{contact.phone_number}"
@@ -633,8 +662,8 @@ class TelegramMessageConverter:
puppet = await pu.Puppet.get_by_tgid(TelegramID(contact.user_id))
if not puppet.displayname:
try:
entity = await source.client.get_entity(PeerUser(contact.user_id))
await puppet.update_info(source, entity)
entity = await client.get_entity(PeerUser(contact.user_id))
await puppet.update_info(source, entity, client_override=client)
except Exception as e:
source.log.warning(f"Failed to sync puppet info of received contact: {e}")
else:
mautrix_telegram/puppet.py
+29 -13
View File
@@ -29,8 +29,6 @@ from telethon.tl.types import (
PeerChat,
PeerUser,
TypeChatPhoto,
TypeInputPeer,
TypeInputUser,
TypePeer,
TypeUserProfilePhoto,
UpdateUserName,
@@ -48,6 +46,7 @@ from mautrix.util.simple_template import SimpleTemplate
from . import abstract_user as au, portal as p, util
from .config import Config
from .db import Puppet as DBPuppet
from .tgclient import MautrixTelegramClient
from .types import TelegramID
if TYPE_CHECKING:
@@ -147,9 +146,6 @@ class Puppet(DBPuppet, BasePuppet):
def plain_displayname(self) -> str:
return self.displayname_template.parse(self.displayname) or self.displayname
def get_input_entity(self, user: au.AbstractUser) -> Awaitable[TypeInputPeer | TypeInputUser]:
return user.client.get_input_entity(self.peer)
def intent_for(self, portal: p.Portal) -> IntentAPI:
if portal.tgid == self.tgid:
return self.default_mxid_intent
@@ -255,7 +251,12 @@ class Puppet(DBPuppet, BasePuppet):
except Exception:
source.log.exception(f"Failed to update info of {self.tgid}")
async def update_info(self, source: au.AbstractUser, info: User | Channel) -> None:
async def update_info(
self,
source: au.AbstractUser,
info: User | Channel,
client_override: MautrixTelegramClient | None = None,
) -> None:
is_bot = False if isinstance(info, Channel) else info.bot
is_premium = False if isinstance(info, Channel) else info.premium
is_channel = isinstance(info, Channel)
@@ -277,8 +278,14 @@ class Puppet(DBPuppet, BasePuppet):
if not self.disable_updates:
try:
changed = await self.update_displayname(source, info) or changed
changed = await self.update_avatar(source, info.photo) or changed
changed = (
await self.update_displayname(source, info, client_override=client_override)
or changed
)
changed = (
await self.update_avatar(source, info.photo, client_override=client_override)
or changed
)
except Exception:
self.log.exception(f"Failed to update info from source {source.tgid}")
@@ -293,7 +300,10 @@ class Puppet(DBPuppet, BasePuppet):
await portal.update_info_from_puppet(self)
async def update_displayname(
self, source: au.AbstractUser, info: User | Channel | UpdateUserName
self,
source: au.AbstractUser,
info: User | Channel | UpdateUserName,
client_override: MautrixTelegramClient | None = None,
) -> bool:
if self.disable_updates:
return False
@@ -320,7 +330,7 @@ class Puppet(DBPuppet, BasePuppet):
return False
if isinstance(info, UpdateUserName):
info = await source.client.get_entity(self.peer)
info = await (client_override or source.client).get_entity(self.peer)
if isinstance(info, Channel) or not info.contact:
self.displayname_contact = False
elif not self.displayname_contact:
@@ -357,7 +367,10 @@ class Puppet(DBPuppet, BasePuppet):
return False
async def update_avatar(
self, source: au.AbstractUser, photo: TypeUserProfilePhoto | TypeChatPhoto
self,
source: au.AbstractUser,
photo: TypeUserProfilePhoto | TypeChatPhoto,
client_override: MautrixTelegramClient | None = None,
) -> bool:
if self.disable_updates:
return False
@@ -376,11 +389,14 @@ class Puppet(DBPuppet, BasePuppet):
self.photo_id = ""
self.avatar_url = None
elif self.photo_id != photo_id or not self.avatar_url:
client = client_override or source.client
file = await util.transfer_file_to_matrix(
client=source.client,
client=client,
intent=self.default_mxid_intent,
location=InputPeerPhotoFileLocation(
peer=await self.get_input_entity(source), photo_id=photo.photo_id, big=True
peer=await client.get_input_entity(self.peer),
photo_id=photo.photo_id,
big=True,
),
async_upload=self.config["homeserver.async_media"],
)
mautrix_telegram/util/file_transfer.py
+5 -3
View File
@@ -229,8 +229,10 @@ class UnicodeCustomEmoji(NamedTuple):
async def transfer_custom_emojis_to_matrix(
source: au.AbstractUser, emoji_ids: list[int]
source: au.AbstractUser, emoji_ids: list[int], client: MautrixTelegramClient | None = None
) -> dict[int, DBTelegramFile | UnicodeCustomEmoji]:
if not client:
client = source.client
emoji_ids = set(emoji_ids)
existing_unicode = {}
for emoji_id in emoji_ids:
@@ -249,7 +251,7 @@ async def transfer_custom_emojis_to_matrix(
if not_existing_ids:
log.debug(f"Transferring custom emojis through {source.mxid}: {not_existing_ids}")
documents: list[Document] = await source.client(
documents: list[Document] = await client(
GetCustomEmojiDocumentsRequest(document_id=not_existing_ids)
)
@@ -261,7 +263,7 @@ async def transfer_custom_emojis_to_matrix(
async def transfer(document: Document) -> None:
async with transfer_sema:
file_map[document.id] = await transfer_file_to_matrix(
source.client,
client,
source.bridge.az.intent,
document,
is_sticker=True,