Switch from SQLAlchemy to asyncpg/aiosqlite
This commit is contained in:
Tulir Asokan
@@ -1,88 +0,0 @@
|
||||
from typing import Union
|
||||
import argparse
|
||||
|
||||
from sqlalchemy import orm
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
import sqlalchemy as sql
|
||||
|
||||
from alchemysession import AlchemySessionContainer
|
||||
|
||||
parser = argparse.ArgumentParser(description="mautrix-telegram dbms migration script",
|
||||
prog="python -m mautrix_telegram.scripts.dbms_migrate")
|
||||
parser.add_argument("-f", "--from-url", type=str, required=True, metavar="<url>",
|
||||
help="the old database path")
|
||||
parser.add_argument("-t", "--to-url", type=str, required=True, metavar="<url>",
|
||||
help="the new database path")
|
||||
parser.add_argument("-v", "--verbose", action="store_true", help="Verbose logs while migrating")
|
||||
args = parser.parse_args()
|
||||
verbose = args.verbose or False
|
||||
|
||||
|
||||
def log(message, end="\n"):
|
||||
if verbose:
|
||||
print(message, end=end, flush=True)
|
||||
|
||||
|
||||
def connect(to):
|
||||
from mautrix.util.db import Base
|
||||
from mautrix.client.state_store.sqlalchemy import RoomState, UserProfile
|
||||
from mautrix_telegram.db import (Portal, Message, UserPortal, User, Contact, Puppet, BotChat,
|
||||
TelegramFile)
|
||||
|
||||
db_engine = sql.create_engine(to)
|
||||
db_factory = orm.sessionmaker(bind=db_engine)
|
||||
db_session: Union[orm.Session, orm.scoped_session] = orm.scoped_session(db_factory)
|
||||
Base.metadata.bind = db_engine
|
||||
|
||||
new_base = declarative_base()
|
||||
new_base.metadata.bind = db_engine
|
||||
session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
|
||||
table_base=new_base, table_prefix="telethon_",
|
||||
manage_tables=False)
|
||||
|
||||
return db_session, {
|
||||
"Version": session_container.Version,
|
||||
"Session": session_container.Session,
|
||||
"Entity": session_container.Entity,
|
||||
"SentFile": session_container.SentFile,
|
||||
"UpdateState": session_container.UpdateState,
|
||||
"Portal": Portal,
|
||||
"Message": Message,
|
||||
"Puppet": Puppet,
|
||||
"User": User,
|
||||
"UserPortal": UserPortal,
|
||||
"RoomState": RoomState,
|
||||
"UserProfile": UserProfile,
|
||||
"Contact": Contact,
|
||||
"BotChat": BotChat,
|
||||
"TelegramFile": TelegramFile,
|
||||
}
|
||||
|
||||
|
||||
log("Connecting to old database")
|
||||
session, tables = connect(args.from_url)
|
||||
|
||||
data = {}
|
||||
for name, table in tables.items():
|
||||
log("Reading table {name}...".format(name=name), end=" ")
|
||||
data[name] = session.query(table).all()
|
||||
log("Done!")
|
||||
|
||||
log("Connecting to new database")
|
||||
session, tables = connect(args.to_url)
|
||||
|
||||
for name, table in tables.items():
|
||||
log("Writing table {name}".format(name=name), end="")
|
||||
length = len(data[name])
|
||||
n = 0
|
||||
for row in data[name]:
|
||||
session.merge(row)
|
||||
n += 5
|
||||
if n >= length:
|
||||
log(".", end="")
|
||||
n = 0
|
||||
log(" Done!")
|
||||
|
||||
log("Committing changes to database...", end=" ")
|
||||
session.commit()
|
||||
log("Done!")
|
||||
@@ -1,125 +0,0 @@
|
||||
# mautrix-telegram - A Matrix-Telegram puppeting bridge
|
||||
# Copyright (C) 2019 Tulir Asokan
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
from typing import Dict
|
||||
import argparse
|
||||
|
||||
from sqlalchemy import orm
|
||||
import sqlalchemy as sql
|
||||
|
||||
from mautrix.util.db import Base
|
||||
|
||||
from mautrix_telegram.db import Portal, Message, Puppet, BotChat
|
||||
from mautrix_telegram.config import Config
|
||||
|
||||
from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as TelematrixBase
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="mautrix-telegram telematrix import script",
|
||||
prog="python -m mautrix_telegram.scripts.telematrix_import")
|
||||
parser.add_argument("-c", "--config", type=str, default="config.yaml",
|
||||
metavar="<path>", help="the path to your mautrix-telegram config file")
|
||||
parser.add_argument("-b", "--bot-id", type=int, required=True,
|
||||
metavar="<id>", help="the telegram user ID of your relay bot")
|
||||
parser.add_argument("-t", "--telematrix-database", type=str, default="sqlite:///database.db",
|
||||
metavar="<url>", help="your telematrix database URL")
|
||||
args = parser.parse_args()
|
||||
|
||||
config = Config(args.config, None, None)
|
||||
config.load()
|
||||
|
||||
mxtg_db_engine = sql.create_engine(config["appservice.database"])
|
||||
mxtg = orm.sessionmaker(bind=mxtg_db_engine)()
|
||||
Base.metadata.bind = mxtg_db_engine
|
||||
|
||||
telematrix_db_engine = sql.create_engine(args.telematrix_database)
|
||||
telematrix = orm.sessionmaker(bind=telematrix_db_engine)()
|
||||
TelematrixBase.metadata.bind = telematrix_db_engine
|
||||
|
||||
chat_links = telematrix.query(ChatLink).all()
|
||||
tg_users = telematrix.query(TgUser).all()
|
||||
mx_users = telematrix.query(MatrixUser).all()
|
||||
tm_messages = telematrix.query(TMMessage).all()
|
||||
|
||||
telematrix.close()
|
||||
telematrix_db_engine.dispose()
|
||||
|
||||
portals_by_tgid: Dict[int, Portal] = {}
|
||||
portals_by_mxid: Dict[str, Portal] = {}
|
||||
chats: Dict[int, BotChat] = {}
|
||||
messages: Dict[str, Message] = {}
|
||||
puppets: Dict[int, Puppet] = {}
|
||||
|
||||
for chat_link in chat_links:
|
||||
if type(chat_link.tg_room) is str:
|
||||
print(f"Expected tg_room to be a number, got a string. Ignoring {chat_link.tg_room}")
|
||||
continue
|
||||
if chat_link.tg_room >= 0:
|
||||
print(f"Unexpected unprefixed telegram chat ID: {chat_link.tg_room}, ignoring...")
|
||||
continue
|
||||
tgid = str(chat_link.tg_room)
|
||||
if tgid.startswith("-100"):
|
||||
tgid = int(tgid[4:])
|
||||
peer_type = "channel"
|
||||
megagroup = True
|
||||
else:
|
||||
tgid = -chat_link.tg_room
|
||||
peer_type = "chat"
|
||||
megagroup = False
|
||||
|
||||
portal = Portal(tgid=tgid, tg_receiver=tgid, peer_type=peer_type, megagroup=megagroup,
|
||||
mxid=chat_link.matrix_room)
|
||||
chats[tgid] = BotChat(id=tgid, type=peer_type)
|
||||
if chat_link.tg_room in portals_by_tgid:
|
||||
print(f"Warning: Ignoring bridge from {portal.tgid} to {portal.mxid} "
|
||||
f"in favor of {portals_by_tgid[portal.tgid].mxid}")
|
||||
continue
|
||||
elif chat_link.matrix_room in portals_by_mxid:
|
||||
print(f"Warning: Ignoring bridge from {portal.mxid} to {portal.tgid} "
|
||||
f"in favor of {portals_by_mxid[portal.mxid].tgid}")
|
||||
continue
|
||||
portals_by_tgid[portal.tgid] = portal
|
||||
portals_by_mxid[portal.mxid] = portal
|
||||
|
||||
for tm_msg in tm_messages:
|
||||
try:
|
||||
portal = portals_by_tgid[tm_msg.tg_group_id]
|
||||
except KeyError:
|
||||
print(f"Found message entry {tm_msg.tg_message_id} in unlinked chat {tm_msg.tg_group_id},"
|
||||
" ignoring...")
|
||||
continue
|
||||
if tm_msg.matrix_room_id != portal.mxid:
|
||||
print(f"Found message entry {tm_msg.tg_message_id} with "
|
||||
f"mismatching matrix room ID {tm_msg.matrix_room_id} (expected {portal.mxid})")
|
||||
continue
|
||||
tg_space = portal.tgid if portal.peer_type == "channel" else args.bot_id
|
||||
message = Message(mxid=tm_msg.matrix_event_id, mx_room=tm_msg.matrix_room_id,
|
||||
tgid=tm_msg.tg_message_id, tg_space=tg_space)
|
||||
messages[tm_msg.matrix_event_id] = message
|
||||
|
||||
for user in tg_users:
|
||||
puppets[user.tg_id] = Puppet(id=user.tg_id, displayname=user.name,
|
||||
displayname_source=args.bot_id)
|
||||
|
||||
for k, v in portals_by_tgid.items():
|
||||
mxtg.add(v)
|
||||
for k, v in chats.items():
|
||||
mxtg.add(v)
|
||||
for k, v in messages.items():
|
||||
mxtg.add(v)
|
||||
for k, v in puppets.items():
|
||||
mxtg.add(v)
|
||||
|
||||
mxtg.commit()
|
||||
@@ -1,44 +0,0 @@
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class ChatLink(Base):
|
||||
__tablename__ = "chat_link"
|
||||
|
||||
id = sa.Column(sa.Integer, primary_key=True)
|
||||
matrix_room = sa.Column(sa.String)
|
||||
tg_room = sa.Column(sa.BigInteger)
|
||||
active = sa.Column(sa.Boolean)
|
||||
|
||||
|
||||
class TgUser(Base):
|
||||
__tablename__ = "tg_user"
|
||||
|
||||
id = sa.Column(sa.Integer, primary_key=True)
|
||||
tg_id = sa.Column(sa.BigInteger)
|
||||
name = sa.Column(sa.String)
|
||||
profile_pic_id = sa.Column(sa.String, nullable=True)
|
||||
|
||||
|
||||
class MatrixUser(Base):
|
||||
__tablename__ = "matrix_user"
|
||||
|
||||
id = sa.Column(sa.Integer, primary_key=True)
|
||||
matrix_id = sa.Column(sa.String)
|
||||
name = sa.Column(sa.String)
|
||||
|
||||
|
||||
class Message(Base):
|
||||
"""Describes a message in a room bridged between Telegram and Matrix"""
|
||||
__tablename__ = "message"
|
||||
|
||||
id = sa.Column(sa.Integer, primary_key=True)
|
||||
tg_group_id = sa.Column(sa.BigInteger)
|
||||
tg_message_id = sa.Column(sa.BigInteger)
|
||||
|
||||
matrix_room_id = sa.Column(sa.String)
|
||||
matrix_event_id = sa.Column(sa.String)
|
||||
|
||||
displayname = sa.Column(sa.String)
|
||||
Reference in New Issue
Block a user