From cbba340da62bff88fbfded85989b74a2f2861348 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Mon, 1 Jul 2024 13:24:51 -0600 Subject: [PATCH] db: add telegram_file table Signed-off-by: Sumner Evans --- pkg/connector/client.go | 2 +- pkg/connector/connector.go | 6 +- pkg/connector/download/document.go | 7 +-- pkg/connector/login.go | 2 +- pkg/connector/msgconv/tomatrix.go | 3 +- pkg/connector/telegram.go | 13 +++- pkg/store/container.go | 14 +++-- pkg/store/telegramfile.go | 96 ++++++++++++++++++++++++++++++ pkg/store/upgrades/00-latest.sql | 15 +++++ 9 files changed, 141 insertions(+), 17 deletions(-) create mode 100644 pkg/store/telegramfile.go diff --git a/pkg/connector/client.go b/pkg/connector/client.go index dea77820..f078b59f 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -101,7 +101,7 @@ func NewTelegramClient(ctx context.Context, tc *TelegramConnector, login *bridge dispatcher.OnDeleteMessages(client.onDeleteMessages) dispatcher.OnEditMessage(client.onMessageEdit) - store := tc.store.GetScopedStore(loginID) + store := tc.Store.GetScopedStore(loginID) updatesManager := updates.New(updates.Config{ OnChannelTooLong: func(channelID int64) { diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index bce8f994..af2ff0cd 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -36,8 +36,8 @@ type TelegramConfig struct { type TelegramConnector struct { Bridge *bridgev2.Bridge Config *TelegramConfig + Store *store.Container - store *store.Container useDirectMedia bool } @@ -54,12 +54,12 @@ func NewConnector() *TelegramConnector { func (tg *TelegramConnector) Init(bridge *bridgev2.Bridge) { // TODO - tg.store = store.NewStore(bridge.DB.Database, dbutil.ZeroLogger(bridge.Log.With().Str("db_section", "telegram").Logger())) + tg.Store = store.NewStore(bridge.DB.Database, dbutil.ZeroLogger(bridge.Log.With().Str("db_section", "telegram").Logger())) tg.Bridge = bridge } func (tg *TelegramConnector) Start(ctx context.Context) error { - return tg.store.Upgrade(ctx) + return tg.Store.Upgrade(ctx) } func (tc *TelegramConnector) LoadUserLogin(ctx context.Context, login *bridgev2.UserLogin) (err error) { diff --git a/pkg/connector/download/document.go b/pkg/connector/download/document.go index a2cc6b72..422c0379 100644 --- a/pkg/connector/download/document.go +++ b/pkg/connector/download/document.go @@ -8,14 +8,13 @@ import ( "github.com/gotd/td/tg" ) -func DownloadDocument(ctx context.Context, client downloader.Client, document *tg.Document) (data []byte, err error) { +func DownloadDocument(ctx context.Context, client downloader.Client, document *tg.Document) ([]byte, error) { file := tg.InputDocumentFileLocation{ ID: document.GetID(), AccessHash: document.GetAccessHash(), FileReference: document.GetFileReference(), } var buf bytes.Buffer - _, err = downloader.NewDownloader().Download(client, &file).Stream(ctx, &buf) - data = buf.Bytes() - return + _, err := downloader.NewDownloader().Download(client, &file).Stream(ctx, &buf) + return buf.Bytes(), err } diff --git a/pkg/connector/login.go b/pkg/connector/login.go index e2900526..5f22ca18 100644 --- a/pkg/connector/login.go +++ b/pkg/connector/login.go @@ -178,7 +178,7 @@ func (p *PhoneLogin) SubmitUserInput(ctx context.Context, input map[string]strin func (p *PhoneLogin) handleAuthSuccess(ctx context.Context, authorization *tg.AuthAuthorization) (*bridgev2.LoginStep, error) { // Now that we have the Telegram user ID, store it in the database and // close the login client. - sessionStore := p.main.store.GetScopedStore(authorization.User.GetID()) + sessionStore := p.main.Store.GetScopedStore(authorization.User.GetID()) var sessionData []byte sessionData, err := p.storage.Bytes(sessionData) if err != nil { diff --git a/pkg/connector/msgconv/tomatrix.go b/pkg/connector/msgconv/tomatrix.go index de9b59e1..7c40f594 100644 --- a/pkg/connector/msgconv/tomatrix.go +++ b/pkg/connector/msgconv/tomatrix.go @@ -134,9 +134,8 @@ func (mc *MessageConverter) webpageToBeeperLinkPreview(ctx context.Context, inte } if pc, ok := webpage.GetPhoto(); ok && pc.TypeID() == tg.PhotoTypeID { - photo := pc.(*tg.Photo) var data []byte - data, preview.ImageWidth, preview.ImageHeight, preview.ImageType, err = download.DownloadPhoto(ctx, mc.client.API(), photo) + data, preview.ImageWidth, preview.ImageHeight, preview.ImageType, err = download.DownloadPhoto(ctx, mc.client.API(), pc.(*tg.Photo)) if err != nil { return nil, err } diff --git a/pkg/connector/telegram.go b/pkg/connector/telegram.go index a8f93443..97a36687 100644 --- a/pkg/connector/telegram.go +++ b/pkg/connector/telegram.go @@ -328,13 +328,22 @@ func (t *TelegramClient) getReactionLimit(ctx context.Context, sender networkid. // TODO move this to emojis package func (t *TelegramClient) transferEmojisToMatrix(ctx context.Context, customEmojiIDs []int64) (result map[networkid.EmojiID]string, err error) { result, customEmojiIDs = emojis.ConvertKnownEmojis(customEmojiIDs) - fmt.Printf("leftover custom emoji ids %+v\n", customEmojiIDs) + for _, customEmojiID := range customEmojiIDs { + fmt.Printf("customEmojiID %d\n", customEmojiID) + locationID := fmt.Sprintf("%d", customEmojiID) + if file, err := t.main.Store.TelegramFile.GetByLocationID(ctx, locationID); err != nil { + return nil, fmt.Errorf("failed to search for Telegram file by location ID") + } else if file == nil { + // TODO download shit + } else { + result[ids.MakeEmojiIDFromDocumentID(customEmojiID)] = file.MXC + } + } return } func (t *TelegramClient) handleTelegramParsedReactionsLocked(ctx context.Context, msg *database.Message, reactions map[networkid.UserID][]tg.MessagePeerReaction, customEmojiIDs []int64, isFull bool, onlyUserID *networkid.UserID, timestamp *time.Time) error { // TODO deal with the custom emoji IDs - fmt.Printf("custom emoji IDs %v\n", customEmojiIDs) customEmojis, err := t.transferEmojisToMatrix(ctx, customEmojiIDs) if err != nil { return err diff --git a/pkg/store/container.go b/pkg/store/container.go index 98a99278..9494a04e 100644 --- a/pkg/store/container.go +++ b/pkg/store/container.go @@ -25,17 +25,23 @@ import ( ) type Container struct { - db *dbutil.Database + *dbutil.Database + + TelegramFile *TelegramFileQuery } func NewStore(db *dbutil.Database, log dbutil.DatabaseLogger) *Container { - return &Container{db: db.Child("telegram_version", upgrades.Table, log)} + return &Container{ + Database: db.Child("telegram_version", upgrades.Table, log), + + TelegramFile: &TelegramFileQuery{dbutil.MakeQueryHelper(db, newTelegramFile)}, + } } func (c *Container) Upgrade(ctx context.Context) error { - return c.db.Upgrade(ctx) + return c.Database.Upgrade(ctx) } func (c *Container) GetScopedStore(telegramUserID int64) *scopedStore { - return &scopedStore{c.db, telegramUserID} + return &scopedStore{c.Database, telegramUserID} } diff --git a/pkg/store/telegramfile.go b/pkg/store/telegramfile.go new file mode 100644 index 00000000..027c329f --- /dev/null +++ b/pkg/store/telegramfile.go @@ -0,0 +1,96 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "time" + + "go.mau.fi/util/dbutil" +) + +const ( + insertTelegramFileQuery = ` + INSERT INTO telegram_file ( + id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail, decryption_info) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ` + getTelegramFileSelect = ` + SELECT id, mxc, mime_type, was_converted, timestamp, size, width, height, thumbnail, decryption_info + FROM telegram_file + ` + getTelegramFileByLocationIDQuery = getTelegramFileSelect + "WHERE id=$1" + getTelegramFileByMXCQuery = getTelegramFileSelect + "WHERE mxc=$1" +) + +type TelegramFileQuery struct { + *dbutil.QueryHelper[*TelegramFile] +} + +type TelegramFile struct { + qh *dbutil.QueryHelper[*TelegramFile] + + LocationID string + MXC string + MimeType string + WasConverted bool + Timestamp time.Time + Size int64 + Width int + Height int + ThumbnailID string + DecryptionInfo json.RawMessage +} + +var _ dbutil.DataStruct[*TelegramFile] = (*TelegramFile)(nil) + +func newTelegramFile(qh *dbutil.QueryHelper[*TelegramFile]) *TelegramFile { + return &TelegramFile{qh: qh} +} + +func (fq *TelegramFileQuery) GetByLocationID(ctx context.Context, locationID string) (*TelegramFile, error) { + return fq.QueryOne(ctx, getTelegramFileByLocationIDQuery, locationID) +} + +func (fq *TelegramFileQuery) GetByMXC(ctx context.Context, mxc string) (*TelegramFile, error) { + return fq.QueryOne(ctx, getTelegramFileByMXCQuery, mxc) +} + +func (f *TelegramFile) sqlVariables() []any { + return []any{ + f.LocationID, + f.MXC, + f.MimeType, + f.WasConverted, + f.Timestamp.UnixMilli(), + f.Size, + f.Width, + f.Height, + f.ThumbnailID, + f.DecryptionInfo, + } +} + +func (f *TelegramFile) Insert(ctx context.Context) error { + return f.qh.Exec(ctx, insertTelegramFileQuery, f.sqlVariables()...) +} + +func (f *TelegramFile) Scan(row dbutil.Scannable) (*TelegramFile, error) { + var thumbnailID sql.NullString + var timestamp int64 + err := row.Scan( + &f.LocationID, + &f.MXC, + &f.MimeType, + &f.WasConverted, + ×tamp, + &f.Size, + &f.Width, + &f.Height, + &thumbnailID, + &f.DecryptionInfo, + ) + f.Timestamp = time.UnixMilli(timestamp) + f.ThumbnailID = thumbnailID.String + return f, err +} diff --git a/pkg/store/upgrades/00-latest.sql b/pkg/store/upgrades/00-latest.sql index 327130d3..e7886eb4 100644 --- a/pkg/store/upgrades/00-latest.sql +++ b/pkg/store/upgrades/00-latest.sql @@ -30,3 +30,18 @@ CREATE TABLE telegram_channel_access_hashes ( PRIMARY KEY (user_id, channel_id) ); + +CREATE TABLE telegram_file ( + id TEXT PRIMARY KEY, + mxc TEXT NOT NULL, + mime_type TEXT, + was_converted BOOLEAN NOT NULL DEFAULT false, + timestamp BIGINT NOT NULL DEFAULT 0, + size BIGINT, + width INTEGER, + height INTEGER, + thumbnail TEXT, + decryption_info jsonb, + FOREIGN KEY (thumbnail) REFERENCES telegram_file(id) + ON UPDATE CASCADE ON DELETE SET NULL +);