From 664d6050dffe0d3cbd6447f481c9f129e546411e Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 9 Jan 2025 17:41:24 -0700 Subject: [PATCH] backfill: manually skip too-new messages in backwards backfill For some reason, even though we provide an offset, Telegram sometimes sends us more events than we request, including newer events than the offset ID. Messages beyond the offset are then chopped off by the bridgev2 code, but we continue trying to backfill the portal thinking that there is more to backfill. This causes infinite backfill loops. Signed-off-by: Sumner Evans --- pkg/connector/backfill.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pkg/connector/backfill.go b/pkg/connector/backfill.go index 6c04efbd..6a1e99ad 100644 --- a/pkg/connector/backfill.go +++ b/pkg/connector/backfill.go @@ -246,20 +246,29 @@ func (t *TelegramClient) FetchMessages(ctx context.Context, fetchParams bridgev2 } var stopAt int - if fetchParams.AnchorMessage != nil && fetchParams.Forward { + if fetchParams.AnchorMessage != nil { _, stopAt, err = ids.ParseMessageID(fetchParams.AnchorMessage.ID) if err != nil { return nil, err } + log = log.With().Int("stop_at", stopAt).Logger() } var backfillMessages []*bridgev2.BackfillMessage for _, msg := range messages { - // If we are doing forward backfill and we get to the anchor message, - // don't convert any more messages. - if stopAt > 0 && msg.GetID() <= stopAt { - log.Debug().Int("stop_at", stopAt).Int("message_id", msg.GetID()).Msg("stopping at anchor message") - break + log := log.With().Int("message_id", msg.GetID()).Logger() + if stopAt > 0 { + if fetchParams.Forward && msg.GetID() <= stopAt { + // If we are doing forward backfill and we get to the anchor + // message, don't convert any more messages. + log.Debug().Msg("stopping at anchor message") + break + } else if msg.GetID() >= stopAt { + // If we are doing backwards backfill and we get a message more + // recent than the anchor message, skip it. + log.Debug().Msg("skipping message past anchor message") + continue + } } message, ok := msg.(*tg.Message)