Split forward backfill limits by chat type

This commit is contained in:
Tulir Asokan
2023-05-08 17:46:08 +03:00
parent cb5c51cd27
commit 5f7d3ac8c1
3 changed files with 42 additions and 12 deletions
+18 -2
View File
@@ -174,8 +174,24 @@ class Config(BaseBridgeConfig):
copy("bridge.backfill.double_puppet_backfill")
copy("bridge.backfill.normal_groups")
copy("bridge.backfill.unread_hours_threshold")
copy("bridge.backfill.forward.initial_limit")
copy("bridge.backfill.forward.sync_limit")
if "bridge.backfill.forward" in self:
initial_limit = self.get("bridge.backfill.forward.initial_limit", 10)
sync_limit = self.get("bridge.backfill.forward.sync_limit", 100)
base["bridge.backfill.forward_limits.initial.user"] = initial_limit
base["bridge.backfill.forward_limits.initial.normal_group"] = initial_limit
base["bridge.backfill.forward_limits.initial.supergroup"] = initial_limit
base["bridge.backfill.forward_limits.initial.channel"] = initial_limit
base["bridge.backfill.forward_limits.sync.user"] = sync_limit
base["bridge.backfill.forward_limits.sync.normal_group"] = sync_limit
base["bridge.backfill.forward_limits.sync.supergroup"] = sync_limit
base["bridge.backfill.forward_limits.sync.channel"] = sync_limit
else:
copy("bridge.backfill.forward_limits.initial.user")
copy("bridge.backfill.forward_limits.initial.chat")
copy("bridge.backfill.forward_limits.initial.channel")
copy("bridge.backfill.forward_limits.sync.user")
copy("bridge.backfill.forward_limits.sync.chat")
copy("bridge.backfill.forward_limits.sync.channel")
copy("bridge.backfill.incremental.messages_per_batch")
copy("bridge.backfill.incremental.post_batch_delay")
copy("bridge.backfill.incremental.max_batches.user")
+11 -3
View File
@@ -399,11 +399,19 @@ bridge:
#
# Using a negative initial limit is not recommended, as it would try to backfill everything in a single batch.
# MSC2716 and the incremental settings are meant for backfilling everything incrementally rather than at once.
forward:
forward_limits:
# Number of messages to backfill immediately after creating a portal.
initial_limit: 10
initial:
user: 50
normal_group: 100
supergroup: 10
channel: 10
# Number of messages to backfill when syncing chats.
sync_limit: 100
sync:
user: 100
normal_group: 100
supergroup: 100
channel: 100
# Settings for incremental backfill of history. These only apply when using MSC2716.
incremental:
+13 -7
View File
@@ -2800,16 +2800,19 @@ class Portal(DBPortal, BasePortal):
await DBMessage.replace_temp_mxid(temporary_identifier, self.mxid, event_id)
@property
def _default_max_batches(self) -> int:
def _backfill_config_type(self) -> str:
if self.peer_type == "user":
own_type = "user"
return "user"
elif self.peer_type == "chat":
own_type = "normal_group"
return "normal_group"
elif self.megagroup:
own_type = "supergroup"
return "supergroup"
else:
own_type = "channel"
return self.config[f"bridge.backfill.incremental.max_batches.{own_type}"]
return "channel"
@property
def _default_max_batches(self) -> int:
return self.config[f"bridge.backfill.incremental.max_batches.{self._backfill_config_type}"]
async def enqueue_backfill(
self,
@@ -2853,7 +2856,10 @@ class Portal(DBPortal, BasePortal):
if not client:
client = source.client
type = "initial" if initial else "sync"
limit = override_limit or self.config[f"bridge.backfill.forward.{type}_limit"]
limit = (
override_limit
or self.config[f"bridge.backfill.forward_limits.{type}.{self._backfill_config_type}"]
)
if limit == 0:
return "Limit is zero, not backfilling"
with self.backfill_lock: