dont need to fetch if its older than 24 hours thats crazy
Build Images and Deploy / Update-PROD-Stack (push) Successful in 1m26s
Build Images and Deploy / Update-PROD-Stack (push) Successful in 1m26s
This commit is contained in:
+7
-2
@@ -74,8 +74,9 @@ export async function getPostsPerHour(tag: string): Promise<number> {
|
|||||||
* - Fetch pages until the oldest post in a batch falls before the 1-hour cutoff
|
* - Fetch pages until the oldest post in a batch falls before the 1-hour cutoff
|
||||||
* (the horizon), OR the timeline is exhausted, OR MAX_PAGES_PER_HASHTAG is reached.
|
* (the horizon), OR the timeline is exhausted, OR MAX_PAGES_PER_HASHTAG is reached.
|
||||||
* - When we first cross the horizon, keep fetching additional pages as long as each
|
* - When we first cross the horizon, keep fetching additional pages as long as each
|
||||||
* new page contributes at least one post within the cutoff. Only stop when a page
|
* new page contributes at least one post within the cutoff and not beyond 24 hours.
|
||||||
* adds nothing new to the in-window count — at that point the window is stable.
|
* Only stop when a page adds nothing new to the in-window count — at that point the
|
||||||
|
* window is stable.
|
||||||
* This handles out-of-order federation: Mastodon timelines are ordered by post ID
|
* This handles out-of-order federation: Mastodon timelines are ordered by post ID
|
||||||
* (local receive time), not created_at. A post authored at 10:45 on a remote server
|
* (local receive time), not created_at. A post authored at 10:45 on a remote server
|
||||||
* may arrive at 11:05, get a recent ID and appear near the top of the stream — but
|
* may arrive at 11:05, get a recent ID and appear near the top of the stream — but
|
||||||
@@ -131,6 +132,10 @@ export async function getPostsData(
|
|||||||
const oldestInBatch = Math.min(...posts.map((p) => new Date(p.created_at).getTime()))
|
const oldestInBatch = Math.min(...posts.map((p) => new Date(p.created_at).getTime()))
|
||||||
if (oldestInBatch < cutoff) crossedHorizon = true
|
if (oldestInBatch < cutoff) crossedHorizon = true
|
||||||
|
|
||||||
|
// If the oldest post is more than 24 hours old the window is well covered — no need
|
||||||
|
// to keep fetching for federation stragglers this far back
|
||||||
|
if (oldestInBatch < now - 24 * ONE_HOUR_MS) break
|
||||||
|
|
||||||
maxId = nextMaxId
|
maxId = nextMaxId
|
||||||
|
|
||||||
// Only mark as hit-cap when we never found old enough data (true burst scenario)
|
// Only mark as hit-cap when we never found old enough data (true burst scenario)
|
||||||
|
|||||||
Reference in New Issue
Block a user