From 5b98a71044d54dcc85cdd8d15c80ecacf69ed11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 13:20:17 +0200 Subject: [PATCH 01/17] fix: retry rejected capture batches and self-heal newly-gapped repos (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../jobs/starSnapshotHealthReporting.job.ts | 33 ++++++----- .../star_snapshot_worker/src/activities.ts | 2 + .../src/activities/index.ts | 16 +++++ .../src/bin/star-snapshot-backfill.ts | 58 ++++++++++++++++--- .../src/workflows/captureStarSnapshots.ts | 48 +++++++++++---- .../src/workflows/selfHealStarBackfill.ts | 15 ++++- .../src/repositoryStarSnapshots/index.ts | 37 ++++++++++++ 7 files changed, 174 insertions(+), 35 deletions(-) diff --git a/services/apps/cron_service/src/jobs/starSnapshotHealthReporting.job.ts b/services/apps/cron_service/src/jobs/starSnapshotHealthReporting.job.ts index 5edf705562..b085b24c5b 100644 --- a/services/apps/cron_service/src/jobs/starSnapshotHealthReporting.job.ts +++ b/services/apps/cron_service/src/jobs/starSnapshotHealthReporting.job.ts @@ -4,8 +4,8 @@ import { IS_DEV_ENV, IS_PROD_ENV } from '@crowd/common' import { IRepoStarSnapshotGapDays, countDeadLetteredStarBackfillFailures, + findAllRepoIdsWithStarSnapshotGaps, findDeadLetteredStarBackfillFailures, - findRepoIdsWithStarSnapshotGaps, findReposForStarSnapshot, findStarSnapshotGapDaysForRepos, getDeadLetterReportCursor, @@ -25,8 +25,7 @@ import { IJobDefinition } from '../types' const LAST_DEAD_LETTER_REPORTED_AT_KEY = 'star-snapshot-health-reporting:last-dead-letter-reported-at' const SAMPLE_SIZE = 20 -// Keeps each gap-check query's IN-list bounded as the eligible repo count grows. -const GAP_CHECK_BATCH_SIZE = 5_000 +const GAP_DAYS_BATCH_SIZE = 5_000 const job: IJobDefinition = { name: 'star-snapshot-health-reporting', @@ -55,19 +54,21 @@ const job: IJobDefinition = { const repoUrlById = new Map(allRepos.map((repo) => [repo.repositoryId, repo.repoUrl])) const allRepoIds = allRepos.map((repo) => repo.repositoryId) - const gappedRepoIds: string[] = [] - for (let i = 0; i < allRepoIds.length; i += GAP_CHECK_BATCH_SIZE) { - const batch = allRepoIds.slice(i, i + GAP_CHECK_BATCH_SIZE) - gappedRepoIds.push(...(await findRepoIdsWithStarSnapshotGaps(qx, batch))) - } + const gappedRepoIds = await findAllRepoIdsWithStarSnapshotGaps(qx, allRepoIds) const gapDays: IRepoStarSnapshotGapDays[] = [] - for (let i = 0; i < gappedRepoIds.length; i += GAP_CHECK_BATCH_SIZE) { - const batch = gappedRepoIds.slice(i, i + GAP_CHECK_BATCH_SIZE) + for (let i = 0; i < gappedRepoIds.length; i += GAP_DAYS_BATCH_SIZE) { + const batch = gappedRepoIds.slice(i, i + GAP_DAYS_BATCH_SIZE) gapDays.push(...(await findStarSnapshotGapDaysForRepos(qx, batch))) } const missingDaysByRepoId = new Map(gapDays.map((gap) => [gap.repositoryId, gap.missingDays])) const totalMissingDays = gapDays.reduce((sum, gap) => sum + gap.missingDays, 0) + // The two queries above run seconds apart, so a repo can close its gap in between + // (e.g. self-heal writing yesterday's snapshot) and come back with 0 missing days - + // drop those instead of reporting a gap count the day list can't back up. + const currentlyGappedRepoIds = gapDays + .filter((gap) => gap.missingDays > 0) + .map((gap) => gap.repositoryId) const sections: SlackMessageSection[] = [ { @@ -75,7 +76,7 @@ const job: IJobDefinition = { text: [ `🪦 New repos GitHub gave up retrying (3 failures in a row, excl. repo-gone/IP-allowlist): *${newlyDeadLettered.length}*`, `📉 Total repos GitHub gave up retrying: *${totalDeadLettered}*`, - `📅 Repos with a snapshot gap right now: *${gappedRepoIds.length}*`, + `📅 Repos with a snapshot gap right now: *${currentlyGappedRepoIds.length}*`, `📆 Total missing snapshot-days across those repos: *${totalMissingDays}*`, ].join('\n'), }, @@ -93,8 +94,8 @@ const job: IJobDefinition = { }) } - if (gappedRepoIds.length > 0) { - const sortedByMissingDays = [...gappedRepoIds].sort( + if (currentlyGappedRepoIds.length > 0) { + const sortedByMissingDays = [...currentlyGappedRepoIds].sort( (a, b) => (missingDaysByRepoId.get(b) ?? 0) - (missingDaysByRepoId.get(a) ?? 0), ) const shown = sortedByMissingDays.slice(0, SAMPLE_SIZE) @@ -104,13 +105,13 @@ const job: IJobDefinition = { return `• \`${url}\` - missing ${days} day${days === 1 ? '' : 's'}` }) sections.push({ - title: `Snapshot Gaps (top ${shown.length} of ${gappedRepoIds.length}, most days missing first)`, + title: `Snapshot Gaps (top ${shown.length} of ${currentlyGappedRepoIds.length}, most days missing first)`, text: lines.join('\n'), }) } const persona = - newlyDeadLettered.length > 0 || gappedRepoIds.length > 0 + newlyDeadLettered.length > 0 || currentlyGappedRepoIds.length > 0 ? SlackPersona.WARNING_PROPAGATOR : SlackPersona.INFO_NOTIFIER @@ -128,7 +129,7 @@ const job: IJobDefinition = { } ctx.log.info( - `Star snapshot health report sent: newlyDeadLettered=${newlyDeadLettered.length}, totalDeadLettered=${totalDeadLettered}, gaps=${gappedRepoIds.length}`, + `Star snapshot health report sent: newlyDeadLettered=${newlyDeadLettered.length}, totalDeadLettered=${totalDeadLettered}, gaps=${currentlyGappedRepoIds.length}`, ) }, } diff --git a/services/apps/star_snapshot_worker/src/activities.ts b/services/apps/star_snapshot_worker/src/activities.ts index e213973e15..2de3a5fe88 100644 --- a/services/apps/star_snapshot_worker/src/activities.ts +++ b/services/apps/star_snapshot_worker/src/activities.ts @@ -2,6 +2,7 @@ import { backfillRepoStarHistory, fetchAndSaveStarSnapshotBatch, findReposForStarSnapshot, + findReposNeedingGapHeal, findReposNeedingStarBackfill, } from './activities/index' @@ -9,5 +10,6 @@ export { backfillRepoStarHistory, fetchAndSaveStarSnapshotBatch, findReposForStarSnapshot, + findReposNeedingGapHeal, findReposNeedingStarBackfill, } diff --git a/services/apps/star_snapshot_worker/src/activities/index.ts b/services/apps/star_snapshot_worker/src/activities/index.ts index 93c5d9a26a..ddc5fe7a39 100644 --- a/services/apps/star_snapshot_worker/src/activities/index.ts +++ b/services/apps/star_snapshot_worker/src/activities/index.ts @@ -2,6 +2,8 @@ import { ApplicationFailure } from '@temporalio/client' import { getGithubInstallationToken } from '@crowd/common_services' import { + findAllRepoIdsWithStarSnapshotGaps, + findCompletedReposEligibleForGapHeal, findReposForStarSnapshot as findReposForStarSnapshotQx, findReposNeedingStarBackfill as findReposNeedingStarBackfillQx, recordStarBackfillFailure, @@ -395,6 +397,20 @@ export async function findReposNeedingStarBackfill( return findReposNeedingStarBackfillQx(qx, limit, afterUrl) } +// A completed repo can still pick up a fresh gap (e.g. a dropped capture batch) - this +// finds those so selfHealStarBackfill re-sweeps them too, without a manual backfill (CM-1441). +export async function findReposNeedingGapHeal(): Promise { + const qx = pgpQx(svc.postgres.reader.connection()) + const eligible = await findCompletedReposEligibleForGapHeal(qx) + const gappedIds = new Set( + await findAllRepoIdsWithStarSnapshotGaps( + qx, + eligible.map((repo) => repo.repositoryId), + ), + ) + return eligible.filter((repo) => gappedIds.has(repo.repositoryId)) +} + // A leftover claim only costs a repo one skipped run before the TTL clears it - not worth // forcing a full activity retry (re-fetching the entire stargazer history) over. async function releaseInflightClaim(cache: RedisCache, repositoryId: string): Promise { diff --git a/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts b/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts index 63a91ee0b8..62286e8b57 100644 --- a/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts +++ b/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts @@ -2,7 +2,11 @@ import { randomUUID } from 'crypto' import { mkdir, readFile, rename, rm, writeFile } from 'fs/promises' import { dirname } from 'path' -import { findRepoIdsWithStarSnapshotGaps } from '@crowd/data-access-layer' +import { + findAllRepoIdsWithStarSnapshotGaps, + findRepoIdsWithStarSnapshotGaps, + findReposForStarSnapshot, +} from '@crowd/data-access-layer' import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' import { getServiceLogger } from '@crowd/logging' @@ -29,6 +33,12 @@ const DEFAULT_RESERVED_CORE_RATE_LIMIT = 2_000 const DEFAULT_CONCURRENCY = 5 const DEFAULT_CHECKPOINT_FILE = '/var/lib/star-snapshot-worker/backfill-checkpoint.json' const DEFAULT_COMPLETED_REPOS_FILE = '/var/lib/star-snapshot-worker/backfill-completed-repos.json' +// --gapped-only keeps its own progress files so it can't clobber a normal full-sweep run's +// checkpoint/completed-repos state (or vice versa) if the two are ever run against each other. +const DEFAULT_GAPPED_ONLY_CHECKPOINT_FILE = + '/var/lib/star-snapshot-worker/backfill-gapped-only-checkpoint.json' +const DEFAULT_GAPPED_ONLY_COMPLETED_REPOS_FILE = + '/var/lib/star-snapshot-worker/backfill-gapped-only-completed-repos.json' interface Checkpoint { afterUrl: string @@ -106,11 +116,18 @@ async function clearCompletedRepoIds(path: string): Promise { const main = async () => { const dryRun = process.argv.includes('--dry-run') const fresh = process.argv.includes('--fresh') + // Targets only currently-gapped repos instead of sweeping every eligible one - fixes a + // specific gap on demand without a full backfill pass (CM-1441). + const gappedOnly = process.argv.includes('--gapped-only') const afterUrlOverride = readFlagValue('--after-url') - const checkpointFile = - process.env.STAR_SNAPSHOT_BACKFILL_CHECKPOINT_FILE ?? DEFAULT_CHECKPOINT_FILE - const completedReposFile = - process.env.STAR_SNAPSHOT_BACKFILL_COMPLETED_REPOS_FILE ?? DEFAULT_COMPLETED_REPOS_FILE + const checkpointFile = gappedOnly + ? process.env.STAR_SNAPSHOT_BACKFILL_GAPPED_ONLY_CHECKPOINT_FILE ?? + DEFAULT_GAPPED_ONLY_CHECKPOINT_FILE + : process.env.STAR_SNAPSHOT_BACKFILL_CHECKPOINT_FILE ?? DEFAULT_CHECKPOINT_FILE + const completedReposFile = gappedOnly + ? process.env.STAR_SNAPSHOT_BACKFILL_GAPPED_ONLY_COMPLETED_REPOS_FILE ?? + DEFAULT_GAPPED_ONLY_COMPLETED_REPOS_FILE + : process.env.STAR_SNAPSHOT_BACKFILL_COMPLETED_REPOS_FILE ?? DEFAULT_COMPLETED_REPOS_FILE const reservedCoreRateLimit = readIntEnv( 'STAR_SNAPSHOT_BACKFILL_RESERVED_CORE_RATE_LIMIT', DEFAULT_RESERVED_CORE_RATE_LIMIT, @@ -119,12 +136,14 @@ const main = async () => { const concurrency = readIntEnv('STAR_SNAPSHOT_BACKFILL_CONCURRENCY', DEFAULT_CONCURRENCY, false) let afterUrl = afterUrlOverride + let resumedFromCheckpoint = false if (!afterUrl && fresh) { await clearCheckpoint(checkpointFile) } else if (!afterUrl) { const checkpoint = await readCheckpoint(checkpointFile) if (checkpoint) { afterUrl = checkpoint.afterUrl + resumedFromCheckpoint = true log.info({ checkpoint }, 'resuming star snapshot backfill from checkpoint') } } @@ -142,6 +161,7 @@ const main = async () => { { dryRun, fresh, + gappedOnly, afterUrl, checkpointFile, completedReposFile, @@ -157,9 +177,31 @@ const main = async () => { await qx.selectOne('SELECT 1') log.info('Connected to database.') - // A repo can be marked complete yet still have a gap (e.g. an earlier interrupted - // run) - re-check and reprocess it instead of trusting the flag forever. - if (completedRepoIds.size > 0) { + if (gappedOnly && !resumedFromCheckpoint) { + // Everyone not currently gapped is pre-marked "completed" so the scan/skip loop below + // flies through them, touching only the repos actually gapped right now (CM-1441). + const allRepos = await findReposForStarSnapshot(qx) + const gappedRepoIds = new Set( + await findAllRepoIdsWithStarSnapshotGaps( + qx, + allRepos.map((repo) => repo.repositoryId), + ), + ) + completedRepoIds.clear() + for (const repo of allRepos) { + if (!gappedRepoIds.has(repo.repositoryId)) { + completedRepoIds.add(repo.repositoryId) + } + } + log.info( + { totalRepos: allRepos.length, gappedRepoCount: gappedRepoIds.size }, + 'gapped-only backfill targeting currently-gapped repos', + ) + } + + // A repo can be marked complete yet still have a gap (e.g. an interrupted run) - re-check + // it. Skipped in gapped-only mode, which already re-derives completedRepoIds every run. + if (!gappedOnly && completedRepoIds.size > 0) { const gappedRepoIds = await findRepoIdsWithStarSnapshotGaps(qx, [...completedRepoIds]) const unmarked = gappedRepoIds.filter((id) => completedRepoIds.delete(id)).length if (unmarked > 0) { diff --git a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts index 49506e3e33..cb26eb085c 100644 --- a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts +++ b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts @@ -19,6 +19,9 @@ const { findReposForStarSnapshot, fetchAndSaveStarSnapshotBatch } = proxyActivit const GRAPHQL_BATCH_SIZE = 100 const CONCURRENCY = 5 const PAGE_SIZE = 2_000 +// A rejected batch usually means a transient blip - one retry pass after a cooldown +// recovers most of them instead of leaving a permanent gap for that day. +const REJECTED_BATCH_RETRY_DELAY_MS = 30_000 export interface ICaptureStarSnapshotsArgs { capturedAt?: string @@ -39,12 +42,15 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) let succeeded = args.succeededSoFar ?? 0 let failed = args.failedSoFar ?? 0 - let rejectedBatches = 0 + let rejectedBatches: (typeof batches)[number][] = [] - for (let i = 0; i < batches.length; i += CONCURRENCY) { - // Rate-limited batches retry in place via a durable workflow sleep, not a blocking - // activity call - mirrors backfillStarHistoryBatch's handling of the same quota problem. - let window = batches.slice(i, i + CONCURRENCY) + // Runs one window of batches to completion, handling rate-limit backoff in place. + // Returns the batches that were still rejected (activity retries exhausted) when done. + const runWindow = async ( + initialWindow: (typeof batches)[number][], + ): Promise<(typeof batches)[number][]> => { + let window = initialWindow + const rejected: (typeof batches)[number][] = [] while (window.length > 0) { const results = await Promise.allSettled( @@ -56,8 +62,7 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) for (const [idx, result] of results.entries()) { if (result.status === 'rejected') { - rejectedBatches++ - failed += window[idx].length + rejected.push(window[idx]) log.warn('Failed to capture star snapshot batch', { repoCount: window[idx].length, error: (result.reason as Error)?.message ?? result.reason, @@ -93,13 +98,36 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) await sleep(waitMs) } } + + return rejected + } + + for (let i = 0; i < batches.length; i += CONCURRENCY) { + rejectedBatches.push(...(await runWindow(batches.slice(i, i + CONCURRENCY)))) + } + + // One retry pass over whatever's still rejected before giving up on it (CM-1441). + if (rejectedBatches.length > 0 && rejectedBatches.length < batches.length) { + await sleep(REJECTED_BATCH_RETRY_DELAY_MS) + const stillRejected: (typeof batches)[number][] = [] + for (let i = 0; i < rejectedBatches.length; i += CONCURRENCY) { + stillRejected.push(...(await runWindow(rejectedBatches.slice(i, i + CONCURRENCY)))) + } + for (const batch of stillRejected) { + failed += batch.length + } + rejectedBatches = stillRejected + } else { + for (const batch of rejectedBatches) { + failed += batch.length + } } const total = (args.totalSoFar ?? 0) + repos.length // A few rejected batches self-heal (next run's diff, or the gap backfill) - only a fully // wiped-out page signals something systemic (auth/token/outage) worth failing the run over. - if (batches.length > 0 && rejectedBatches === batches.length) { + if (batches.length > 0 && rejectedBatches.length === batches.length) { // A plain Error only fails the workflow task (infinite replay); ApplicationFailure // is required to fail the execution so the schedule's retry policy engages. throw ApplicationFailure.create({ @@ -108,9 +136,9 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) }) } - if (rejectedBatches > 0) { + if (rejectedBatches.length > 0) { log.error('star snapshot capture had partial batch failures on this page', { - rejectedBatches, + rejectedBatches: rejectedBatches.length, totalBatches: batches.length, succeeded, failed, diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 5fff1829b2..97dc57cf02 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -11,7 +11,9 @@ import { import * as activities from '../activities' import { backfillStarHistoryBatch } from './backfillStarHistoryBatch' -const { findReposNeedingStarBackfill } = proxyActivities({ +const { findReposNeedingStarBackfill, findReposNeedingGapHeal } = proxyActivities< + typeof activities +>({ startToCloseTimeout: '2 minutes', retry: { maximumAttempts: 3, backoffCoefficient: 2 }, }) @@ -83,6 +85,17 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) batchesDispatched++ } + // Runs once per full sweep, not per page - args.afterUrl is only unset on the true first + // page, and this covers completed repos, a population the paged scan above never touches. + if (args.afterUrl === undefined) { + const gapped = await findReposNeedingGapHeal() + for (let i = 0; i < gapped.length; i += BATCH_SIZE) { + const batch = gapped.slice(i, i + BATCH_SIZE) + await startBatchChild(batch, batchWorkflowId(batch)) + batchesDispatched++ + } + } + if (repos.length === PAGE_SIZE) { await continueAsNew({ afterUrl: repos[repos.length - 1].repoUrl, diff --git a/services/libs/data-access-layer/src/repositoryStarSnapshots/index.ts b/services/libs/data-access-layer/src/repositoryStarSnapshots/index.ts index 576a3ecc1d..54eef9bad2 100644 --- a/services/libs/data-access-layer/src/repositoryStarSnapshots/index.ts +++ b/services/libs/data-access-layer/src/repositoryStarSnapshots/index.ts @@ -97,6 +97,43 @@ export async function findRepoIdsWithStarSnapshotGaps( return (rows || []).map((row) => row.repositoryId) } +const DEFAULT_GAP_CHECK_BATCH_SIZE = 5_000 + +// Chunks the IN-list so it stays bounded as the eligible repo count grows. +export async function findAllRepoIdsWithStarSnapshotGaps( + qx: QueryExecutor, + repositoryIds: string[], + batchSize: number = DEFAULT_GAP_CHECK_BATCH_SIZE, +): Promise { + const gapped: string[] = [] + for (let i = 0; i < repositoryIds.length; i += batchSize) { + const batch = repositoryIds.slice(i, i + batchSize) + gapped.push(...(await findRepoIdsWithStarSnapshotGaps(qx, batch))) + } + return gapped +} + +// Completed, still-retryable repos - the population findReposNeedingStarBackfill skips. +// Cheap index lookup; the actual gap check happens via findAllRepoIdsWithStarSnapshotGaps. +export async function findCompletedReposEligibleForGapHeal( + qx: QueryExecutor, +): Promise { + const repos: IRepoForStarSnapshot[] = await qx.select(` + select + r.id as "repositoryId", + r.url as "repoUrl" + from public.repositories r + join public."repositoryStarBackfillStatus" f on f."repositoryId" = r.id + where r."deletedAt" is null + and r."excluded" = false + and r.url like 'https://github.com%' + and f."completedAt" is not null + and f."deadLetteredAt" is null + `) + + return repos || [] +} + export interface IRepoStarSnapshotGapDays { repositoryId: string missingDays: number From 94f3a5a1162c086a78c343825690356552426c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 13:27:58 +0200 Subject: [PATCH 02/17] style: fix formatting (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../src/bin/star-snapshot-backfill.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts b/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts index 62286e8b57..c865c711dd 100644 --- a/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts +++ b/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts @@ -121,13 +121,13 @@ const main = async () => { const gappedOnly = process.argv.includes('--gapped-only') const afterUrlOverride = readFlagValue('--after-url') const checkpointFile = gappedOnly - ? process.env.STAR_SNAPSHOT_BACKFILL_GAPPED_ONLY_CHECKPOINT_FILE ?? - DEFAULT_GAPPED_ONLY_CHECKPOINT_FILE - : process.env.STAR_SNAPSHOT_BACKFILL_CHECKPOINT_FILE ?? DEFAULT_CHECKPOINT_FILE + ? (process.env.STAR_SNAPSHOT_BACKFILL_GAPPED_ONLY_CHECKPOINT_FILE ?? + DEFAULT_GAPPED_ONLY_CHECKPOINT_FILE) + : (process.env.STAR_SNAPSHOT_BACKFILL_CHECKPOINT_FILE ?? DEFAULT_CHECKPOINT_FILE) const completedReposFile = gappedOnly - ? process.env.STAR_SNAPSHOT_BACKFILL_GAPPED_ONLY_COMPLETED_REPOS_FILE ?? - DEFAULT_GAPPED_ONLY_COMPLETED_REPOS_FILE - : process.env.STAR_SNAPSHOT_BACKFILL_COMPLETED_REPOS_FILE ?? DEFAULT_COMPLETED_REPOS_FILE + ? (process.env.STAR_SNAPSHOT_BACKFILL_GAPPED_ONLY_COMPLETED_REPOS_FILE ?? + DEFAULT_GAPPED_ONLY_COMPLETED_REPOS_FILE) + : (process.env.STAR_SNAPSHOT_BACKFILL_COMPLETED_REPOS_FILE ?? DEFAULT_COMPLETED_REPOS_FILE) const reservedCoreRateLimit = readIntEnv( 'STAR_SNAPSHOT_BACKFILL_RESERVED_CORE_RATE_LIMIT', DEFAULT_RESERVED_CORE_RATE_LIMIT, From ba9aed1630f83388c98a30dd9352bf63cc71f614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 13:39:22 +0200 Subject: [PATCH 03/17] fix: paginate self-heal gap detection, fix capture retry guard (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../src/activities/index.ts | 22 ++++++++-- .../src/workflows/captureStarSnapshots.ts | 9 ++-- .../src/workflows/selfHealStarBackfill.ts | 42 +++++++++++++------ .../src/repositoryStarSnapshots/index.ts | 34 +++++++++------ 4 files changed, 71 insertions(+), 36 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/activities/index.ts b/services/apps/star_snapshot_worker/src/activities/index.ts index ddc5fe7a39..3963a119fd 100644 --- a/services/apps/star_snapshot_worker/src/activities/index.ts +++ b/services/apps/star_snapshot_worker/src/activities/index.ts @@ -397,18 +397,32 @@ export async function findReposNeedingStarBackfill( return findReposNeedingStarBackfillQx(qx, limit, afterUrl) } +export interface IGapHealPage { + gappedRepos: IRepoForStarSnapshot[] + pageSize: number + lastUrl?: string +} + // A completed repo can still pick up a fresh gap (e.g. a dropped capture batch) - this // finds those so selfHealStarBackfill re-sweeps them too, without a manual backfill (CM-1441). -export async function findReposNeedingGapHeal(): Promise { +// Paginated like findReposNeedingStarBackfill, so each call and its gap check stay bounded. +export async function findReposNeedingGapHeal( + limit: number, + afterUrl?: string, +): Promise { const qx = pgpQx(svc.postgres.reader.connection()) - const eligible = await findCompletedReposEligibleForGapHeal(qx) + const page = await findCompletedReposEligibleForGapHeal(qx, limit, afterUrl) const gappedIds = new Set( await findAllRepoIdsWithStarSnapshotGaps( qx, - eligible.map((repo) => repo.repositoryId), + page.map((repo) => repo.repositoryId), ), ) - return eligible.filter((repo) => gappedIds.has(repo.repositoryId)) + return { + gappedRepos: page.filter((repo) => gappedIds.has(repo.repositoryId)), + pageSize: page.length, + lastUrl: page.length > 0 ? page[page.length - 1].repoUrl : undefined, + } } // A leftover claim only costs a repo one skipped run before the TTL clears it - not worth diff --git a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts index cb26eb085c..b273be1f2b 100644 --- a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts +++ b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts @@ -106,8 +106,9 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) rejectedBatches.push(...(await runWindow(batches.slice(i, i + CONCURRENCY)))) } - // One retry pass over whatever's still rejected before giving up on it (CM-1441). - if (rejectedBatches.length > 0 && rejectedBatches.length < batches.length) { + // One retry pass over whatever's still rejected before giving up on it, even if every + // batch on this page rejected (small/last page - still worth one cooldown retry) (CM-1441). + if (rejectedBatches.length > 0) { await sleep(REJECTED_BATCH_RETRY_DELAY_MS) const stillRejected: (typeof batches)[number][] = [] for (let i = 0; i < rejectedBatches.length; i += CONCURRENCY) { @@ -117,10 +118,6 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) failed += batch.length } rejectedBatches = stillRejected - } else { - for (const batch of rejectedBatches) { - failed += batch.length - } } const total = (args.totalSoFar ?? 0) + repos.length diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 97dc57cf02..604dcf8b02 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -23,6 +23,9 @@ const BATCH_SIZE = 100 export interface ISelfHealStarBackfillArgs { afterUrl?: string + mainScanDone?: boolean + gapHealAfterUrl?: string + gapHealDone?: boolean batchesDispatchedSoFar?: number } @@ -75,30 +78,43 @@ async function startBatchChild( // Fans out to abandoned child workflows so each batch's own rate-limit retry loop doesn't // hold up other pages waiting on GitHub's reset. +// Runs two independent paged scans per tick - the never-completed repos (existing behavior) +// and, separately, completed repos that picked up a fresh gap (CM-1441) - each bounded to +// PAGE_SIZE per activity call and carried across continueAsNew via its own cursor. export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}): Promise { - const repos = await findReposNeedingStarBackfill(PAGE_SIZE, args.afterUrl) let batchesDispatched = args.batchesDispatchedSoFar ?? 0 + const mainScanDone = args.mainScanDone ?? false + const gapHealDone = args.gapHealDone ?? false - for (let i = 0; i < repos.length; i += BATCH_SIZE) { - const batch = repos.slice(i, i + BATCH_SIZE) - await startBatchChild(batch, batchWorkflowId(batch)) - batchesDispatched++ + let repos: Awaited> = [] + if (!mainScanDone) { + repos = await findReposNeedingStarBackfill(PAGE_SIZE, args.afterUrl) + for (let i = 0; i < repos.length; i += BATCH_SIZE) { + const batch = repos.slice(i, i + BATCH_SIZE) + await startBatchChild(batch, batchWorkflowId(batch)) + batchesDispatched++ + } } - // Runs once per full sweep, not per page - args.afterUrl is only unset on the true first - // page, and this covers completed repos, a population the paged scan above never touches. - if (args.afterUrl === undefined) { - const gapped = await findReposNeedingGapHeal() - for (let i = 0; i < gapped.length; i += BATCH_SIZE) { - const batch = gapped.slice(i, i + BATCH_SIZE) + let gapHealPage: Awaited> | undefined + if (!gapHealDone) { + gapHealPage = await findReposNeedingGapHeal(PAGE_SIZE, args.gapHealAfterUrl) + for (let i = 0; i < gapHealPage.gappedRepos.length; i += BATCH_SIZE) { + const batch = gapHealPage.gappedRepos.slice(i, i + BATCH_SIZE) await startBatchChild(batch, batchWorkflowId(batch)) batchesDispatched++ } } - if (repos.length === PAGE_SIZE) { + const nextMainScanDone = mainScanDone || repos.length < PAGE_SIZE + const nextGapHealDone = gapHealDone || (gapHealPage?.pageSize ?? 0) < PAGE_SIZE + + if (!nextMainScanDone || !nextGapHealDone) { await continueAsNew({ - afterUrl: repos[repos.length - 1].repoUrl, + afterUrl: nextMainScanDone ? undefined : repos[repos.length - 1].repoUrl, + mainScanDone: nextMainScanDone, + gapHealAfterUrl: nextGapHealDone ? undefined : gapHealPage!.lastUrl, + gapHealDone: nextGapHealDone, batchesDispatchedSoFar: batchesDispatched, }) return diff --git a/services/libs/data-access-layer/src/repositoryStarSnapshots/index.ts b/services/libs/data-access-layer/src/repositoryStarSnapshots/index.ts index 54eef9bad2..6633967d94 100644 --- a/services/libs/data-access-layer/src/repositoryStarSnapshots/index.ts +++ b/services/libs/data-access-layer/src/repositoryStarSnapshots/index.ts @@ -114,22 +114,30 @@ export async function findAllRepoIdsWithStarSnapshotGaps( } // Completed, still-retryable repos - the population findReposNeedingStarBackfill skips. -// Cheap index lookup; the actual gap check happens via findAllRepoIdsWithStarSnapshotGaps. +// Paginated the same way, so a caller can bound each page's gap check (findAllRepoIdsWithStarSnapshotGaps). export async function findCompletedReposEligibleForGapHeal( qx: QueryExecutor, + limit: number | null = null, + afterUrl: string | null = null, ): Promise { - const repos: IRepoForStarSnapshot[] = await qx.select(` - select - r.id as "repositoryId", - r.url as "repoUrl" - from public.repositories r - join public."repositoryStarBackfillStatus" f on f."repositoryId" = r.id - where r."deletedAt" is null - and r."excluded" = false - and r.url like 'https://github.com%' - and f."completedAt" is not null - and f."deadLetteredAt" is null - `) + const repos: IRepoForStarSnapshot[] = await qx.select( + ` + select + r.id as "repositoryId", + r.url as "repoUrl" + from public.repositories r + join public."repositoryStarBackfillStatus" f on f."repositoryId" = r.id + where r."deletedAt" is null + and r."excluded" = false + and r.url like 'https://github.com%' + and ($(afterUrl)::text is null or r.url > $(afterUrl)) + and f."completedAt" is not null + and f."deadLetteredAt" is null + order by r.url asc + limit $(limit) + `, + { limit, afterUrl }, + ) return repos || [] } From db835a18a51f34a68094f0705b518c38e4a5072f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 13:52:17 +0200 Subject: [PATCH 04/17] fix: version new workflow branches with patched() for safe rolling deploy (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../src/workflows/captureStarSnapshots.ts | 6 +++++- .../src/workflows/selfHealStarBackfill.ts | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts index b273be1f2b..cd76fd78ee 100644 --- a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts +++ b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts @@ -2,6 +2,7 @@ import { ApplicationFailure, continueAsNew, log, + patched, proxyActivities, sleep, workflowInfo, @@ -108,7 +109,10 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) // One retry pass over whatever's still rejected before giving up on it, even if every // batch on this page rejected (small/last page - still worth one cooldown retry) (CM-1441). - if (rejectedBatches.length > 0) { + // Gated by patched() - an execution already in flight when this shipped must keep replaying + // its old command sequence (straight to the page decision below) or it'll hit a nondeterminism + // error; only executions that start fresh after the deploy take the new retry-then-decide path. + if (rejectedBatches.length > 0 && patched('CM-1441-retry-rejected-batches')) { await sleep(REJECTED_BATCH_RETRY_DELAY_MS) const stillRejected: (typeof batches)[number][] = [] for (let i = 0; i < rejectedBatches.length; i += CONCURRENCY) { diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 604dcf8b02..6db8dd4dea 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -3,6 +3,7 @@ import { WorkflowIdReusePolicy, continueAsNew, log, + patched, proxyActivities, startChild, workflowInfo, @@ -96,8 +97,11 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) } } + // Gated by patched() - an execution already in flight when this shipped must keep replaying + // its old command sequence (skip straight to continueAsNew) or it'll hit a nondeterminism + // error; only executions that start fresh after the deploy take the new gap-heal scan. let gapHealPage: Awaited> | undefined - if (!gapHealDone) { + if (!gapHealDone && patched('CM-1441-gap-heal-scan')) { gapHealPage = await findReposNeedingGapHeal(PAGE_SIZE, args.gapHealAfterUrl) for (let i = 0; i < gapHealPage.gappedRepos.length; i += BATCH_SIZE) { const batch = gapHealPage.gappedRepos.slice(i, i + BATCH_SIZE) From d640f9a33c0d02de14e192a3890554aae8fe540d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 13:53:06 +0200 Subject: [PATCH 05/17] style: trim patched() comments to 2 lines (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../src/workflows/captureStarSnapshots.ts | 5 ++--- .../src/workflows/selfHealStarBackfill.ts | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts index cd76fd78ee..397bb78e72 100644 --- a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts +++ b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts @@ -109,9 +109,8 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) // One retry pass over whatever's still rejected before giving up on it, even if every // batch on this page rejected (small/last page - still worth one cooldown retry) (CM-1441). - // Gated by patched() - an execution already in flight when this shipped must keep replaying - // its old command sequence (straight to the page decision below) or it'll hit a nondeterminism - // error; only executions that start fresh after the deploy take the new retry-then-decide path. + // patched() keeps an execution already in flight on its old command sequence so a + // mid-deploy replay doesn't hit a nondeterminism error (CM-1441). if (rejectedBatches.length > 0 && patched('CM-1441-retry-rejected-batches')) { await sleep(REJECTED_BATCH_RETRY_DELAY_MS) const stillRejected: (typeof batches)[number][] = [] diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 6db8dd4dea..52dbe36453 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -97,9 +97,8 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) } } - // Gated by patched() - an execution already in flight when this shipped must keep replaying - // its old command sequence (skip straight to continueAsNew) or it'll hit a nondeterminism - // error; only executions that start fresh after the deploy take the new gap-heal scan. + // patched() keeps an execution already in flight on its old command sequence so a + // mid-deploy replay doesn't hit a nondeterminism error (CM-1441). let gapHealPage: Awaited> | undefined if (!gapHealDone && patched('CM-1441-gap-heal-scan')) { gapHealPage = await findReposNeedingGapHeal(PAGE_SIZE, args.gapHealAfterUrl) From d0b911d16be06893418fb2edb2f925ab07030b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 13:56:59 +0200 Subject: [PATCH 06/17] style: drop ticket id from patched() marker names (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../star_snapshot_worker/src/workflows/captureStarSnapshots.ts | 2 +- .../star_snapshot_worker/src/workflows/selfHealStarBackfill.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts index 397bb78e72..ae8faedc87 100644 --- a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts +++ b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts @@ -111,7 +111,7 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) // batch on this page rejected (small/last page - still worth one cooldown retry) (CM-1441). // patched() keeps an execution already in flight on its old command sequence so a // mid-deploy replay doesn't hit a nondeterminism error (CM-1441). - if (rejectedBatches.length > 0 && patched('CM-1441-retry-rejected-batches')) { + if (rejectedBatches.length > 0 && patched('retry-rejected-batches')) { await sleep(REJECTED_BATCH_RETRY_DELAY_MS) const stillRejected: (typeof batches)[number][] = [] for (let i = 0; i < rejectedBatches.length; i += CONCURRENCY) { diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 52dbe36453..d928926589 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -100,7 +100,7 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) // patched() keeps an execution already in flight on its old command sequence so a // mid-deploy replay doesn't hit a nondeterminism error (CM-1441). let gapHealPage: Awaited> | undefined - if (!gapHealDone && patched('CM-1441-gap-heal-scan')) { + if (!gapHealDone && patched('gap-heal-scan')) { gapHealPage = await findReposNeedingGapHeal(PAGE_SIZE, args.gapHealAfterUrl) for (let i = 0; i < gapHealPage.gappedRepos.length; i += BATCH_SIZE) { const batch = gapHealPage.gappedRepos.slice(i, i + BATCH_SIZE) From 23251cc7140721edd318396ca37b8c105a5f1d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 14:03:51 +0200 Subject: [PATCH 07/17] fix: count rejected batches as failed on non-patched path, trim stale comments (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../src/workflows/captureStarSnapshots.ts | 6 ++++-- .../src/workflows/selfHealStarBackfill.ts | 5 +---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts index ae8faedc87..cf0650e59d 100644 --- a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts +++ b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts @@ -107,8 +107,6 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) rejectedBatches.push(...(await runWindow(batches.slice(i, i + CONCURRENCY)))) } - // One retry pass over whatever's still rejected before giving up on it, even if every - // batch on this page rejected (small/last page - still worth one cooldown retry) (CM-1441). // patched() keeps an execution already in flight on its old command sequence so a // mid-deploy replay doesn't hit a nondeterminism error (CM-1441). if (rejectedBatches.length > 0 && patched('retry-rejected-batches')) { @@ -121,6 +119,10 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) failed += batch.length } rejectedBatches = stillRejected + } else { + for (const batch of rejectedBatches) { + failed += batch.length + } } const total = (args.totalSoFar ?? 0) + repos.length diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index d928926589..bdfe70c7b3 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -78,10 +78,7 @@ async function startBatchChild( } // Fans out to abandoned child workflows so each batch's own rate-limit retry loop doesn't -// hold up other pages waiting on GitHub's reset. -// Runs two independent paged scans per tick - the never-completed repos (existing behavior) -// and, separately, completed repos that picked up a fresh gap (CM-1441) - each bounded to -// PAGE_SIZE per activity call and carried across continueAsNew via its own cursor. +// hold up other pages. Runs two independently-cursored paged scans per tick (CM-1441). export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}): Promise { let batchesDispatched = args.batchesDispatchedSoFar ?? 0 const mainScanDone = args.mainScanDone ?? false From 5cd44e83d6d52b60f6ae9b902466c3316ddb4cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 14:07:46 +0200 Subject: [PATCH 08/17] style: drop ticket refs from code comments (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- services/apps/star_snapshot_worker/src/activities/index.ts | 3 +-- .../star_snapshot_worker/src/bin/star-snapshot-backfill.ts | 4 ++-- .../src/workflows/captureStarSnapshots.ts | 2 +- .../src/workflows/selfHealStarBackfill.ts | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/activities/index.ts b/services/apps/star_snapshot_worker/src/activities/index.ts index 3963a119fd..78b2603690 100644 --- a/services/apps/star_snapshot_worker/src/activities/index.ts +++ b/services/apps/star_snapshot_worker/src/activities/index.ts @@ -404,8 +404,7 @@ export interface IGapHealPage { } // A completed repo can still pick up a fresh gap (e.g. a dropped capture batch) - this -// finds those so selfHealStarBackfill re-sweeps them too, without a manual backfill (CM-1441). -// Paginated like findReposNeedingStarBackfill, so each call and its gap check stay bounded. +// finds those so selfHealStarBackfill re-sweeps them too, paginated like the backfill scan. export async function findReposNeedingGapHeal( limit: number, afterUrl?: string, diff --git a/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts b/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts index c865c711dd..6b0e758f1c 100644 --- a/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts +++ b/services/apps/star_snapshot_worker/src/bin/star-snapshot-backfill.ts @@ -117,7 +117,7 @@ const main = async () => { const dryRun = process.argv.includes('--dry-run') const fresh = process.argv.includes('--fresh') // Targets only currently-gapped repos instead of sweeping every eligible one - fixes a - // specific gap on demand without a full backfill pass (CM-1441). + // specific gap on demand without a full backfill pass. const gappedOnly = process.argv.includes('--gapped-only') const afterUrlOverride = readFlagValue('--after-url') const checkpointFile = gappedOnly @@ -179,7 +179,7 @@ const main = async () => { if (gappedOnly && !resumedFromCheckpoint) { // Everyone not currently gapped is pre-marked "completed" so the scan/skip loop below - // flies through them, touching only the repos actually gapped right now (CM-1441). + // flies through them, touching only the repos actually gapped right now. const allRepos = await findReposForStarSnapshot(qx) const gappedRepoIds = new Set( await findAllRepoIdsWithStarSnapshotGaps( diff --git a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts index cf0650e59d..b3a6dd300f 100644 --- a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts +++ b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts @@ -108,7 +108,7 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) } // patched() keeps an execution already in flight on its old command sequence so a - // mid-deploy replay doesn't hit a nondeterminism error (CM-1441). + // mid-deploy replay doesn't hit a nondeterminism error. if (rejectedBatches.length > 0 && patched('retry-rejected-batches')) { await sleep(REJECTED_BATCH_RETRY_DELAY_MS) const stillRejected: (typeof batches)[number][] = [] diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index bdfe70c7b3..9b3c8d1fe5 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -78,7 +78,7 @@ async function startBatchChild( } // Fans out to abandoned child workflows so each batch's own rate-limit retry loop doesn't -// hold up other pages. Runs two independently-cursored paged scans per tick (CM-1441). +// hold up other pages. Runs two independently-cursored paged scans per tick. export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}): Promise { let batchesDispatched = args.batchesDispatchedSoFar ?? 0 const mainScanDone = args.mainScanDone ?? false @@ -95,7 +95,7 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) } // patched() keeps an execution already in flight on its old command sequence so a - // mid-deploy replay doesn't hit a nondeterminism error (CM-1441). + // mid-deploy replay doesn't hit a nondeterminism error. let gapHealPage: Awaited> | undefined if (!gapHealDone && patched('gap-heal-scan')) { gapHealPage = await findReposNeedingGapHeal(PAGE_SIZE, args.gapHealAfterUrl) From 1557d30af5fe6dae5061feb83c40ff0ce54563b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 14:47:53 +0200 Subject: [PATCH 09/17] style: trim gap-count comment to 2 lines (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../cron_service/src/jobs/starSnapshotHealthReporting.job.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/apps/cron_service/src/jobs/starSnapshotHealthReporting.job.ts b/services/apps/cron_service/src/jobs/starSnapshotHealthReporting.job.ts index b085b24c5b..080518c695 100644 --- a/services/apps/cron_service/src/jobs/starSnapshotHealthReporting.job.ts +++ b/services/apps/cron_service/src/jobs/starSnapshotHealthReporting.job.ts @@ -63,9 +63,8 @@ const job: IJobDefinition = { } const missingDaysByRepoId = new Map(gapDays.map((gap) => [gap.repositoryId, gap.missingDays])) const totalMissingDays = gapDays.reduce((sum, gap) => sum + gap.missingDays, 0) - // The two queries above run seconds apart, so a repo can close its gap in between - // (e.g. self-heal writing yesterday's snapshot) and come back with 0 missing days - - // drop those instead of reporting a gap count the day list can't back up. + // The two queries above run seconds apart, so a repo can close its gap in between and + // come back with 0 missing days - drop those instead of over-reporting the gap count. const currentlyGappedRepoIds = gapDays .filter((gap) => gap.missingDays > 0) .map((gap) => gap.repositoryId) From 974953fa3e405c00371e90283de0bfba6b2f26ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 16:45:32 +0200 Subject: [PATCH 10/17] fix: don't advance gapHealDone when patched() suppresses the scan (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../src/workflows/selfHealStarBackfill.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 9b3c8d1fe5..a6adc24c5a 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -96,8 +96,9 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) // patched() keeps an execution already in flight on its old command sequence so a // mid-deploy replay doesn't hit a nondeterminism error. + const gapHealPatched = patched('gap-heal-scan') let gapHealPage: Awaited> | undefined - if (!gapHealDone && patched('gap-heal-scan')) { + if (!gapHealDone && gapHealPatched) { gapHealPage = await findReposNeedingGapHeal(PAGE_SIZE, args.gapHealAfterUrl) for (let i = 0; i < gapHealPage.gappedRepos.length; i += BATCH_SIZE) { const batch = gapHealPage.gappedRepos.slice(i, i + BATCH_SIZE) @@ -107,7 +108,9 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) } const nextMainScanDone = mainScanDone || repos.length < PAGE_SIZE - const nextGapHealDone = gapHealDone || (gapHealPage?.pageSize ?? 0) < PAGE_SIZE + // Only advance gapHealDone once the scan actually ran (gapHealPatched) - otherwise a + // pre-deploy replay would bake gapHealDone: true into continueAsNew and never run it. + const nextGapHealDone = gapHealDone || (gapHealPatched && (gapHealPage?.pageSize ?? 0) < PAGE_SIZE) if (!nextMainScanDone || !nextGapHealDone) { await continueAsNew({ From 4aea84d5fff98c8158f3f6a083a53ef4a987ac99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 16:47:53 +0200 Subject: [PATCH 11/17] style: run oxfmt on selfHealStarBackfill (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../star_snapshot_worker/src/workflows/selfHealStarBackfill.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index a6adc24c5a..19dd187111 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -110,7 +110,8 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) const nextMainScanDone = mainScanDone || repos.length < PAGE_SIZE // Only advance gapHealDone once the scan actually ran (gapHealPatched) - otherwise a // pre-deploy replay would bake gapHealDone: true into continueAsNew and never run it. - const nextGapHealDone = gapHealDone || (gapHealPatched && (gapHealPage?.pageSize ?? 0) < PAGE_SIZE) + const nextGapHealDone = + gapHealDone || (gapHealPatched && (gapHealPage?.pageSize ?? 0) < PAGE_SIZE) if (!nextMainScanDone || !nextGapHealDone) { await continueAsNew({ From d896af151975145b2c5bb68e163ed40d2170bdfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 16:54:13 +0200 Subject: [PATCH 12/17] fix: preserve gapHealAfterUrl cursor when patched() suppresses the scan (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../star_snapshot_worker/src/workflows/selfHealStarBackfill.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 19dd187111..6ca9c3dad2 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -117,7 +117,7 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) await continueAsNew({ afterUrl: nextMainScanDone ? undefined : repos[repos.length - 1].repoUrl, mainScanDone: nextMainScanDone, - gapHealAfterUrl: nextGapHealDone ? undefined : gapHealPage!.lastUrl, + gapHealAfterUrl: nextGapHealDone ? undefined : (gapHealPage?.lastUrl ?? args.gapHealAfterUrl), gapHealDone: nextGapHealDone, batchesDispatchedSoFar: batchesDispatched, }) From 4838b155f8ccb6eba18bbbe7b2e84b7812aa6980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 16:59:31 +0200 Subject: [PATCH 13/17] fix: keep continue/complete decision unaffected by gap heal when unpatched (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../src/workflows/selfHealStarBackfill.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 6ca9c3dad2..511cbacab5 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -112,8 +112,11 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) // pre-deploy replay would bake gapHealDone: true into continueAsNew and never run it. const nextGapHealDone = gapHealDone || (gapHealPatched && (gapHealPage?.pageSize ?? 0) < PAGE_SIZE) + // While unpatched, gap healing doesn't exist yet - the continue/complete decision must + // depend on nextMainScanDone alone, exactly like the pre-gap-heal command sequence. + const shouldContinue = gapHealPatched ? !nextMainScanDone || !nextGapHealDone : !nextMainScanDone - if (!nextMainScanDone || !nextGapHealDone) { + if (shouldContinue) { await continueAsNew({ afterUrl: nextMainScanDone ? undefined : repos[repos.length - 1].repoUrl, mainScanDone: nextMainScanDone, From bab10a2fb37beb61dd3d6e9d200b51c6473e9e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 17:29:03 +0200 Subject: [PATCH 14/17] fix: restore original Temporal patch ID for rejected-batch retry (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../star_snapshot_worker/src/workflows/captureStarSnapshots.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts index b3a6dd300f..458bec2f7d 100644 --- a/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts +++ b/services/apps/star_snapshot_worker/src/workflows/captureStarSnapshots.ts @@ -109,7 +109,7 @@ export async function captureStarSnapshots(args: ICaptureStarSnapshotsArgs = {}) // patched() keeps an execution already in flight on its old command sequence so a // mid-deploy replay doesn't hit a nondeterminism error. - if (rejectedBatches.length > 0 && patched('retry-rejected-batches')) { + if (rejectedBatches.length > 0 && patched('CM-1441-retry-rejected-batches')) { await sleep(REJECTED_BATCH_RETRY_DELAY_MS) const stillRejected: (typeof batches)[number][] = [] for (let i = 0; i < rejectedBatches.length; i += CONCURRENCY) { From 149742bd9c5087cbcf5c1f7558f79f6f07693837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 18:26:35 +0200 Subject: [PATCH 15/17] fix: restore original Temporal patch ID for gap-heal scan (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../star_snapshot_worker/src/workflows/selfHealStarBackfill.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 511cbacab5..615f22c42b 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -96,7 +96,7 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) // patched() keeps an execution already in flight on its old command sequence so a // mid-deploy replay doesn't hit a nondeterminism error. - const gapHealPatched = patched('gap-heal-scan') + const gapHealPatched = patched('CM-1441-gap-heal-scan') let gapHealPage: Awaited> | undefined if (!gapHealDone && gapHealPatched) { gapHealPage = await findReposNeedingGapHeal(PAGE_SIZE, args.gapHealAfterUrl) From ad62b89f5a0466be7f42ab6e039c23129e012550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 19:07:30 +0200 Subject: [PATCH 16/17] fix: namespace gap-heal batch child-workflow IDs to avoid main-scan collision (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../src/workflows/selfHealStarBackfill.ts | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 615f22c42b..6dc799cc9d 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -42,15 +42,25 @@ function fnv1a32Hex(input: string): string { } // Content-addressed (not positional) so a workflow retry with a reshuffled candidate list can't -// collide two different batches under REJECT_DUPLICATE and silently skip one. -function batchWorkflowId(batch: Awaited>): string { +// collide two different batches under REJECT_DUPLICATE and silently skip one. Namespaced by scan +// kind so a gap-heal batch can never collide with a main-scan batch under REJECT_DUPLICATE and +// get silently skipped as an already-started duplicate. The main-scan ID format is already live, +// so `namespaced` (gated by patched()) keeps an in-flight main-scan batch dispatched under the +// old, un-namespaced format from getting a different-looking ID on replay. +function batchWorkflowId( + scanKind: 'main' | 'gap-heal', + batch: Awaited>, + namespaced: boolean, +): string { const digest = fnv1a32Hex( batch .map((repo) => repo.repositoryId) .sort() .join(','), ) - return `${workflowInfo().workflowId}/batch-${digest}` + return namespaced + ? `${workflowInfo().workflowId}/${scanKind}-batch-${digest}` + : `${workflowInfo().workflowId}/batch-${digest}` } async function startBatchChild( @@ -84,12 +94,17 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) const mainScanDone = args.mainScanDone ?? false const gapHealDone = args.gapHealDone ?? false + // patched() keeps a main-scan batch already dispatched in this run's history on its old, + // un-namespaced child-workflow ID so a replay after this deploy doesn't compute a + // different-looking ID for that call and hit a nondeterminism error. + const namespacedBatchIds = patched('CM-1441-namespaced-batch-ids') + let repos: Awaited> = [] if (!mainScanDone) { repos = await findReposNeedingStarBackfill(PAGE_SIZE, args.afterUrl) for (let i = 0; i < repos.length; i += BATCH_SIZE) { const batch = repos.slice(i, i + BATCH_SIZE) - await startBatchChild(batch, batchWorkflowId(batch)) + await startBatchChild(batch, batchWorkflowId('main', batch, namespacedBatchIds)) batchesDispatched++ } } @@ -102,7 +117,9 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) gapHealPage = await findReposNeedingGapHeal(PAGE_SIZE, args.gapHealAfterUrl) for (let i = 0; i < gapHealPage.gappedRepos.length; i += BATCH_SIZE) { const batch = gapHealPage.gappedRepos.slice(i, i + BATCH_SIZE) - await startBatchChild(batch, batchWorkflowId(batch)) + // Gap-heal batches are a brand-new call site introduced by this PR (no prior deployed + // format to preserve), so they're always namespaced. + await startBatchChild(batch, batchWorkflowId('gap-heal', batch, true)) batchesDispatched++ } } From b61e841b0decbd951d68389c90f19f5d0f2667dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 23 Sep 2026 20:04:38 +0200 Subject: [PATCH 17/17] fix: trim comments over 2-line limit (CM-1441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../src/workflows/selfHealStarBackfill.ts | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts index 6dc799cc9d..62e304670b 100644 --- a/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts +++ b/services/apps/star_snapshot_worker/src/workflows/selfHealStarBackfill.ts @@ -41,12 +41,8 @@ function fnv1a32Hex(input: string): string { return (hash >>> 0).toString(16).padStart(8, '0') } -// Content-addressed (not positional) so a workflow retry with a reshuffled candidate list can't -// collide two different batches under REJECT_DUPLICATE and silently skip one. Namespaced by scan -// kind so a gap-heal batch can never collide with a main-scan batch under REJECT_DUPLICATE and -// get silently skipped as an already-started duplicate. The main-scan ID format is already live, -// so `namespaced` (gated by patched()) keeps an in-flight main-scan batch dispatched under the -// old, un-namespaced format from getting a different-looking ID on replay. +// Content-addressed so a retry with a reshuffled candidate list can't collide two batches +// under REJECT_DUPLICATE; namespaced by scanKind so gap-heal can't collide with main-scan. function batchWorkflowId( scanKind: 'main' | 'gap-heal', batch: Awaited>, @@ -94,9 +90,7 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) const mainScanDone = args.mainScanDone ?? false const gapHealDone = args.gapHealDone ?? false - // patched() keeps a main-scan batch already dispatched in this run's history on its old, - // un-namespaced child-workflow ID so a replay after this deploy doesn't compute a - // different-looking ID for that call and hit a nondeterminism error. + // Keeps an in-flight main-scan batch's ID stable across the deploy that added namespacing. const namespacedBatchIds = patched('CM-1441-namespaced-batch-ids') let repos: Awaited> = [] @@ -109,8 +103,7 @@ export async function selfHealStarBackfill(args: ISelfHealStarBackfillArgs = {}) } } - // patched() keeps an execution already in flight on its old command sequence so a - // mid-deploy replay doesn't hit a nondeterminism error. + // Keeps an in-flight execution on its old command sequence through a mid-deploy replay. const gapHealPatched = patched('CM-1441-gap-heal-scan') let gapHealPage: Awaited> | undefined if (!gapHealDone && gapHealPatched) {