fix restore process being fully cancelled, insted try for 5 times then give up.

This commit is contained in:
Blake Kaufman
2026-07-12 08:59:48 -04:00
parent 00529933c3
commit 85d5099f1d
2 changed files with 122 additions and 22 deletions
@@ -96,17 +96,79 @@ describe('fullRestoreSparkState — restore-complete gating', () => {
jest.clearAllMocks();
});
it('does NOT mark restore complete when the transaction fetch fails', async () => {
mockGetSparkTransactions.mockResolvedValue({
transfers: [],
success: false,
});
it('does NOT mark restore complete when the transaction fetch keeps failing', async () => {
jest.useFakeTimers();
try {
mockGetSparkTransactions.mockResolvedValue({
transfers: [],
success: false,
});
await runRestore();
const restorePromise = runRestore();
// A persistent failure retries the same offset with a backoff before
// giving up; drive the fake timers so the retries resolve without waiting.
await jest.runAllTimersAsync();
await restorePromise;
expect(markedComplete()).toBe(false);
// A failed fetch must not even reach the balance sanity check.
expect(mockGetSparkBalance).not.toHaveBeenCalled();
expect(markedComplete()).toBe(false);
// A failed fetch must not even reach the balance sanity check.
expect(mockGetSparkBalance).not.toHaveBeenCalled();
// Retries the same offset up to the consecutive-failure cap, then throws.
expect(mockGetSparkTransactions).toHaveBeenCalledTimes(3);
} finally {
jest.useRealTimers();
}
});
it('recovers within a single run when a transient fetch failure is followed by success', async () => {
jest.useFakeTimers();
try {
// Fail once, then return a real batch, then an empty batch (end of history).
mockGetSparkTransactions
.mockResolvedValueOnce({ transfers: [], success: false })
.mockResolvedValueOnce({
transfers: [{ id: 'tx-1', transferDirection: 'INCOMING' }],
success: true,
})
.mockResolvedValue({ transfers: [], success: true });
mockGetSparkBalance.mockResolvedValue({ didWork: true, balance: 0n });
const restorePromise = runRestore();
await jest.runAllTimersAsync();
await restorePromise;
// The transient failure did not abort the run — it completed normally.
expect(markedComplete()).toBe(true);
} finally {
jest.useRealTimers();
}
});
it('retries the suspicious empty-with-balance batch instead of aborting', async () => {
jest.useFakeTimers();
try {
// First a successful-but-empty batch while the wallet reports a balance
// (suspicious → retry), then a real batch, then end of history.
mockGetSparkTransactions
.mockResolvedValueOnce({ transfers: [], success: true })
.mockResolvedValueOnce({
transfers: [{ id: 'tx-1', transferDirection: 'INCOMING' }],
success: true,
})
.mockResolvedValue({ transfers: [], success: true });
// Positive balance on the first (suspicious) check, then zero at the end.
mockGetSparkBalance
.mockResolvedValueOnce({ didWork: true, balance: 5000n })
.mockResolvedValue({ didWork: true, balance: 0n });
const restorePromise = runRestore();
await jest.runAllTimersAsync();
await restorePromise;
expect(markedComplete()).toBe(true);
} finally {
jest.useRealTimers();
}
});
it('DOES mark restore complete on a successful empty batch for a zero-balance wallet', async () => {
@@ -122,15 +184,23 @@ describe('fullRestoreSparkState — restore-complete gating', () => {
});
it('does NOT mark restore complete when zero txs are returned but the wallet has a balance', async () => {
mockGetSparkTransactions.mockResolvedValue({
transfers: [],
success: true,
});
mockGetSparkBalance.mockResolvedValue({ didWork: true, balance: 5000n });
jest.useFakeTimers();
try {
mockGetSparkTransactions.mockResolvedValue({
transfers: [],
success: true,
});
mockGetSparkBalance.mockResolvedValue({ didWork: true, balance: 5000n });
await runRestore();
const restorePromise = runRestore();
// Persistent empty-with-balance is retried before giving up; drive timers.
await jest.runAllTimersAsync();
await restorePromise;
expect(markedComplete()).toBe(false);
expect(markedComplete()).toBe(false);
} finally {
jest.useRealTimers();
}
});
it('marks complete on an empty batch when the balance is unknown (didWork:false), staying conservative', async () => {
+36 -6
View File
@@ -40,6 +40,10 @@ const RESTORE_STATE_KEY = 'spark_tx_restore_state';
const MAX_BATCH_SIZE = 400;
const DEFAULT_BATCH_SIZE = 5;
const INCREMENTAL_SAVE_THRESHOLD = 200;
// Max consecutive failed/suspicious page fetches before a restore run gives up.
const MAX_RESTORE_FETCH_RETRIES = 5;
// Base backoff between retries, scaled by the current consecutive-failure count.
const RESTORE_RETRY_DELAY_MS = 1500;
/**
* Get the current restore state for an account
@@ -203,6 +207,26 @@ const restoreSparkTxState = async (
let batchCounter = 0;
let foundOverlap = false;
// Track consecutive failed/suspicious fetches so a single transient error
// doesn't discard the whole run. We retry the SAME offset with a short
// backoff and only give up (throw) after MAX_RESTORE_FETCH_RETRIES in a row,
// keeping the loop bounded so it can't hang.
let consecutiveFailures = 0;
const handleRetryableFailure = async reason => {
consecutiveFailures++;
if (consecutiveFailures >= MAX_RESTORE_FETCH_RETRIES) {
// Give up. Throw (NOT markRestoreComplete) so isFullyRestored stays
// false and the restore re-runs on the next connect/launch — same
// safety as before.
throw new Error(
`Failed to fetch transactions during restore after ${consecutiveFailures} attempts: ${reason}`,
);
}
await new Promise(r =>
setTimeout(r, RESTORE_RETRY_DELAY_MS * consecutiveFailures),
);
};
while (true) {
const txs = await getSparkTransactions(localBatchSize, offset, mnemonic);
@@ -211,9 +235,10 @@ const restoreSparkTxState = async (
// from an empty wallet at the .transfers level, so we must NOT fall
// through to markRestoreComplete — doing so would persist
// isFullyRestored:true and stop the restore poller from ever
// re-scanning this session. Throw so the outer catch reports the run as
// incomplete and the caller retries.
throw new Error('Failed to fetch transactions during restore');
// re-scanning this session. Retry the same offset rather than aborting
// the whole run on the first transient failure.
await handleRetryableFailure('fetch failed');
continue;
}
const batchTxs = txs.transfers || [];
@@ -223,15 +248,16 @@ const restoreSparkTxState = async (
// history. But if we've discovered zero transactions in total and the
// wallet still reports a positive balance, the empty result is almost
// certainly a bad/incomplete fetch — you cannot hold a balance with no
// transactions. Treat that as a failure and retry rather than marking
// transactions. Treat that as a retryable failure rather than marking
// the restore complete on a wallet that clearly has history.
const noTxsDiscovered = savedIds.size === 0 && !restoredTxs.length;
if (noTxsDiscovered) {
const { didWork, balance } = await getBalanceWithTimeout(mnemonic);
if (didWork && BigInt(balance || 0) > 0n) {
throw new Error(
'Restore returned 0 transactions but wallet has a balance; retrying',
await handleRetryableFailure(
'empty batch but wallet has a balance',
);
continue;
}
}
console.log('No more transactions found, ending restore.');
@@ -239,6 +265,10 @@ const restoreSparkTxState = async (
break;
}
// Genuine batch of transfers — reset the consecutive-failure streak so the
// cap only ever counts failures that happen back-to-back.
consecutiveFailures = 0;
// Process batch and check for overlap simultaneously
const newBatchTxs = [];
for (const tx of batchTxs) {