diff --git a/common/changes/@microsoft/rush/fix-http-build-cache-retries_2026-10-30-00-30.json b/common/changes/@microsoft/rush/fix-http-build-cache-retries_2026-10-30-00-30.json new file mode 100644 index 00000000000..e8e20bf4273 --- /dev/null +++ b/common/changes/@microsoft/rush/fix-http-build-cache-retries_2026-10-30-00-30.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Retry transient HTTP build cache transport failures and file-stream uploads with bounded exponential backoff, Retry-After support, and retry diagnostics.", + "type": "patch" + } + ], + "packageName": "@microsoft/rush" +} diff --git a/rush-plugins/rush-http-build-cache-plugin/README.md b/rush-plugins/rush-http-build-cache-plugin/README.md index d8468855d6f..35534585579 100644 --- a/rush-plugins/rush-http-build-cache-plugin/README.md +++ b/rush-plugins/rush-http-build-cache-plugin/README.md @@ -33,6 +33,25 @@ Then customize the `httpConfiguration` block. For typical use cases, where you'l (For more detail on the above properties and additional optional properties, consult the default `build-cache.json` file.) +## Retries + +Cache requests make up to three attempts for transient network failures and HTTP +408, 429, 500, 502, 503, 504, and 520-524 responses. Buffered and file-stream uploads +use the same retry policy; each file upload attempt opens a fresh stream. +Authentication failures, cache misses, and other permanent failures are not +retried, apart from the existing credential challenge flow. + +Retries use exponential backoff with jitter, starting at 2.5-5 seconds, with a +maximum delay of 60 seconds. A valid `Retry-After` header can increase the delay. +If the server requests a delay longer than 60 seconds, Rush reports the failure +instead of retrying early or stalling the build. Retry settings are not configurable. + +Run Rush with `--verbose` to see retry attempts, delays, and the `cf-ray` response +header when present. Exhausted retries preserve the existing failure reporting. +Cache writes must be idempotent for a given cache key, including when a custom +`uploadMethod` of POST or PATCH is used: a server may store an upload even if its +response is lost. + ## Authorization Details The HTTP build cache plugin offloads authorization to an external executable script that you define. A typical use case would be to create a simple script, for example `common/scripts/cache-auth.js`, that prints an Authorization header value when executed. diff --git a/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts b/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts index cb15370149f..b37c2f690a1 100644 --- a/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts +++ b/rush-plugins/rush-http-build-cache-plugin/src/HttpBuildCacheProvider.ts @@ -68,6 +68,20 @@ export interface IHttpBuildCacheProviderOptions { const MAX_HTTP_CACHE_ATTEMPTS: number = 3; const DEFAULT_MIN_HTTP_RETRY_DELAY_MS: number = 2500; +const MAX_HTTP_RETRY_DELAY_MS: number = 60 * 1000; +const RETRYABLE_HTTP_STATUS_CODES: ReadonlySet = new Set([ + 408, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524 +]); +const RETRYABLE_ERROR_CODES: ReadonlySet = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ECONNABORTED', + 'ETIMEDOUT', + 'EPIPE', + 'EAI_AGAIN', + 'ENETUNREACH', + 'EHOSTUNREACH' +]); export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { readonly #pluginName: string; @@ -203,22 +217,16 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { try { const { size } = await FileSystem.getStatisticsAsync(localFilePath); - const entryStream: FileSystemReadStream = FileSystem.createReadStream(localFilePath); const result: IWebClientStreamResponse | false = await this.#makeHttpStreamRequestAsync({ terminal, relUrl: `${this.#cacheKeyPrefix}${cacheId}`, method: this.#uploadMethod, - body: entryStream, + body: (): FileSystemReadStream => FileSystem.createReadStream(localFilePath), headers: { 'Content-Length': `${size}` }, warningText: 'Could not write cache entry', - // maxAttempts is 1 because the file read stream is consumed after the first attempt - // and cannot be replayed. Downloads use MAX_HTTP_CACHE_ATTEMPTS since each retry - // issues a fresh GET with no request body. - maxAttempts: 1, - // Pre-resolve credentials for stream uploads because the credential fallback path - // in _makeHttpCoreRequestAsync cannot replay a consumed stream body. + maxAttempts: MAX_HTTP_CACHE_ATTEMPTS, credentialOptions: CredentialsOptions.Required }); @@ -356,7 +364,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { terminal: ITerminal; relUrl: string; method: 'GET' | UploadMethod; - body: Readable | undefined; + body: (() => Readable) | undefined; headers?: Record; warningText: string; maxAttempts: number; @@ -387,7 +395,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { terminal: ITerminal; relUrl: string; method: 'GET' | UploadMethod; - body: Buffer | Readable | undefined; + body: Buffer | (() => Readable) | undefined; headers?: Record; warningText: string; maxAttempts: number; @@ -402,6 +410,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { headers: requestHeaders, warningText, credentialOptions, + maxAttempts, stream } = options; const safeCredentialOptions: CredentialsOptions = credentialOptions ?? CredentialsOptions.Optional; @@ -427,66 +436,117 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { const bodyLengthDesc: string = Buffer.isBuffer(body) ? `${body.length} bytes` : 'unknown length'; - terminal.writeDebugLine(`[http-build-cache] request: ${method} ${url} ${bodyLengthDesc}`); + for (let attempt: number = 1; ; attempt++) { + terminal.writeDebugLine(`[http-build-cache] request: ${method} ${url} ${bodyLengthDesc}`); + + let requestBody: Buffer | Readable | undefined; + let response: IWebClientResponse | IWebClientStreamResponse | undefined; + let errorCode: string | undefined; + try { + requestBody = typeof body === 'function' ? body() : body; + const fetchOptions: IGetFetchOptions | IFetchOptionsWithBody = { + verb: method, + headers, + body: requestBody, + redirect: 'follow', + timeoutMs: 0 // Disable timeout for streaming transfers of large cache entries + }; + + response = stream + ? await this.#webClient.fetchStreamAsync(url, fetchOptions) + : await this.#webClient.fetchAsync(url, fetchOptions); + } catch (error) { + const code: unknown = error && typeof error === 'object' && 'code' in error ? error.code : undefined; + if (typeof code !== 'string' || !RETRYABLE_ERROR_CODES.has(code) || attempt >= maxAttempts) { + throw error; + } - const fetchOptions: IGetFetchOptions | IFetchOptionsWithBody = { - verb: method, - headers, - body, - redirect: 'follow', - timeoutMs: 0 // Disable timeout for streaming transfers of large cache entries - }; - - const response: IWebClientResponse | IWebClientStreamResponse = stream - ? await this.#webClient.fetchStreamAsync(url, fetchOptions) - : await this.#webClient.fetchAsync(url, fetchOptions); - - if (!response.ok) { - // Drain the response body on stream responses so the connection can be reused - if ('stream' in response) { - response.stream.resume(); + errorCode = code; + } finally { + if (requestBody && !Buffer.isBuffer(requestBody) && !response?.ok) { + requestBody.destroy(); + } } - const isNonCredentialResponse: boolean = response.status >= 500 && response.status < 600; - - if ( - !isNonCredentialResponse && - typeof credentials !== 'string' && - safeCredentialOptions === CredentialsOptions.Optional - ) { - // Skip credential fallback for stream bodies since the stream has already been consumed - // by the first attempt and cannot be replayed. - const isStreamBody: boolean = !!body && typeof (body as Readable).pipe === 'function'; - if (!isStreamBody) { - // If we don't already have credentials yet, and we got a response from the server - // that is a "normal" failure (4xx), then we assume that credentials are probably - // required. Re-attempt the request, requiring credentials this time. - // - // This counts as part of the "first attempt", so it is not included in the max attempts + if (response) { + if (response.ok) { + return response; + } + + if ('stream' in response) { + response.stream.destroy(); + } + + const isCredentialResponse: boolean = + response.status === 401 || response.status === 403 || response.status === 407; + if ( + isCredentialResponse && + !(method === 'GET' && response.redirected) && + typeof credentials !== 'string' && + safeCredentialOptions === CredentialsOptions.Optional + ) { + // A credential challenge is part of the same attempt, not a transient failure. return await this.#makeHttpCoreRequestAsync({ ...options, - credentialOptions: CredentialsOptions.Required + credentialOptions: CredentialsOptions.Required, + maxAttempts: maxAttempts - attempt + 1 }); } + + if (!RETRYABLE_HTTP_STATUS_CODES.has(response.status) || attempt >= maxAttempts) { + this.#reportFailure(terminal, method, response, false, warningText); + return false; + } } - if (options.maxAttempts > 1) { - // Pause a bit before retrying in case the server is busy - // Add some random jitter to the retry so we can spread out load on the remote service - // A proper solution might add exponential back off in case the retry count is high (10 or more) - const factor: number = 1.0 + Math.random(); // A random number between 1.0 and 2.0 - const retryDelay: number = Math.floor(factor * this.#minHttpRetryDelayMs); + const retryDelay: number | undefined = this.#getRetryDelayMs( + attempt, + response?.headers?.['retry-after'] + ); + if (retryDelay === undefined && response) { + terminal.writeVerboseLine( + `[http-build-cache] Not retrying: Retry-After exceeds ${MAX_HTTP_RETRY_DELAY_MS} ms.` + ); + this.#reportFailure(terminal, method, response, false, warningText); + return false; + } + if (retryDelay !== undefined) { + const failure: string = response ? this.#describeResponse(response) : `${errorCode}`; + terminal.writeVerboseLine( + `[http-build-cache] ${method} attempt ${attempt}/${maxAttempts} failed: ${failure}. ` + + `Retrying attempt ${attempt + 1}/${maxAttempts} in ${retryDelay} ms.` + ); await Async.sleepAsync(retryDelay); - - return await this.#makeHttpCoreRequestAsync({ ...options, maxAttempts: options.maxAttempts - 1 }); } + } + } - this.#reportFailure(terminal, method, response, false, warningText); - return false; + #getRetryDelayMs(attempt: number, retryAfterHeader: string | string[] | undefined): number | undefined { + const retryAfter: string | undefined = ( + Array.isArray(retryAfterHeader) ? retryAfterHeader[0] : retryAfterHeader + )?.trim(); + const retryAfterMs: number = retryAfter + ? /^\d+$/.test(retryAfter) + ? Number(retryAfter) * 1000 + : Date.parse(retryAfter) - Date.now() + : 0; + // Do not retry earlier than requested, or stall a build for an unbounded server-specified delay. + if (retryAfterMs > MAX_HTTP_RETRY_DELAY_MS) { + return undefined; } - return response; + const backoffMs: number = Math.min( + MAX_HTTP_RETRY_DELAY_MS, + Math.floor(this.#minHttpRetryDelayMs * 2 ** (attempt - 1) * (1 + Math.random())) + ); + return Math.max(backoffMs, Number.isFinite(retryAfterMs) ? retryAfterMs : 0); + } + + #describeResponse(response: IWebClientResponse | IWebClientStreamResponse): string { + const { status, statusText, headers } = response; + const cfRay: string | string[] | undefined = headers?.['cf-ray']; + return `HTTP ${status}: ${statusText}${cfRay ? ` (cf-ray: ${cfRay})` : ''}`; } async #tryGetCredentialsAsync(options: CredentialsOptions.Required): Promise; @@ -599,31 +659,32 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider { isRedirect: boolean, message: string ): void { + const failureMessage: string = `${message}: ${this.#describeResponse(response)}`; switch (this.#getFailureType(requestMethod, response, isRedirect)) { default: { - terminal.writeErrorLine(`${message}: HTTP ${response.status}: ${response.statusText}`); + terminal.writeErrorLine(failureMessage); break; } case FailureType.Warning: { - terminal.writeWarningLine(`${message}: HTTP ${response.status}: ${response.statusText}`); + terminal.writeWarningLine(failureMessage); break; } case FailureType.Informational: { - terminal.writeLine(`${message}: HTTP ${response.status}: ${response.statusText}`); + terminal.writeLine(failureMessage); break; } case FailureType.None: { - terminal.writeDebugLine(`${message}: HTTP ${response.status}: ${response.statusText}`); + terminal.writeDebugLine(failureMessage); break; } case FailureType.Authentication: { throw new Error( [ - `${this.#url.href} responded with ${response.status}: ${response.statusText}.`, + `${this.#url.href} responded with ${this.#describeResponse(response)}.`, `Credentials may be misconfigured or have expired.`, `In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.`, ``, diff --git a/rush-plugins/rush-http-build-cache-plugin/src/test/HttpBuildCacheProvider.test.ts b/rush-plugins/rush-http-build-cache-plugin/src/test/HttpBuildCacheProvider.test.ts index 3fd8389d50d..257799062e2 100644 --- a/rush-plugins/rush-http-build-cache-plugin/src/test/HttpBuildCacheProvider.test.ts +++ b/rush-plugins/rush-http-build-cache-plugin/src/test/HttpBuildCacheProvider.test.ts @@ -9,14 +9,19 @@ jest.mock('node:stream/promises', () => ({ pipeline: jest.fn().mockResolvedValue(undefined) })); +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { type RushSession, EnvironmentConfiguration } from '@rushstack/rush-sdk'; import { type ICredentialCacheEntry, CredentialCache } from '@rushstack/credential-cache'; -import { FileSystem } from '@rushstack/node-core-library'; +import { Async, FileSystem, type FileSystemReadStream } from '@rushstack/node-core-library'; import { StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; -import { WebClient } from '@rushstack/rush-sdk/lib/utilities/WebClient'; +import { + WebClient, + type IWebClientResponse, + type IWebClientStreamResponse +} from '@rushstack/rush-sdk/lib/utilities/WebClient'; import { HttpBuildCacheProvider, type IHttpBuildCacheProviderOptions } from '../HttpBuildCacheProvider'; @@ -41,7 +46,7 @@ const WRITE_ALLOWED_OPTIONS: IHttpBuildCacheProviderOptions = { type FetchFnType = Parameters[0]; type StreamFetchFnType = Parameters[0]; -describe('HttpBuildCacheProvider', () => { +describe(HttpBuildCacheProvider.name, () => { let terminalBuffer: StringBufferTerminalProvider; let terminal!: Terminal; let fetchFn: jest.Mock; @@ -56,7 +61,7 @@ describe('HttpBuildCacheProvider', () => { WebClient.mockStreamRequestFn(streamFetchFn as unknown as StreamFetchFnType); jest .spyOn(FileSystem, 'createReadStream') - .mockReturnValue({ pipe: jest.fn() } as unknown as ReturnType); + .mockImplementation((): FileSystemReadStream => new Readable({ read() {} }) as FileSystemReadStream); jest .spyOn(FileSystem, 'getStatisticsAsync') .mockResolvedValue({ size: 123 } as Awaited>); @@ -64,6 +69,8 @@ describe('HttpBuildCacheProvider', () => { .spyOn(FileSystem, 'createWriteStreamAsync') .mockResolvedValue({} as unknown as Awaited>); jest.spyOn(FileSystem, 'ensureFolderAsync').mockResolvedValue(); + jest.spyOn(Async, 'sleepAsync').mockResolvedValue(); + jest.spyOn(Math, 'random').mockReturnValue(0); }); afterEach(() => { @@ -164,7 +171,9 @@ Array [ expect(terminalBuffer.getAllOutputAsChunks({ asLines: true })).toMatchInlineSnapshot(` Array [ "[ debug] [http-build-cache] request: GET https://buildcache.example.acme.com/some-key unknown length[n]", + "[verbose] [http-build-cache] GET attempt 1/3 failed: HTTP 500: InternalServiceError. Retrying attempt 2/3 in 1 ms.[n]", "[ debug] [http-build-cache] request: GET https://buildcache.example.acme.com/some-key unknown length[n]", + "[verbose] [http-build-cache] GET attempt 2/3 failed: HTTP 503: ServiceUnavailable. Retrying attempt 3/3 in 2 ms.[n]", "[ debug] [http-build-cache] request: GET https://buildcache.example.acme.com/some-key unknown length[n]", "[warning] Could not get cache entry: HTTP 504: Gateway Timeout[n]", ] @@ -424,7 +433,7 @@ Array [ ); }); - it('does not retry on failure (file stream already consumed)', async () => { + it('retries with a fresh file stream on failure', async () => { jest.spyOn(EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue('token123'); const session: RushSession = {} as RushSession; @@ -445,8 +454,17 @@ Array [ ); expect(result).toBe(false); - // maxAttempts is 1 for file-based uploads, so only 1 call - expect(streamFetchFn).toHaveBeenCalledTimes(1); + expect(streamFetchFn).toHaveBeenCalledTimes(3); + expect(FileSystem.createReadStream).toHaveBeenCalledTimes(3); + const streams: FileSystemReadStream[] = jest + .mocked(FileSystem.createReadStream) + .mock.results.map((streamResult: jest.MockResult) => streamResult.value); + expect(new Set(streams).size).toBe(3); + for (const stream of streams) { + expect(stream.destroyed).toBe(true); + } + + expect(responseStream.destroyed).toBe(true); }); it('skips credential fallback for file-based uploads on 4xx', async () => { @@ -487,5 +505,338 @@ Array [ // Should only be called once (no credential fallback retry with consumed stream) expect(streamFetchFn).toHaveBeenCalledTimes(1); }); + + describe.each(['buffer', 'file'] as const)('%s upload retries', (transferMode: 'buffer' | 'file') => { + let provider: HttpBuildCacheProvider; + let requestMock: jest.Mock; + + beforeEach(() => { + jest.spyOn(EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue('token123'); + provider = new HttpBuildCacheProvider(WRITE_ALLOWED_OPTIONS, {} as RushSession); + requestMock = transferMode === 'buffer' ? fetchFn : streamFetchFn; + }); + + function createResponse( + status: number, + headers: Record = {} + ): IWebClientResponse | IWebClientStreamResponse { + const response: IWebClientResponse = { + ok: status >= 200 && status < 300, + status, + statusText: 'test response', + redirected: false, + headers, + getBufferAsync: jest.fn(), + getTextAsync: jest.fn(), + getJsonAsync: jest.fn() + }; + return transferMode === 'file' ? { ...response, stream: new Readable({ read() {} }) } : response; + } + + async function uploadAsync(): Promise { + return transferMode === 'buffer' + ? await provider.trySetCacheEntryBufferAsync(terminal, 'some-key', Buffer.from('cache contents')) + : await provider.tryUploadCacheEntryFromFileAsync( + terminal, + 'some-key', + `${__dirname}/../../temp/cache-entry` + ); + } + + it.each([408, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524])( + 'recovers from HTTP %s without warnings', + async (status: number) => { + requestMock + .mockResolvedValueOnce(createResponse(status)) + .mockResolvedValueOnce(createResponse(200)); + + expect(await uploadAsync()).toBe(true); + expect(requestMock).toHaveBeenCalledTimes(2); + expect(Async.sleepAsync).toHaveBeenCalledTimes(1); + expect(terminalBuffer.getWarningOutput()).toBe(''); + } + ); + + it('exhausts the three-attempt HTTP budget and reports the final failure', async () => { + requestMock.mockResolvedValue(createResponse(520, { 'cf-ray': 'ray-exhausted' })); + + expect(await uploadAsync()).toBe(false); + expect(requestMock).toHaveBeenCalledTimes(3); + expect(Async.sleepAsync).toHaveBeenCalledTimes(2); + expect(terminalBuffer.getWarningOutput()).toContain( + 'Could not write cache entry: HTTP 520: test response (cf-ray: ray-exhausted)' + ); + }); + + it.each([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ECONNABORTED', + 'ETIMEDOUT', + 'EPIPE', + 'EAI_AGAIN', + 'ENETUNREACH', + 'EHOSTUNREACH' + ])('recovers from %s without warnings', async (code: string) => { + requestMock + .mockRejectedValueOnce(Object.assign(new Error('transient transport error'), { code })) + .mockResolvedValueOnce(createResponse(200)); + + expect(await uploadAsync()).toBe(true); + expect(requestMock).toHaveBeenCalledTimes(2); + expect(terminalBuffer.getWarningOutput()).toBe(''); + if (transferMode === 'file') { + const [first, second] = requestMock.mock.calls; + expect(first[1].body).not.toBe(second[1].body); + expect(first[1].body.destroyed).toBe(true); + } + }); + + it('shares the attempt budget between HTTP and transport failures', async () => { + requestMock + .mockRejectedValueOnce(Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' })) + .mockResolvedValueOnce(createResponse(520)) + .mockResolvedValueOnce(createResponse(200)); + + expect(await uploadAsync()).toBe(true); + expect(requestMock).toHaveBeenCalledTimes(3); + expect(Async.sleepAsync).toHaveBeenNthCalledWith(1, 1); + expect(Async.sleepAsync).toHaveBeenNthCalledWith(2, 2); + }); + + it('reports the final transport error after exhausting retries', async () => { + requestMock.mockRejectedValue(Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' })); + + expect(await uploadAsync()).toBe(false); + expect(requestMock).toHaveBeenCalledTimes(3); + expect(Async.sleepAsync).toHaveBeenCalledTimes(2); + expect(terminalBuffer.getWarningOutput()).toContain( + 'Error uploading cache entry: Error: read ECONNRESET' + ); + }); + + it.each([400, 401, 403, 404, 405, 407, 413, 501, 505, 526])( + 'does not retry permanent HTTP %s failures', + async (status: number) => { + requestMock.mockResolvedValue(createResponse(status)); + + expect(await uploadAsync()).toBe(false); + expect(requestMock).toHaveBeenCalledTimes(1); + expect(Async.sleepAsync).not.toHaveBeenCalled(); + expect(terminalBuffer.getWarningOutput()).not.toBe(''); + } + ); + + it.each(['ENOENT', 'EACCES', 'ENOTFOUND', 'ERR_TLS_CERT_ALTNAME_INVALID'])( + 'does not retry permanent %s errors', + async (code: string) => { + requestMock.mockRejectedValue(Object.assign(new Error('permanent error'), { code })); + + expect(await uploadAsync()).toBe(false); + expect(requestMock).toHaveBeenCalledTimes(1); + expect(Async.sleepAsync).not.toHaveBeenCalled(); + expect(terminalBuffer.getWarningOutput()).toContain('permanent error'); + } + ); + + it('does not retry an unclassified exception', async () => { + requestMock.mockRejectedValue(new Error('unexpected failure')); + + expect(await uploadAsync()).toBe(false); + expect(requestMock).toHaveBeenCalledTimes(1); + expect(terminalBuffer.getWarningOutput()).toContain('unexpected failure'); + }); + + it('uses exponential backoff with jitter', async () => { + jest.mocked(Math.random).mockReturnValue(0.5); + requestMock.mockResolvedValue(createResponse(520)); + + expect(await uploadAsync()).toBe(false); + expect(Async.sleepAsync).toHaveBeenNthCalledWith(1, 1); + expect(Async.sleepAsync).toHaveBeenNthCalledWith(2, 3); + }); + + it('caps the exponential backoff', async () => { + provider = new HttpBuildCacheProvider( + { ...WRITE_ALLOWED_OPTIONS, minHttpRetryDelayMs: 40000 }, + {} as RushSession + ); + requestMock.mockResolvedValue(createResponse(520)); + + expect(await uploadAsync()).toBe(false); + expect(Async.sleepAsync).toHaveBeenNthCalledWith(1, 40000); + expect(Async.sleepAsync).toHaveBeenNthCalledWith(2, 60000); + }); + + it('honors Retry-After delta seconds', async () => { + requestMock + .mockResolvedValueOnce(createResponse(429, { 'retry-after': '7' })) + .mockResolvedValueOnce(createResponse(200)); + + expect(await uploadAsync()).toBe(true); + expect(Async.sleepAsync).toHaveBeenCalledWith(7000); + }); + + it('honors Retry-After HTTP dates', async () => { + jest.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-10-29T12:00:00Z')); + requestMock + .mockResolvedValueOnce(createResponse(503, { 'retry-after': 'Thu, 29 Oct 2026 12:00:09 GMT' })) + .mockResolvedValueOnce(createResponse(200)); + + expect(await uploadAsync()).toBe(true); + expect(Async.sleepAsync).toHaveBeenCalledWith(9000); + }); + + it.each(['invalid', '-1', 'Thu, 01 Jan 1970 00:00:00 GMT', '0'])( + 'uses backoff for expired or invalid Retry-After %s', + async (retryAfter: string) => { + requestMock + .mockResolvedValueOnce(createResponse(503, { 'retry-after': retryAfter })) + .mockResolvedValueOnce(createResponse(200)); + + expect(await uploadAsync()).toBe(true); + expect(Async.sleepAsync).toHaveBeenCalledWith(1); + } + ); + + it('accepts an array-valued Retry-After header', async () => { + requestMock + .mockResolvedValueOnce(createResponse(503, { 'retry-after': ['5'] })) + .mockResolvedValueOnce(createResponse(200)); + + expect(await uploadAsync()).toBe(true); + expect(Async.sleepAsync).toHaveBeenCalledWith(5000); + }); + + it('does not retry earlier than a Retry-After exceeding the wait limit', async () => { + requestMock.mockResolvedValue(createResponse(503, { 'retry-after': '61' })); + + expect(await uploadAsync()).toBe(false); + expect(requestMock).toHaveBeenCalledTimes(1); + expect(Async.sleepAsync).not.toHaveBeenCalled(); + expect(terminalBuffer.getVerboseOutput()).toContain('Retry-After exceeds 60000 ms'); + expect(terminalBuffer.getOutput()).toContain('Could not write cache entry: HTTP 503'); + }); + + it('logs attempts and cf-ray without logging authorization headers', async () => { + requestMock + .mockResolvedValueOnce(createResponse(520, { 'cf-ray': 'ray-to-correlate' })) + .mockResolvedValueOnce(createResponse(200)); + + expect(await uploadAsync()).toBe(true); + const output: string = terminalBuffer.getVerboseOutput(); + expect(output).toContain( + 'POST attempt 1/3 failed: HTTP 520: test response (cf-ray: ray-to-correlate)' + ); + expect(output).toContain('Retrying attempt 2/3 in 1 ms'); + expect(output).not.toContain('token123'); + }); + + if (transferMode === 'file') { + it('reports a local file error without retrying', async () => { + jest.mocked(FileSystem.createReadStream).mockImplementation((): never => { + throw Object.assign(new Error('file missing'), { code: 'ENOENT' }); + }); + + expect(await uploadAsync()).toBe(false); + expect(requestMock).not.toHaveBeenCalled(); + expect(Async.sleepAsync).not.toHaveBeenCalled(); + expect(terminalBuffer.getWarningOutput()).toContain('file missing'); + }); + } + }); + + describe('stream upload transport', () => { + it.each(['HTTP 520', 'early HTTP 520', 'ECONNRESET'])( + 'replays the complete file after %s using the real HTTP client', + async (failure: string) => { + WebClient.resetMockStreamRequestFn(); + jest.spyOn(EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue('token123'); + const content: Buffer = Buffer.alloc(123, 'cache contents'); + const streams: Readable[] = []; + jest.mocked(FileSystem.createReadStream).mockImplementation((): FileSystemReadStream => { + let chunkSent: boolean = false; + const stream: Readable = + failure === 'early HTTP 520' && streams.length === 0 + ? new Readable({ + read(): void { + if (!chunkSent) { + chunkSent = true; + this.push(content.subarray(0, 1)); + } + } + }) + : Readable.from([content]); + streams.push(stream); + return stream as FileSystemReadStream; + }); + + const uploads: Buffer[] = []; + let requests: number = 0; + const server: Server = createServer((request: IncomingMessage, response: ServerResponse): void => { + requests++; + if (requests === 1 && failure === 'ECONNRESET') { + request.socket.destroy(); + return; + } + + if (requests === 1 && failure === 'early HTTP 520') { + response.writeHead(520); + response.end(); + return; + } + + const chunks: Buffer[] = []; + request.on('data', (chunk: Buffer): void => { + chunks.push(chunk); + }); + request.on('end', (): void => { + uploads.push(Buffer.concat(chunks)); + response.writeHead(requests === 1 ? 520 : 200, { 'cf-ray': 'local-test-ray' }); + response.end(); + }); + }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => resolve()); + }); + + try { + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Expected a TCP server address'); + } + + const provider: HttpBuildCacheProvider = new HttpBuildCacheProvider( + { ...WRITE_ALLOWED_OPTIONS, url: `http://127.0.0.1:${address.port}/cache/` }, + {} as RushSession + ); + expect( + await provider.tryUploadCacheEntryFromFileAsync( + terminal, + 'some-key', + `${__dirname}/../../temp/cache-entry` + ) + ).toBe(true); + expect(requests).toBe(2); + expect(streams).toHaveLength(2); + expect(streams[0]).not.toBe(streams[1]); + expect(streams[0]?.destroyed).toBe(true); + expect(uploads).toHaveLength(failure === 'HTTP 520' ? 2 : 1); + for (const upload of uploads) { + expect(upload).toEqual(content); + } + + expect(terminalBuffer.getWarningOutput()).toBe(''); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => { + server.close((error: Error | undefined) => (error ? reject(error) : resolve())); + }); + } + } + ); + }); }); });