From 543b891c9574579dd7a2b3dbb8993ccc69264cd6 Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Tue, 3 Nov 2026 15:54:20 +0100 Subject: [PATCH] feat(fetch): add --retries to fetch and extract, following the shared retry policy, off by default A busy API answer or a connection that never reached the API made agents write their own retry loops. With --retries , fetch and extract retry a 429, 502, 503 or 504 and a network error before the request was sent, waiting Retry-After or full jitter, and start no retry after 90 s. They never retry a client timeout, a connection closed after sending, a target status or extract prepare. Without --retries, nothing changes. --- README.md | 14 ++ skills/extract/SKILL.md | 1 + skills/protected-fetch/SKILL.md | 5 + skills/trace-debug/SKILL.md | 5 + src/adapters/protected-fetch.ts | 4 +- src/cli/commands/extract.ts | 4 + src/cli/commands/fetch.ts | 17 ++ src/core/http.ts | 150 +++++++++++---- tests/fetch-retry.test.ts | 323 ++++++++++++++++++++++++++++++++ 9 files changed, 487 insertions(+), 36 deletions(-) create mode 100644 tests/fetch-retry.test.ts diff --git a/README.md b/README.md index 4d96f6f..1028eb1 100644 --- a/README.md +++ b/README.md @@ -125,8 +125,22 @@ zenrows fetch # Adaptive Stealth Mode (mode=auto) zenrows fetch --output markdown zenrows fetch --manual --js-render --premium-proxy zenrows fetch --proxy-country us --wait-for ".price" +zenrows fetch --retries 2 # retry a busy answer up to 2 times; off by default ``` +`fetch` and `extract` send each request once. With `--retries `, they retry +up to `n` times on a 429, 502, 503 or 504 from the API, and on a network error +before the request reached the API (refused connection, DNS failure, connect +timeout, unreachable network). Each wait is the `Retry-After` header in whole +seconds (0.25 s to 30 s) or a random delay that grows with each retry, and no +retry starts more than 90 s after the first request. They never retry a client +timeout (`--timeout`) or a connection that closed after the request was sent, +because the API may still finish and charge it; a status the target site sent +(`--original-status`); or any request of `zenrows extract prepare`. If a 502, +503 or 504 is still the answer after one or more retries, the error says how +many retries ran and to wait a minute and run the same command again, without +suggesting costlier options. A 429 keeps the error it has without `--retries`. + ## 8. Extract ```bash diff --git a/skills/extract/SKILL.md b/skills/extract/SKILL.md index 6937e28..4de84ca 100644 --- a/skills/extract/SKILL.md +++ b/skills/extract/SKILL.md @@ -68,6 +68,7 @@ zenrows extract https://www.example.com/products/a # 3. extract again `limit` / `remaining`. - `EXTRACT_PREPARE_OUT_OF_ALLOWANCE`: no slots left this month; use `--css`. `EXTRACT_PREPARE_UNAVAILABLE`: retry in a few minutes; do not loop. + `prepare` sends each request once: `--retries` does not apply to it. A `failed` status prints the next step for its failure code and says whether the attempt used an allowance slot. - A domain that is already prepared answers `ready` at once diff --git a/skills/protected-fetch/SKILL.md b/skills/protected-fetch/SKILL.md index 0448f46..43a745a 100644 --- a/skills/protected-fetch/SKILL.md +++ b/skills/protected-fetch/SKILL.md @@ -47,6 +47,11 @@ zenrows fetch --manual --js-render --premium-proxy # the most expensive needs `--premium-proxy` (else `PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM`). - You are billed only for the configuration that succeeds. - On failure, read `zenrows trace explain ` before escalating. +- The CLI sends each request once. `--retries ` retries a 429, 502, 503 or + 504 and a network error before the request reached the API, up to `n` times, + within 90 s; prefer it to your own retry loop. It never retries a timeout, a + connection closed after the request was sent, or a target status under + `--original-status`. Every run is saved under `.zenrows/runs//` (no secrets). See [[extract]] for structured output and [[cost-control]] for the cost model. diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index d1ddab1..30f6db0 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -26,6 +26,11 @@ zenrows trace export # JSON for sharing `--manual --js-render` run; auto mode may ignore it. Each step multiplies the cost of the request, and both together are the most expensive configuration available ([[cost-control]]). +- `FETCH_FAILED` "The Zenrows API answered HTTP 502/503/504 after N retries" → + the CLI already retried a busy answer (`--retries`). Wait a minute and run the + same command again; do not escalate, a costlier configuration does not fix + it. A 429 ("Rate limited") keeps its advice with or without `--retries`: wait + a few seconds and retry; if it persists, lower concurrency. - `REQUEST_TIMEOUT` → the CLI stopped waiting; the API was reached. Raise `--timeout` (default 120000ms, above the API's own 90s budget), or drop `--wait-for` so the request finishes inside that budget and the API returns diff --git a/src/adapters/protected-fetch.ts b/src/adapters/protected-fetch.ts index 7b68c25..22121be 100644 --- a/src/adapters/protected-fetch.ts +++ b/src/adapters/protected-fetch.ts @@ -51,6 +51,8 @@ export interface FetchOptions { * gateway's own request budget on purpose. */ timeoutMs?: number; + /** Retries for a busy answer or a request that never reached the API. Not an API parameter. */ + retries?: number; } const RESPONSE_TYPE: Partial> = { @@ -152,6 +154,6 @@ export async function runFetch( validateAutoManual(opts, config); const params = buildParams(opts, config); const mode: "auto" | "manual" = params.mode === "auto" ? "auto" : "manual"; - const result = await scrape(config.apiBase, apiKey, params, { timeoutMs: opts.timeoutMs, projectRoot }); + const result = await scrape(config.apiBase, apiKey, params, { timeoutMs: opts.timeoutMs, retries: opts.retries, projectRoot }); return { result, params, mode }; } diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index a5f4876..6cd9167 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -21,6 +21,7 @@ import { formatRequestCost } from "../../core/http.ts"; import { parse, asString, asNumber, type Command, type RunContext } from "../command.ts"; import { printError, writeOut } from "../output.ts"; import { refreshClaimAdvice } from "../../core/claim-link.ts"; +import { normalizeRetries } from "./fetch.ts"; import { PREPARE_HELP, PREPARE_USAGE, runPrepare } from "./extract-prepare.ts"; export const extract: Command = { @@ -42,6 +43,7 @@ export const extract: Command = { "Shared:", " --manual --js-render --premium-proxy manual fetch controls", " --validate fail if the result is not valid JSON", + " --retries retry 429/502/503/504 and failed connects up to n times (default 0)", " --out write output to a file", " --no-signup do not auto-create a Free plan account if no key exists", " --json structured result", @@ -66,6 +68,7 @@ export const extract: Command = { wait: { type: "string" }, "wait-for": { type: "string" }, validate: { type: "boolean" }, + retries: { type: "string" }, out: { type: "string" }, "no-signup": { type: "boolean" }, json: { type: "boolean" }, @@ -116,6 +119,7 @@ export const extract: Command = { wait: asNumber(values.wait), waitFor: asString(values["wait-for"]), validate: values.validate === true, + retries: normalizeRetries(values.retries), }; assertUsable("extract"); diff --git a/src/cli/commands/fetch.ts b/src/cli/commands/fetch.ts index f5437a3..01c3b04 100644 --- a/src/cli/commands/fetch.ts +++ b/src/cli/commands/fetch.ts @@ -37,6 +37,7 @@ export const fetch_: Command = { " --screenshot capture an above-the-fold screenshot", " --out write the response body to a file", " --timeout client-side timeout (default 120000; above the API's own 90s budget)", + " --retries retry 429/502/503/504 and failed connects up to n times (default 0)", " --no-signup do not auto-create a Free plan account if no key exists", " --json print a structured result", "", @@ -59,6 +60,7 @@ export const fetch_: Command = { screenshot: { type: "boolean" }, out: { type: "string" }, timeout: { type: "string" }, + retries: { type: "string" }, "no-signup": { type: "boolean" }, json: { type: "boolean" }, }); @@ -101,6 +103,7 @@ export const fetch_: Command = { output: normalizeOutput(asString(values.output)), screenshot: values.screenshot === true, timeoutMs: normalizeTimeout(values.timeout), + retries: normalizeRetries(values.retries), }; const runId = newRunId(); @@ -209,6 +212,20 @@ export function normalizeTimeout(v: unknown): number | undefined { return ms; } +/** `--retries `: a whole number, 0 or more. Like `--timeout`, a bad value is an error. */ +export function normalizeRetries(v: unknown): number | undefined { + if (v === undefined) return undefined; + if (typeof v !== "string" || !/^\d+$/.test(v)) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --retries value '${String(v)}'.`, + likely_cause: "--retries takes a whole number, 0 or more.", + next_action: "Pass a count, e.g. --retries 2. 0, the default, sends the request once.", + }); + } + return Number(v); +} + export function normalizeOutput(v?: string): ResponseFormat | undefined { if (!v) return undefined; const map: Record = { diff --git a/src/core/http.ts b/src/core/http.ts index 3cfbf23..e55eb2a 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -9,7 +9,7 @@ import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./ import { storedClaimHint, usableClaimUrl } from "./claim-link.ts"; import { readAccount } from "./agent-account.ts"; import { ENV_KEY, resolveApiKey } from "./auth.ts"; -import { registerSecret } from "./logger.ts"; +import { log, registerSecret } from "./logger.ts"; import { CLI_USER_AGENT } from "./config.ts"; import { agentClientHeader } from "./agent-client.ts"; import { normalizeDomain, PAGE_NOT_RECOGNISED_STEPS, prepareCommand } from "./prepared-domains-api.ts"; @@ -68,6 +68,47 @@ export const DEFAULT_TIMEOUT_MS = 120_000; /** The gateway's own request budget. Kept here only to justify the default above. */ export const SERVER_BUDGET_MS = 90_000; +/** No retry starts later than this after the first request. */ +const RETRY_BUDGET_MS = 90_000; +/** Busy answers from the API, retried with `--retries`. */ +const RETRY_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); +/** Network error codes that mean the request never reached the API. */ +const NEVER_SENT_CODES: ReadonlySet = new Set([ + "ECONNREFUSED", + "ENOTFOUND", + "EAI_AGAIN", + "ENETUNREACH", + "EHOSTUNREACH", + "UND_ERR_CONNECT_TIMEOUT", +]); + +/** `Retry-After` in whole seconds, clamped to 0.25-30 s; undefined for any other value. */ +export function retryAfterMs(value: string | null): number | undefined { + if (!value || !/^\d+$/.test(value.trim())) return undefined; + return Math.min(Math.max(Number(value) * 1000, 250), 30_000); +} + +/** Full jitter before retry `n` (0-based): uniform(0.25 s, min(30 s, 2^n s)). */ +export function jitterMs(n: number, random: () => number = Math.random): number { + return 250 + random() * (Math.min(30_000, 1000 * 2 ** n) - 250); +} + +/** + * True when a fetch error's `cause` means the request never reached the API. + * ETIMEDOUT counts only on connect. A happy-eyeballs AggregateError counts + * only when every address failed to connect. + */ +export function neverSent(cause: unknown): boolean { + if (cause instanceof AggregateError) return cause.errors.length > 0 && cause.errors.every(neverSent); + const { code, syscall } = (cause ?? {}) as { code?: unknown; syscall?: unknown }; + return code === "ETIMEDOUT" ? syscall === "connect" : NEVER_SENT_CODES.has(String(code)); +} + +/** A busy answer from the API. A status the target site sent is never retried. */ +function isRetryable(status: number, params: ScraperParams): boolean { + return RETRY_STATUSES.has(status) && !params.original_status && params.allowed_status_codes === undefined; +} + function buildUrl(apiBase: string, apiKey: string, params: ScraperParams): { full: string; redacted: string } { const u = new URL(apiBase); u.searchParams.set("apikey", apiKey); @@ -86,48 +127,77 @@ export async function scrape( apiBase: string, apiKey: string, params: ScraperParams, - opts: { timeoutMs?: number; projectRoot?: string } = {}, + opts: { + timeoutMs?: number; + projectRoot?: string; + /** Retries after the first request (`--retries`). 0, the default, sends it once. */ + retries?: number; + /** Test seams for the waits and the retry budget. */ + sleep?: (ms: number) => Promise; + now?: () => number; + } = {}, ): Promise { registerSecret(apiKey); const { full, redacted } = buildUrl(apiBase, apiKey, params); const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; - const controller = new AbortController(); - // Our own timer is the only thing that aborts this controller, so this flag — - // not `err.name === "AbortError"` — is what tells a client-side give-up apart - // from a genuine transport failure. The two must never share an error code. - let timedOut = false; - const timeout = setTimeout(() => { - timedOut = true; - controller.abort(); - }, timeoutMs); - const startedAt = Date.now(); + const retries = opts.retries ?? 0; + const now = opts.now ?? Date.now; + const firstSentAt = now(); + /** Waits before retry `n` and returns true, unless no retry is left or it would start after the budget. */ + const retry = async (n: number, delayMs: number, reason: string): Promise => { + if (n >= retries || now() - firstSentAt + delayMs > RETRY_BUDGET_MS) return false; + log.dim(`${reason}; retry ${n + 1}/${retries} in ${formatMs(delayMs)}`); + await (opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms))))(delayMs); + return true; + }; let res: Response; - try { - res = await fetch(full, { - method: "GET", - headers: { "User-Agent": CLI_USER_AGENT, "Accept-Encoding": "gzip, deflate", ...agentClientHeader({ projectRoot: opts.projectRoot }) }, - signal: controller.signal, - }); - } catch (err) { + let raw: Buffer; + let retriesUsed = 0; + for (; ; retriesUsed++) { + const controller = new AbortController(); + // Our own timer is the only thing that aborts this controller, so this flag — + // not `err.name === "AbortError"` — is what tells a client-side give-up apart + // from a genuine transport failure. The two must never share an error code. + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + controller.abort(); + }, timeoutMs); + const startedAt = Date.now(); + + try { + res = await fetch(full, { + method: "GET", + headers: { "User-Agent": CLI_USER_AGENT, "Accept-Encoding": "gzip, deflate", ...agentClientHeader({ projectRoot: opts.projectRoot }) }, + signal: controller.signal, + }); + } catch (err) { + clearTimeout(timeout); + const elapsedMs = Date.now() - startedAt; + // A timeout, or an error after the request was sent, is never retried: + // the API may still finish and charge that request. + if (timedOut) throw requestTimeout(params.url, timeoutMs, elapsedMs); + const retryable = err instanceof Error && neverSent(err.cause); + if (retryable && (await retry(retriesUsed, jitterMs(retriesUsed), "Network error before the request reached the API"))) continue; + const cause = err instanceof Error ? err.message : String(err); + throw new ToolkitError({ + code: "BACKEND_UNAVAILABLE", + message: `Could not reach the Zenrows API.`, + likely_cause: `Network error after ${formatMs(elapsedMs)}: ${cause}`, + next_action: "Check connectivity and retry. Verify api base in `zenrows config show`.", + suggested_commands: ["zenrows status", "zenrows config show"], + }); + } clearTimeout(timeout); - const elapsedMs = Date.now() - startedAt; - if (timedOut) throw requestTimeout(params.url, timeoutMs, elapsedMs); - const cause = err instanceof Error ? err.message : String(err); - throw new ToolkitError({ - code: "BACKEND_UNAVAILABLE", - message: `Could not reach the Zenrows API.`, - likely_cause: `Network error after ${formatMs(elapsedMs)}: ${cause}`, - next_action: "Check connectivity and retry. Verify api base in `zenrows config show`.", - suggested_commands: ["zenrows status", "zenrows config show"], - }); - } - clearTimeout(timeout); - // Read the raw bytes once. `body` (UTF-8) drives all error detection and text - // output; `raw` preserves the exact bytes so binary responses (screenshot / - // PDF) can be written without the UTF-8 round-trip that corrupts them. - const raw = Buffer.from(await res.arrayBuffer()); + // Read the raw bytes once. `body` (UTF-8) drives all error detection and text + // output; `raw` preserves the exact bytes so binary responses (screenshot / + // PDF) can be written without the UTF-8 round-trip that corrupts them. + raw = Buffer.from(await res.arrayBuffer()); + const delayMs = retryAfterMs(res.headers.get("retry-after")) ?? jitterMs(retriesUsed); + if (!isRetryable(res.status, params) || !(await retry(retriesUsed, delayMs, `HTTP ${res.status}`))) break; + } const body = raw.toString("utf8"); // A binary body is only ever produced by a successful screenshot/PDF request. // Zenrows error responses are always a JSON envelope (text), so exclude those. @@ -280,6 +350,16 @@ export async function scrape( : "Verify the URL. Rendering or premium proxies will not turn a 404 into content — do not retry; it will fail identically and still cost credits.", }); } + if (retriesUsed > 0 && isRetryable(res.status, params)) { + // A busy answer that outlasted the retries: a costlier configuration does not fix it. + throw new ToolkitError({ + code: "FETCH_FAILED", + message: `The Zenrows API answered HTTP ${res.status} after ${retriesUsed} ${retriesUsed === 1 ? "retry" : "retries"}.`, + likely_cause: snippet(body) || "The Zenrows API is busy.", + next_action: + "Wait a minute, then run the same command again. Do not add --js-render or --premium-proxy: they cost more and do not help here.", + }); + } if (res.status === 422 || res.status >= 500 || (res.status >= 400 && !looksLikeContent(result))) { throw new ToolkitError({ code: "FETCH_FAILED", diff --git a/tests/fetch-retry.test.ts b/tests/fetch-retry.test.ts new file mode 100644 index 0000000..2252653 --- /dev/null +++ b/tests/fetch-retry.test.ts @@ -0,0 +1,323 @@ +/** + * `--retries` on fetch and extract, following the shared retry policy. Fetch + * is stubbed and the waits are injected, so only the command tests wait (the + * 250 ms floor). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { scrape, retryAfterMs, jitterMs, neverSent } from "../src/core/http.ts"; +import { fetch_, normalizeRetries } from "../src/cli/commands/fetch.ts"; +import { extract } from "../src/cli/commands/extract.ts"; +import { createWorkspace } from "../src/core/workspace.ts"; +import { savePolicy, defaultPolicy } from "../src/core/policy.ts"; +import { saveApiKey } from "../src/core/auth.ts"; +import { ToolkitError } from "../src/core/errors.ts"; +import type { Command } from "../src/cli/command.ts"; +import { tempRoot } from "./helpers.ts"; + +const API = "https://api.zenrows.com/v1/"; +const RESP009 = JSON.stringify({ code: "RESP009", status: 503, title: "Service temporarily at capacity (RESP009)" }); + +/** An answer, or undici's rejection for a transport failure with this `cause.code`. */ +type Step = { status: number; body?: string; headers?: Record } | { network: string }; +const OK: Step = { status: 200, body: "ok", headers: { "content-type": "text/html" } }; + +const sysErr = (code: string, syscall?: string) => Object.assign(new Error(`${syscall ?? "op"} ${code}`), { code, syscall }); +const fetchFailed = (cause: unknown) => new TypeError("fetch failed", { cause }); + +/** Run `fn` with a global fetch that answers `steps` in order (repeating the last); returns the call count. */ +async function withSteps(steps: Step[], fn: () => Promise, onCall?: () => void): Promise { + let n = 0; + const orig = globalThis.fetch; + globalThis.fetch = (async () => { + onCall?.(); + const step = steps[Math.min(n++, steps.length - 1)]!; + if ("network" in step) throw fetchFailed(sysErr(step.network, "connect")); + return new Response(step.body ?? "", { status: step.status, headers: { "content-type": "application/json", ...step.headers } }); + }) as typeof fetch; + try { + await fn(); + } finally { + globalThis.fetch = orig; + } + return n; +} + +/** Records each wait instead of sleeping. */ +function waits(): { sleep: (ms: number) => Promise; list: number[] } { + const list: number[] = []; + return { list, sleep: async (ms) => void list.push(ms) }; +} + +const errorOf = async (p: Promise): Promise => { + try { + await p; + } catch (err) { + return err as ToolkitError; + } + assert.fail("expected an error"); +}; + +// --- shared conformance vectors ------------------------------------------------- + +test("conformance: Retry-After is whole seconds, clamped to 0.25-30 s; anything else means jitter", () => { + const vectors: Array<[string, number | undefined]> = [ + ["5", 5000], + ["0", 250], + [" 7 ", 7000], + ["99999", 30_000], + ["1.5", undefined], + ["-1", undefined], + ["1e3", undefined], + ["0x10", undefined], + ["", undefined], + ["abc", undefined], + ["Wed, 21 Oct 2026 07:28:00 GMT", undefined], + ]; + for (const [header, expected] of vectors) assert.equal(retryAfterMs(header), expected, JSON.stringify(header)); +}); + +test("conformance: full jitter is uniform(0.25 s, min(30 s, 2^attempt s))", () => { + for (const [attempt, high] of [[0, 1000], [1, 2000], [2, 4000], [5, 30_000], [9, 30_000]] as const) { + assert.equal(jitterMs(attempt, () => 0), 250, `attempt ${attempt}`); + assert.equal(jitterMs(attempt, () => 1), high, `attempt ${attempt}`); + } +}); + +test("conformance: only a network error before the request reached the API is retryable", () => { + const agg = (...errors: Error[]) => new AggregateError(errors); + const vectors: Array<[string, unknown, boolean]> = [ + ...["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ENETUNREACH", "EHOSTUNREACH", "UND_ERR_CONNECT_TIMEOUT"].map( + (code) => [code, sysErr(code), true] as [string, unknown, boolean], + ), + ...["UND_ERR_SOCKET", "ECONNRESET", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT", "EPIPE"].map( + (code) => [code, sysErr(code), false] as [string, unknown, boolean], + ), + ["no cause", undefined, false], + ["ETIMEDOUT on connect", sysErr("ETIMEDOUT", "connect"), true], + ["ETIMEDOUT on read", sysErr("ETIMEDOUT", "read"), false], + ["ETIMEDOUT, no syscall", sysErr("ETIMEDOUT"), false], + ["happy eyeballs, every address refused", agg(sysErr("ECONNREFUSED", "connect"), sysErr("ECONNREFUSED", "connect")), true], + ["happy eyeballs, refused and connect timeout", agg(sysErr("ECONNREFUSED", "connect"), sysErr("ETIMEDOUT", "connect")), true], + ["happy eyeballs, one address reset", agg(sysErr("ECONNREFUSED", "connect"), sysErr("ECONNRESET", "read")), false], + ["happy eyeballs, one read timeout", agg(sysErr("ETIMEDOUT", "connect"), sysErr("ETIMEDOUT", "read")), false], + ["happy eyeballs, no errors", agg(), false], + ]; + for (const [name, cause, expected] of vectors) assert.equal(neverSent(cause), expected, name); +}); + +// --- which answers are retried -------------------------------------------------- + +test("429, 502, 503 and 504 are retried, a 429 whatever its body says", async () => { + const credits = JSON.stringify({ code: "AUTH004", title: "No credit available. Upgrade your plan." }); + for (const [status, body] of [[429, "{}"], [429, credits], [502, ""], [503, RESP009], [504, ""]] as const) { + const w = waits(); + const calls = await withSteps([{ status, body }, OK], async () => { + assert.equal((await scrape(API, "k", { url: "https://x" }, { retries: 2, sleep: w.sleep })).status, 200); + }); + assert.equal(calls, 2, `${status} ${body}`); + assert.equal(w.list.length, 1); + } +}); + +test("other statuses, a target status and a client timeout are sent once", async () => { + const cases: Array<[string, Step, Record]> = [ + ...[400, 401, 402, 403, 404, 422, 500].map((status) => [`HTTP ${status}`, { status, body: "{}" }, {}] as [string, Step, Record]), + ["--original-status 503", { status: 503, body: "target" }, { original_status: true }], + ["allowed_status_codes 503", { status: 503, body: "target" }, { allowed_status_codes: "503" }], + ]; + for (const [name, step, params] of cases) { + const w = waits(); + const calls = await withSteps([step, OK], () => scrape(API, "k", { url: "https://x", ...params }, { retries: 2, sleep: w.sleep }).catch(() => {})); + assert.equal(calls, 1, name); + assert.deepEqual(w.list, [], name); + } + let hung = 0; + const orig = globalThis.fetch; + globalThis.fetch = ((_u: unknown, init?: RequestInit) => { + hung++; + return new Promise((_r, reject) => init?.signal?.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError")))); + }) as typeof fetch; + try { + const err = await errorOf(scrape(API, "k", { url: "https://x" }, { retries: 5, timeoutMs: 20, sleep: waits().sleep })); + assert.equal(err.code, "REQUEST_TIMEOUT"); + } finally { + globalThis.fetch = orig; + } + assert.equal(hung, 1); +}); + +test("each wait is Retry-After, capped at 30 s, or full jitter", async () => { + for (const [header, check] of [ + ["7", (ms: number) => ms === 7000], + ["3600", (ms: number) => ms === 30_000], + ["abc", (ms: number) => ms >= 250 && ms <= 1000], + ] as const) { + const w = waits(); + await withSteps([{ status: 503, headers: { "retry-after": header } }, OK], () => scrape(API, "k", { url: "https://x" }, { retries: 2, sleep: w.sleep })); + assert.equal(w.list.length, 1); + assert.ok(check(w.list[0]!), `Retry-After ${header}: waited ${w.list[0]}`); + } +}); + +test("no retry starts more than 90 s after the first request", async () => { + // Each request takes 40 s. A 503 waits 5 s: retries start at 45 s and 90 s, and one at 135 s would be late. + // A refused connection waits at most 1 s, then 2 s: the third retry would start after 120 s. + for (const step of [{ status: 503, headers: { "retry-after": "5" } }, { network: "ECONNREFUSED" }] as Step[]) { + let clock = 0; + const calls = await withSteps( + [step], + () => scrape(API, "k", { url: "https://x" }, { retries: 10, now: () => clock, sleep: async (ms) => void (clock += ms) }).catch(() => {}), + () => (clock += 40_000), + ); + assert.equal(calls, 3, JSON.stringify(step)); + } +}); + +test("after the retries, a 502, 503 or 504 says to run the same command again; a 429 keeps its error", async () => { + for (const status of [502, 503, 504]) { + const calls = await withSteps([{ status, body: RESP009 }], async () => { + const err = await errorOf(scrape(API, "k", { url: "https://x.com/p" }, { retries: 2, sleep: waits().sleep })); + assert.equal(err.code, "FETCH_FAILED"); + assert.equal(err.message, `The Zenrows API answered HTTP ${status} after 2 retries.`); + assert.match(err.next_action, /^Wait a minute, then run the same command again\./); + assert.deepEqual(err.suggested_commands, []); + }); + assert.equal(calls, 3); + } + await withSteps([{ status: 429, body: "{}" }], async () => { + const err = await errorOf(scrape(API, "k", { url: "https://x.com/p" }, { retries: 1, sleep: waits().sleep })); + assert.equal(err.message, "Rate limited (HTTP 429)."); + assert.deepEqual(err.suggested_commands, ["zenrows fetch https://x.com/p"]); + }); +}); + +test("real sockets: a refused port is retried; a connection closed after the request was sent is not", async () => { + let hits = 0; + const server = createServer((req) => { + hits++; + req.resume(); + req.on("end", () => req.socket.destroy()); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const { port } = server.address() as AddressInfo; + const err = await errorOf(scrape(`http://127.0.0.1:${port}/v1/`, "k", { url: "https://x" }, { retries: 5, sleep: waits().sleep })); + assert.equal(err.code, "BACKEND_UNAVAILABLE"); + assert.equal(hits, 1); + server.closeAllConnections(); + await new Promise((r) => server.close(() => r())); + // Now nothing listens. localhost may resolve to several addresses (happy eyeballs). + for (const host of ["127.0.0.1", "localhost"]) { + const w = waits(); + const e = await errorOf(scrape(`http://${host}:${port}/v1/`, "k", { url: "https://x" }, { retries: 2, sleep: w.sleep })); + assert.equal(e.code, "BACKEND_UNAVAILABLE"); + assert.equal(w.list.length, 2, host); + } +}); + +test("--retries takes a whole number, 0 or more", () => { + assert.equal(normalizeRetries(undefined), undefined); + assert.equal(normalizeRetries("0"), 0); + assert.equal(normalizeRetries("3"), 3); + for (const bad of [true, "", "-1", "1.5", "many", "1e1", "0x2", " 3", "+1"]) { + assert.throws(() => normalizeRetries(bad), (err: unknown) => (err as ToolkitError).code === "INVALID_USAGE", String(bad)); + } +}); + +// --- the commands --------------------------------------------------------------- + +/** Run a command with --json in a fresh workspace; every request answers `steps`. */ +async function runCommand(cmd: Command, args: string[], steps: Step[]): Promise<{ exit: number; out: any; calls: number }> { + const { root, cleanup } = tempRoot(); + const cwd = process.cwd(); + createWorkspace(root); + savePolicy(defaultPolicy(), root); + saveApiKey("0".repeat(41), root); + process.chdir(root); + const write = process.stdout.write.bind(process.stdout); + let buf = ""; + process.stdout.write = ((s: string | Uint8Array, ...rest: unknown[]) => + typeof s === "string" ? ((buf += s), true) : (write as (...a: unknown[]) => boolean)(s, ...rest)) as typeof process.stdout.write; + let exit = -1; + try { + const calls = await withSteps(steps, async () => { + exit = await cmd.run(args, { json: true, yes: false }); + }); + return { exit, out: JSON.parse(buf), calls }; + } finally { + process.stdout.write = write; + process.chdir(cwd); + cleanup(); + } +} + +// What 1.12.0 prints for each failure. Without --retries, nothing changes. +const UNCHANGED: Array<[string, Step, Record]> = [ + [ + "503 RESP009", + { status: 503, body: RESP009, headers: { "retry-after": "0" } }, + { + code: "FETCH_FAILED", + message: "Protected Fetch failed with HTTP 503.", + likely_cause: RESP009, + next_action: "Retry with --manual --js-render --premium-proxy, or inspect the trace for the exact reason.", + suggested_commands: ["zenrows fetch https://x.com/p --manual --js-render --premium-proxy"], + }, + ], + [ + "429", + { status: 429, body: "{}" }, + { + code: "FETCH_FAILED", + message: "Rate limited (HTTP 429).", + likely_cause: "{}. This looks like a concurrency cap or a target-site rate limit, not Free-plan credit exhaustion.", + next_action: "Wait a few seconds and retry. If it persists, lower concurrency / parallel requests.", + suggested_commands: ["zenrows fetch https://x.com/p"], + }, + ], + [ + "refused connection", + { network: "ECONNREFUSED" }, + { + code: "BACKEND_UNAVAILABLE", + message: "Could not reach the Zenrows API.", + likely_cause: "Network error after 0s: fetch failed", + next_action: "Check connectivity and retry. Verify api base in `zenrows config show`.", + suggested_commands: ["zenrows status", "zenrows config show"], + }, + ], +]; + +for (const [name, step, expected] of UNCHANGED) { + test(`without --retries, fetch sends a ${name} once, with the 1.12.0 error`, async () => { + const { exit, out, calls } = await runCommand(fetch_, ["https://x.com/p"], [step, OK]); + assert.equal(calls, 1); + assert.equal(exit, 1); + const { code, message, likely_cause, next_action, suggested_commands } = out.error; + assert.deepEqual({ code, message, likely_cause, next_action, suggested_commands }, expected); + }); +} + +test("extract without --retries sends a 503 once; fetch and extract with --retries 1 send it again", async () => { + const busy: Step = { status: 503, body: RESP009, headers: { "retry-after": "0" } }; + assert.equal((await runCommand(extract, ["https://x.com/p"], [busy, OK])).calls, 1); + for (const [cmd, ok] of [[fetch_, OK], [extract, { status: 200, body: '{"title":"ok"}' }]] as const) { + const { exit, out, calls } = await runCommand(cmd, ["https://x.com/p", "--retries", "1"], [busy, ok]); + assert.equal(calls, 2); + assert.equal(exit, 0); + assert.equal(out.ok, true); + } +}); + +test("extract prepare is never retried: a 429 and a 503 are answers", async () => { + for (const [status, code] of [[429, "out_of_allowance"], [503, "overloaded"]] as const) { + const body = JSON.stringify({ code, status, title: code, detail: code }); + const step: Step = { status, body, headers: { "content-type": "application/problem+json", "retry-after": "0" } }; + const { exit, out, calls } = await runCommand(extract, ["prepare", "x.com", "--url", "https://x.com/p"], [step]); + assert.equal(exit, 1); + assert.match(out.error.code, /^EXTRACT_PREPARE_/); + assert.equal(calls, 1, code); + } +});