From 31c126789f8287a1a442336ee75911de3286bb35 Mon Sep 17 00:00:00 2001 From: Sami Ahmed Siddiqui Date: Fri, 2 Oct 2026 18:31:26 +0500 Subject: [PATCH 1/3] Add onPageLimit to keep crawling past maxPages When everything up to maxPages is checked and more pages are waiting, call onPageLimit and crawl more pages if it says so, instead of stopping. --- CHANGELOG.md | 6 ++++ README.md | 1 + index.d.ts | 11 +++++++ index.js | 1 + lib/checker.js | 69 +++++++++++++++++++++++++++++++++++++++++--- test-d/usage.ts | 3 +- test/checker.test.js | 47 ++++++++++++++++++++++++++++++ 7 files changed, 133 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7af8d26..a98e6b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # check-http-status changelog +## Unreleased + +### New + +- `onPageLimit` option: when `maxPages` is reached, ask whether to keep crawling instead of stopping (for apps built on the library) + ## 2.0.1 - Oct 2, 2026 ### Fixed diff --git a/README.md b/README.md index bfe27ea..6a3b95d 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,7 @@ checkHttpStatus({ | `signal` | AbortSignal | | Stop the crawl. Everything checked so far is returned; unfinished URLs are "Not checked". | | `onResult` | Function | | Called with each result as soon as its URL is checked (live results). | | `onProgress` | Function | | Called with progress stats after each URL. | +| `onPageLimit` | Function | | Called when everything up to `maxPages` is checked but more pages are waiting, instead of stopping. Return (or resolve) `true` to crawl up to `maxPages` more, a number to crawl that many more (`Infinity` for all), or `false` to finish. Receives the progress stats plus `waitingPages`. | Provide at least one of `crawl`, `urls` or `sitemaps`. For a full site audit, `crawl` on its own is enough. diff --git a/index.d.ts b/index.d.ts index 3d6815a..a0fce55 100644 --- a/index.d.ts +++ b/index.d.ts @@ -139,6 +139,12 @@ declare namespace checkHttpStatus { onResult?: (result: Result) => void; /** Called with progress after each URL. */ onProgress?: (progress: Progress) => void; + /** + * Called once everything up to `maxPages` is checked and more pages are waiting. + * Resolve `true` to crawl up to `maxPages` more, a number to crawl that many more + * (`Infinity` for all), or `false` to finish. Asked again each time the new limit is reached. + */ + onPageLimit?: (progress: PageLimitProgress) => boolean | number | Promise; } interface FoundOn { @@ -203,6 +209,11 @@ declare namespace checkHttpStatus { pageLimitReached: boolean; } + interface PageLimitProgress extends Progress { + /** Pages found after `maxPages` was reached. */ + waitingPages: number; + } + interface Summary extends Progress { crawl: string[]; sitemaps: string[]; diff --git a/index.js b/index.js index 8fc538b..8d00490 100644 --- a/index.js +++ b/index.js @@ -69,6 +69,7 @@ function normalizeConfig(config) { 'maxDepth': config.maxDepth === undefined || config.maxDepth === null || config.maxDepth === '' ? null : toInt(config.maxDepth, null, 0, 10000), 'maxPages': toInt(config.maxPages, 1000, 1, 1000000), 'maxRedirects': toInt(config.maxRedirects, 10, 0, 50), + 'onPageLimit': typeof config.onPageLimit === 'function' ? config.onPageLimit : null, 'onProgress': typeof config.onProgress === 'function' ? config.onProgress : null, 'onResult': typeof config.onResult === 'function' ? config.onResult : null, // CLI: print the URL list even when reports are saved. diff --git a/lib/checker.js b/lib/checker.js index 4ba23e5..2f6d371 100644 --- a/lib/checker.js +++ b/lib/checker.js @@ -52,6 +52,7 @@ class Checker { this.include = compilePatterns(config.include); this.exclude = compilePatterns(config.exclude); this.crawlMode = config.crawl.length > 0; + this.initialMaxPages = config.maxPages; // In list mode "internal" means the hosts of the URLs the user supplied. this.rootHosts = new Set((this.crawlMode ? config.crawl : config.urls).map((url) => stripWww(new URL(url).host))); @@ -63,6 +64,8 @@ class Checker { this.pagesCrawled = 0; this.checked = 0; this.stopped = false; + // True while `onPageLimit` decides whether to crawl the pages past `maxPages`. + this.askingPageLimit = false; this.state = 'idle'; this.startedAt = null; this.finishedAt = null; @@ -306,11 +309,69 @@ class Checker { return; } - if (this.stopped || (this.pageQueue.length === 0 && this.checkQueue.length === 0)) { - this.state = this.stopped ? 'stopped' : 'done'; - this.finishedAt = Date.now(); - this.resolveDone(); + if (this.stopped) { + this.finish(); + } else if (this.pageQueue.length === 0 && this.checkQueue.length === 0 && !this.askingPageLimit) { + if (this.config.onPageLimit && this.waitingPages() > 0) { + this.askPageLimit(); + } else { + this.finish(); + } + } + } + + finish() { + this.askingPageLimit = false; + this.state = this.stopped ? 'stopped' : 'done'; + this.finishedAt = Date.now(); + this.resolveDone(); + } + + // Pages found after `maxPages` was reached, in the order they were found. + waitingPages() { + let count = 0; + for (const record of this.records.values()) { + if (record.scope === 'limit') { + count++; + } } + + return count; + } + + // Everything up to `maxPages` is done: ask whether to crawl the pages found after it. + // `true` crawls up to `maxPages` more, a number crawls that many more, `Infinity` all of them. + askPageLimit() { + this.askingPageLimit = true; + const progress = { ...this.progress(), 'waitingPages': this.waitingPages() }; + + Promise.resolve() + .then(() => this.config.onPageLimit(progress)) + .then((answer) => answer, () => false) + .then((answer) => { + if (this.state !== 'running') { + return; + } + + this.askingPageLimit = false; + const more = answer === true ? this.initialMaxPages : Number(answer); + if (this.stopped || !answer || !(more > 0)) { + this.finish(); + return; + } + + this.config.maxPages += more; + for (const record of this.records.values()) { + if (record.scope === 'limit' && this.crawlQueued < this.config.maxPages) { + record.scope = 'crawl'; + this.crawlQueued++; + this.enqueue(record); + } + } + + this.pump(); + this.maybeFinish(); + }); } async process(record) { diff --git a/test-d/usage.ts b/test-d/usage.ts index 2a972e0..0a6096d 100644 --- a/test-d/usage.ts +++ b/test-d/usage.ts @@ -26,7 +26,8 @@ async function main(): Promise { onProgress: (progress) => { const state: 'idle' | 'running' | 'done' | 'stopped' = progress.state; void state; - } + }, + onPageLimit: async (progress) => progress.waitingPages > 100 ? 1000 : true }); const notFound = report.results.filter((result) => result.category === 'Not Found'); diff --git a/test/checker.test.js b/test/checker.test.js index d72e5dd..9cb8b41 100644 --- a/test/checker.test.js +++ b/test/checker.test.js @@ -139,6 +139,53 @@ describe('crawl limits and rules', () => { assert.strictEqual(report.summary.pageLimitReached, true); }); + test('onPageLimit can keep crawling past maxPages, in chunks or all at once', async () => { + const asked = []; + const full = await checkHttpStatus({ 'crawl': site.origin, 'silent': true }); + const fullPages = full.results.filter((result) => result.scope === 'crawl').length; + + const chunked = await checkHttpStatus({ + 'crawl': site.origin, + 'maxPages': 3, + 'onPageLimit': (progress) => { + asked.push(progress); + return asked.length === 1; + }, + 'silent': true + }); + assert.strictEqual(asked.length, 2); + assert.strictEqual(asked[0].pagesCrawled, 3); + assert.ok(asked[0].waitingPages > 0); + assert.strictEqual(chunked.results.filter((result) => result.scope === 'crawl').length, 6); + assert.ok(chunked.results.some((result) => result.scope === 'limit')); + assert.strictEqual(chunked.summary.state, 'done'); + + const all = await checkHttpStatus({ 'crawl': site.origin, 'maxPages': 3, 'onPageLimit': async () => Infinity, 'silent': true }); + assert.strictEqual(all.results.filter((result) => result.scope === 'crawl').length, fullPages); + assert.ok(!all.results.some((result) => result.scope === 'limit')); + }); + + test('onPageLimit answering false, or aborting while asked, keeps the v2 result', async () => { + const declined = await checkHttpStatus({ 'crawl': site.origin, 'maxPages': 3, 'onPageLimit': () => false, 'silent': true }); + assert.strictEqual(declined.results.filter((result) => result.scope === 'crawl').length, 3); + assert.strictEqual(declined.summary.pageLimitReached, true); + + const controller = new AbortController(); + const aborted = await checkHttpStatus({ + 'crawl': site.origin, + 'maxPages': 3, + // Never answers: the abort has to end the crawl. + 'onPageLimit': () => { + setTimeout(() => controller.abort(), 20); + return new Promise(() => {}); + }, + 'signal': controller.signal, + 'silent': true + }); + assert.strictEqual(aborted.summary.state, 'stopped'); + assert.strictEqual(aborted.results.filter((result) => result.scope === 'crawl').length, 3); + }); + test('checkExternal: false skips external links but still checks excluded pages and assets', async () => { const report = await checkHttpStatus({ 'checkAssets': true, 'checkExternal': false, 'crawl': site.origin, 'exclude': ['/docs'], 'silent': true }); assert.strictEqual(find(report, '/partners', site.externalOrigin).category, 'Not checked'); From c92ea49a8e0ddba6533a862c9f3c931376f43bce Mon Sep 17 00:00:00 2001 From: Sami Ahmed Siddiqui Date: Fri, 2 Oct 2026 18:32:44 +0500 Subject: [PATCH 2/3] Default maxPages to 5000 --- CHANGELOG.md | 4 ++++ README.md | 2 +- bin/check-http-status.js | 2 +- docs/index.html | 2 +- index.d.ts | 2 +- index.js | 2 +- schema.json | 2 +- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a98e6b6..d3f935a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ - `onPageLimit` option: when `maxPages` is reached, ask whether to keep crawling instead of stopping (for apps built on the library) +### Changed + +- `maxPages` (`--max-pages`) now defaults to 5000 instead of 1000 + ## 2.0.1 - Oct 2, 2026 ### Fixed diff --git a/README.md b/README.md index 6a3b95d..9492e6a 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ checkHttpStatus({ | `sitemaps` | Array | | *Optional.* Sitemap / sitemap index URLs. With `crawl`, their pages are crawled too and orphans are flagged; without it, their URLs are checked. | | `include` | Array / String | | Only crawl URLs matching one of these rules. | | `exclude` | Array / String | | Don't crawl URLs matching these rules. They are still listed and checked. | -| `maxPages` | Number | `1000` | Maximum pages to crawl. Later pages are listed as "Skipped (max pages)". | +| `maxPages` | Number | `5000` | Maximum pages to crawl. Later pages are listed as "Skipped (max pages)". | | `maxDepth` | Number | unlimited | Maximum link depth from the start URL. | | `checkExternal` | Boolean | `true` | Check the status of external links. Excluded pages and assets are always checked. | | `checkAssets` | Boolean | `false` | Also check images, scripts, stylesheets, icons and iframes. | diff --git a/bin/check-http-status.js b/bin/check-http-status.js index e5b81e1..c712b9f 100755 --- a/bin/check-http-status.js +++ b/bin/check-http-status.js @@ -25,7 +25,7 @@ Crawl options -e, --exclude Don't crawl URLs containing ; they are still listed and their status is checked (repeatable) Rules may use * wildcards or /regex/ - --max-pages Max pages to crawl (default 1000) + --max-pages Max pages to crawl (default 5000) --max-depth Max link depth from the start URL (default unlimited) --no-external Don't check the status of external links (excluded pages and assets are still checked) diff --git a/docs/index.html b/docs/index.html index 5281a71..c81d370 100644 --- a/docs/index.html +++ b/docs/index.html @@ -228,7 +228,7 @@

Options

-s, --sitemapsitemapsOptional. Sitemap or sitemap index URL(s). Not needed for crawling. Gzip supported. -i, --includeincludeOnly crawl URLs matching one of these rules. -e, --excludeexcludeDon't crawl matching URLs. They're still listed and checked. - --max-pagesmaxPages1000Maximum pages to crawl. + --max-pagesmaxPages5000Maximum pages to crawl. --max-depthmaxDepth∞Maximum link depth from the start URL. --no-externalcheckExternal: falsecheckedDon't check external links. Excluded pages and assets are still checked. --assetscheckAssetsoffAlso check images, scripts, CSS and iframes. diff --git a/index.d.ts b/index.d.ts index a0fce55..abf5fa9 100644 --- a/index.d.ts +++ b/index.d.ts @@ -96,7 +96,7 @@ declare namespace checkHttpStatus { include?: string | string[]; /** Don't crawl matching URLs; they're still listed and checked. */ exclude?: string | string[]; - /** Default 1000. */ + /** Default 5000. */ maxPages?: number; /** Maximum link depth from the start URL. Default unlimited. */ maxDepth?: number | null; diff --git a/index.js b/index.js index 8d00490..dde82f5 100644 --- a/index.js +++ b/index.js @@ -67,7 +67,7 @@ function normalizeConfig(config) { 'ignoreQuery': !!config.ignoreQuery, 'include': rules(config.include), 'maxDepth': config.maxDepth === undefined || config.maxDepth === null || config.maxDepth === '' ? null : toInt(config.maxDepth, null, 0, 10000), - 'maxPages': toInt(config.maxPages, 1000, 1, 1000000), + 'maxPages': toInt(config.maxPages, 5000, 1, 1000000), 'maxRedirects': toInt(config.maxRedirects, 10, 0, 50), 'onPageLimit': typeof config.onPageLimit === 'function' ? config.onPageLimit : null, 'onProgress': typeof config.onProgress === 'function' ? config.onProgress : null, diff --git a/schema.json b/schema.json index 28822e4..967522d 100644 --- a/schema.json +++ b/schema.json @@ -29,7 +29,7 @@ "sitemaps": { "$ref": "#/definitions/stringList", "description": "Optional sitemap / sitemap index URLs." }, "include": { "$ref": "#/definitions/stringList", "description": "Only crawl URLs matching one of these rules (substring, * wildcard or /regex/)." }, "exclude": { "$ref": "#/definitions/stringList", "description": "Don't crawl URLs matching these rules. They are still listed and checked." }, - "maxPages": { "type": "integer", "minimum": 1, "default": 1000 }, + "maxPages": { "type": "integer", "minimum": 1, "default": 5000 }, "maxDepth": { "type": ["integer", "null"], "minimum": 0, "description": "Maximum link depth from the start URL. Unlimited when omitted." }, "maxRedirects": { "type": "integer", "minimum": 0, "maximum": 50, "default": 10 }, "checkExternal": { "type": "boolean", "default": true, "description": "Check the status of external links. Excluded pages and assets are always checked." }, From 7d1ec450f570a72eeba5f2675747839720b2743c Mon Sep 17 00:00:00 2001 From: Sami Ahmed Siddiqui Date: Fri, 2 Oct 2026 18:45:20 +0500 Subject: [PATCH 3/3] Release v2.1.0 --- CHANGELOG.md | 2 +- index.d.ts | 6 +----- lib/checker.js | 5 +---- package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3f935a..e55fa7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # check-http-status changelog -## Unreleased +## 2.1.0 - Oct 2, 2026 ### New diff --git a/index.d.ts b/index.d.ts index abf5fa9..df79f9a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -139,11 +139,7 @@ declare namespace checkHttpStatus { onResult?: (result: Result) => void; /** Called with progress after each URL. */ onProgress?: (progress: Progress) => void; - /** - * Called once everything up to `maxPages` is checked and more pages are waiting. - * Resolve `true` to crawl up to `maxPages` more, a number to crawl that many more - * (`Infinity` for all), or `false` to finish. Asked again each time the new limit is reached. - */ + /** At `maxPages`: `true` crawls `maxPages` more, a number that many (`Infinity` all), `false` stops. */ onPageLimit?: (progress: PageLimitProgress) => boolean | number | Promise; } diff --git a/lib/checker.js b/lib/checker.js index 2f6d371..ed3ef5e 100644 --- a/lib/checker.js +++ b/lib/checker.js @@ -64,7 +64,6 @@ class Checker { this.pagesCrawled = 0; this.checked = 0; this.stopped = false; - // True while `onPageLimit` decides whether to crawl the pages past `maxPages`. this.askingPageLimit = false; this.state = 'idle'; this.startedAt = null; @@ -327,7 +326,6 @@ class Checker { this.resolveDone(); } - // Pages found after `maxPages` was reached, in the order they were found. waitingPages() { let count = 0; for (const record of this.records.values()) { @@ -339,8 +337,7 @@ class Checker { return count; } - // Everything up to `maxPages` is done: ask whether to crawl the pages found after it. - // `true` crawls up to `maxPages` more, a number crawls that many more, `Infinity` all of them. + // `true`: `maxPages` more, a number: that many more, `Infinity`: all. askPageLimit() { this.askingPageLimit = true; const progress = { ...this.progress(), 'waitingPages': this.waitingPages() }; diff --git a/package-lock.json b/package-lock.json index 46a10fb..b17484b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "check-http-status", - "version": "2.0.1", + "version": "2.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "check-http-status", - "version": "2.0.1", + "version": "2.1.0", "license": "MIT", "dependencies": { "cheerio": "^1.0.0", diff --git a/package.json b/package.json index 9dc5c6e..11bbc7e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "check-http-status", "description": "Crawl a website or check a list of URLs: HTTP status codes, redirect chains, broken links and the pages they were found on. Export to Excel, CSV, JSON or an HTML report.", - "version": "2.0.1", + "version": "2.1.0", "main": "index.js", "types": "index.d.ts", "bin": {