diff --git a/CHANGELOG.md b/CHANGELOG.md index 7af8d26..e55fa7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # check-http-status changelog +## 2.1.0 - Oct 2, 2026 + +### New + +- `onPageLimit` option: when `maxPages` is reached, ask whether to keep crawling instead of stopping (for apps built on the library) + +### Changed + +- `maxPages` (`--max-pages`) now defaults to 5000 instead of 1000 + ## 2.0.1 - Oct 2, 2026 ### Fixed diff --git a/README.md b/README.md index bfe27ea..9492e6a 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ checkHttpStatus({ | `sitemaps` | Array | | *Optional.* Sitemap / sitemap index URLs. With `crawl`, their pages are crawled too and orphans are flagged; without it, their URLs are checked. | | `include` | Array / String | | Only crawl URLs matching one of these rules. | | `exclude` | Array / String | | Don't crawl URLs matching these rules. They are still listed and checked. | -| `maxPages` | Number | `1000` | Maximum pages to crawl. Later pages are listed as "Skipped (max pages)". | +| `maxPages` | Number | `5000` | Maximum pages to crawl. Later pages are listed as "Skipped (max pages)". | | `maxDepth` | Number | unlimited | Maximum link depth from the start URL. | | `checkExternal` | Boolean | `true` | Check the status of external links. Excluded pages and assets are always checked. | | `checkAssets` | Boolean | `false` | Also check images, scripts, stylesheets, icons and iframes. | @@ -183,6 +183,7 @@ checkHttpStatus({ | `signal` | AbortSignal | | Stop the crawl. Everything checked so far is returned; unfinished URLs are "Not checked". | | `onResult` | Function | | Called with each result as soon as its URL is checked (live results). | | `onProgress` | Function | | Called with progress stats after each URL. | +| `onPageLimit` | Function | | Called when everything up to `maxPages` is checked but more pages are waiting, instead of stopping. Return (or resolve) `true` to crawl up to `maxPages` more, a number to crawl that many more (`Infinity` for all), or `false` to finish. Receives the progress stats plus `waitingPages`. | Provide at least one of `crawl`, `urls` or `sitemaps`. For a full site audit, `crawl` on its own is enough. diff --git a/bin/check-http-status.js b/bin/check-http-status.js index e5b81e1..c712b9f 100755 --- a/bin/check-http-status.js +++ b/bin/check-http-status.js @@ -25,7 +25,7 @@ Crawl options -e, --exclude Don't crawl URLs containing ; they are still listed and their status is checked (repeatable) Rules may use * wildcards or /regex/ - --max-pages Max pages to crawl (default 1000) + --max-pages Max pages to crawl (default 5000) --max-depth Max link depth from the start URL (default unlimited) --no-external Don't check the status of external links (excluded pages and assets are still checked) diff --git a/docs/index.html b/docs/index.html index 5281a71..c81d370 100644 --- a/docs/index.html +++ b/docs/index.html @@ -228,7 +228,7 @@

Options

-s, --sitemapsitemapsOptional. Sitemap or sitemap index URL(s). Not needed for crawling. Gzip supported. -i, --includeincludeOnly crawl URLs matching one of these rules. -e, --excludeexcludeDon't crawl matching URLs. They're still listed and checked. - --max-pagesmaxPages1000Maximum pages to crawl. + --max-pagesmaxPages5000Maximum pages to crawl. --max-depthmaxDepth∞Maximum link depth from the start URL. --no-externalcheckExternal: falsecheckedDon't check external links. Excluded pages and assets are still checked. --assetscheckAssetsoffAlso check images, scripts, CSS and iframes. diff --git a/index.d.ts b/index.d.ts index 3d6815a..df79f9a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -96,7 +96,7 @@ declare namespace checkHttpStatus { include?: string | string[]; /** Don't crawl matching URLs; they're still listed and checked. */ exclude?: string | string[]; - /** Default 1000. */ + /** Default 5000. */ maxPages?: number; /** Maximum link depth from the start URL. Default unlimited. */ maxDepth?: number | null; @@ -139,6 +139,8 @@ declare namespace checkHttpStatus { onResult?: (result: Result) => void; /** Called with progress after each URL. */ onProgress?: (progress: Progress) => void; + /** At `maxPages`: `true` crawls `maxPages` more, a number that many (`Infinity` all), `false` stops. */ + onPageLimit?: (progress: PageLimitProgress) => boolean | number | Promise; } interface FoundOn { @@ -203,6 +205,11 @@ declare namespace checkHttpStatus { pageLimitReached: boolean; } + interface PageLimitProgress extends Progress { + /** Pages found after `maxPages` was reached. */ + waitingPages: number; + } + interface Summary extends Progress { crawl: string[]; sitemaps: string[]; diff --git a/index.js b/index.js index 8fc538b..dde82f5 100644 --- a/index.js +++ b/index.js @@ -67,8 +67,9 @@ function normalizeConfig(config) { 'ignoreQuery': !!config.ignoreQuery, 'include': rules(config.include), 'maxDepth': config.maxDepth === undefined || config.maxDepth === null || config.maxDepth === '' ? null : toInt(config.maxDepth, null, 0, 10000), - 'maxPages': toInt(config.maxPages, 1000, 1, 1000000), + 'maxPages': toInt(config.maxPages, 5000, 1, 1000000), 'maxRedirects': toInt(config.maxRedirects, 10, 0, 50), + 'onPageLimit': typeof config.onPageLimit === 'function' ? config.onPageLimit : null, 'onProgress': typeof config.onProgress === 'function' ? config.onProgress : null, 'onResult': typeof config.onResult === 'function' ? config.onResult : null, // CLI: print the URL list even when reports are saved. diff --git a/lib/checker.js b/lib/checker.js index 4ba23e5..ed3ef5e 100644 --- a/lib/checker.js +++ b/lib/checker.js @@ -52,6 +52,7 @@ class Checker { this.include = compilePatterns(config.include); this.exclude = compilePatterns(config.exclude); this.crawlMode = config.crawl.length > 0; + this.initialMaxPages = config.maxPages; // In list mode "internal" means the hosts of the URLs the user supplied. this.rootHosts = new Set((this.crawlMode ? config.crawl : config.urls).map((url) => stripWww(new URL(url).host))); @@ -63,6 +64,7 @@ class Checker { this.pagesCrawled = 0; this.checked = 0; this.stopped = false; + this.askingPageLimit = false; this.state = 'idle'; this.startedAt = null; this.finishedAt = null; @@ -306,11 +308,67 @@ class Checker { return; } - if (this.stopped || (this.pageQueue.length === 0 && this.checkQueue.length === 0)) { - this.state = this.stopped ? 'stopped' : 'done'; - this.finishedAt = Date.now(); - this.resolveDone(); + if (this.stopped) { + this.finish(); + } else if (this.pageQueue.length === 0 && this.checkQueue.length === 0 && !this.askingPageLimit) { + if (this.config.onPageLimit && this.waitingPages() > 0) { + this.askPageLimit(); + } else { + this.finish(); + } + } + } + + finish() { + this.askingPageLimit = false; + this.state = this.stopped ? 'stopped' : 'done'; + this.finishedAt = Date.now(); + this.resolveDone(); + } + + waitingPages() { + let count = 0; + for (const record of this.records.values()) { + if (record.scope === 'limit') { + count++; + } } + + return count; + } + + // `true`: `maxPages` more, a number: that many more, `Infinity`: all. + askPageLimit() { + this.askingPageLimit = true; + const progress = { ...this.progress(), 'waitingPages': this.waitingPages() }; + + Promise.resolve() + .then(() => this.config.onPageLimit(progress)) + .then((answer) => answer, () => false) + .then((answer) => { + if (this.state !== 'running') { + return; + } + + this.askingPageLimit = false; + const more = answer === true ? this.initialMaxPages : Number(answer); + if (this.stopped || !answer || !(more > 0)) { + this.finish(); + return; + } + + this.config.maxPages += more; + for (const record of this.records.values()) { + if (record.scope === 'limit' && this.crawlQueued < this.config.maxPages) { + record.scope = 'crawl'; + this.crawlQueued++; + this.enqueue(record); + } + } + + this.pump(); + this.maybeFinish(); + }); } async process(record) { diff --git a/package-lock.json b/package-lock.json index 46a10fb..b17484b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "check-http-status", - "version": "2.0.1", + "version": "2.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "check-http-status", - "version": "2.0.1", + "version": "2.1.0", "license": "MIT", "dependencies": { "cheerio": "^1.0.0", diff --git a/package.json b/package.json index 9dc5c6e..11bbc7e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "check-http-status", "description": "Crawl a website or check a list of URLs: HTTP status codes, redirect chains, broken links and the pages they were found on. Export to Excel, CSV, JSON or an HTML report.", - "version": "2.0.1", + "version": "2.1.0", "main": "index.js", "types": "index.d.ts", "bin": { diff --git a/schema.json b/schema.json index 28822e4..967522d 100644 --- a/schema.json +++ b/schema.json @@ -29,7 +29,7 @@ "sitemaps": { "$ref": "#/definitions/stringList", "description": "Optional sitemap / sitemap index URLs." }, "include": { "$ref": "#/definitions/stringList", "description": "Only crawl URLs matching one of these rules (substring, * wildcard or /regex/)." }, "exclude": { "$ref": "#/definitions/stringList", "description": "Don't crawl URLs matching these rules. They are still listed and checked." }, - "maxPages": { "type": "integer", "minimum": 1, "default": 1000 }, + "maxPages": { "type": "integer", "minimum": 1, "default": 5000 }, "maxDepth": { "type": ["integer", "null"], "minimum": 0, "description": "Maximum link depth from the start URL. Unlimited when omitted." }, "maxRedirects": { "type": "integer", "minimum": 0, "maximum": 50, "default": 10 }, "checkExternal": { "type": "boolean", "default": true, "description": "Check the status of external links. Excluded pages and assets are always checked." }, diff --git a/test-d/usage.ts b/test-d/usage.ts index 2a972e0..0a6096d 100644 --- a/test-d/usage.ts +++ b/test-d/usage.ts @@ -26,7 +26,8 @@ async function main(): Promise { onProgress: (progress) => { const state: 'idle' | 'running' | 'done' | 'stopped' = progress.state; void state; - } + }, + onPageLimit: async (progress) => progress.waitingPages > 100 ? 1000 : true }); const notFound = report.results.filter((result) => result.category === 'Not Found'); diff --git a/test/checker.test.js b/test/checker.test.js index d72e5dd..9cb8b41 100644 --- a/test/checker.test.js +++ b/test/checker.test.js @@ -139,6 +139,53 @@ describe('crawl limits and rules', () => { assert.strictEqual(report.summary.pageLimitReached, true); }); + test('onPageLimit can keep crawling past maxPages, in chunks or all at once', async () => { + const asked = []; + const full = await checkHttpStatus({ 'crawl': site.origin, 'silent': true }); + const fullPages = full.results.filter((result) => result.scope === 'crawl').length; + + const chunked = await checkHttpStatus({ + 'crawl': site.origin, + 'maxPages': 3, + 'onPageLimit': (progress) => { + asked.push(progress); + return asked.length === 1; + }, + 'silent': true + }); + assert.strictEqual(asked.length, 2); + assert.strictEqual(asked[0].pagesCrawled, 3); + assert.ok(asked[0].waitingPages > 0); + assert.strictEqual(chunked.results.filter((result) => result.scope === 'crawl').length, 6); + assert.ok(chunked.results.some((result) => result.scope === 'limit')); + assert.strictEqual(chunked.summary.state, 'done'); + + const all = await checkHttpStatus({ 'crawl': site.origin, 'maxPages': 3, 'onPageLimit': async () => Infinity, 'silent': true }); + assert.strictEqual(all.results.filter((result) => result.scope === 'crawl').length, fullPages); + assert.ok(!all.results.some((result) => result.scope === 'limit')); + }); + + test('onPageLimit answering false, or aborting while asked, keeps the v2 result', async () => { + const declined = await checkHttpStatus({ 'crawl': site.origin, 'maxPages': 3, 'onPageLimit': () => false, 'silent': true }); + assert.strictEqual(declined.results.filter((result) => result.scope === 'crawl').length, 3); + assert.strictEqual(declined.summary.pageLimitReached, true); + + const controller = new AbortController(); + const aborted = await checkHttpStatus({ + 'crawl': site.origin, + 'maxPages': 3, + // Never answers: the abort has to end the crawl. + 'onPageLimit': () => { + setTimeout(() => controller.abort(), 20); + return new Promise(() => {}); + }, + 'signal': controller.signal, + 'silent': true + }); + assert.strictEqual(aborted.summary.state, 'stopped'); + assert.strictEqual(aborted.results.filter((result) => result.scope === 'crawl').length, 3); + }); + test('checkExternal: false skips external links but still checks excluded pages and assets', async () => { const report = await checkHttpStatus({ 'checkAssets': true, 'checkExternal': false, 'crawl': site.origin, 'exclude': ['/docs'], 'silent': true }); assert.strictEqual(find(report, '/partners', site.externalOrigin).category, 'Not checked');