From 671ff50227c218975f58613b9b59ca1524f17598 Mon Sep 17 00:00:00 2001 From: Chris Portscheller Date: Tue, 17 Nov 2026 20:54:32 -0600 Subject: [PATCH] feat(registry): add six crawlers from the 2026-11-18 sweep Regenerated agents table and parity vectors from pkg/agents. --- sdk/src/registry/agents.generated.php | 80 +++++++++++++++++- .../registry/parity-vectors.generated.json | 84 +++++++++++++++++++ 2 files changed, 163 insertions(+), 1 deletion(-) diff --git a/sdk/src/registry/agents.generated.php b/sdk/src/registry/agents.generated.php index ae71625..4a36a30 100644 --- a/sdk/src/registry/agents.generated.php +++ b/sdk/src/registry/agents.generated.php @@ -14,9 +14,22 @@ return [ 'schema' => 2, - 'agent_count' => 245, + 'agent_count' => 251, 'categories' => ['training_crawler', 'ai_search_crawler', 'ai_agent', 'ai_assistant', 'search_crawler', 'seo_crawler', 'security_scanner', 'generic_scraper', 'archiver', 'fetcher', 'monitoring', 'headless_browser', 'feed_reader', 'commercial_scraper', 'none'], 'agents' => [ + [ + 'id' => 'hunyuan', + 'name' => 'Hunyuan', + 'category' => 'training_crawler', + 'behavior' => 'training', + 'organization' => 'Unknown', + 'patterns' => ['hunyuan/'], + 'robots_name' => 'Hunyuan', + 'website' => '', + 'base_score' => 70, + 'respects_robots' => false, + 'http_client' => false, + ], [ 'id' => 'speridlabsresearchcrawler', 'name' => 'SperidLabsResearchCrawler', @@ -927,6 +940,32 @@ 'respects_robots' => false, 'http_client' => false, ], + [ + 'id' => '360spider', + 'name' => '360Spider', + 'category' => 'search_crawler', + 'behavior' => 'search', + 'organization' => 'Qihoo 360', + 'patterns' => ['360spider'], + 'robots_name' => '360Spider', + 'website' => 'https://www.so.com', + 'base_score' => 40, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'cgmsearchbot', + 'name' => 'CGMSearchBot', + 'category' => 'search_crawler', + 'behavior' => 'search', + 'organization' => 'CGMIMM', + 'patterns' => ['cgmsearchbot'], + 'robots_name' => 'CGMSearchBot', + 'website' => 'https://www.cgmimm.com/bot', + 'base_score' => 40, + 'respects_robots' => true, + 'http_client' => false, + ], [ 'id' => 'iboubot', 'name' => 'IbouBot', @@ -2838,6 +2877,45 @@ 'respects_robots' => true, 'http_client' => false, ], + [ + 'id' => 'spiderling', + 'name' => 'SpiderLing', + 'category' => 'commercial_scraper', + 'behavior' => '', + 'organization' => 'Lexical Computing', + 'patterns' => ['spiderling', 'sketchengine'], + 'robots_name' => 'SpiderLing', + 'website' => 'https://www.sketchengine.eu/crawler/', + 'base_score' => 55, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'netapi', + 'name' => 'NetAPI Bot', + 'category' => 'commercial_scraper', + 'behavior' => '', + 'organization' => 'NetAPI', + 'patterns' => ['netapi/'], + 'robots_name' => 'NetAPI', + 'website' => 'https://netapi.com/bot.html', + 'base_score' => 50, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'anticlonebot', + 'name' => 'AntiCloneBot', + 'category' => 'commercial_scraper', + 'behavior' => '', + 'organization' => 'AntiClone', + 'patterns' => ['anticlonebot'], + 'robots_name' => 'AntiCloneBot', + 'website' => 'https://anticlone.de/bot', + 'base_score' => 45, + 'respects_robots' => true, + 'http_client' => false, + ], [ 'id' => 'terncrawler', 'name' => 'TernCrawler', diff --git a/sdk/src/registry/parity-vectors.generated.json b/sdk/src/registry/parity-vectors.generated.json index e7edbbc..8b70ae7 100644 --- a/sdk/src/registry/parity-vectors.generated.json +++ b/sdk/src/registry/parity-vectors.generated.json @@ -1,4 +1,16 @@ [ + { + "userAgent": "360spider", + "id": "360spider", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; 360spider/1.0; +http://example.com/bot)", + "id": "360spider", + "category": "search_crawler", + "matched": true + }, { "userAgent": "ai2bot", "id": "ai2bot", @@ -131,6 +143,18 @@ "category": "training_crawler", "matched": true }, + { + "userAgent": "anticlonebot", + "id": "anticlonebot", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; anticlonebot/1.0; +http://example.com/bot)", + "id": "anticlonebot", + "category": "commercial_scraper", + "matched": true + }, { "userAgent": "apache-httpclient", "id": "apache-httpclient", @@ -467,6 +491,18 @@ "category": "training_crawler", "matched": true }, + { + "userAgent": "cgmsearchbot", + "id": "cgmsearchbot", + "category": "search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; cgmsearchbot/1.0; +http://example.com/bot)", + "id": "cgmsearchbot", + "category": "search_crawler", + "matched": true + }, { "userAgent": "censys", "id": "censys", @@ -1271,6 +1307,18 @@ "category": "seo_crawler", "matched": true }, + { + "userAgent": "hunyuan/", + "id": "hunyuan", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; hunyuan//1.0; +http://example.com/bot)", + "id": "hunyuan", + "category": "training_crawler", + "matched": true + }, { "userAgent": "isscyberriskcrawler", "id": "isscyberriskcrawler", @@ -1799,6 +1847,18 @@ "category": "security_scanner", "matched": true }, + { + "userAgent": "netapi/", + "id": "netapi", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; netapi//1.0; +http://example.com/bot)", + "id": "netapi", + "category": "commercial_scraper", + "matched": true + }, { "userAgent": "netsparker", "id": "netsparker", @@ -2591,6 +2651,30 @@ "category": "training_crawler", "matched": true }, + { + "userAgent": "spiderling", + "id": "spiderling", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; spiderling/1.0; +http://example.com/bot)", + "id": "spiderling", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "sketchengine", + "id": "spiderling", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; sketchengine/1.0; +http://example.com/bot)", + "id": "spiderling", + "category": "commercial_scraper", + "matched": true + }, { "userAgent": "splash", "id": "splash",