From 4485847c4ec8ef4edea592680d29ed88653da36b Mon Sep 17 00:00:00 2001 From: Chris Portscheller Date: Mon, 9 Nov 2026 20:02:39 -0600 Subject: [PATCH] Regenerate the agent registry: Meta-WebIndexer, ShapBot, Shap-User, ExaUser, Manus-User --- sdk/src/registry/agents.generated.php | 73 ++++++++++++++++++- .../registry/parity-vectors.generated.json | 64 +++++++++++++++- 2 files changed, 131 insertions(+), 6 deletions(-) diff --git a/sdk/src/registry/agents.generated.php b/sdk/src/registry/agents.generated.php index 3b28205..507dc59 100644 --- a/sdk/src/registry/agents.generated.php +++ b/sdk/src/registry/agents.generated.php @@ -14,7 +14,7 @@ return [ 'schema' => 2, - 'agent_count' => 186, + 'agent_count' => 191, 'categories' => ['training_crawler', 'ai_search_crawler', 'ai_agent', 'ai_assistant', 'search_crawler', 'seo_crawler', 'security_scanner', 'generic_scraper', 'archiver', 'fetcher', 'monitoring', 'headless_browser', 'feed_reader', 'commercial_scraper', 'none'], 'agents' => [ [ @@ -504,9 +504,35 @@ 'category' => 'ai_search_crawler', 'behavior' => 'search', 'organization' => 'Exa AI', - 'patterns' => ['exa.ai', 'exabot'], - 'robots_name' => 'Exa', - 'website' => 'https://exa.ai', + 'patterns' => ['exasearchbot', 'exabot'], + 'robots_name' => 'ExaSearchBot', + 'website' => 'https://crawler.exa.ai', + 'base_score' => 70, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'shapbot', + 'name' => 'ShapBot', + 'category' => 'ai_search_crawler', + 'behavior' => 'search', + 'organization' => 'Parallel Web Systems', + 'patterns' => ['shapbot'], + 'robots_name' => 'ShapBot', + 'website' => 'https://parallel.ai/parallel-web-systems-bots', + 'base_score' => 65, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'meta-webindexer', + 'name' => 'Meta-WebIndexer', + 'category' => 'ai_search_crawler', + 'behavior' => 'search', + 'organization' => 'Meta', + 'patterns' => ['meta-webindexer'], + 'robots_name' => 'Meta-WebIndexer', + 'website' => 'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers', 'base_score' => 70, 'respects_robots' => true, 'http_client' => false, @@ -654,6 +680,19 @@ 'respects_robots' => false, 'http_client' => false, ], + [ + 'id' => 'manus-user', + 'name' => 'Manus-User', + 'category' => 'ai_agent', + 'behavior' => 'agent', + 'organization' => 'Manus', + 'patterns' => ['manus-user'], + 'robots_name' => 'Manus-User', + 'website' => 'https://manus.im', + 'base_score' => 70, + 'respects_robots' => false, + 'http_client' => false, + ], [ 'id' => 'multion', 'name' => 'MultiOn', @@ -2370,6 +2409,32 @@ 'respects_robots' => true, 'http_client' => false, ], + [ + 'id' => 'exauser', + 'name' => 'ExaUser', + 'category' => 'ai_assistant', + 'behavior' => 'agent', + 'organization' => 'Exa AI', + 'patterns' => ['exauser'], + 'robots_name' => 'ExaUser', + 'website' => 'https://exauser.crawler.exa.ai', + 'base_score' => 70, + 'respects_robots' => false, + 'http_client' => false, + ], + [ + 'id' => 'shap-user', + 'name' => 'Shap-User', + 'category' => 'ai_assistant', + 'behavior' => 'agent', + 'organization' => 'Parallel Web Systems', + 'patterns' => ['shap-user'], + 'robots_name' => 'Shap-User', + 'website' => 'https://parallel.ai/parallel-web-systems-bots', + 'base_score' => 65, + 'respects_robots' => false, + 'http_client' => false, + ], [ 'id' => 'meta-externalfetcher', 'name' => 'Meta-ExternalFetcher', diff --git a/sdk/src/registry/parity-vectors.generated.json b/sdk/src/registry/parity-vectors.generated.json index 1182ef5..bc57d09 100644 --- a/sdk/src/registry/parity-vectors.generated.json +++ b/sdk/src/registry/parity-vectors.generated.json @@ -636,13 +636,13 @@ "matched": true }, { - "userAgent": "exa.ai", + "userAgent": "exasearchbot", "id": "exa", "category": "ai_search_crawler", "matched": true }, { - "userAgent": "Mozilla/5.0 (compatible; exa.ai/1.0; +http://example.com/bot)", + "userAgent": "Mozilla/5.0 (compatible; exasearchbot/1.0; +http://example.com/bot)", "id": "exa", "category": "ai_search_crawler", "matched": true @@ -659,6 +659,18 @@ "category": "ai_search_crawler", "matched": true }, + { + "userAgent": "exauser", + "id": "exauser", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; exauser/1.0; +http://example.com/bot)", + "id": "exauser", + "category": "ai_assistant", + "matched": true + }, { "userAgent": "facebookexternalhit", "id": "facebookexternalhit", @@ -1199,6 +1211,18 @@ "category": "search_crawler", "matched": true }, + { + "userAgent": "manus-user", + "id": "manus-user", + "category": "ai_agent", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; manus-user/1.0; +http://example.com/bot)", + "id": "manus-user", + "category": "ai_agent", + "matched": true + }, { "userAgent": "mechanize", "id": "mechanize", @@ -1235,6 +1259,18 @@ "category": "ai_assistant", "matched": true }, + { + "userAgent": "meta-webindexer", + "id": "meta-webindexer", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; meta-webindexer/1.0; +http://example.com/bot)", + "id": "meta-webindexer", + "category": "ai_search_crawler", + "matched": true + }, { "userAgent": "metaphor", "id": "metaphor", @@ -1907,6 +1943,30 @@ "category": "search_crawler", "matched": true }, + { + "userAgent": "shap-user", + "id": "shap-user", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; shap-user/1.0; +http://example.com/bot)", + "id": "shap-user", + "category": "ai_assistant", + "matched": true + }, + { + "userAgent": "shapbot", + "id": "shapbot", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; shapbot/1.0; +http://example.com/bot)", + "id": "shapbot", + "category": "ai_search_crawler", + "matched": true + }, { "userAgent": "shodan", "id": "shodan",