diff --git a/CHANGELOG.md b/CHANGELOG.md index 1974c63..05f5a1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to ### Added +- Add Markdown search and archive output support with `getMd` and + `getMdBySearchId`. - Expose `EngineParameters` type. - Expose `InvalidArgumentError` error. diff --git a/README.md b/README.md index cebd190..9d29f15 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ [![License](https://img.shields.io/github/license/serpapi/serpapi-javascript)](https://github.com/serpapi/serpapi-javascript/blob/master/LICENSE) [![SerpApi Libraries](https://img.shields.io/badge/SerpApi-Libraries-blue)](https://serpapi.com/integrations) -Scrape and parse search engine results using [SerpApi](https://serpapi.com). Get -search results from Google, Bing, Baidu, Yandex, Yahoo, Home Depot, eBay and -more. +Scrape and parse search engine results using [SerpApi](https://serpapi.com). +Retrieve structured JSON, token-efficient Markdown for AI agents, or raw HTML +from Google, Bing, Baidu, Yandex, Yahoo, Home Depot, eBay and more. | 🪧 Coming from `google-search-results-nodejs`?
Check out the [migration document](https://github.com/serpapi/serpapi-javascript/blob/master/docs/migrating_from_google_search_results_nodejs.md) to find out how to upgrade. | | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -94,8 +94,27 @@ console.log(response); [Deno](https://deno.land/x/serpapi). - Promises and async/await support. - Callbacks support. +- JSON, HTML, and token-efficient Markdown response formats. - [Examples in JavaScript/TypeScript on Node.js/Deno using ESM/CommonJS, and more](https://github.com/serpapi/serpapi-javascript/tree/master/examples). +## Markdown output for AI agents + +Use `getMd` to get token-efficient Markdown optimized for LLMs and AI agents: + +```js +import { getMd } from "serpapi"; + +const markdown = await getMd({ + engine: "google", + api_key: API_KEY, + q: "coffee", +}); +``` + +Archived results are also available as Markdown with `getMdBySearchId`. + +Learn more about [SerpApi Markdown output](https://serpapi.com/markdown-output). + ## Configuration You can declare a global `api_key` and `timeout` value by modifying the `config` @@ -176,21 +195,27 @@ for a manual approach: - [getHtml](#gethtml) - [Parameters](#parameters-1) - [Examples](#examples-1) -- [getJsonBySearchId](#getjsonbysearchid) +- [getMd](#getmd) - [Parameters](#parameters-2) - [Examples](#examples-2) -- [getHtmlBySearchId](#gethtmlbysearchid) +- [getJsonBySearchId](#getjsonbysearchid) - [Parameters](#parameters-3) - [Examples](#examples-3) -- [getAccount](#getaccount) +- [getHtmlBySearchId](#gethtmlbysearchid) - [Parameters](#parameters-4) - [Examples](#examples-4) -- [getLocations](#getlocations) +- [getMdBySearchId](#getmdbysearchid) - [Parameters](#parameters-5) - [Examples](#examples-5) -- [uploadImage](#uploadimage) +- [getAccount](#getaccount) - [Parameters](#parameters-6) - [Examples](#examples-6) +- [getLocations](#getlocations) + - [Parameters](#parameters-7) + - [Examples](#examples-7) +- [uploadImage](#uploadimage) + - [Parameters](#parameters-8) + - [Examples](#examples-8) ### getJson @@ -237,6 +262,31 @@ const html = await getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }); getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); ``` +### getMd + +Get a Markdown response based on search parameters. + +#### Parameters + +- `parameters` + **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** + search query parameters for the engine +- `callback` **fn?** optional callback + +#### Examples + +```javascript +// async/await +const markdown = await getMd({ + engine: "google", + api_key: API_KEY, + q: "coffee", +}); + +// callback +getMd({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); +``` + ### getJsonBySearchId Get a JSON response given a search ID. @@ -328,6 +378,51 @@ const html = await getHtmlBySearchId(id, { api_key: API_KEY }); getHtmlBySearchId(id, { api_key: API_KEY }, console.log); ``` +### getMdBySearchId + +Get a Markdown response given a search ID. + +- This search ID can be obtained from the `search_metadata.id` key in the + response. +- Typically used together with the `async` parameter. +- Accepts an optional callback. + +#### Parameters + +- `searchId` + **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** + search ID +- `parameters` + **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** + (optional, default `{}`) + + - `parameters.api_key` + **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** + API key + - `parameters.timeout` + **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)?** + timeout in milliseconds +- `callback` **fn?** optional callback + +#### Examples + +```javascript +const markdown = await getMd({ + engine: "google", + api_key: API_KEY, + q: "coffee", +}); +const idMatch = markdown.match(/^ id:\s*(.+)$/m); +if (!idMatch) throw new Error("Search ID missing from Markdown frontmatter"); +const searchId = idMatch[1].trim(); + +// async/await +const archivedMarkdown = await getMdBySearchId(searchId, { api_key: API_KEY }); + +// callback +getMdBySearchId(searchId, { api_key: API_KEY }, console.log); +``` + ### getAccount Get account information of an API key. diff --git a/mod.ts b/mod.ts index 3b944cd..8006a63 100644 --- a/mod.ts +++ b/mod.ts @@ -23,5 +23,7 @@ export { getJson, getJsonBySearchId, getLocations, + getMd, + getMdBySearchId, uploadImage, } from "./src/serpapi.ts"; diff --git a/smoke_tests/commonjs/commonjs.js b/smoke_tests/commonjs/commonjs.js index fde3f4e..1179e81 100644 --- a/smoke_tests/commonjs/commonjs.js +++ b/smoke_tests/commonjs/commonjs.js @@ -8,8 +8,10 @@ const { config, getJson, getHtml, + getMd, getJsonBySearchId, getHtmlBySearchId, + getMdBySearchId, getAccount, getLocations, } = require("serpapi"); @@ -89,6 +91,12 @@ const run = async () => { }); } + { + console.log("getMd"); + const markdown = await getMd(Object.assign({ engine: "google" }, params)); + if (!markdown.startsWith("---")) throw new Error("Incorrect Markdown"); + } + { console.log("getJsonBySearchId"); config.api_key = apiKey; @@ -111,6 +119,13 @@ const run = async () => { }); } + { + console.log("getMdBySearchId"); + config.api_key = apiKey; + const markdown = await getMdBySearchId(searchId); + if (!markdown.startsWith("---")) throw new Error("Incorrect Markdown"); + } + { console.log("getAccount"); config.api_key = apiKey; diff --git a/smoke_tests/esm/esm.js b/smoke_tests/esm/esm.js index 080eb85..7878076 100644 --- a/smoke_tests/esm/esm.js +++ b/smoke_tests/esm/esm.js @@ -16,6 +16,8 @@ import { getJson, getJsonBySearchId, getLocations, + getMd, + getMdBySearchId, } from "serpapi"; Dotenv.config(); @@ -92,6 +94,12 @@ let searchId; }); } +{ + console.log("getMd"); + const markdown = await getMd(Object.assign({ engine: "google" }, params)); + if (!markdown.startsWith("---")) throw new Error("Incorrect Markdown"); +} + { console.log("getJsonBySearchId"); config.api_key = apiKey; @@ -114,6 +122,13 @@ let searchId; }); } +{ + console.log("getMdBySearchId"); + config.api_key = apiKey; + const markdown = await getMdBySearchId(searchId); + if (!markdown.startsWith("---")) throw new Error("Incorrect Markdown"); +} + { console.log("getAccount"); config.api_key = apiKey; diff --git a/src/serpapi.ts b/src/serpapi.ts index bc4998a..678a89a 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -178,6 +178,89 @@ async function _getHtml( return html; } +/** + * Get Markdown response based on search parameters. + * + * @param {object} parameters Search query parameters for the engine. Refer to https://serpapi.com/search-api for parameter explanations. + * @param {fn=} callback Optional callback. + * @example + * // async/await + * const markdown = await getMd({ engine: "google", api_key: API_KEY, q: "coffee" }); + * + * // callback + * getMd({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); + */ +export function getMd( + parameters: EngineParameters, + callback?: (markdown: string) => void, +): Promise; + +/** + * Get Markdown response based on search parameters. + * + * @param {string} engine Engine name. Refer to https://serpapi.com/search-api for valid engines. + * @param {object} parameters Search query parameters for the engine. Refer to https://serpapi.com/search-api for parameter explanations. + * @param {fn=} callback Optional callback. + * @example + * // async/await + * const markdown = await getMd("google", { api_key: API_KEY, q: "coffee" }); + * + * // callback + * getMd("google", { api_key: API_KEY, q: "coffee" }, console.log); + */ +export function getMd( + engine: string, + parameters: EngineParameters, + callback?: (markdown: string) => void, +): Promise; + +export function getMd( + ...args: + | [ + parameters: EngineParameters, + callback?: (markdown: string) => void, + ] + | [ + engine: string, + parameters: EngineParameters, + callback?: (markdown: string) => void, + ] +): Promise { + if (typeof args[0] === "string" && typeof args[1] === "object") { + const [engine, parameters, callback] = args; + const newParameters = { ...parameters, engine } as EngineParameters; + return _getMd(newParameters, callback); + } else if ( + typeof args[0] === "object" && + typeof args[1] !== "object" && + (typeof args[1] === "undefined" || typeof args[1] === "function") + ) { + const [parameters, callback] = args; + return _getMd(parameters, callback); + } else { + throw new InvalidArgumentError(); + } +} + +async function _getMd( + parameters: EngineParameters, + callback?: (markdown: string) => void, +): Promise { + const key = validateApiKey(parameters.api_key, true); + const timeout = validateTimeout(parameters.timeout); + const markdown = await _internals.execute( + SEARCH_PATH, + { + ...parameters, + api_key: key, + output: "md", + }, + timeout, + ); + callback?.(markdown); + return markdown; +} + /** * Get a JSON response given a search ID. * - This search ID can be obtained from the `search_metadata.id` key in the response. @@ -259,6 +342,47 @@ export async function getHtmlBySearchId( return html; } +/** + * Get a Markdown response given a search ID. + * - This search ID can be obtained from the `search_metadata.id` key in the response. + * - Typically used together with the `async` parameter. + * + * @param {string} searchId Search ID. + * @param {object} parameters + * @param {string=} [parameters.api_key] API key. + * @param {number=} [parameters.timeout] Timeout in milliseconds. + * @param {fn=} callback Optional callback. + * @example + * const markdown = await getMd({ engine: "google", api_key: API_KEY, q: "coffee" }); + * const idMatch = markdown.match(/^ id:\s*(.+)$/m); + * if (!idMatch) throw new Error("Search ID missing from Markdown frontmatter"); + * const searchId = idMatch[1].trim(); + * + * // async/await + * const archivedMarkdown = await getMdBySearchId(searchId, { api_key: API_KEY }); + * + * // callback + * getMdBySearchId(searchId, { api_key: API_KEY }, console.log); + */ +export async function getMdBySearchId( + searchId: string, + parameters: GetBySearchIdParameters = {}, + callback?: (markdown: string) => void, +) { + const key = validateApiKey(parameters.api_key); + const timeout = validateTimeout(parameters.timeout); + const markdown = await _internals.execute( + `${SEARCH_ARCHIVE_PATH}/${searchId}`, + { + api_key: key, + output: "md", + }, + timeout, + ); + callback?.(markdown); + return markdown; +} + /** * Get account information of an API key. * diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index b6b3f0a..99cc821 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -26,6 +26,8 @@ import { getJson, getJsonBySearchId, getLocations, + getMd, + getMdBySearchId, InvalidArgumentError, InvalidTimeoutError, MissingApiKeyError, @@ -670,6 +672,198 @@ describe( }, ); +describe( + "getMd", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); + + afterEach(() => { + config.api_key = null; + }); + + afterAll(() => { + urlStub.restore(); + }); + + it("with no api_key", () => { + assertRejects( + async () => await getMd({ engine: "google", q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getMd("google", { q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getMd({}), + MissingApiKeyError, + ); + }); + + it("with invalid arguments", () => { + assertRejects( + // @ts-ignore testing invalid usage + async () => await getMd("google"), + InvalidArgumentError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getMd(), + InvalidArgumentError, + ); + }); + + it("with invalid timeout", () => { + config.api_key = "test_api_key"; + assertRejects( + async () => await getMd({ engine: "google", q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getMd({ engine: "google", q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getMd("google", { q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getMd("google", { q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + }); + + it( + "async/await", + { + ignore: !HAS_API_KEY, + }, + async () => { + const markdown = await getMd({ + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assert(markdown.startsWith("---")); + }, + ); + + it("returns Markdown with async/await and callbacks", async () => { + const markdownResponse = "---\n## Organic Results\n"; + const executeStub = stub( + _internals, + "execute", + () => Promise.resolve(markdownResponse), + ); + config.api_key = "test_api_key"; + + try { + const markdown = await getMd({ + engine: "google", + q: "Paris", + output: "json", + }); + assertEquals(markdown, markdownResponse); + + const markdownFromOldApi = await getMd("google", { q: "Paris" }); + assertEquals(markdownFromOldApi, markdownResponse); + + const markdownFromCallback = await new Promise((done) => { + getMd({ engine: "google", q: "Paris" }, done); + }); + assertEquals(markdownFromCallback, markdownResponse); + + const markdownFromOldApiCallback = await new Promise((done) => { + getMd("google", { q: "Paris" }, done); + }); + assertEquals(markdownFromOldApiCallback, markdownResponse); + } finally { + executeStub.restore(); + } + + assertSpyCalls(executeStub, 4); + assertSpyCallArg(executeStub, 0, 0, "/search"); + assertSpyCallArg(executeStub, 0, 1, { + api_key: "test_api_key", + engine: "google", + output: "md", + q: "Paris", + }); + }); + }, +); + +describe( + "getMdBySearchId", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + afterEach(() => { + config.api_key = null; + }); + + it( + "async/await", + { + ignore: !HAS_API_KEY, + }, + async () => { + const response = await getJson({ + engine: "google", + api_key: SERPAPI_TEST_KEY, + q: "Paris", + }); + const markdown = await getMdBySearchId(response.search_metadata.id, { + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assert(markdown.startsWith("---")); + }, + ); + + it("returns archived Markdown with async/await and callbacks", async () => { + const markdownResponse = "---\n## Organic Results\n"; + const executeStub = stub( + _internals, + "execute", + () => Promise.resolve(markdownResponse), + ); + config.api_key = "test_api_key"; + + try { + const markdown = await getMdBySearchId("search-id"); + assertEquals(markdown, markdownResponse); + + const markdownFromCallback = await new Promise((done) => { + getMdBySearchId("search-id", {}, done); + }); + assertEquals(markdownFromCallback, markdownResponse); + } finally { + executeStub.restore(); + } + + assertSpyCalls(executeStub, 2); + assertSpyCallArg(executeStub, 0, 0, "/searches/search-id"); + assertSpyCallArg(executeStub, 0, 1, { + api_key: "test_api_key", + output: "md", + }); + }); + }, +); + describe( "getJsonBySearchId", {