From 38a449b6351a484e81a9737f087b4115813936ab Mon Sep 17 00:00:00 2001 From: Dan Zucker Date: Thu, 5 Nov 2026 09:12:16 -0800 Subject: [PATCH] docs: publish domain llms.txt files for agents crawling the site. Give each extension surface its own text file and point llms.txt at that map so an agent loads one product area. --- website/docusaurus.config.js | 46 ++++++++++++- website/package.json | 2 + website/scripts/llms-domains.mjs | 106 +++++++++++++++++++++++++++++ website/scripts/llms-postbuild.mjs | 65 ++++++++++++++++++ website/src/remark/variables.mjs | 47 +++++++++++++ website/yarn.lock | 30 +++++++- 6 files changed, 294 insertions(+), 2 deletions(-) create mode 100644 website/scripts/llms-domains.mjs create mode 100644 website/scripts/llms-postbuild.mjs create mode 100644 website/src/remark/variables.mjs diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 17819b6e..2fe51fc4 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -2,6 +2,8 @@ // Note: type annotations allow type checking and IDEs autocompletion import {themes as prismThemes} from 'prism-react-renderer'; +import remarkVariables from './src/remark/variables.mjs'; +import { llmsDomains, llmsPluginFiles } from './scripts/llms-domains.mjs'; // const lightCodeTheme = require('prism-react-renderer/themes/github'); const lightCodeTheme = require('prism-react-renderer').themes.github; @@ -16,6 +18,20 @@ const getConfig = async () => { if (isInofficial) title = 'Unofficial Extensions API'; else if (isUpcomingVersion) title = 'Pre-Release Extensions API'; + // Site root for this deploy. Official Pages uses /extensions-api/. + // llms.txt lives there. llms-full.txt is generated but not linked. + // Agents should open one domain file. + const baseUrl = (process.env.GITHUB_BASE_PATH ?? '') + '/'; + const llmsHeadTags = ['llms.txt', ...llmsDomains.map((domain) => domain.filename)].map((file) => ({ + tagName: 'link', + attributes: { + rel: 'alternate', + type: 'text/plain', + href: `${baseUrl}${file}`, + title: file, + }, + })); + /** @type {import('@docusaurus/types').Config} */ return { title: title, @@ -24,7 +40,10 @@ const getConfig = async () => { // Pick up URL and baseUrl from config parameters set by the Github action runner url: process.env.GITHUB_ORIGIN ?? 'http://localhost/', - baseUrl: (process.env.GITHUB_BASE_PATH ?? '') + '/', + baseUrl, + + // llms.txt links are on every build so a crawler can find them. + headTags: llmsHeadTags, // We don't want to have the preview version or clones of the Github repository // indexed by search engines @@ -58,6 +77,10 @@ const getConfig = async () => { // Remove this to remove the "edit this page" links. editUrl: 'https://github.com/tableau/extensions-api/blob/gh-pages-dev/website', + // Content variables (`%product_name%`, ...) must run before the + // default remark plugins so the built-in link resolver doesn't treat + // a `%`-containing slug in a URL as an invalid percent-escape. + beforeDefaultRemarkPlugins: [remarkVariables], remarkPlugins: [remarkDefList], sidebarCollapsed: true, }, @@ -156,6 +179,14 @@ const getConfig = async () => { label: 'Tutorial', href: 'https://github.com/tableau/extensions-api/blob/main/Tutorial/Dashboard/readme.md', }, + { + label: 'llms.txt', + to: 'pathname:///llms.txt', + }, + ...llmsDomains.map((domain) => ({ + label: domain.navLabel, + to: `pathname:///${domain.filename}`, + })), ], }, @@ -221,6 +252,19 @@ const getConfig = async () => { }), plugins: [ + [ + 'docusaurus-plugin-llms', + { + generateLLMsTxt: true, + // Archive of every guide. The postbuild step replaces llms.txt with a + // domain map and does not link this file from the site. + generateLLMsFullTxt: true, + generateMarkdownFiles: false, + excludeImports: true, + removeDuplicateHeadings: true, + customLLMFiles: llmsPluginFiles(), + }, + ], [ '@docusaurus/plugin-google-tag-manager', { diff --git a/website/package.json b/website/package.json index 1eb5488d..81233b82 100644 --- a/website/package.json +++ b/website/package.json @@ -7,6 +7,7 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "docusaurus build", + "postbuild": "node scripts/llms-postbuild.mjs", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", @@ -40,6 +41,7 @@ "@docusaurus/module-type-aliases": "^3.7.0", "@docusaurus/types": "^3.7.0", "@tsconfig/docusaurus": "^1.0.5", + "docusaurus-plugin-llms": "^0.6.1", "prettier": "^2.8.4", "ts-node": "^10.9.1", "typescript": "^4.7.4" diff --git a/website/scripts/llms-domains.mjs b/website/scripts/llms-domains.mjs new file mode 100644 index 00000000..4618749b --- /dev/null +++ b/website/scripts/llms-domains.mjs @@ -0,0 +1,106 @@ +// Domain files for agents. llms.txt is rewritten after the build into a map of +// these files. llms-full.txt stays on disk as an archive and is not linked. +// Workspace extensions are omitted: this site has no docs/workspaceext pages, +// and the plugin does not emit a file when nothing matches. + +export const llmsDomains = [ + { + filename: 'llms-dataapps.txt', + navLabel: 'Data Apps', + title: 'Tableau Data Apps', + description: + 'Tableau Cloud data apps only. Dashboard, viz, and workspace extension guides are out of scope.', + rootContent: + 'Use this file for Data Apps. The Dashboard and Viz manifest pages are included only for packaging a data app. Do not apply other Dashboard, Viz, or Workspace extension instructions.', + includePatterns: [ + 'docs/dataapps/**', + 'docs/installation.md', + 'docs/dashext/trex_manifest.md', + 'docs/vizext/trex_viz_manifest.md', + ], + }, + { + filename: 'llms-dashext.txt', + navLabel: 'Dashboard Extensions', + title: 'Tableau Dashboard Extensions', + description: 'Dashboard extensions only. Viz, workspace, and data app guides are out of scope.', + rootContent: + 'Use this file for Dashboard Extensions. Do not apply Viz, Workspace, or Data Apps instructions.', + includePatterns: ['docs/dashext/**'], + }, + { + filename: 'llms-vizext.txt', + navLabel: 'Viz Extensions', + title: 'Tableau Viz Extensions', + description: 'Viz extensions only. Dashboard, workspace, and data app guides are out of scope.', + rootContent: + 'Use this file for Viz Extensions. Do not apply Dashboard, Workspace, or Data Apps instructions.', + includePatterns: ['docs/vizext/**'], + }, + { + filename: 'llms-core.txt', + navLabel: 'Core', + title: 'Tableau Extensions API Core', + description: 'Installation, API overview, debugging, and security.', + rootContent: + 'Shared setup and security for every extension type. Product-specific guides are in the other domain files.', + includePatterns: [ + 'docs/index.md', + 'docs/installation.md', + 'docs/trex_api_about.md', + 'docs/debug/**', + 'docs/security/**', + ], + }, + { + filename: 'llms-fundamentals.txt', + navLabel: 'Extension Fundamentals', + title: 'Extension Fundamentals', + description: 'Data access, dialogs, events, formatting, and TypeScript for extensions.', + rootContent: + 'Shared extension mechanics. Data Apps packaging and administration are in llms-dataapps.txt.', + includePatterns: ['docs/core/**'], + }, + { + filename: 'llms-design.txt', + navLabel: 'Design Guidelines', + title: 'Extension Design Guidelines', + description: 'Interaction and style guidelines for extensions.', + rootContent: 'Design guidance. API behavior and product setup are in the other domain files.', + includePatterns: [ + 'docs/ux_design.md', + 'docs/ux_extension_gallery.md', + 'docs/Interaction_Guidelines/**', + 'docs/Style_Guidelines/**', + ], + }, + { + filename: 'llms-reference.txt', + navLabel: 'Reference', + title: 'Extensions API Reference Guides', + description: 'Publishing, Tableau Viz input specs, release notes, and known issues.', + rootContent: 'Reference material. Use a product domain file for how to build that extension type.', + includePatterns: [ + 'docs/publish/**', + 'docs/trex_tableau_viz_ref.md', + 'docs/trex_tableau_viz_ref_v1.md', + 'docs/trex_tableau_viz_ref_v2.md', + 'docs/trex_release-notes.md', + 'docs/trex_known_issues.md', + 'docs/trex_tableau_help.md', + ], + }, +]; + +export function llmsPluginFiles() { + return llmsDomains.map((domain) => ({ + filename: domain.filename, + title: domain.title, + description: domain.description, + rootContent: domain.rootContent, + fullContent: true, + includePatterns: domain.includePatterns, + orderPatterns: domain.includePatterns, + includeUnmatchedLast: false, + })); +} diff --git a/website/scripts/llms-postbuild.mjs b/website/scripts/llms-postbuild.mjs new file mode 100644 index 00000000..b2e1f1be --- /dev/null +++ b/website/scripts/llms-postbuild.mjs @@ -0,0 +1,65 @@ +// Post-build step for the llms text files. +// docusaurus-plugin-llms reads docs source, so %product_name% and similar +// tokens are still literal, and ```mdx-code-block fences (imports, DocCardList, +// Tabs) are kept because they look like code. Expand the tokens with the same +// map the HTML build uses, drop those fences, then replace the plugin's flat +// page list with a domain map. llms-full.txt stays as an unlinked archive. + +import { promises as fs } from 'fs'; +import path from 'path'; +import { expandVariableText } from '../src/remark/variables.mjs'; +import { llmsDomains } from './llms-domains.mjs'; + +const buildDir = path.resolve('build'); + +const mdxBlock = /```mdx-code-block\r?\n[\s\S]*?```\r?\n?/g; + +function siteRoot() { + const origin = (process.env.GITHUB_ORIGIN ?? 'http://localhost').replace(/\/+$/, ''); + const base = process.env.GITHUB_BASE_PATH ?? ''; + const prefix = base === '' || base === '/' ? '/' : (base.endsWith('/') ? base : `${base}/`); + return `${origin}${prefix}`; +} + +function domainIndex() { + const root = siteRoot(); + const official = process.env.GITHUB_ORIGIN === 'https://tableau.github.io'; + const upcoming = !!process.env.IS_UPCOMING; + const title = !official + ? 'Unofficial Extensions API' + : upcoming + ? 'Pre-Release Extensions API' + : 'Tableau Extensions API'; + const links = llmsDomains.map((domain) => + `- [${domain.navLabel}](${root}${domain.filename}): ${domain.description}`, + ); + return `# ${title} + +> Create dashboard and viz extensions for Tableau. + +Read exactly one domain file for the task. For Data Apps, read llms-dataapps.txt and do not load the other domain files. + +## Domains + +${links.join('\n')} +`; +} + +async function expandLlmsVariables() { + const names = (await fs.readdir(buildDir)).filter((name) => /^llms.*\.txt$/.test(name)); + if (names.length === 0) { + console.warn('[llms] no llms*.txt files found; skipping.'); + return; + } + for (const name of names) { + const target = path.join(buildDir, name); + const text = await fs.readFile(target, 'utf8'); + const updated = expandVariableText(text).replace(mdxBlock, ''); + if (updated !== text) await fs.writeFile(target, updated); + console.log(`[llms] expanded variables in ${name}`); + } + await fs.writeFile(path.join(buildDir, 'llms.txt'), domainIndex()); + console.log('[llms] wrote domain index llms.txt'); +} + +expandLlmsVariables().catch((e) => { console.error('[llms] failed:', e); process.exit(1); }); diff --git a/website/src/remark/variables.mjs b/website/src/remark/variables.mjs new file mode 100644 index 00000000..3440c63d --- /dev/null +++ b/website/src/remark/variables.mjs @@ -0,0 +1,47 @@ +// Minimal build-time string-variable substitution for Markdown/MDX content. +// +// Single source of truth for content variables. Author a slug in the form +// `%variable_name%` anywhere in page prose and it is replaced with the value +// below at build time. Percent delimiters are used (not `{{ }}`) because braces +// are interpreted as JSX expressions in MDX; `%…%` is inert text that survives +// parsing and is safe to rewrite in `text` nodes. +// +// This mirrors the mechanism used by the tableau-mcp docs site +// (docs/src/remark/variables.mjs). +export const VARIABLES = { + product_name: 'Tableau Extensions API', + // Upcoming named release. Author `%coming_release%` in prose that names it + // (never a hardcoded "2026.3.1" or "November 2026 (2026.3.1)"). + coming_release: 'November 2026 (2026.3.1)', +}; + +const TOKEN = /%([a-zA-Z0-9_]+)%/g; + +export function expandVariableText(value) { + return value.replace(TOKEN, (match, key) => + Object.prototype.hasOwnProperty.call(VARIABLES, key) ? VARIABLES[key] : match, + ); +} + +function walk(node) { + // Prose text nodes (e.g. `%product_name%`). + if (node.type === 'text' && typeof node.value === 'string') { + node.value = expandVariableText(node.value); + } + // Link/image URLs are a separate `url` field, not a text child — cover them so a + // wording slug works inside a link target. + if ((node.type === 'link' || node.type === 'image' || node.type === 'definition') && typeof node.url === 'string') { + node.url = expandVariableText(node.url); + } + if (Array.isArray(node.children)) { + for (const child of node.children) { + walk(child); + } + } +} + +export default function remarkVariables() { + return tree => { + walk(tree); + }; +} diff --git a/website/yarn.lock b/website/yarn.lock index 8e2aac2d..2a8cf3c6 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -5756,6 +5756,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.1.7" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.1.7.tgz#14ea3836731823bb24740225ea43b941b436f9d1" + integrity sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g== + dependencies: + balanced-match "^1.0.0" + braces@^3.0.2, braces@~3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz" @@ -6979,6 +6986,15 @@ dns-packet@^5.2.2: dependencies: "@leichtgewicht/ip-codec" "^2.0.1" +docusaurus-plugin-llms@^0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/docusaurus-plugin-llms/-/docusaurus-plugin-llms-0.6.1.tgz#08e67711885addc9138c433ef49eaf025a1a877d" + integrity sha512-elqTn7httrRG20g1mfzTtWZjLVpXMNKO2IS9R84dxiIp7QjAcDuubLp6/bvpe3umHdPnEXHJKhNOpTY+wnQa1Q== + dependencies: + gray-matter "4.0.3" + minimatch "9.0.3" + yaml "2.8.1" + dom-converter@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz" @@ -7744,7 +7760,7 @@ graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== -gray-matter@^4.0.3: +gray-matter@4.0.3, gray-matter@^4.0.3: version "4.0.3" resolved "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz" integrity sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q== @@ -9826,6 +9842,13 @@ minimatch@3.1.2, minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1: dependencies: brace-expansion "^1.1.7" +minimatch@9.0.3: + version "9.0.3" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.3.tgz#a6e00c3de44c3a542bfaae70abfc22420a6da825" + integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== + dependencies: + brace-expansion "^2.0.1" + minimist@^1.2.0: version "1.2.8" resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" @@ -12957,6 +12980,11 @@ yallist@^4.0.0: resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== +yaml@2.8.1: + version "2.8.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.1.tgz#1870aa02b631f7e8328b93f8bc574fac5d6c4d79" + integrity sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw== + yaml@^1.7.2: version "1.10.2" resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz"