test(e2e): check every widget state with axe in light, dark and RTL

CONTRIBUTING.md

@@ -221,7 +221,9 @@ pnpm exec playwright install chromium   # once
 pnpm e2e
 ```
 
-They cover starting from the script tag, streaming, Escape and the tab order, the phone layout, accessibility, a reply that fails or is cut off, a slow tool kept alive by pings, session tokens, and a chat API on another origin. CI runs them in the `e2e` job. Add a test there when you change how the widget behaves in a browser.
+With Google Chrome installed you can skip the Chromium download and run `E2E_CHANNEL=chrome pnpm e2e`.
+
+They cover starting from the script tag, streaming, Escape and the tab order, the phone layout, a reply that fails or is cut off, a slow tool kept alive by pings, session tokens, and a chat API on another origin. `e2e/tests/states.spec.ts` puts the widget in each of its states (launcher, unread, welcome bubble, suggestions, pre-chat form, streaming, sources, errors, rate limit, offline, conversation limit, phone) in light, dark and right-to-left, and checks each one with axe against WCAG 2.2 AA. When you add a state, add it there too. CI runs them all in the `e2e` job. Add a test there when you change how the widget behaves in a browser.
 
 ## Adding a new AI provider
 

e2e/playwright.config.ts

@@ -13,7 +13,8 @@ export default defineConfig({
     baseURL: `http://localhost:${port}`,
     trace: 'retain-on-failure',
   },
-  projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }],
+  // E2E_CHANNEL=chrome runs the tests in your installed Chrome, without downloading Chromium.
+  projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'], ...(process.env.E2E_CHANNEL ? { channel: process.env.E2E_CHANNEL } : {}) } }],
   webServer: [
     {
       command: 'node e2e/server.mjs page',

e2e/server.mjs

@@ -60,6 +60,8 @@ const SLOW_TOOL_MS = 4000;
  * - "cut off": some text, then nothing (the bridge below ends the response)
  * - "slow tool": calls `slow_lookup`, then answers once it returns
  * - "long reply": about 4,000 words of markdown in small pieces, for the render cost test
+ * - "still streaming": some text, then nothing for a minute, with the response open
+ * - "with sources": a reply that cites the pricing page, found by the knowledge source below
  * - anything else: a normal streamed reply
  */
 /**
@@ -121,10 +123,16 @@ const provider = {
       return;
     }
 
-    const words = /slow tool/i.test(askedText) ? ['Found', ' it.'] : /fail midway|cut off/i.test(askedText) ? REPLY.slice(0, 3) : REPLY;
+    const words = /slow tool/i.test(askedText)
+      ? ['Found', ' it.']
+      : /with sources/i.test(askedText)
+        ? ['Plans start', ' at $19', ' a month [1].']
+        : /fail midway|cut off|still streaming/i.test(askedText)
+          ? REPLY.slice(0, 3)
+          : REPLY;
     yield* text(words);
     if (/fail midway/i.test(askedText)) throw new Error('The e2e provider failed on purpose.');
-    if (/cut off/i.test(askedText)) await sleep(60_000, signal);
+    if (/cut off|still streaming/i.test(askedText)) await sleep(60_000, signal);
     yield {
       type: 'finish',
       reason: 'stop',
@@ -144,10 +152,28 @@ const slowLookup = {
   },
 };
 
+/** Finds the pricing page for "with sources", and nothing otherwise. */
+const pricingPage = {
+  type: 'e2e',
+  async getContext() {
+    return '';
+  },
+  async retrieve({ text }) {
+    if (!/with sources/i.test(text)) return [];
+    return [{
+      id: 'pricing#0',
+      text: 'Plans start at $19 a month.',
+      score: 1,
+      source: { id: 'pricing', url: 'https://example.com/pricing', title: 'Pricing' },
+    }];
+  },
+};
+
 function chatEndpoint(overrides = {}) {
   const config = {
     provider,
     systemPrompt: 'e2e',
+    knowledge: [pricingPage],
     tools: [slowLookup],
     toolTimeoutMs: 30_000,
     budgets: false,

e2e/tests/states.spec.ts

@@ -0,0 +1,200 @@
+import { expect, test, type Page } from '@playwright/test';
+import AxeBuilder from '@axe-core/playwright';
+
+// Every widget state in F-05's list that exists today, each checked with axe
+// against WCAG 2.2 AA in light, dark and right-to-left. States still to come:
+// S8 tool confirmation (F-08), S12 handoff (F-10), S17 waiting for consent.
+
+const fab = '.cc-fab';
+const panel = '.cc-panel';
+const input = '.cc-input';
+const send = '.cc-send-btn';
+
+const looks = [
+  { name: 'light', config: { theme: { mode: 'light' } } },
+  { name: 'dark', config: { theme: { mode: 'dark' } } },
+  { name: 'rtl', config: { theme: { mode: 'light' }, dir: 'rtl' } },
+] as const;
+
+type Look = (typeof looks)[number];
+type Api = {
+  destroy(): void;
+  init(config: object): void;
+  sendMessage(text: string, options?: { open?: boolean }): Promise<void>;
+};
+
+/** The single-file build, set up again with this look and `extra`. */
+async function start(page: Page, look: Look, extra: object = {}) {
+  await page.emulateMedia({ colorScheme: look.config.theme.mode, reducedMotion: 'reduce' });
+  await page.goto('/full.html');
+  await page.evaluate((config) => {
+    const chat = (window as unknown as { ChatCops: Api }).ChatCops;
+    chat.destroy();
+    chat.init(config);
+  }, { apiUrl: '/api/chat', ...look.config, ...extra });
+}
+
+async function openChat(page: Page) {
+  await page.locator(fab).click();
+  await expect(page.locator(panel)).toHaveClass(/cc-visible/);
+}
+
+async function ask(page: Page, text: string) {
+  await page.locator(input).fill(text);
+  await page.locator(send).click();
+}
+
+/**
+ * Every WCAG 2.2 A and AA rule axe has, at any impact. It waits for the
+ * widget's animations first: axe skips the contrast of text that is still
+ * fading in, so a check mid-animation passes whatever the colors.
+ */
+async function expectAccessible(page: Page) {
+  await page.evaluate(() =>
+    Promise.all(
+      [...document.querySelectorAll('.chatcops-root')].flatMap((host) =>
+        (host.shadowRoot?.getAnimations() ?? []).map((animation) => animation.finished.catch(() => undefined))
+      )
+    )
+  );
+  const results = await new AxeBuilder({ page })
+    .include('.chatcops-root')
+    .withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'wcag22aa'])
+    .analyze();
+  const found = results.violations.map(
+    (v) => `${v.id} (${v.impact}): ${v.nodes.map((n) => n.target.join(' > ')).join(', ')}`
+  );
+  expect(found).toEqual([]);
+}
+
+/** A protocol v1 error answer for the next chat request. */
+async function failNextRequest(page: Page, status: number, error: object) {
+  await page.route('**/api/chat', (route) =>
+    route.request().method() === 'POST'
+      ? route.fulfill({ status, contentType: 'application/json', body: JSON.stringify({ error }) })
+      : route.fallback()
+  );
+}
+
+for (const look of looks) {
+  test.describe(`states, ${look.name}`, () => {
+    test('S1 launcher', async ({ page }) => {
+      await start(page, look);
+      await expect(page.locator(fab)).toBeVisible();
+      if (look.name === 'rtl') await expect(page.locator('.chatcops-root')).toHaveAttribute('dir', 'rtl');
+      await expectAccessible(page);
+    });
+
+    test('S2 launcher with an unread reply', async ({ page }) => {
+      await start(page, look);
+      await page.evaluate(() => (window as unknown as { ChatCops: Api }).ChatCops.sendMessage('Hi', { open: false }));
+      await expect(page.locator('.cc-fab-badge')).toHaveClass(/cc-visible/);
+      await expect(page.locator(fab)).toHaveAttribute('aria-label', /1/);
+      await expectAccessible(page);
+    });
+
+    test('S3 welcome bubble', async ({ page }) => {
+      await start(page, look, { welcomeBubble: { text: 'Need help choosing a plan?', delay: 0 } });
+      await expect(page.locator('.cc-bubble')).toHaveClass(/cc-visible/);
+      await expectAccessible(page);
+    });
+
+    test('S4 empty panel with suggestions', async ({ page }) => {
+      await start(page, look, { suggestions: ['Pricing', 'Talk to sales'] });
+      await openChat(page);
+      await expect(page.locator('.cc-suggestion')).toHaveCount(2);
+      await expect(page.locator('.cc-disclosure')).toBeVisible();
+      await expectAccessible(page);
+    });
+
+    test('S5 pre-chat form', async ({ page }) => {
+      await start(page, look, {
+        preChatForm: {
+          enabled: true,
+          fields: [
+            { name: 'name', label: 'Name', type: 'text', required: true },
+            { name: 'email', label: 'Email', type: 'email', required: true },
+            { name: 'topic', label: 'Topic', type: 'select', options: ['Sales', 'Support'] },
+            { name: 'message', label: 'Message', type: 'textarea' },
+          ],
+        },
+      });
+      await openChat(page);
+      await expect(page.locator('.cc-prechat')).toBeVisible();
+      await expectAccessible(page);
+      // A wrong email: the error shows next to the field, and is still accessible.
+      await page.getByRole('textbox', { name: 'Name' }).fill('Sam');
+      await page.getByRole('textbox', { name: 'Email' }).fill('not an email');
+      await page.locator('.cc-prechat button[type="submit"]').click();
+      await expect(page.getByRole('textbox', { name: 'Email' })).toHaveAttribute('aria-invalid', 'true');
+      await expect(page.locator('#cc-prechat-email-error')).not.toBeEmpty();
+      await expectAccessible(page);
+    });
+
+    test('S6 reply streaming in', async ({ page }) => {
+      await start(page, look);
+      await openChat(page);
+      await ask(page, 'Please keep it still streaming');
+      await expect(page.locator('.cc-message-assistant').last()).toContainText('Hello from the');
+      await expectAccessible(page);
+    });
+
+    test('S7 reply with sources', async ({ page }) => {
+      await start(page, look);
+      await openChat(page);
+      await ask(page, 'What does it cost, with sources?');
+      const sources = page.locator('.cc-message-sources a');
+      await expect(sources).toHaveText(['Pricing']);
+      await expect(sources).toHaveAttribute('href', 'https://example.com/pricing');
+      await expectAccessible(page);
+    });
+
+    test('S9 error that can be retried', async ({ page }) => {
+      await start(page, look);
+      await openChat(page);
+      await ask(page, 'Please fail midway');
+      await expect(page.locator('.cc-message-retry')).toBeVisible();
+      await expectAccessible(page);
+    });
+
+    test('S10 rate limited', async ({ page }) => {
+      await start(page, look);
+      await openChat(page);
+      await failNextRequest(page, 429, { code: 'rate_limited', message: 'Too many requests', retryAfterMs: 20_000 });
+      await ask(page, 'Hi');
+      await expect(page.locator('.cc-message-assistant').last()).toContainText('Please try again in 20 seconds.');
+      await expectAccessible(page);
+    });
+
+    test('S11 offline', async ({ page, context }) => {
+      await start(page, look);
+      await openChat(page);
+      await context.setOffline(true);
+      await ask(page, 'Hi');
+      await expect(page.locator('.cc-message-retry')).toBeVisible();
+      await context.setOffline(false);
+      await expectAccessible(page);
+    });
+
+    test('S13 conversation limit reached', async ({ page }) => {
+      await start(page, look);
+      await openChat(page);
+      await failNextRequest(page, 429, { code: 'budget_exceeded', message: 'Limit reached', scope: 'conversation' });
+      await ask(page, 'Hi');
+      await expect(page.locator('.cc-message-assistant').last()).toContainText('This chat has reached its limit.');
+      await expectAccessible(page);
+    });
+
+    test.describe('S14 on a phone', () => {
+      test.use({ viewport: { width: 390, height: 844 }, hasTouch: true, isMobile: true });
+
+      test('full screen panel with a reply', async ({ page }) => {
+        await start(page, look);
+        await openChat(page);
+        await ask(page, 'Hi there');
+        await expect(page.locator('.cc-message-assistant').last()).toContainText('Hello from the test server.');
+        await expectAccessible(page);
+      });
+    });
+  });
+}