diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef3ea9bc..c58d0b18 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -221,7 +221,9 @@ pnpm exec playwright install chromium # once pnpm e2e ``` -They cover starting from the script tag, streaming, Escape and the tab order, the phone layout, accessibility, a reply that fails or is cut off, a slow tool kept alive by pings, session tokens, and a chat API on another origin. CI runs them in the `e2e` job. Add a test there when you change how the widget behaves in a browser. +With Google Chrome installed you can skip the Chromium download and run `E2E_CHANNEL=chrome pnpm e2e`. + +They cover starting from the script tag, streaming, Escape and the tab order, the phone layout, a reply that fails or is cut off, a slow tool kept alive by pings, session tokens, and a chat API on another origin. `e2e/tests/states.spec.ts` puts the widget in each of its states (launcher, unread, welcome bubble, suggestions, pre-chat form, streaming, sources, errors, rate limit, offline, conversation limit, phone) in light, dark and right-to-left, and checks each one with axe against WCAG 2.2 AA. When you add a state, add it there too. CI runs them all in the `e2e` job. Add a test there when you change how the widget behaves in a browser. ## Adding a new AI provider diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 953973ae..ec1a60e6 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -13,7 +13,8 @@ export default defineConfig({ baseURL: `http://localhost:${port}`, trace: 'retain-on-failure', }, - projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // E2E_CHANNEL=chrome runs the tests in your installed Chrome, without downloading Chromium. + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'], ...(process.env.E2E_CHANNEL ? { channel: process.env.E2E_CHANNEL } : {}) } }], webServer: [ { command: 'node e2e/server.mjs page', diff --git a/e2e/server.mjs b/e2e/server.mjs index 9563cf3a..d5ca7987 100644 --- a/e2e/server.mjs +++ b/e2e/server.mjs @@ -60,6 +60,8 @@ const SLOW_TOOL_MS = 4000; * - "cut off": some text, then nothing (the bridge below ends the response) * - "slow tool": calls `slow_lookup`, then answers once it returns * - "long reply": about 4,000 words of markdown in small pieces, for the render cost test + * - "still streaming": some text, then nothing for a minute, with the response open + * - "with sources": a reply that cites the pricing page, found by the knowledge source below * - anything else: a normal streamed reply */ /** @@ -121,10 +123,16 @@ const provider = { return; } - const words = /slow tool/i.test(askedText) ? ['Found', ' it.'] : /fail midway|cut off/i.test(askedText) ? REPLY.slice(0, 3) : REPLY; + const words = /slow tool/i.test(askedText) + ? ['Found', ' it.'] + : /with sources/i.test(askedText) + ? ['Plans start', ' at $19', ' a month [1].'] + : /fail midway|cut off|still streaming/i.test(askedText) + ? REPLY.slice(0, 3) + : REPLY; yield* text(words); if (/fail midway/i.test(askedText)) throw new Error('The e2e provider failed on purpose.'); - if (/cut off/i.test(askedText)) await sleep(60_000, signal); + if (/cut off|still streaming/i.test(askedText)) await sleep(60_000, signal); yield { type: 'finish', reason: 'stop', @@ -144,10 +152,28 @@ const slowLookup = { }, }; +/** Finds the pricing page for "with sources", and nothing otherwise. */ +const pricingPage = { + type: 'e2e', + async getContext() { + return ''; + }, + async retrieve({ text }) { + if (!/with sources/i.test(text)) return []; + return [{ + id: 'pricing#0', + text: 'Plans start at $19 a month.', + score: 1, + source: { id: 'pricing', url: 'https://example.com/pricing', title: 'Pricing' }, + }]; + }, +}; + function chatEndpoint(overrides = {}) { const config = { provider, systemPrompt: 'e2e', + knowledge: [pricingPage], tools: [slowLookup], toolTimeoutMs: 30_000, budgets: false, diff --git a/e2e/tests/states.spec.ts b/e2e/tests/states.spec.ts new file mode 100644 index 00000000..b7bbbb7f --- /dev/null +++ b/e2e/tests/states.spec.ts @@ -0,0 +1,200 @@ +import { expect, test, type Page } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; + +// Every widget state in F-05's list that exists today, each checked with axe +// against WCAG 2.2 AA in light, dark and right-to-left. States still to come: +// S8 tool confirmation (F-08), S12 handoff (F-10), S17 waiting for consent. + +const fab = '.cc-fab'; +const panel = '.cc-panel'; +const input = '.cc-input'; +const send = '.cc-send-btn'; + +const looks = [ + { name: 'light', config: { theme: { mode: 'light' } } }, + { name: 'dark', config: { theme: { mode: 'dark' } } }, + { name: 'rtl', config: { theme: { mode: 'light' }, dir: 'rtl' } }, +] as const; + +type Look = (typeof looks)[number]; +type Api = { + destroy(): void; + init(config: object): void; + sendMessage(text: string, options?: { open?: boolean }): Promise; +}; + +/** The single-file build, set up again with this look and `extra`. */ +async function start(page: Page, look: Look, extra: object = {}) { + await page.emulateMedia({ colorScheme: look.config.theme.mode, reducedMotion: 'reduce' }); + await page.goto('/full.html'); + await page.evaluate((config) => { + const chat = (window as unknown as { ChatCops: Api }).ChatCops; + chat.destroy(); + chat.init(config); + }, { apiUrl: '/api/chat', ...look.config, ...extra }); +} + +async function openChat(page: Page) { + await page.locator(fab).click(); + await expect(page.locator(panel)).toHaveClass(/cc-visible/); +} + +async function ask(page: Page, text: string) { + await page.locator(input).fill(text); + await page.locator(send).click(); +} + +/** + * Every WCAG 2.2 A and AA rule axe has, at any impact. It waits for the + * widget's animations first: axe skips the contrast of text that is still + * fading in, so a check mid-animation passes whatever the colors. + */ +async function expectAccessible(page: Page) { + await page.evaluate(() => + Promise.all( + [...document.querySelectorAll('.chatcops-root')].flatMap((host) => + (host.shadowRoot?.getAnimations() ?? []).map((animation) => animation.finished.catch(() => undefined)) + ) + ) + ); + const results = await new AxeBuilder({ page }) + .include('.chatcops-root') + .withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'wcag22aa']) + .analyze(); + const found = results.violations.map( + (v) => `${v.id} (${v.impact}): ${v.nodes.map((n) => n.target.join(' > ')).join(', ')}` + ); + expect(found).toEqual([]); +} + +/** A protocol v1 error answer for the next chat request. */ +async function failNextRequest(page: Page, status: number, error: object) { + await page.route('**/api/chat', (route) => + route.request().method() === 'POST' + ? route.fulfill({ status, contentType: 'application/json', body: JSON.stringify({ error }) }) + : route.fallback() + ); +} + +for (const look of looks) { + test.describe(`states, ${look.name}`, () => { + test('S1 launcher', async ({ page }) => { + await start(page, look); + await expect(page.locator(fab)).toBeVisible(); + if (look.name === 'rtl') await expect(page.locator('.chatcops-root')).toHaveAttribute('dir', 'rtl'); + await expectAccessible(page); + }); + + test('S2 launcher with an unread reply', async ({ page }) => { + await start(page, look); + await page.evaluate(() => (window as unknown as { ChatCops: Api }).ChatCops.sendMessage('Hi', { open: false })); + await expect(page.locator('.cc-fab-badge')).toHaveClass(/cc-visible/); + await expect(page.locator(fab)).toHaveAttribute('aria-label', /1/); + await expectAccessible(page); + }); + + test('S3 welcome bubble', async ({ page }) => { + await start(page, look, { welcomeBubble: { text: 'Need help choosing a plan?', delay: 0 } }); + await expect(page.locator('.cc-bubble')).toHaveClass(/cc-visible/); + await expectAccessible(page); + }); + + test('S4 empty panel with suggestions', async ({ page }) => { + await start(page, look, { suggestions: ['Pricing', 'Talk to sales'] }); + await openChat(page); + await expect(page.locator('.cc-suggestion')).toHaveCount(2); + await expect(page.locator('.cc-disclosure')).toBeVisible(); + await expectAccessible(page); + }); + + test('S5 pre-chat form', async ({ page }) => { + await start(page, look, { + preChatForm: { + enabled: true, + fields: [ + { name: 'name', label: 'Name', type: 'text', required: true }, + { name: 'email', label: 'Email', type: 'email', required: true }, + { name: 'topic', label: 'Topic', type: 'select', options: ['Sales', 'Support'] }, + { name: 'message', label: 'Message', type: 'textarea' }, + ], + }, + }); + await openChat(page); + await expect(page.locator('.cc-prechat')).toBeVisible(); + await expectAccessible(page); + // A wrong email: the error shows next to the field, and is still accessible. + await page.getByRole('textbox', { name: 'Name' }).fill('Sam'); + await page.getByRole('textbox', { name: 'Email' }).fill('not an email'); + await page.locator('.cc-prechat button[type="submit"]').click(); + await expect(page.getByRole('textbox', { name: 'Email' })).toHaveAttribute('aria-invalid', 'true'); + await expect(page.locator('#cc-prechat-email-error')).not.toBeEmpty(); + await expectAccessible(page); + }); + + test('S6 reply streaming in', async ({ page }) => { + await start(page, look); + await openChat(page); + await ask(page, 'Please keep it still streaming'); + await expect(page.locator('.cc-message-assistant').last()).toContainText('Hello from the'); + await expectAccessible(page); + }); + + test('S7 reply with sources', async ({ page }) => { + await start(page, look); + await openChat(page); + await ask(page, 'What does it cost, with sources?'); + const sources = page.locator('.cc-message-sources a'); + await expect(sources).toHaveText(['Pricing']); + await expect(sources).toHaveAttribute('href', 'https://example.com/pricing'); + await expectAccessible(page); + }); + + test('S9 error that can be retried', async ({ page }) => { + await start(page, look); + await openChat(page); + await ask(page, 'Please fail midway'); + await expect(page.locator('.cc-message-retry')).toBeVisible(); + await expectAccessible(page); + }); + + test('S10 rate limited', async ({ page }) => { + await start(page, look); + await openChat(page); + await failNextRequest(page, 429, { code: 'rate_limited', message: 'Too many requests', retryAfterMs: 20_000 }); + await ask(page, 'Hi'); + await expect(page.locator('.cc-message-assistant').last()).toContainText('Please try again in 20 seconds.'); + await expectAccessible(page); + }); + + test('S11 offline', async ({ page, context }) => { + await start(page, look); + await openChat(page); + await context.setOffline(true); + await ask(page, 'Hi'); + await expect(page.locator('.cc-message-retry')).toBeVisible(); + await context.setOffline(false); + await expectAccessible(page); + }); + + test('S13 conversation limit reached', async ({ page }) => { + await start(page, look); + await openChat(page); + await failNextRequest(page, 429, { code: 'budget_exceeded', message: 'Limit reached', scope: 'conversation' }); + await ask(page, 'Hi'); + await expect(page.locator('.cc-message-assistant').last()).toContainText('This chat has reached its limit.'); + await expectAccessible(page); + }); + + test.describe('S14 on a phone', () => { + test.use({ viewport: { width: 390, height: 844 }, hasTouch: true, isMobile: true }); + + test('full screen panel with a reply', async ({ page }) => { + await start(page, look); + await openChat(page); + await ask(page, 'Hi there'); + await expect(page.locator('.cc-message-assistant').last()).toContainText('Hello from the test server.'); + await expectAccessible(page); + }); + }); + }); +}