test(e2e): screenshot tests for the widget's states, with a workflow that makes the baselines

.github/workflows/ci.yml

@@ -5,6 +5,9 @@ on:
     branches: [dev]
   pull_request:
     branches: [dev, production]
+  # The Update screenshots workflow starts this on the branch it committed to,
+  # since its own pushes don't trigger workflows.
+  workflow_dispatch:
 
 jobs:
   typecheck-test-build:

.github/workflows/update-screenshots.yml

@@ -0,0 +1,73 @@
+name: Update screenshots
+
+# Makes the screenshot baselines in e2e/screenshots/ on CI's Linux Chromium,
+# the only place they're compared. Run it from the Actions tab on a branch
+# in this repo after a change to how the widget looks. For dev it opens a PR
+# instead, since dev only takes changes through PRs.
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: Branch to update (a branch in this repo, or dev for a new PR)
+        required: true
+        default: dev
+
+permissions:
+  contents: write
+  pull-requests: write
+  actions: write
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+
+      - uses: pnpm/action-setup@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+
+      - run: pnpm install --frozen-lockfile
+
+      - name: Build the widget and the server
+        run: pnpm --filter @chatcops/widget --filter "@chatcops/server..." build
+
+      - name: Install Chromium
+        run: pnpm exec playwright install --with-deps chromium
+
+      - name: Make the screenshots
+        run: pnpm e2e visual.spec.ts --update-snapshots
+        env:
+          E2E_UPDATE_SCREENSHOTS: '1'
+
+      - name: Commit them
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BRANCH: ${{ inputs.branch }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          git add e2e/screenshots
+          if git diff --cached --quiet; then
+            echo "The screenshots are up to date."
+            exit 0
+          fi
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -m "test(e2e): update screenshot baselines"
+          target="$BRANCH"
+          if [ "$BRANCH" = dev ] || [ "$BRANCH" = production ]; then
+            target="screenshots/$RUN_ID"
+            git push origin "HEAD:refs/heads/$target"
+            gh pr create --base "$BRANCH" --head "$target" \
+              --title "test(e2e): update screenshot baselines" \
+              --body "Made by the Update screenshots workflow, run $RUN_ID. Check the images in the Files tab before merging."
+          else
+            git push origin "HEAD:refs/heads/$target"
+          fi
+          # Pushes made with this workflow's token don't start CI on their own.
+          gh workflow run ci.yml --ref "$target"

CONTRIBUTING.md

@@ -224,7 +224,9 @@ pnpm e2e
 
 With Google Chrome installed you can skip the Chromium download and run `E2E_CHANNEL=chrome pnpm e2e`.
 
-They cover starting from the script tag, streaming, Escape and the tab order, the phone layout, a reply that fails or is cut off, a slow tool kept alive by pings, session tokens, and a chat API on another origin. `e2e/tests/states.spec.ts` puts the widget in each of its states (launcher, unread, welcome bubble, suggestions, pre-chat form, streaming, sources, errors, rate limit, offline, conversation limit, phone) in light, dark and right-to-left, and checks each one with axe against WCAG 2.2 AA. When you add a state, add it there too. CI runs them all in the `e2e` job. Add a test there when you change how the widget behaves in a browser.
+They cover starting from the script tag, streaming, Escape and the tab order, the phone layout, a reply that fails or is cut off, a slow tool kept alive by pings, session tokens, and a chat API on another origin. `e2e/tests/states.spec.ts` puts the widget in each of its states (launcher, unread, welcome bubble, suggestions, pre-chat form, streaming, sources, errors, rate limit, offline, conversation limit, phone) in light, dark and right-to-left, and checks each one with axe against WCAG 2.2 AA. When you add a state, add it there too. CI runs them all in the `e2e` job.
+
+`e2e/tests/visual.spec.ts` compares screenshots of the main states, in light, dark and right-to-left, with the baselines in `e2e/screenshots/`. Fonts render differently on every OS, so the baselines are made on CI's Linux Chromium and the spec skips anywhere else, your machine included. When you change how the widget looks, a maintainer runs the "Update screenshots" workflow (Actions tab, Run workflow) on your branch. It commits the new images there and starts CI again. Look at them in the PR's Files tab: the diff is the review. For a PR from a fork, which the workflow can't push to, the maintainer updates them after merging, and the workflow opens a PR against `dev` for that. Add a test there when you change how the widget behaves in a browser.
 
 ## Adding a new AI provider
 

e2e/playwright.config.ts

@@ -9,6 +9,9 @@ export default defineConfig({
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 1 : 0,
   reporter: process.env.CI ? [['github'], ['html', { open: 'never' }]] : 'list',
+  // Screenshot baselines (visual.spec.ts), made on CI's Linux Chromium only.
+  snapshotPathTemplate: '{testDir}/../screenshots/{testFilePath}/{arg}{ext}',
+  expect: { toHaveScreenshot: { maxDiffPixelRatio: 0.01, animations: 'disabled', caret: 'hide' } },
   use: {
     baseURL: `http://localhost:${port}`,
     trace: 'retain-on-failure',

e2e/tests/visual.spec.ts

@@ -0,0 +1,124 @@
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { expect, test, type Locator, type Page } from '@playwright/test';
+
+// Screenshots of the widget's states in light, dark and right-to-left,
+// compared with the baselines in e2e/screenshots/. They're made on CI's
+// Linux Chromium, so they're skipped anywhere else (a Mac renders fonts
+// differently). To make or update them, run the "Update screenshots"
+// workflow on your branch; see CONTRIBUTING.md.
+
+const baselines = join(__dirname, '..', 'screenshots', 'visual.spec.ts');
+const updating = process.env.E2E_UPDATE_SCREENSHOTS === '1';
+const comparable = process.platform === 'linux' && !process.env.E2E_CHANNEL;
+
+test.skip(!comparable, 'Screenshots are compared on CI Linux Chromium only.');
+test.skip(!updating && !existsSync(baselines), 'No baselines yet: run the Update screenshots workflow.');
+
+const looks = [
+  { name: 'light', config: { theme: { mode: 'light' } } },
+  { name: 'dark', config: { theme: { mode: 'dark' } } },
+  { name: 'rtl', config: { theme: { mode: 'light' }, dir: 'rtl' } },
+] as const;
+
+type Look = (typeof looks)[number];
+type Api = { destroy(): void; init(config: object): void };
+
+async function start(page: Page, look: Look, extra: object = {}) {
+  await page.emulateMedia({ colorScheme: look.config.theme.mode, reducedMotion: 'reduce' });
+  await page.goto('/full.html');
+  await page.evaluate((config) => {
+    const chat = (window as unknown as { ChatCops: Api }).ChatCops;
+    chat.destroy();
+    chat.init(config);
+  }, { apiUrl: '/api/chat', branding: { name: 'Acme help', subtitle: 'Online' }, ...look.config, ...extra });
+}
+
+async function settled(page: Page) {
+  await page.evaluate(() =>
+    Promise.all(
+      [...document.querySelectorAll('.chatcops-root')].flatMap((host) =>
+        (host.shadowRoot?.getAnimations() ?? []).map((animation) => animation.finished.catch(() => undefined))
+      )
+    )
+  );
+}
+
+async function openChat(page: Page) {
+  await page.locator('.cc-fab').click();
+  await expect(page.locator('.cc-panel')).toHaveClass(/cc-visible/);
+}
+
+async function ask(page: Page, text: string) {
+  await page.locator('.cc-input').fill(text);
+  await page.locator('.cc-send-btn').click();
+}
+
+async function snap(page: Page, target: Locator | Page, name: string) {
+  await settled(page);
+  await expect(target).toHaveScreenshot(`${name}.png`);
+}
+
+for (const look of looks) {
+  test.describe(`screenshots, ${look.name}`, () => {
+    test('S1 launcher', async ({ page }) => {
+      await start(page, look);
+      await snap(page, page.locator('.cc-fab'), `s1-launcher-${look.name}`);
+    });
+
+    test('S3 welcome bubble', async ({ page }) => {
+      await start(page, look, { welcomeBubble: { text: 'Need help choosing a plan?', delay: 0 } });
+      await expect(page.locator('.cc-bubble')).toHaveClass(/cc-visible/);
+      await snap(page, page.locator('.cc-bubble'), `s3-bubble-${look.name}`);
+    });
+
+    test('S4 empty panel with suggestions', async ({ page }) => {
+      await start(page, look, { suggestions: ['Pricing', 'Talk to sales'] });
+      await openChat(page);
+      await snap(page, page.locator('.cc-panel'), `s4-suggestions-${look.name}`);
+    });
+
+    test('S5 pre-chat form', async ({ page }) => {
+      await start(page, look, {
+        preChatForm: {
+          enabled: true,
+          fields: [
+            { name: 'name', label: 'Name', type: 'text', required: true },
+            { name: 'email', label: 'Email', type: 'email', required: true },
+          ],
+        },
+      });
+      await openChat(page);
+      await expect(page.locator('.cc-prechat')).toBeVisible();
+      await snap(page, page.locator('.cc-panel'), `s5-prechat-${look.name}`);
+    });
+
+    test('S7 reply with sources', async ({ page }) => {
+      await start(page, look);
+      await openChat(page);
+      await ask(page, 'What does it cost, with sources?');
+      await expect(page.locator('.cc-message-sources a')).toHaveText(['Pricing']);
+      await snap(page, page.locator('.cc-panel'), `s7-sources-${look.name}`);
+    });
+
+    test('S9 error that can be retried', async ({ page }) => {
+      await start(page, look);
+      await openChat(page);
+      await ask(page, 'Please fail midway');
+      await expect(page.locator('.cc-message-retry')).toBeVisible();
+      await snap(page, page.locator('.cc-panel'), `s9-error-${look.name}`);
+    });
+
+    test.describe('S14 on a phone', () => {
+      test.use({ viewport: { width: 390, height: 844 }, hasTouch: true, isMobile: true });
+
+      test('full screen panel with a reply', async ({ page }) => {
+        await start(page, look);
+        await openChat(page);
+        await ask(page, 'Hi there');
+        await expect(page.locator('.cc-message-assistant').last()).toContainText('Hello from the test server.');
+        await snap(page, page, `s14-phone-${look.name}`);
+      });
+    });
+  });
+}