diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 889e33081..b7dd9ceb1 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -120,16 +120,32 @@ the workspace with `pnpm -r run build`, and fetch the datasets with ```sh pnpm -F webapp run test:e2e:local +pnpm -F webapp run test:e2e:federated pnpm -F webapp run test:e2e:decentralized ``` The local command runs both Titanic and LUS COVID with a mocked server. +The federated command runs both tasks with a real server and a separate Node +participant for each task. Locally, it defaults to Firefox because headless Electron +can stall on the LUS COVID image task without WebGL support; Firefox must therefore +be installed locally. The decentralized command runs MNIST across Chrome, Edge and Firefox participants with a real server; install Chrome, Edge and Firefox (plus `xvfb` and `xauth` on headless Linux). -Both commands start and stop their servers and Vite automatically. +All three commands start and stop their servers and Vite automatically. Federated training tests run in separate CI jobs for Titanic and LUS COVID. -Each job starts a real server and a Node participant configured for its task; -see `test-e2e-federated` in `.github/workflows/_test.yml` for the setup. +Each job starts a real server and a Node participant configured for its task, +while the package command runs both tasks together; see `test-e2e-federated` +in `.github/workflows/_test.yml` for the CI setup. CI runs Cypress in its default +Electron environment under `xvfb`. + +##### Test environment variables + +The end-to-end training test suite uses the following environment variables: + +- `DISCO_TRAINING_E2E`: Set to `1` to include training specs (`cypress/e2e/training/{local,federated,decentralized}/**/*.cy.ts`), which are otherwise excluded by default from the general test suite. +- `DISCO_SCRIPTED_SERVER_E2E`: Set to `1` to include scripted server specs (`cypress/e2e/training/scripted/**/*.cy.ts`) and resolve the `server` test host to `127.0.0.1`. +- `DISCO_E2E_FEDERATED_TASK`: Controls which task(s) the federated peer serves and trains: `titanic`, `lus_covid`, or `all` (defaults to `titanic`). +- `VITE_SERVER_URL`: Sets the DISCO server endpoint that the webapp connects to (e.g., `http://server` for mock tests or `http://localhost:8080` for real server tests). #### Cypress and Github Actions diff --git a/webapp/cypress/e2e/training/federated/titanic.cy.ts b/webapp/cypress/e2e/training/federated/titanic.cy.ts index cf818f197..d37980529 100644 --- a/webapp/cypress/e2e/training/federated/titanic.cy.ts +++ b/webapp/cypress/e2e/training/federated/titanic.cy.ts @@ -1,10 +1,10 @@ -import { assertNoErrorToast } from "../../../support/training"; +import { + assertNoErrorToast, + goToTaskOverview, +} from "../../../support/training"; function goToTitanicTraining(): void { - cy.visit("/"); - cy.contains("a", "Start training").click(); - cy.get(".driver-popover-close-btn").click(); - cy.contains("button", "participate").click(); + goToTaskOverview("titanic"); cy.contains("button", "next").click(); cy.get('[data-testid="select-tabular-button"]') .first() diff --git a/webapp/cypress/support/training/federated/peer.ts b/webapp/cypress/support/training/federated/peer.ts index 8b011bc44..b951a93db 100644 --- a/webapp/cypress/support/training/federated/peer.ts +++ b/webapp/cypress/support/training/federated/peer.ts @@ -1,5 +1,5 @@ -// Serves the selected task and trains on it as a Node participant so that the -// browser driven by cypress/e2e/training/federated has a federated peer. +// Serves the selected task(s) and trains on each as a Node participant so that +// the browser driven by cypress/e2e/training/federated has a federated peer. // Started next to vite by the test runner, which kills it once Cypress is done. import "@tensorflow/tfjs-node"; @@ -15,7 +15,7 @@ import { Server } from "server"; import { withTrainingConfig } from "../index.ts"; const taskName = process.env.DISCO_E2E_FEDERATED_TASK ?? "titanic"; -if (taskName !== "titanic" && taskName !== "lus_covid") { +if (taskName !== "titanic" && taskName !== "lus_covid" && taskName !== "all") { throw new Error(`Unknown federated E2E task: ${taskName}`); } @@ -28,9 +28,8 @@ const reducedLusCovidTask = withTrainingConfig(defaultTasks.lusCovid, { async function train( provider: TaskProvider, dataset: Dataset, + url: URL, ): Promise { - const server = await Server.with([provider.modelCard], [provider]); - const [, url] = await server.serve(8080); const task = await provider.getTask(); const disco = new Disco(task, url, { preprocessOnce: true, @@ -60,17 +59,34 @@ async function nodeHalfOfImages(directory: string) { } try { - if (taskName === "titanic") { - await train( - defaultTasks.titanic, - loadCSV( - path.join( - import.meta.dirname, - "../../../../../datasets/titanic_train.csv", + const providers = + taskName === "all" + ? [defaultTasks.titanic, reducedLusCovidTask] + : taskName === "titanic" + ? [defaultTasks.titanic] + : [reducedLusCovidTask]; + const server = await Server.with( + providers.map((provider) => provider.modelCard), + providers, + ); + const [, url] = await server.serve(8080); + + const trainings: (() => Promise)[] = []; + if (taskName === "titanic" || taskName === "all") { + trainings.push(() => + train( + defaultTasks.titanic, + loadCSV( + path.join( + import.meta.dirname, + "../../../../../datasets/titanic_train.csv", + ), ), + url, ), ); - } else { + } + if (taskName === "lus_covid" || taskName === "all") { const folder = path.join( import.meta.dirname, "../../../../../datasets/lus_covid", @@ -81,8 +97,11 @@ try { const negative = (await nodeHalfOfImages(path.join(folder, "COVID-"))).zip( Repeat("COVID-Negative"), ); - await train(reducedLusCovidTask, positive.chain(negative)); + trainings.push(() => + train(reducedLusCovidTask, positive.chain(negative), url), + ); } + await Promise.all(trainings.map((run) => run())); } catch (error) { // Take the server down so that the browser fails instead of waiting forever. console.error("federated peer failed", error); diff --git a/webapp/package.json b/webapp/package.json index 3463a7ade..3b0517f9b 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -10,6 +10,7 @@ "test:e2e": "VITE_SERVER_URL=http://server start-server-and-test \"vite\" http://localhost:1351 'pnpm exec cypress run --e2e'", "test:e2e:local": "DISCO_TRAINING_E2E=1 VITE_SERVER_URL=http://server start-server-and-test 'vite --mode test --port 1351 --strictPort' http://localhost:1351 'pnpm exec cypress run --e2e --spec \"cypress/e2e/training/local/*.cy.ts\"'", "test:e2e:scripted": "DISCO_SCRIPTED_SERVER_E2E=1 VITE_SERVER_URL=http://server:8080 start-server-and-test 'vite --mode test --port 1351 --strictPort' http://localhost:1351 'pnpm exec cypress run --e2e --spec \"cypress/e2e/training/scripted/*.cy.ts\"'", + "test:e2e:federated": "DISCO_TRAINING_E2E=1 DISCO_E2E_FEDERATED_TASK=all VITE_SERVER_URL=http://localhost:8080 start-server-and-test 'node --experimental-strip-types cypress/support/training/federated/peer.ts' http://localhost:8080 'vite --mode test --port 1351 --strictPort' http://localhost:1351 'pnpm exec cypress run --e2e --browser firefox --spec \"cypress/e2e/training/federated/*.cy.ts\"'", "test:e2e:decentralized": "DISCO_TRAINING_E2E=1 VITE_SERVER_URL=http://localhost:8080 start-server-and-test 'node ../server/dist/main.js' http://localhost:8080 'vite --mode test --port 1351 --strictPort' http://localhost:1351 'node --experimental-strip-types cypress/support/training/decentralized/browsers.ts'", "test:e2e-interactive": "VITE_SERVER_URL=http://server start-server-and-test \"pnpm start\" http://localhost:1351 'pnpm exec cypress open --e2e'" },