From 9c278d652a1e6e06b8c195c947eab1a64dcbd709 Mon Sep 17 00:00:00 2001 From: Okechi Jones-Williams Date: Mon, 12 Oct 2026 19:47:11 +0100 Subject: [PATCH 1/7] Updated run_local_isolated script to use the Supabase CLI installed inside the project --- scripts/run_local_isolated.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run_local_isolated.sh b/scripts/run_local_isolated.sh index c803a8851..6a0b825a6 100755 --- a/scripts/run_local_isolated.sh +++ b/scripts/run_local_isolated.sh @@ -35,13 +35,13 @@ export $(cat .env.local | grep -v '^#' | xargs) export $(cat .env.test | grep -v '^#' | xargs) # Ensure Supabase local stack is running; if not, reset/start it -STATUS_JSON=$(supabase status --output json 2>/dev/null || echo '') +STATUS_JSON=$(npx supabase status --output json 2>/dev/null || echo '') API_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL // empty') if [ -z "$API_URL" ]; then echo "Supabase is not running. Bootstrapping local stack with: yarn test:db:reset" yarn test:db:reset - STATUS_JSON=$(supabase status --output json) + STATUS_JSON=$(npx supabase status --output json) fi export NEXT_PUBLIC_SUPABASE_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL') From 23c377b1042e1c3e8dd67210b767670ef20fcd4d Mon Sep 17 00:00:00 2001 From: Okechi Jones-Williams Date: Tue, 13 Oct 2026 14:48:51 +0100 Subject: [PATCH 2/7] Updated the running the app section of README.md, to be easier to understand --- README.md | 76 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 45cff17e6..f353e7318 100644 --- a/README.md +++ b/README.md @@ -196,45 +196,33 @@ Then, install the dependencies for this project: yarn install --frozen-lockfile ``` -### Tests +### Running the app -Make sure the Jest tests pass: +There are two ways to run Compass locally: -```bash -yarn test -``` +| | Quick start (`yarn dev`) | Full isolation (`yarn dev:isolated`) | +| --------------- | ------------------------------------- | ---------------------------------------------------- | +| Database & auth | Shared remote dev environment | Local Supabase + Firebase emulators | +| Extra setup | None | Docker, Java 21+, Firebase CLI | +| Best for | UI changes, wording fixes, first look | Database, auth, or anything that writes lots of data | -If they don't and you can't find out why, simply raise an issue! Sometimes it's something on our end that we overlooked. +If you're unsure, start with the quick start. You can switch to full isolation later. -### Running the Development Server +After pulling new changes, run `yarn install --frozen-lockfile` before launching either one. -Start the development server: +#### Option A: Quick start (shared dev database) ```bash yarn dev ``` -[//]: # 'Note: the dev remote server is not available until further notice (to save money). The above command will start a local' -[//]: # "server instead (same as `yarn dev:isolated` below). You'll have to set up Firebase and Supabase first (see below)." -[//]: # 'Contact us if you need help; we may spin up the dev remote server for you.' - -Once the server is running, visit http://localhost:3000 to start using the app. You can sign up and visit the profiles; -you should see a few synthetic profiles. - -Note: it's normal if page loading locally is much slower than the deployed version. It can take up to 10 seconds, it -would be great to improve that though! +> ⚠️ The remote dev database is shared by all contributors. Don't reset or wipe it. If you need a clean or disposable +> database, use full isolation. -If you update the database schema (in the SQL files) or the database is not super clean or too large (e.g., someone -clogged it up), you can reset it to a clean state by running: +#### Option B: Full isolation (local database) -```bash -yarn dev:db:reset -``` - -#### Full isolation - -Running `yarn dev:isolated` spins up a local Supabase and Firebase emulator instead of pointing at the shared remote -database. This is strongly recommended for day-to-day development: +`yarn dev:isolated` runs a local Supabase database and Firebase emulators instead of using the shared remote +environment: - **Freedom** — Reset, wipe, and reseed your local database as many times as you want without affecting other contributors. @@ -242,7 +230,7 @@ database. This is strongly recommended for day-to-day development: - **Works offline** — No internet required once services are started locally. - **Faster** — No network latency on every database query. -However, running in full isolation requires installing several heavy dependencies: +The trade-off is heavier setup: - **Docker** (~500MB) — runs the Supabase Postgres container - **Supabase CLI** — manages the local Supabase stack (10+ Docker containers, ~1-2GB RAM when running) @@ -252,11 +240,7 @@ However, running in full isolation requires installing several heavy dependencie First startup is slow (30-60s) and the stack uses significant memory. If your machine has less than 8GB RAM, you may notice slowdowns. -If this feels like too much, you can skip isolation entirely — `yarn dev` works out of the box against the shared remote -and is perfectly fine for most contributions, especially UI changes, wording fixes, or anything that doesn't touch the -database or authentication. - -###### Setup instructions +##### Prerequisites As always, don't hesitate to raise an issue if you run into any problems! @@ -305,14 +289,34 @@ npm install -g firebase-tools firebase --version ``` -Run in isolation +##### Run + +Start each session with a fresh local database that has all the latest migrations: ```bash +yarn test:db:reset yarn dev:isolated ``` -Visit http://localhost:3000 as usual. Your local database comes preloaded with synthetic test profiles so the app -looks and feels like the real thing. +`yarn test:db:reset` only affects your local database and never touches the shared dev one. The Firebase emulator +doesn't keep users between runs, so resetting the database each time keeps the two in sync. + +#### What to expect + +Visit http://localhost:3000. You can sign up and browse a few synthetic profiles. + +Local page loads can be much slower than the deployed site, up to 10 seconds. That's expected (though improvements are +welcome!). + +### Tests + +Before opening a pull request, make sure the Jest tests pass: + +```bash +yarn test +``` + +If they fail and you can't work out why, raise an issue. Sometimes it's something on our end that we overlooked. ### Contributing From 5e72f4fec84ecd7465ccb3a88f1b66e3803f058e Mon Sep 17 00:00:00 2001 From: Okechi Jones-Williams Date: Fri, 23 Oct 2026 18:09:51 +0100 Subject: [PATCH 3/7] This commit is for adding load test func Updated the testing.md with load test info Added load-testing.md focused on explaining load testing and how to interact with it Added JMeter test plan with tests for a singular endpoint Added accounts.csv so that tests can be carried out locally Added workflows to cover smoke and full load tests Added check_results.py, which fails the CI run when error rate or p95 responce time exceeds its limits Updated gitignore to cover the files created by JMeter that wont be commited --- .github/workflows/load-full-test.yml | 22 + .github/workflows/load-smoke-test.yml | 28 + .github/workflows/load-test.yml | 122 +++ .gitignore | 7 + CLAUDE.md | 1 + docs/load-testing.md | 1166 +++++++++++++++++++++++++ docs/testing.md | 10 +- tests/load/accounts.csv | 200 +++++ tests/load/check_results.py | 83 ++ tests/load/compass-testPlan.jmx | 519 +++++++++++ 10 files changed, 2157 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/load-full-test.yml create mode 100644 .github/workflows/load-smoke-test.yml create mode 100644 .github/workflows/load-test.yml create mode 100644 docs/load-testing.md create mode 100644 tests/load/accounts.csv create mode 100644 tests/load/check_results.py create mode 100644 tests/load/compass-testPlan.jmx diff --git a/.github/workflows/load-full-test.yml b/.github/workflows/load-full-test.yml new file mode 100644 index 000000000..7448cd90e --- /dev/null +++ b/.github/workflows/load-full-test.yml @@ -0,0 +1,22 @@ +name: Load test (full) + +# The full load test against the dev API, once a week: 100 users for 120 seconds. +# 100 users is Firebase's limit of 100 new accounts per hour from one IP address. Raising it needs a +# higher sign-up quota in the Firebase console first. +on: + schedule: + - cron: '0 6 * * 1' # Mondays at 06:00 UTC + workflow_dispatch: # adds a "Run workflow" button in the Actions tab + +permissions: + contents: read + +jobs: + full: + uses: ./.github/workflows/load-test.yml + with: + users: 100 + duration: 120 + max_error_rate: 1 + max_p95: 800 + report_name: load-full-report diff --git a/.github/workflows/load-smoke-test.yml b/.github/workflows/load-smoke-test.yml new file mode 100644 index 000000000..d50f59d34 --- /dev/null +++ b/.github/workflows/load-smoke-test.yml @@ -0,0 +1,28 @@ +name: Load test (smoke) + +# A small JMeter run when a pull request changes API-related files, to check the test plan still works. +# It tests the API deployed to dev, not the pull request's code: dev is only updated by running +# backend/api/deploy-api.sh dev by hand. The weekly full run is in load-full-test.yml. +on: + pull_request: + branches: [main] + paths: + - 'backend/api/**' + - 'backend/shared/**' + - 'common/src/api/**' + - 'tests/load/**' + - '.github/workflows/load-*.yml' + workflow_dispatch: # adds a "Run workflow" button in the Actions tab + +permissions: + contents: read + +jobs: + smoke: + uses: ./.github/workflows/load-test.yml + with: + users: 3 + duration: 30 + max_error_rate: 0 + max_p95: 2000 + report_name: load-smoke-report diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml new file mode 100644 index 000000000..1794f8140 --- /dev/null +++ b/.github/workflows/load-test.yml @@ -0,0 +1,122 @@ +name: Load test + +# The steps shared by the smoke and full load tests. It doesn't run on its own: load-smoke-test.yml and +# load-full-test.yml call it with their own settings. See docs/load-testing.md. +# +# It tests the API deployed to dev (the plan's default apiHost), not the code in this checkout. +on: + workflow_call: + inputs: + users: + description: Threads for account creation and the load test (one account each) + type: number + required: true + duration: + description: Seconds the load test runs + type: number + required: true + max_error_rate: + description: Highest error rate allowed for any request, in percent + type: number + required: true + max_p95: + description: Highest p95 response time allowed for TEST requests, in milliseconds + type: number + required: true + report_name: + description: Name of the uploaded report artifact + type: string + required: true + +env: + JMETER_VERSION: '5.6.3' + PLAN: compass-testPlan.jmx + USERS: ${{ inputs.users }} + DURATION: ${{ inputs.duration }} + MAX_ERROR_RATE: ${{ inputs.max_error_rate }} + MAX_P95: ${{ inputs.max_p95 }} + +jobs: + load-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + # Never run two load tests at once: they hit the same dev API, so each would skew the other's results. + concurrency: + group: load-test + cancel-in-progress: false + defaults: + run: + working-directory: tests/load # folder holding the .jmx and check_results.py + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + + - name: Cache JMeter + id: jmeter-cache + uses: actions/cache@v4 + with: + path: ~/jmeter + key: jmeter-${{ env.JMETER_VERSION }} + + - name: Install JMeter + if: steps.jmeter-cache.outputs.cache-hit != 'true' + run: | + cd "$HOME" + base="https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-${JMETER_VERSION}.tgz" + curl -fsSLO "$base" + # make sure the download wasn't corrupted or tampered with + echo "$(curl -fsSL "$base.sha512" | cut -d' ' -f1) apache-jmeter-${JMETER_VERSION}.tgz" | sha512sum -c - + mkdir -p jmeter + tar -xzf "apache-jmeter-${JMETER_VERSION}.tgz" -C jmeter --strip-components=1 + + - name: Add JMeter to PATH + run: echo "$HOME/jmeter/bin" >> "$GITHUB_PATH" + + - name: Make test accounts for this run + run: | + # one row per user: email,password,username,name (no header, no commas inside values) + # The run ID and attempt number make the accounts unique, including when a failed job is re-run. + id="${GITHUB_RUN_ID: -5}${GITHUB_RUN_ATTEMPT}" + rm -f accounts.csv + for i in $(seq 1 "$USERS"); do + echo "loadTest${id}x${i}@example.com,$(openssl rand -hex 8),loadUser${id}x${i},load Name${id}x${i}" >> accounts.csv + done + echo "Made $(wc -l < accounts.csv) accounts" + + - name: Run load test + id: jmeter + run: | + mkdir -p results + jmeter -n -t "$PLAN" \ + -JsetupThreads="$USERS" -Jthreads="$USERS" -Jduration="$DURATION" \ + -l results/results.jtl -j jmeter.log -e -o report + + - name: Check results + run: python3 check_results.py results/results.jtl --max-error-rate "$MAX_ERROR_RATE" --max-p95 "$MAX_P95" + + # tearDown normally deletes the accounts. This only runs if JMeter crashed or the job was cancelled. + - name: Clean up accounts if the test didn't finish + if: always() && steps.jmeter.outcome != 'success' + run: | + [ -s created_accounts.txt ] || { echo "No accounts to clean up"; exit 0; } + jmeter -n -t "$PLAN" \ + -JresetFile=0 -JsetupThreads=0 -Jthreads=0 \ + -l results/cleanup.jtl -j cleanup.log + + - name: Save report + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.report_name }} + path: | + tests/load/report + tests/load/results + tests/load/errors.xml + tests/load/jmeter.log + tests/load/cleanup.log + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 34ff2b9fb..b9497a10a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,13 @@ /tests/reports/playwright-report /tests/e2e/web/.auth/ +# JMeter load tests (generated by each run, see docs/load-testing.md) +/tests/load/created_accounts.txt +/tests/load/errors.xml +/tests/load/jmeter.log +/tests/load/results/ +/tests/load/report/ + # next.js /.next/ /out/ diff --git a/CLAUDE.md b/CLAUDE.md index 1c8f4ee2c..14da4eea9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,6 +132,7 @@ Cross-cutting: [`docs/compatibility-questions.md`](docs/compatibility-questions. compatibility prompts), [`docs/feed.md`](docs/feed.md), [`docs/internationalization.md`](docs/internationalization.md), [`docs/profile-fields.md`](docs/profile-fields.md), [`docs/testing.md`](docs/testing.md), +[`docs/load-testing.md`](docs/load-testing.md) (JMeter load tests), [`docs/logging-monitoring.md`](docs/logging-monitoring.md), [`docs/troubleshooting.md`](docs/troubleshooting.md). Mobile: [`android/README.md`](android/README.md), [`ios/README.md`](ios/README.md), diff --git a/docs/load-testing.md b/docs/load-testing.md new file mode 100644 index 000000000..7f5fb1f0f --- /dev/null +++ b/docs/load-testing.md @@ -0,0 +1,1166 @@ +# JMeter (Load) Testing Guide + +Load tests check how the API holds up when many people use it at the same time. They measure response times, +throughput (requests per second) and error rate, so slowdowns and failures under load show up before real users find +them. They don't check that features work correctly; unit, integration and E2E tests do that (see +[testing.md](testing.md)). + +Load tests use [Apache JMeter](https://jmeter.apache.org/). The test plan, `tests/load/compass-testPlan.jmx`, load +tests the `/me` endpoint with real, temporary accounts. Each run has three stages: + +1. **Create** a set of temporary accounts. +2. **Load test:** all the accounts call the API at the same time, for a set amount of time. +3. **Delete** the accounts again. + +**In this guide:** + +- [Placeholders](#placeholders) and [Prerequisites](#prerequisites), including + [setups for Windows, WSL, Linux and macOS](#choose-your-setup) +- [JMeter basics](#jmeter-basics): the terms and rules the rest of the guide relies on +- [How the plan works](#how-the-plan-works): what happens, stage by stage +- [accounts.csv](#accountscsv): the accounts the plan creates +- [Running the test](#running-the-test): commands, stopping early, flags and properties +- [Why it is structured this way](#why-it-is-structured-this-way) +- [Element reference](#element-reference): what each element does, why it's used and how to change it +- [Adding a new test to this plan](#adding-a-new-test-to-this-plan) +- [Reading the results](#reading-the-results) +- [Automated runs](#automated-runs-github-actions): the smoke and weekly full tests in GitHub Actions +- [Troubleshooting](#troubleshooting) + +--- + +## Placeholders + +Values in angle brackets are placeholders. Replace them with the real values when you copy a command. + +| Placeholder | What it is | Where to find the real value | +| -------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| `` | The API host the plan sends requests to | `apiHost` in the plan's User Defined Variables | +| `` | The Firebase key used to sign up and sign in | `firebaseApiKey` in the plan's User Defined Variables (the dev key is also in `common/src/envs/dev.ts`) | +| `` | The email domain of the test accounts | `tests/load/accounts.csv` | +| `` | The password of the test accounts | `tests/load/accounts.csv` | + +--- + +## Prerequisites + +- **Java 8 or newer** (17+ recommended), installed on the system where JMeter runs. JMeter runs on Java. +- **[Apache JMeter 5.6+](https://jmeter.apache.org/download_jmeter.cgi)**, set up for your system as described in + [Choose your setup](#choose-your-setup). + +The plan's files live in `tests/load/`: + +| File | What it is | +| ---------------------- | ------------------------------------------------------------------------------------------- | +| `compass-testPlan.jmx` | The test plan | +| `accounts.csv` | The accounts to create (see [accounts.csv](#accountscsv)) | +| `check_results.py` | Checks a results file against limits, used by CI (see [check_results.py](#check_resultspy)) | + +A run creates these files. The [commands](#commands) are run from `tests/load/`, so they all land there. They are +generated output and are git-ignored: + +| File or folder | What it is | +| ---------------------- | ------------------------------------------------------------------- | +| `created_accounts.txt` | The accounts created in this run, passed between stages | +| `errors.xml` | Details of failed requests | +| `jmeter.log` | JMeter's own log, including the warnings the plan's scripts write | +| `results/` | The raw results of each run ([`.jtl` files](#the-results-file-jtl)) | +| `report/` | The HTML report | + +### Choose your setup + +The [commands](#commands) use short paths such as `compass-testPlan.jmx` and `results/smoke.jtl`, which only work when +JMeter runs **inside `tests/load/`** on the same system as the project. Find the row that matches your machine, then +follow its section below. + +| Your setup | Run the commands in | Open the GUI with | +| ------------------------------------------- | ---------------------------------------------------------- | ----------------------------------------------- | +| Project and JMeter on the same system | A terminal on that system (PowerShell on Windows) | JMeter on that system | +| Project in WSL, JMeter installed on Windows | A WSL terminal, using the Windows install's Linux launcher | JMeter on Windows, with the plan's Windows path | +| Project in WSL, JMeter installed in WSL | A WSL terminal | JMeter in WSL, shown as a window on Windows | + +Common to every setup: + +- **A WSL terminal** (needed for both WSL setups) is a terminal running Linux inside WSL. Open one with the **Ubuntu** + app from the Start menu, an Ubuntu tab in Windows Terminal, or `wsl` in PowerShell. If the project is open in VS Code + through WSL (the bottom-left corner shows **WSL: Ubuntu**), VS Code's own terminal (**Terminal → New Terminal**) is + already one. +- **Java** must be installed on the system that runs the commands. For both WSL setups that means **in WSL**, even if + Java is also installed on Windows. Check with `java -version`. To install it in WSL, use the full package (the + `-headless` one can't show the GUI): + + ```bash + sudo apt install openjdk-21-jre + ``` + +- **Check your setup** by running `jmeter --version` in the terminal you'll run the commands from. +- **Commands** are then run exactly as written under [Commands](#commands), from `tests/load/`. +- **To open the plan in the GUI:** start JMeter, choose **File → Open**, paste the full path of + `compass-testPlan.jmx` into the file name box and open it. When JMeter runs on the same system as the project, you + can instead run `jmeter -t compass-testPlan.jmx` from `tests/load/`, which opens the plan straight away. + +#### Project and JMeter on the same system + +For example, everything on Linux, on macOS, or on Windows without WSL. + +1. Install JMeter. +2. Add JMeter's `bin/` folder to your PATH, so the `jmeter` command works from any folder. + +To start the GUI, run `jmeter`. On Windows, you can also double-click `jmeter.bat` in JMeter's `bin/` folder. + +#### Project in WSL, JMeter installed on Windows + +The JMeter download is the same for every system, so its `bin/` folder has a launcher for each: `jmeter.bat` for +Windows and `jmeter` (a shell script) for Linux and macOS. WSL sees your Windows drive under `/mnt/c`, so it can run the +Linux launcher from your Windows install, using WSL's Java. + +Put JMeter's `bin/` folder on your WSL PATH in **one** of these two ways. Either way, JMeter's `shutdown.sh` also +becomes available, which you need to [stop a run early](#stopping-a-run-early). + +**Option A: add it to your Windows PATH.** WSL adds the Windows PATH to its own, so this makes `jmeter` work in WSL +**and** in PowerShell. + +1. In the Start menu, search for **Edit environment variables for your account** and open it. +2. Select **Path**, click **Edit**, then **New**, and paste the `bin/` folder's Windows path, e.g. + `C:\tools\apache-jmeter-5.6.3\bin`. Click **OK** in both windows. +3. Close and reopen your terminals and VS Code, because WSL only sees the new PATH in a fresh session. If `jmeter` still + isn't found in WSL, run `wsl --shutdown` in PowerShell and open WSL again. + +This doesn't work if WSL is set not to include the Windows PATH (`appendWindowsPath = false` in `/etc/wsl.conf`). In +that case, use option B. + +**Option B: add it to your WSL PATH only.** Run these in a WSL terminal, from any folder. + +1. Find the WSL path of your JMeter folder, using the Windows path where you unpacked it: + + ```bash + wslpath 'C:\tools\apache-jmeter-5.6.3' + ``` + + This prints e.g. `/mnt/c/tools/apache-jmeter-5.6.3`. + +2. Add its `bin/` folder to your WSL PATH, using the path from step 1: + + ```bash + echo 'export PATH="$PATH:/mnt/c/tools/apache-jmeter-5.6.3/bin"' >> ~/.bashrc && source ~/.bashrc + ``` + +In this setup, `jmeter` starts about a second slower than with JMeter installed in WSL (the very first start of the day +can take a few seconds), because its files are read from the Windows drive. Only start-up is affected: a running test +is just as fast. + +**GUI:** start JMeter on Windows (double-click `jmeter.bat` in its `bin/` folder, or type `jmeter` in PowerShell if you +used option A) and open the plan with its Windows path. It looks like `\\wsl.localhost\\home\\…\tests\load\compass-testPlan.jmx`. To get it, run this from +`tests/load/` in WSL and copy the output: + +```bash +wslpath -w compass-testPlan.jmx +``` + +You can also open the folder in File Explorer with `explorer.exe .` and copy the path from the address bar. + +> **Don't run the commands with `jmeter.bat`.** Batch files run in `cmd.exe`, which can't use a WSL folder (a +> `\\wsl.localhost\…` path) as its working folder. It switches to `C:\Windows` instead, so the short paths in the +> commands point to the wrong place and the run fails. + +#### Project in WSL, JMeter installed in WSL + +Everything runs inside WSL. JMeter starts faster than in the setup above, and its GUI opens as a normal window on +Windows through WSLg, which is built into current versions of WSL. + +Run these in a WSL terminal, from any folder (the first command moves to your home folder itself): + +1. Download and unpack JMeter into your WSL home folder: + + ```bash + cd ~ && curl -LO https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.6.3.tgz && tar -xzf apache-jmeter-5.6.3.tgz + ``` + +2. Add its `bin/` folder to the **front** of your PATH: + + ```bash + echo 'export PATH="$HOME/apache-jmeter-5.6.3/bin:$PATH"' >> ~/.bashrc && source ~/.bashrc + ``` + + Putting it first means this JMeter is used even if another one is on your Windows PATH, because WSL places the + Windows PATH in the middle of its own. + +3. Check which JMeter runs with `command -v jmeter`. It should print a path in your home folder, e.g. + `/home//apache-jmeter-5.6.3/bin/jmeter`. + +**GUI:** run `jmeter`. For **File → Open**, use the plan's WSL path, e.g. +`/home//…/tests/load/compass-testPlan.jmx`. To print it, run `realpath compass-testPlan.jmx` in `tests/load/`. + +--- + +## JMeter basics + +A JMeter test plan is a **tree** of elements. Each element type has one job: + +| Term | What it means | Example in this plan | +| ---------------------- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | +| **Test plan** | The whole file and its top-level settings | [`compass-testPlan.jmx`](#test-plan) | +| **Thread group** | A group of threads that all run the same steps | [`1 - Create accounts`](#thread-groups) | +| **Thread** | One simulated user (see the note below) | One of the 100 threads in a thread group | +| **Sampler** | Something a thread _does_, usually one request | [`SETUP \| Firebase Signup POST`](#http-request) | +| **Config element** | Settings or data used by samplers | [CSV Data Set Config](#csv-data-set-config), [HTTP Header Manager](#http-header-manager) | +| **Pre/Post-processor** | Runs before/after a sampler, e.g. to pull a value out of a response | [JSON Extractor](#json-extractor) | +| **Assertion** | Checks a response and marks the request as failed if it's wrong | [Response Assertion](#response-assertion) (code must be `200`) | +| **Timer** | Adds a pause before requests | [Think time](#constant-timer) | +| **Logic controller** | Controls _how often_ or _whether_ its children run | [Runtime Controller](#runtime-controller) | +| **Listener** | Collects or shows results | [Simple Data Writer](#listeners) | + +> **Words used in this guide:** +> +> - **Thread:** one simulated user that JMeter runs. JMeter's own screens sometimes call these "users", e.g. +> "Number of Threads (users)". +> - **Account:** a login (email, password, token) that a thread uses. +> - **User:** only ever means a real person using the app. + +### Variables and properties + +- **Variables** belong to **one thread**. You read them with `${name}`, e.g. `${authToken}`. Each thread has its own + copy, so 100 threads can each hold a different token. A variable that a thread sets while it runs (like + `authToken`) **doesn't carry over** to other thread groups. +- The exception is variables defined in the Test Plan or in [User Defined Variables](#user-defined-variables). JMeter + gives every thread its own copy of those when the thread starts, so threads in every group can read them. That's why + `${apiHost}` works everywhere. +- **Properties** are **global** to the whole run. You read them with `${__P(name,default)}` and set them from the + command line with `-Jname=value`. This plan uses properties for anything you might want to change per run (thread + count, duration, host). See [Properties](#properties) and [Functions](#functions). + +### Scope: where an element applies + +An element applies to its **parent and everything under its parent** in the tree: + +- A Header Manager placed **under one request** only affects that request. +- A timer placed under a controller applies before **every** request in that controller. +- A CSV Data Set placed under a thread group feeds **every thread** in that group. + +### Order of execution for each request + +For every sampler, JMeter runs the elements in scope in this order: + +```text +config elements → pre-processors → timers → the sampler (request) → post-processors → assertions → listeners +``` + +This matters when one element depends on another. For example, post-processors run **before** assertions, so a script +in a post-processor can't see whether an assertion passed. The [Save account script](#jsr223-scripts-groovy) is an +example. + +--- + +## How the plan works + +The plan has one thread group per stage: + +```text +compass-testPlan.jmx +├── User Defined Variables hosts and endpoints shared by all requests +├── 0 - Reset shared file (setUp) empties created_accounts.txt +├── 1 - Create accounts (setUp) creates accounts, saves them to created_accounts.txt +├── 2 - /me load test (Thread Group) the load test, reads created_accounts.txt +├── 3 - Delete accounts (tearDown) reads created_accounts.txt, deletes each account +└── Listeners Summary Report, Simple Data Writer, View Results Tree +``` + +Account data moves through the plan like this: + +```text +accounts.csv ──► 1 - Create accounts ──► created_accounts.txt ──► 2 - /me load test + └────────► 3 - Delete accounts +``` + +### Stage by stage + +The stages run one after another (see [Test Plan](#test-plan)). The elements in brackets are explained in the +[Element reference](#element-reference). + +**0 - Reset shared file** (setUp, 1 thread) + +- `SETUP | Reset created_accounts.txt` ([JSR223 Sampler](#jsr223-scripts-groovy)) empties `created_accounts.txt`, so + only this run's accounts end up in it. The stage is skipped when `resetFile=0` (used by + [Cleanup only](#cleanup-only)). + +**1 - Create accounts** (setUp, `setupThreads` threads, one account each) + +Each thread takes the next row of `accounts.csv` ([CSV Data Set Config](#csv-data-set-config)), then: + +1. `SETUP | Firebase Signup POST` creates the Firebase login. A [JSON Extractor](#json-extractor) saves the sign-in + token from the response as `authToken`. +2. `SETUP | create-user-and-profile POST` creates the account's app user and profile, signed in with `authToken`. +3. **Save account for test and teardown** ([JSR223 PostProcessor](#jsr223-scripts-groovy)) adds the account to + `created_accounts.txt` as one line: `email|password|token|name`. + +A disabled [Debug Sampler](#debug-sampler) sits at the end of this stage, for checking the thread's variables while +debugging. + +**2 - /me load test** (Thread Group, `meThreads` threads) + +Each thread takes one line of `created_accounts.txt`, so it gets one account. A +[Runtime Controller](#runtime-controller) then repeats this request for `meDuration` seconds: + +- `TEST | /me GET` calls `/me` with that account's token. It passes if the response code is `200` + ([Response Assertion](#response-assertion)) **and** the returned `name` matches the account + ([JSON Assertion](#json-assertion)). + +**Think time** ([Constant Timer](#constant-timer)) can add a pause before each request. It's 0 by default. + +**3 - Delete accounts** (tearDown, 1 thread) + +The thread goes through `created_accounts.txt` line by line until the file runs out. For each account: + +1. `TEARDOWN | Firebase Sign in POST` signs in again to get a fresh token (saved by a JSON Extractor). Tokens expire + after 1 hour, so the one from setUp may be too old. +2. `TEARDOWN | Delete app user` calls `/me/delete`, which deletes the app user, their files and their Firebase login. + It sends the reason `jmeter-load-test`, which the API stores in the `deleted_users` table, so load-test deletions + can be told apart from real users' deletions. + +--- + +## accounts.csv + +### Why it's needed + +The API only accepts most requests from **signed-in accounts**, so the test needs real accounts. Each thread needs its +**own** account: + +- Each row holds one account, so each thread acts like a separate person using the app, as in real traffic. +- Sign-up rejects an email that's already in use, so every row needs its own email. + +### How the plan uses it + +- The file is read by the **CSV - Accounts** element in `1 - Create accounts`. +- Each thread in that group takes the **next unused row**. The sharing mode is _All threads_, so no two threads get + the same row. +- The four columns become the variables `${userEmail}`, `${userPassword}`, `${username}` and `${name}`. These are used + in the bodies of the sign-up and profile requests. +- If there are more threads than rows, the extra threads **stop** without doing anything ("Stop thread on EOF"). + +### Format + +Four columns, separated by commas, with **no header row**: + +```text +email,password,username,name +``` + +Example row: + +```text +loadtest001@,,loadtest001,Load Test 001 +``` + +Rules for each column: + +| Column | Rule | +| ---------- | ------------------------------------------------------------------------------------------------------------------------ | +| `email` | Unique, and not already used by an account in the environment you're testing | +| `password` | At least 6 characters (Firebase's minimum) | +| `username` | Unique, 3-25 letters, numbers or underscores. If it's already taken, the API quietly picks a similar one | +| `name` | At most 30 characters. Stick to letters, numbers and single spaces, and avoid staff words such as "admin" or "moderator" | + +**No commas or `|` inside any value.** A comma always starts a new column, and the accounts are saved to +`created_accounts.txt` with `|` between values. + +**Why the name rules matter:** the API cleans names before saving them. It squeezes repeated spaces, removes emoji and +symbols, cuts names to 30 characters, and rejects names that look like staff. The `/me` test checks that the returned +name matches `accounts.csv` exactly, so if the API changes a name, every `/me` request for that account fails. + +### How to change it + +- **Support more threads:** you need **at least as many rows as `setupThreads`** (100 by default). To make a new file + with more rows, run this from `tests/load/`, using the domain and password from the current file: + + ```bash + domain=''; password=''; count=200 + for i in $(seq -f '%03g' 1 "$count"); do echo "loadtest$i@$domain,$password,loadtest$i,Load Test $i"; done > accounts.csv + ``` + + ```powershell + $domain = ''; $password = ''; $count = 200 + 1..$count | ForEach-Object { $n = '{0:D3}' -f $_; "loadtest$n@$domain,$password,loadtest$n,Load Test $n" } | Set-Content -Encoding ascii accounts.csv + ``` + +- **Keep values unique:** no two rows can share an email or username. +- **Reusing rows is fine:** the accounts are deleted at the end of every run, so the same file works run after run. If + a run fails to clean up, see [Troubleshooting](#sign-up-fails-with-email_exists). +- **Adding, removing or reordering columns:** the columns map to variables **by position**. If you change them, also + update: + 1. **CSV - Accounts → Variable Names**, so the names are in the same order as the columns. + 2. Every request body that uses the variables (e.g. `SETUP | Firebase Signup POST`). + 3. The **Save account for test and teardown** script, if the new value should be passed on to later stages. + +--- + +## Running the test + +### Why non-GUI mode + +Always run load tests from the command line (**non-GUI mode**, the `-n` flag). Use the GUI only to build and debug the +plan. + +- **It uses far less CPU and memory.** The GUI draws results on screen and keeps them in memory while the test runs. + That takes resources away from sending requests, so the machine running JMeter can become the bottleneck, and the + results get less accurate. +- **It works in CI and scripts,** since no screen is needed. + +JMeter's own docs give the same advice, and JMeter prints a warning about it whenever the GUI opens. + +### Commands + +Run these from `tests/load/`, in the terminal for your [setup](#choose-your-setup) (a WSL terminal if the project is in +WSL): + +```bash +cd tests/load +``` + +The results (`results/…`) and report (`report`) paths are relative to that folder. + +#### Smoke test + +1 thread for 10 seconds. Run this 1hr(`Firebase creation limit resets every hour`) before every full run to check that each step works. + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/smoke.jtl -e -o report -Jthreads=1 -JsetupRamp=1 -Jramp=1 -Jduration=10 +``` + +#### Full run with HTML report + +Uses the defaults (100 threads, 120 seconds). + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report +``` + +#### Report with only the load test requests + +Hides the account setup and cleanup requests from the report. The filter is a regular expression that keeps only the +labels starting with `TEST |`. + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report '-Jjmeter.reportgenerator.sample_filter=^TEST \|.*' +``` + +```powershell +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report "-Jjmeter.reportgenerator.sample_filter=^TEST \|.*" +``` + +#### Run against a different environment + +The plan tests the dev environment by default. An API only accepts sign-in tokens from its own Firebase project, so +when you change the API host, also pass the Firebase key of the same environment: + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -JapiHost= -JfirebaseApiKey= +``` + +```powershell +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl "-JapiHost=" -JfirebaseApiKey= +``` + +> **PowerShell note:** wrap any `-J` argument whose value contains a dot in double quotes, as above. Without the +> quotes, PowerShell can split the argument at the dot, and JMeter gets a broken value. + +#### Cleanup only + +Deletes the accounts still listed in `created_accounts.txt`, without creating accounts or running the test. Use it +when a run stopped before cleanup finished. + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/cleanup.jtl -JresetFile=0 -Jthreads=0 +``` + +- `-JresetFile=0` gives `0 - Reset shared file` zero threads, so the file isn't emptied. +- `-Jthreads=0` gives account creation and the load test zero threads, so they're skipped. +- `3 - Delete accounts` always runs with 1 thread, so cleanup is all that runs. + +#### Rebuild a report from an existing results file + +Builds the HTML report again without running the test. The `report` folder must be empty or not exist yet. + +```bash +jmeter -g results/results.jtl -o report +``` + +### Stopping a run early + +To stop a run before it finishes **and still delete the accounts**, stop it gracefully. From another terminal of the +same kind as the one running the test (a WSL terminal if the project is in WSL), run JMeter's shutdown script. It's +in JMeter's `bin/` folder, so it's on your PATH: + +```bash +shutdown.sh +``` + +```powershell +shutdown.cmd +``` + +Type the full name: plain `shutdown` is your operating system's command to turn the computer off. + +JMeter lets the requests in progress finish, then runs `3 - Delete accounts` as usual. In the GUI, use +**Run → Shutdown** for the same effect. + +Pressing **Ctrl+C**, running `stoptest.sh`/`stoptest.cmd`, or using **Run → Stop** in the GUI stops JMeter straight +away, and cleanup **doesn't** run. Afterwards, run [Cleanup only](#cleanup-only) to delete the accounts that were +created. + +### Command flags + +| Flag | Meaning | +| ------------- | --------------------------------------------------------------------------- | +| `-n` | Non-GUI mode | +| `-t ` | The test plan to run | +| `-l ` | Where to save the [raw results](#the-results-file-jtl) (`.jtl`, a CSV file) | +| `-e -o ` | After the run, build an HTML report into this folder | +| `-f` | Delete the old results file and report folder first | +| `-g ` | Build a report from an existing `.jtl` without running a test | +| `-J=` | Set a property the plan reads (see below) | + +Without `-f`, `-l` **adds** to an existing results file (mixing runs together), and `-e -o` **fails** if the report +folder isn't empty. + +### Properties + +The plan reads its settings from properties, so you can change a run without editing the `.jmx`. + +| Property | Default | What it controls | +| ---------------- | -------------------- | -------------------------------------------------------------------------------------- | +| `threads` | `100` | Threads for account creation and every load test, unless overridden | +| `ramp` | `10` | Seconds to start all threads in each load test, unless overridden | +| `duration` | `120` | Seconds each load test runs, unless overridden | +| `setupThreads` | `threads` | Number of accounts to create | +| `setupRamp` | `10` | Seconds to start all account-creation threads | +| `meThreads` | `threads` | Threads in the `/me` test | +| `meRamp` | `ramp` | Seconds to start all threads in the `/me` test | +| `meDuration` | `duration` | Seconds the `/me` test runs | +| `thinkTimeMs` | `0` | Pause before each load test request, per thread (0 = none) | +| `apiHost` | `` | API host to test against | +| `firebaseApiKey` | `` | Firebase key for sign-up and sign-in. Must belong to the same environment as `apiHost` | +| `resetFile` | `1` | `0` keeps `created_accounts.txt` instead of emptying it | + +**Specific settings override general ones.** `-Jthreads=50` sets account creation and every load test to 50 threads. +Adding `-JmeThreads=200` changes only the `/me` test. The reset and delete stages always use 1 thread. + +Each load test has its own property prefix (`me` for the `/me` test). +A new test gets its own `Threads`, `Ramp` and `Duration`. + +**Keep `setupThreads` at least as high as the largest load test.** The tests reuse the accounts created in setUp, and +a thread with no account stops. + +> **Ramp-up** is how long JMeter takes to start all the threads. With 100 threads and a 10-second ramp-up, a new +> thread starts every 0.1 seconds. Starting them all at once is less realistic and can overload sign-in. + +--- + +## Why it is structured this way + +- **Stages allow tests to be the main focus.** Creating accounts is slow and only happens once per account. Doing it in a + separate stage keeps it out of the load test results, so they only measure what's being tested. +- **setUp and tearDown thread groups run automatically.** JMeter runs setUp groups before the main test and tearDown + groups after it. tearDown also runs after a [graceful shutdown](#stopping-a-run-early), so test accounts don't pile + up when you stop a run early that way. +- **Stages run in order.** The test plan has **Run Thread Groups consecutively** turned on, so each group waits for the + one before it to finish (see [Test Plan](#test-plan)). +- **One file, one command.** All stages live in one `.jmx`, so the order is guaranteed and CI only needs one command. +- **A shared file passes accounts between stages.** Variables belong to one thread in one thread group, so later stages + can't see what setUp created. setUp writes each account to `created_accounts.txt`, and later stages read it. +- **Properties instead of fixed values.** The same plan works as a smoke test, a full run or a cleanup, just by + changing the command. No edits to the file needed. +- **Paths are relative to the `.jmx`.** The plan looks for `accounts.csv`, `created_accounts.txt` and `errors.xml` + next to the `.jmx`, so JMeter can be started from any folder, on any machine, and still find them. +- **Clear labels.** Every request name starts with `SETUP |`, `TEST |` or `TEARDOWN |`. This makes the stages easy to + tell apart in the results, and lets the report [filter](#report-with-only-the-load-test-requests) to `TEST |` only. + +--- + +## Element reference + +Each element type used in the plan, with: + +- **What it does:** how the element works. +- **Why it's used here:** what the plan needs it for. +- **How it's set up here:** its key settings in this plan, and why. +- **How to change it:** where to edit it, what else has to change with it, and how to add a new one. + +**GUI basics:** to **add** an element, right-click the element it should go under, choose **Add**, then pick the +category and the element. To **change** an element, click it in the tree and edit the fields on the right. To turn an +element off without deleting it, right-click it → **Disable**. Save with **Ctrl+S**. + +### Test Plan + +**What it does:** The top of the tree. It holds settings that apply to the whole run. + +**Why it's used here:** Its settings make the stages run in order, and make cleanup run after a graceful early stop. + +**How it's set up here:** + +- **Run Thread Groups consecutively (i.e. one at a time): on.** Each stage waits for the one before it to finish. + Without it, groups of the same type start together: `0 - Reset shared file` could empty `created_accounts.txt` while + `1 - Create accounts` is saving accounts to it. +- **Run tearDown Thread Groups after shutdown of main threads: on.** Accounts are still deleted after a + [graceful shutdown](#stopping-a-run-early). It doesn't help after Ctrl+C or **Stop**. + +**How to change it:** Click the top element of the tree and tick or untick the boxes. + +### Thread groups + +**What they do:** A thread group is a set of threads that all run the steps under it. There are three types: + +- **setUp Thread Group:** runs **before** all normal thread groups. +- **Thread Group:** the normal kind, used for the load tests. +- **tearDown Thread Group:** runs **after** all normal thread groups. + +**Why they're used here:** Each stage is its own thread group. That keeps the slow, one-off account creation out of the +measured load, and lets JMeter run setUp and cleanup at the right time without any extra logic. + +**How they're set up here:** + +| Thread group | Type | Threads | Loops | +| ----------------------- | ------------ | --------------------------------------- | ------------------------------------------------------ | +| `0 - Reset shared file` | setUp | `resetFile` (1, or 0 to skip the group) | 1 | +| `1 - Create accounts` | setUp | `setupThreads` | 1 (one account per thread) | +| `2 - /me load test` | Thread Group | `meThreads` | 1 (the Runtime Controller inside repeats the requests) | +| `3 - Delete accounts` | tearDown | 1 | Infinite, stops when `created_accounts.txt` runs out | + +- **Number of Threads (users):** how many threads to run. Here it's a property with a fallback, e.g. + `${__P(setupThreads,${__P(threads,100)})}` (see [Functions](#functions)). A group with 0 threads is skipped, which + is how `resetFile=0` and `threads=0` turn stages off. +- **Ramp-up period (seconds):** how long it takes to start all the threads (see the ramp-up note under + [Properties](#properties)). +- **Loop Count:** how many times each thread runs the steps. "Infinite" means until something stops it (here, the end + of `created_accounts.txt`). +- **Action to be taken after a Sampler error:** **Continue** in all groups, so one failed request doesn't stop a + thread. + +**How to change it:** Click the thread group and edit the fields. To change the numbers for one run, use +[properties](#properties) instead of editing the fields. To add a group, right-click the Test Plan → **Add → Threads +(Users)** → pick the type. Within each type, groups run in their order in the tree. + +### User Defined Variables + +**What it does:** Defines named values once, so every element can use them with `${name}`. Every thread gets its own +copy when it starts (see [Variables and properties](#variables-and-properties)). + +**Why it's used here:** Hosts, endpoint paths and the Firebase key are used by many requests. Keeping them in one place +means you change them once instead of in every request. + +**How it's set up here:** + +| Name | Value | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `baseAuthUrl` | Firebase Auth host | +| `apiHost` | `${__P(apiHost,)}` (can be overridden with `-JapiHost`) | +| `firebaseApiKey` | `${__P(firebaseApiKey,)}` (can be overridden with `-JfirebaseApiKey`) | +| `signupUrl`, `signinUrl`, `deleteAccountUrl`, `getUserDataUrl` | Firebase Auth endpoint paths. `deleteAccountUrl` and `getUserDataUrl` aren't used by any request at the moment | + +**How to change it:** Click **User Defined Variables** and edit the table, or add a row for a new value. To make a +value changeable from the command line, wrap it in `__P`, e.g. `${__P(myValue,default)}`, and run with +`-JmyValue=...`. To add another set: right-click the Test Plan → **Add → Config Element → User Defined Variables**. + +### CSV Data Set Config + +**What it does:** Reads a text file line by line. Each line is split into columns, and the columns become variables +for the thread that read it. + +**Why it's used here:** Each thread needs its own account. The CSV configs hand every thread a different line, from +`accounts.csv` in setUp and from `created_accounts.txt` (which passes the accounts on) in the later stages. + +**How it's set up here:** + +| Element | Thread group | File | Variables | Delimiter | Sharing mode | +| ------------------------ | --------------------- | ---------------------- | --------------------------------------- | --------- | -------------------- | +| `CSV - Accounts` | `1 - Create accounts` | `accounts.csv` | `userEmail,userPassword,username,name` | `,` | All threads | +| `CSV - Created accounts` | `2 - /me load test` | `created_accounts.txt` | `userEmail,userPassword,authToken,name` | `\|` | Current thread group | +| `CSV - created accounts` | `3 - Delete accounts` | `created_accounts.txt` | `userEmail,userPassword,authToken,name` | `\|` | Current thread group | + +- **Filename:** a plain file name is looked up **next to the `.jmx`**. +- **Variable Names:** one name per column, in column order, separated by commas. +- **Delimiter:** the character between columns. `created_accounts.txt` uses `|`, because that's how the + [Save account script](#jsr223-scripts-groovy) writes it. +- **Recycle on EOF?:** **False**. When the file runs out, don't start again from the top, because each account is + used once per group. +- **Stop thread on EOF?:** **True**. A thread with no line left stops, instead of running with empty values. +- **Sharing mode:** which threads share one position in the file. + - **All threads:** one shared reader for the whole run, so every thread in every group gets a different line. Used + for `accounts.csv`. + - **Current thread group:** each thread group gets its own reader, starting from line 1. Used for + `created_accounts.txt`, so every load test **and** the cleanup each go through all the accounts. + +**When threads read a line:** each thread reads the next line at the **start of every loop** of its thread group. In +`2 - /me load test`, each thread loops once, so it keeps one account for the whole test. In `3 - Delete accounts`, the +single thread loops until the file runs out, taking the next account each time. + +**How to change it:** Click the CSV config and edit the fields. If you change the columns of either file, update the +Variable Names to match (see [accounts.csv → How to change it](#how-to-change-it), and the +[Save account script](#jsr223-scripts-groovy) for `created_accounts.txt`). To add one: right-click the thread group → +**Add → Config Element → CSV Data Set Config**. + +### HTTP Request + +**What it does:** Sends one HTTP request and records how long the response took. + +**Why it's used here:** Every step of the plan is an HTTP request: signing up, creating the profile, the load test +itself and the cleanup. The plan's requests, in run order, are listed in [Stage by stage](#stage-by-stage). + +**How it's set up here** (Basic tab), using `SETUP | Firebase Signup POST` as the example: + +- **Name** (at the top): the label shown in results. It starts with `SETUP |`, `TEST |` or `TEARDOWN |` (see + [Clear labels](#why-it-is-structured-this-way)). +- **Protocol:** `https`. +- **Server Name or IP:** a variable, here `${baseAuthUrl}` (requests to the API use `${apiHost}`). Don't include + `https://` here. +- **HTTP Request (method):** `POST`. +- **Path:** `${signupUrl}${firebaseApiKey}`. Variables can be combined. +- **Body Data tab:** the JSON body, with variables inside: + `{ "email": "${userEmail}", "password": "${userPassword}", "returnSecureToken": true }`. + +**How to change it:** Click the request and edit the Basic tab. To add one: right-click the thread group or controller +→ **Add → Sampler → HTTP Request**. Give it a name with the right prefix, plus a +[Header Manager](#http-header-manager) and a [Response Assertion](#response-assertion), like the existing requests. + +### HTTP Header Manager + +**What it does:** Adds HTTP headers to the requests in its scope. + +**Why it's used here:** The API and Firebase expect JSON bodies, and signed-in requests must send a token to prove +which account they come from. + +**How it's set up here:** Each request has its own Header Manager underneath it: + +- `Content-Type: application/json` on every request. +- `Authorization: Bearer ${authToken}` on requests that need a signed-in account. + +**How to change it:** Click the Header Manager and edit its table of header **Name** and **Value**. Values can use +variables. To add one: right-click the **request** → **Add → Config Element → HTTP Header Manager**. Placing it under +the request means it only applies to that request. Placing it under a thread group would apply it to **every** request +in the group (see [Scope](#scope-where-an-element-applies)). + +### JSON Extractor + +**What it does:** Reads a value out of a JSON response and saves it as a variable. + +**Why it's used here:** The sign-in token only exists once Firebase sends it back. The extractor catches it, so later +requests can send it in the `Authorization` header. + +**How it's set up here:** There's one under `SETUP | Firebase Signup POST` and one under +`TEARDOWN | Firebase Sign in POST`. Both take `idToken` from the response and save it as `authToken`. + +| Field | Value | Meaning | +| -------------------------- | ----------- | ------------------------------------------------------- | +| Names of created variables | `authToken` | The variable to save into | +| JSON Path expressions | `$.idToken` | Where to find the value. `$` is the top of the response | +| Match No. (0 for Random) | `1` | Use the first match | +| Default Values | (empty) | What to save if nothing is found | + +**How to change it:** Click the extractor and edit the fields. To capture more values, list more variable names and +JSON paths, separated by `;`. To add one: right-click the **request** → **Add → Post Processors → JSON Extractor**. + +### JSR223 scripts (Groovy) + +**What they do:** Run a short script. A **JSR223 Sampler** is a step of its own and shows up in the results. A +**JSR223 PostProcessor** runs right after the request it sits under. This plan uses **Groovy**, the recommended (and +fastest) language for JMeter scripts. + +Objects a script can use: + +| Object | What it is | +| ------- | ------------------------------------------------------------------------ | +| `vars` | This thread's variables: `vars.get("name")`, `vars.put("name", "value")` | +| `props` | Global properties | +| `prev` | The result of the previous request, e.g. `prev.isSuccessful()` | +| `log` | Writes to `jmeter.log`, e.g. `log.warn("...")` | + +**Why they're used here:** Emptying `created_accounts.txt` and adding accounts to it are easiest with a few lines of +script. + +**How they're set up here:** + +- **Language:** `groovy`. +- **Cache compiled script if available:** **on**. The script is compiled once, which matters when it runs thousands + of times. + +`SETUP | Reset created_accounts.txt` (JSR223 Sampler in `0 - Reset shared file`): + +```groovy +import org.apache.jmeter.services.FileServer +new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt").text = "" +``` + +- `FileServer.getFileServer().getBaseDir()` is the folder containing the `.jmx`. Using it means the file is always + found next to the plan, wherever JMeter was started from. +- `.text = ""` empties the file. + +**Save account for test and teardown** (JSR223 PostProcessor under `SETUP | create-user-and-profile POST`): + +```groovy +import org.apache.jmeter.services.FileServer + +def token = vars.get("authToken") +if (!token) { + log.warn("No token for ${vars.get('userEmail')}, not saving") + return +} +if (!prev.isSuccessful()) { + log.warn("Profile creation failed for ${vars.get('userEmail')}, saving anyway so it gets cleaned up") +} + +def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") +synchronized (FileServer.class) { + file.append([vars.get("userEmail"), vars.get("userPassword"), token, vars.get("name")].join("|") + "\n", "UTF-8") +} +``` + +- **No token:** sign-up failed, so there's no account to save. It logs a warning and stops. +- **Profile failed:** `prev` is the profile request. `prev.isSuccessful()` is false when the response code is 4xx or + 5xx. The script runs before the Response Assertion, so the assertion's result isn't included (see + [Order of execution](#order-of-execution-for-each-request)). The Firebase login exists either way, so the account + is **saved anyway**, and cleanup deletes it. +- **`synchronized (FileServer.class)`:** many threads write to the same file at the same time. This lets only one + thread write at a time, so lines never get mixed together. +- **`join("|")`:** joins the values into one line in the format `created_accounts.txt` expects. + +**How to change them:** Click the element and edit the **Script** box. If you add a value to `created_accounts.txt`, +add it to the `join` list here **and** to the Variable Names of both CSV configs that read the file, in the same +position. To add one: right-click a thread group → **Add → Sampler → JSR223 Sampler**, or right-click a request → +**Add → Post Processors → JSR223 PostProcessor**. Check that the language is `groovy`. + +### Response Assertion + +**What it does:** Checks part of a response against a pattern. If it doesn't match, the request is marked as +**failed** in the results. + +**Why it's used here:** On its own, JMeter only marks a request as failed when the response code is 4xx or 5xx. The +assertion makes the check strict: anything other than exactly `200` counts as a failure. + +**How it's set up here:** Every request has one: + +- **Field to Test:** `Response Code` +- **Pattern Matching Rules:** `Equals` +- **Patterns to Test:** `200` + +**How to change it:** Click the assertion and edit **Patterns to Test**, e.g. if a new request should return `201`. +To add one: right-click the **request** → **Add → Assertions → Response Assertion**. + +### JSON Assertion + +**What it does:** Checks that a value exists in a JSON response, and optionally that it equals an expected value. + +**Why it's used here:** A `200` only says the request worked. Checking the returned `name` proves the API sent back +**this** account's data, not someone else's. + +**How it's set up here:** On `TEST | /me GET`: + +- **Assert JSON Path exists:** `$.name` +- **Additionally assert value:** on +- **Expected Value:** `${name}`, the account's name from `created_accounts.txt` +- **Match as regular expression:** **off**. When it's on, characters like `.` or `(` in a name are treated as pattern + symbols, and valid names can fail. + +Because of this check, names in `accounts.csv` must follow the [name rules](#format). + +**How to change it:** Click the assertion and change the JSON path and expected value to something that proves the +response is right. To add one: right-click the **request** → **Add → Assertions → JSON Assertion**. + +### Runtime Controller + +**What it does:** Repeats the elements under it, over and over, until a set number of seconds has passed. + +**Why it's used here:** It makes the load test run for a fixed time, so a test takes the same time however fast or slow +the API responds. + +**How it's set up here:** It wraps `TEST | /me GET`, so each thread keeps sending requests for `meDuration` seconds. + +- **Runtime (seconds):** `${__P(meDuration,${__P(duration,120)})}` + +**How to change it:** Change the time per run with `-JmeDuration` or `-Jduration` (see [Properties](#properties)) +rather than editing the field. To test a sequence of requests, put more requests under the controller. To add one: +right-click the thread group → **Add → Logic Controller → Runtime Controller**, then drag the requests onto it. + +### Constant Timer + +**What it does:** Pauses each thread for a fixed time **before every request in its scope**. + +**Why it's used here:** As **think time**: an optional pause between requests, so threads can act more like real +people. It defaults to `0` (no pause, maximum load). + +**How it's set up here:** It's called **Think time** and sits under the Runtime Controller. + +- **Thread Delay (in milliseconds):** `${__P(thinkTimeMs,0)}` + +**How to change it:** Set the pause per run with `-JthinkTimeMs=`. To add one: right-click the +controller or thread group → **Add → Timer → Constant Timer**. + +### Listeners + +**What they do:** Collect and show results. They apply to every request in their scope; here they sit at the top +level, so they see everything. + +**Why they're used here:** To keep the details of failed requests without slowing the run, and to show results live +when debugging in the GUI. + +**How they're set up here:** + +| Listener | Status | What it does | +| ---------------------- | -------- | ---------------------------------------------------------------------------------------- | +| **Summary Report** | Enabled | A live results table, only useful in the GUI | +| **Simple Data Writer** | Enabled | Saves **failed requests only** to `errors.xml`, with the response body showing why | +| **View Results Tree** | Disabled | Shows every request and response in detail. Great for debugging, too heavy for load runs | + +Simple Data Writer settings: + +- **Filename:** `errors.xml` (relative to the `.jmx`). +- **Log/Display Only: Errors:** ticked, so only failed requests are saved. +- **Configure** button: + - **Save Response Data (XML):** on, to see why a request failed. + - **Save Sampler Data (XML):** **off**. Request bodies contain passwords and tokens. + +The results file from `-l` is separate from these listeners. It always records every request. + +**How to change them:** To debug a smoke test in the GUI, enable **View Results Tree** (right-click → **Enable**), and +disable it again before a load run. To add one: right-click the Test Plan → **Add → Listener**. + +### Debug Sampler + +**What it does:** A sampler that doesn't send a request. Instead, it records the thread's variables at that point in +the steps, so you can read them in **View Results Tree**. + +**Why it's used here:** To check what a thread holds while building or fixing the plan, e.g. whether `authToken` was +extracted, or which row of `accounts.csv` the thread read. It's **disabled** normally, because it adds a row per thread +to the results and isn't needed during a load run. + +**How it's set up here:** It's the last step of `1 - Create accounts`, after `SETUP | create-user-and-profile POST`, so +it shows each thread's variables once its account exists. It shows **JMeter variables** only; JMeter properties and +system properties are turned off. + +**How to change it:** Right-click it → **Enable**, enable **View Results Tree** as well, and run a smoke test in the +GUI. Select the **Debug Sampler** row and open the **Response data** tab to see the variables, which include the +account's password and token. Disable both again before a load run. To check variables somewhere else, add one there: +right-click the thread group → **Add → Sampler → Debug Sampler**, and drag it to the step you want to check. + +### Functions + +**What they do:** Special `${__name(...)}` values that JMeter works out at run time. + +**Why they're used here:** `__P` lets the command line change the plan's settings without editing the `.jmx`. + +**How they're set up here:** The plan only uses `__P`, which reads a property: + +- `${__P(duration,120)}` means the `duration` property, or `120` if it isn't set. +- `${__P(setupThreads,${__P(threads,100)})}` means `setupThreads`; if that isn't set, `threads`; if that isn't set + either, `100`. This is how specific settings override general ones. + +**How to change them:** Edit the function where it's used, e.g. to change a default value. **Tools → Function Helper +Dialog** in the GUI lists every function and builds the syntax for you. + +--- + +## Adding a new test to this plan + +New load tests **reuse the accounts** created by setUp and deleted by tearDown, so you only need to build the test +itself. + +1. **Duplicate the existing test group.** Right-click `2 - /me load test` → **Duplicate**. Drag the copy so it sits + **after the last load test and before `3 - Delete accounts`**. +2. **Rename it** `N - `, where `N` is its position in the run order. +3. **Choose a property prefix** for the new test, e.g. ``, and set these fields: + - Number of Threads: `${__P(Threads,${__P(threads,100)})}` + - Ramp-up period: `${__P(Ramp,${__P(ramp,10)})}` + - Runtime Controller → Runtime: `${__P(Duration,${__P(duration,120)})}` +4. **Keep the `CSV - Created accounts` element** as it is. Its sharing mode, _Current thread group_, gives this test + its own pass through all the accounts. +5. **Change the HTTP Request:** + - Name: `TEST | ` + - Method, Path and Body Data for the new request + - Keep the Header Manager with `Authorization: Bearer ${authToken}` if the request needs a signed-in account +6. **Update the assertions:** + - Keep the Response Assertion if `200` is the expected code, or change it. + - Change or remove the JSON Assertion so it checks something in **this** response. +7. **Optional:** adjust or remove the think time timer, or add more requests inside the Runtime Controller to test a + sequence of requests. +8. **Check account numbers:** `setupThreads` must cover the new test's thread count (see [Properties](#properties)). +9. **Save, then run a [smoke test](#smoke-test).** Check that the new `TEST |` rows show up and pass. + +**Things to keep in mind:** + +- **Tests run one after another and share the same accounts.** Total run time is the sum of all test durations. +- **Long runs need fresh tokens.** All tests use the tokens created in setUp, which expire after 1 hour. If the tests + together run longer than that, later tests start failing with `401`. In that case, add a sign-in request (like + `TEARDOWN | Firebase Sign in POST` and its JSON Extractor) at the start of the new test group, before the Runtime + Controller. +- **Tests that change account data affect later tests.** For example, a test that edits the profile name makes a later + test that checks the name fail. Put such tests last, or make later checks not depend on that data. + +--- + +## Reading the results + +A run gives you three outputs: the results file (`.jtl`), the HTML report built from it, and `errors.xml`. + +### The results file (`.jtl`) + +The file you name with `-l` (e.g. `results/results.jtl`) holds the **raw results** of a run. JMeter writes one row for +every request it sends, in every stage, with how long it took and whether it passed. The HTML report and its statistics +are all calculated from this file. + +For example, a smoke test gives about 50 rows: one for each setup and cleanup request, plus one for every `/me` call +the thread made in its 10 seconds. + +It's a plain CSV file with a header row, so it opens in any spreadsheet. The most useful columns: + +| Column | Meaning | +| ------------------------------------------- | ------------------------------------------------------------------------------ | +| `timeStamp` | When the request was sent (milliseconds since 1970) | +| `elapsed` | Response time in ms, the main number in a load test | +| `label` | Which request it was, e.g. `TEST \| /me GET` | +| `responseCode`, `success`, `failureMessage` | Whether it passed, and why not if it failed | +| `Latency` | Time until the first byte of the response arrived | +| `Connect` | Time spent opening the connection | +| `allThreads` | How many threads were running at that moment, which shows the load at the time | +| `threadName` | Which thread sent it, e.g. `1 - Create accounts 2-1` | + +What it's used for: + +- **Building the HTML report.** `-e -o report` builds it at the end of a run, and + [Rebuild a report](#rebuild-a-report-from-an-existing-results-file) builds it again later from the same file. +- **Looking at single requests.** Sort by `elapsed` to find the slowest ones, or filter on `success` = `false` to see + what failed and when. +- **Comparing runs.** Keep the file from each run, e.g. before and after a change to the API. Because the commands use + `-f`, a run overwrites the file with the same name, so copy or rename the ones you want to keep. + +It doesn't contain passwords or tokens. The `URL` column does include the Firebase API key, but that key isn't secret +(the web app sends it to every browser), and `results/` is git-ignored anyway. + +### The HTML report + +- **Check the run matches what you asked for.** In the **Statistics** table, `#Samples` should fit the threads and + duration you used, e.g. 100 sign-up samples for 100 threads. +- **Error %** shows the share of failed requests per label. Any failures in `SETUP |` rows mean fewer accounts, and so + less load, than intended. +- **Response times:** _Average_ can hide slow outliers. The **90th/95th/99th pct** columns show the time that + 90/95/99% of requests finished within. + +### errors.xml + +Holds only the failed requests, each with the response body that shows why it failed. Unlike the `.jtl`, which `-f` replaces, it +**adds to itself on every run**, so delete it before a run if you only want that run's errors. + +--- + +## Automated runs (GitHub Actions) + +Two GitHub Actions workflows run the plan automatically. Both run the same steps, kept in +`.github/workflows/load-test.yml`, with different settings: + +| | Smoke test | Full test | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | +| Workflow | `load-smoke-test.yml` | `load-full-test.yml` | +| Runs when | A pull request into `main` changes API-related files: `backend/api/`, `backend/shared/`, `common/src/api/`, `tests/load/` or the load test workflows | Every Monday at 06:00 UTC | +| Size | 3 threads for 30 seconds | 100 threads for 120 seconds | +| Passes when | No request fails, and the p95 of `TEST \|` requests is at most 2000 ms | At most 1% of each request fails, and the p95 of `TEST \|` requests is at most 800 ms | +| Report artifact | `load-smoke-report` | `load-full-report` | + +You can also start either one by hand: open the **Actions** tab, pick the workflow and click **Run workflow**. + +> **They test the API deployed to dev, not the pull request's code.** Nothing deploys to dev automatically: dev +> only changes when someone runs `./deploy-api.sh dev` in `backend/api/`. To load test a change, deploy it to dev +> first, then start the smoke or full test by hand. + +### What a run does + +1. Installs JMeter (cached between runs) and checks the download against Apache's SHA-512 checksum. +2. Makes a fresh `accounts.csv` with one account per thread, each with a random password. The run's ID and attempt + number make every account unique, including when a failed run is re-run. The committed `accounts.csv` isn't used. +3. Runs the plan with the workflow's size, and builds the HTML report. +4. Checks the results with [check_results.py](#check_resultspy). This step fails the run when a limit is broken. +5. If JMeter didn't finish (it crashed, or the run was cancelled or timed out), runs [Cleanup only](#cleanup-only), + so the accounts don't stay on dev. +6. Uploads the report, the results, `errors.xml` and the logs. To get them, open the run in the **Actions** tab and + download the artifact under **Artifacts**. + +Only one load test runs at a time, because two tests on the same dev API would skew each other's results. A run that +starts while another is going waits for it to finish; if several are waiting, only the newest one runs. + +The full test's 100 threads match Firebase's limit of 100 new accounts per hour from one IP address. To use more, +raise the sign-up quota in the Firebase console first. + +To change a workflow's size or limits, edit the values under `with:` in `load-smoke-test.yml` or +`load-full-test.yml`. + +### check_results.py + +JMeter exits as if all went well even when requests fail, so CI needs something else to decide pass or fail. +`tests/load/check_results.py` reads a results file and checks two limits for each request label: + +- **Error rate**, for every request: `SETUP |`, `TEST |` and `TEARDOWN |`. A failed setup request means less load than + asked for, and a failed cleanup request leaves accounts behind on dev. +- **p95 response time**, for `TEST |` requests only, because setup and cleanup times aren't what the test measures. + It's only checked when a label has at least 20 samples, since fewer is too little to judge. + +It also fails when there are no `TEST |` results at all, which means no accounts were created. + +You can run it on a local run's results too, from `tests/load/` (on Windows, use `python` instead of `python3`): + +```bash +python3 check_results.py results/results.jtl --max-error-rate 1 --max-p95 800 +``` + +It prints each request's sample count, error rate and p95, then either `PASS` or one `FAIL:` line per broken limit. +It exits with `0` for a pass and `1` for a fail. Without the options, the limits are 1% errors and 800 ms. + +--- + +## Troubleshooting + +### Sign-up fails with `EMAIL_EXISTS` + +A previous run didn't finish cleaning up, so accounts from `accounts.csv` still exist. + +- If `created_accounts.txt` still lists them, run [Cleanup only](#cleanup-only). +- If it doesn't (every run empties it at the start), rebuild it from `accounts.csv`, then run + [Cleanup only](#cleanup-only). Cleanup signs in again with each email and password, so the token column can be any + text: + + ```bash + awk -F, '{print $1 "|" $2 "|x|" $4}' accounts.csv > created_accounts.txt + ``` + + ```powershell + Import-Csv accounts.csv -Header email,password,username,name | ForEach-Object { "$($_.email)|$($_.password)|x|$($_.name)" } | Set-Content -Encoding ascii created_accounts.txt + ``` + + Accounts that no longer exist just show up as failed `TEARDOWN |` requests. + +Don't delete the accounts in the Firebase console instead. That only removes the Firebase login. The app user stays in +the database, and nothing warns you: the next run still works, because the API quietly gives the new accounts +different usernames, so the leftover users pile up. + +### Fewer load test threads than expected + +`created_accounts.txt` has fewer lines than the test has threads, so some threads stop right away. Check the `SETUP |` +rows in the results for failures. Also make sure `setupThreads` is high enough (see [Properties](#properties)), and +that `accounts.csv` has enough rows. + +### `/me` requests fail the JSON Assertion + +The `name` the API returned doesn't match the one in `accounts.csv`. The API cleans names when it saves them, so a +name with emoji, symbols, double spaces or more than 30 characters comes back changed. Fix the names (see the +[name rules](#format)) and run again. + +### `${something}` appears literally in a request + +The variable was never set. Common causes: + +- The name is spelled differently (names are case-sensitive). +- The element that sets it is out of scope (e.g. it's in another thread group). +- The extractor found nothing. + +Enable **View Results Tree** and run a smoke test in the GUI to see the actual requests sent. To see every variable a +thread holds, also enable the [Debug Sampler](#debug-sampler). + +### Report folder error + +JMeter won't write a report into a folder that isn't empty. Add `-f`, or delete the report folder first. diff --git a/docs/testing.md b/docs/testing.md index db10ae481..47f601b64 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -22,7 +22,7 @@ How we apply it here ### Test types at a glance -This project uses three complementary test types. Use the right level for the job: +This project uses four complementary test types. Use the right level for the job: - Unit tests - Purpose: Verify a single function/module in isolation; fast, deterministic. @@ -48,6 +48,14 @@ This project uses three complementary test types. Use the right level for the jo - Naming: `*.e2e.spec.ts`. - When to use: Critical journeys (signup, login, checkout), cross-service interactions, smoke tests for deployments. +- Load tests + - Purpose: Check how the API behaves when many users hit it at once (response times, throughput, error rate). + - Where: `tests/load/` (`compass-testPlan.jmx`). + - Runner: Apache JMeter 5.6+. + - When to use: Before releases or after changes to frequently used endpoints. Not for checking correctness; + unit, integration and E2E tests cover that. + - Guide: [Load testing guide](load-testing.md). + Quick commands ```bash diff --git a/tests/load/accounts.csv b/tests/load/accounts.csv new file mode 100644 index 000000000..283536503 --- /dev/null +++ b/tests/load/accounts.csv @@ -0,0 +1,200 @@ +compassLoadTest001@testExample.com,password123,LoadTestUser001,LoadTestName001 +compassLoadTest002@testExample.com,password123,LoadTestUser002,LoadTestName002 +compassLoadTest003@testExample.com,password123,LoadTestUser003,LoadTestName003 +compassLoadTest004@testExample.com,password123,LoadTestUser004,LoadTestName004 +compassLoadTest005@testExample.com,password123,LoadTestUser005,LoadTestName005 +compassLoadTest006@testExample.com,password123,LoadTestUser006,LoadTestName006 +compassLoadTest007@testExample.com,password123,LoadTestUser007,LoadTestName007 +compassLoadTest008@testExample.com,password123,LoadTestUser008,LoadTestName008 +compassLoadTest009@testExample.com,password123,LoadTestUser009,LoadTestName009 +compassLoadTest010@testExample.com,password123,LoadTestUser010,LoadTestName010 +compassLoadTest011@testExample.com,password123,LoadTestUser011,LoadTestName011 +compassLoadTest012@testExample.com,password123,LoadTestUser012,LoadTestName012 +compassLoadTest013@testExample.com,password123,LoadTestUser013,LoadTestName013 +compassLoadTest014@testExample.com,password123,LoadTestUser014,LoadTestName014 +compassLoadTest015@testExample.com,password123,LoadTestUser015,LoadTestName015 +compassLoadTest016@testExample.com,password123,LoadTestUser016,LoadTestName016 +compassLoadTest017@testExample.com,password123,LoadTestUser017,LoadTestName017 +compassLoadTest018@testExample.com,password123,LoadTestUser018,LoadTestName018 +compassLoadTest019@testExample.com,password123,LoadTestUser019,LoadTestName019 +compassLoadTest020@testExample.com,password123,LoadTestUser020,LoadTestName020 +compassLoadTest021@testExample.com,password123,LoadTestUser021,LoadTestName021 +compassLoadTest022@testExample.com,password123,LoadTestUser022,LoadTestName022 +compassLoadTest023@testExample.com,password123,LoadTestUser023,LoadTestName023 +compassLoadTest024@testExample.com,password123,LoadTestUser024,LoadTestName024 +compassLoadTest025@testExample.com,password123,LoadTestUser025,LoadTestName025 +compassLoadTest026@testExample.com,password123,LoadTestUser026,LoadTestName026 +compassLoadTest027@testExample.com,password123,LoadTestUser027,LoadTestName027 +compassLoadTest028@testExample.com,password123,LoadTestUser028,LoadTestName028 +compassLoadTest029@testExample.com,password123,LoadTestUser029,LoadTestName029 +compassLoadTest030@testExample.com,password123,LoadTestUser030,LoadTestName030 +compassLoadTest031@testExample.com,password123,LoadTestUser031,LoadTestName031 +compassLoadTest032@testExample.com,password123,LoadTestUser032,LoadTestName032 +compassLoadTest033@testExample.com,password123,LoadTestUser033,LoadTestName033 +compassLoadTest034@testExample.com,password123,LoadTestUser034,LoadTestName034 +compassLoadTest035@testExample.com,password123,LoadTestUser035,LoadTestName035 +compassLoadTest036@testExample.com,password123,LoadTestUser036,LoadTestName036 +compassLoadTest037@testExample.com,password123,LoadTestUser037,LoadTestName037 +compassLoadTest038@testExample.com,password123,LoadTestUser038,LoadTestName038 +compassLoadTest039@testExample.com,password123,LoadTestUser039,LoadTestName039 +compassLoadTest040@testExample.com,password123,LoadTestUser040,LoadTestName040 +compassLoadTest041@testExample.com,password123,LoadTestUser041,LoadTestName041 +compassLoadTest042@testExample.com,password123,LoadTestUser042,LoadTestName042 +compassLoadTest043@testExample.com,password123,LoadTestUser043,LoadTestName043 +compassLoadTest044@testExample.com,password123,LoadTestUser044,LoadTestName044 +compassLoadTest045@testExample.com,password123,LoadTestUser045,LoadTestName045 +compassLoadTest046@testExample.com,password123,LoadTestUser046,LoadTestName046 +compassLoadTest047@testExample.com,password123,LoadTestUser047,LoadTestName047 +compassLoadTest048@testExample.com,password123,LoadTestUser048,LoadTestName048 +compassLoadTest049@testExample.com,password123,LoadTestUser049,LoadTestName049 +compassLoadTest050@testExample.com,password123,LoadTestUser050,LoadTestName050 +compassLoadTest051@testExample.com,password123,LoadTestUser051,LoadTestName051 +compassLoadTest052@testExample.com,password123,LoadTestUser052,LoadTestName052 +compassLoadTest053@testExample.com,password123,LoadTestUser053,LoadTestName053 +compassLoadTest054@testExample.com,password123,LoadTestUser054,LoadTestName054 +compassLoadTest055@testExample.com,password123,LoadTestUser055,LoadTestName055 +compassLoadTest056@testExample.com,password123,LoadTestUser056,LoadTestName056 +compassLoadTest057@testExample.com,password123,LoadTestUser057,LoadTestName057 +compassLoadTest058@testExample.com,password123,LoadTestUser058,LoadTestName058 +compassLoadTest059@testExample.com,password123,LoadTestUser059,LoadTestName059 +compassLoadTest060@testExample.com,password123,LoadTestUser060,LoadTestName060 +compassLoadTest061@testExample.com,password123,LoadTestUser061,LoadTestName061 +compassLoadTest062@testExample.com,password123,LoadTestUser062,LoadTestName062 +compassLoadTest063@testExample.com,password123,LoadTestUser063,LoadTestName063 +compassLoadTest064@testExample.com,password123,LoadTestUser064,LoadTestName064 +compassLoadTest065@testExample.com,password123,LoadTestUser065,LoadTestName065 +compassLoadTest066@testExample.com,password123,LoadTestUser066,LoadTestName066 +compassLoadTest067@testExample.com,password123,LoadTestUser067,LoadTestName067 +compassLoadTest068@testExample.com,password123,LoadTestUser068,LoadTestName068 +compassLoadTest069@testExample.com,password123,LoadTestUser069,LoadTestName069 +compassLoadTest070@testExample.com,password123,LoadTestUser070,LoadTestName070 +compassLoadTest071@testExample.com,password123,LoadTestUser071,LoadTestName071 +compassLoadTest072@testExample.com,password123,LoadTestUser072,LoadTestName072 +compassLoadTest073@testExample.com,password123,LoadTestUser073,LoadTestName073 +compassLoadTest074@testExample.com,password123,LoadTestUser074,LoadTestName074 +compassLoadTest075@testExample.com,password123,LoadTestUser075,LoadTestName075 +compassLoadTest076@testExample.com,password123,LoadTestUser076,LoadTestName076 +compassLoadTest077@testExample.com,password123,LoadTestUser077,LoadTestName077 +compassLoadTest078@testExample.com,password123,LoadTestUser078,LoadTestName078 +compassLoadTest079@testExample.com,password123,LoadTestUser079,LoadTestName079 +compassLoadTest080@testExample.com,password123,LoadTestUser080,LoadTestName080 +compassLoadTest081@testExample.com,password123,LoadTestUser081,LoadTestName081 +compassLoadTest082@testExample.com,password123,LoadTestUser082,LoadTestName082 +compassLoadTest083@testExample.com,password123,LoadTestUser083,LoadTestName083 +compassLoadTest084@testExample.com,password123,LoadTestUser084,LoadTestName084 +compassLoadTest085@testExample.com,password123,LoadTestUser085,LoadTestName085 +compassLoadTest086@testExample.com,password123,LoadTestUser086,LoadTestName086 +compassLoadTest087@testExample.com,password123,LoadTestUser087,LoadTestName087 +compassLoadTest088@testExample.com,password123,LoadTestUser088,LoadTestName088 +compassLoadTest089@testExample.com,password123,LoadTestUser089,LoadTestName089 +compassLoadTest090@testExample.com,password123,LoadTestUser090,LoadTestName090 +compassLoadTest091@testExample.com,password123,LoadTestUser091,LoadTestName091 +compassLoadTest092@testExample.com,password123,LoadTestUser092,LoadTestName092 +compassLoadTest093@testExample.com,password123,LoadTestUser093,LoadTestName093 +compassLoadTest094@testExample.com,password123,LoadTestUser094,LoadTestName094 +compassLoadTest095@testExample.com,password123,LoadTestUser095,LoadTestName095 +compassLoadTest096@testExample.com,password123,LoadTestUser096,LoadTestName096 +compassLoadTest097@testExample.com,password123,LoadTestUser097,LoadTestName097 +compassLoadTest098@testExample.com,password123,LoadTestUser098,LoadTestName098 +compassLoadTest099@testExample.com,password123,LoadTestUser099,LoadTestName099 +compassLoadTest100@testExample.com,password123,LoadTestUser100,LoadTestName100 +compassLoadTest101@testExample.com,password123,LoadTestUser101,LoadTestName101 +compassLoadTest102@testExample.com,password123,LoadTestUser102,LoadTestName102 +compassLoadTest103@testExample.com,password123,LoadTestUser103,LoadTestName103 +compassLoadTest104@testExample.com,password123,LoadTestUser104,LoadTestName104 +compassLoadTest105@testExample.com,password123,LoadTestUser105,LoadTestName105 +compassLoadTest106@testExample.com,password123,LoadTestUser106,LoadTestName106 +compassLoadTest107@testExample.com,password123,LoadTestUser107,LoadTestName107 +compassLoadTest108@testExample.com,password123,LoadTestUser108,LoadTestName108 +compassLoadTest109@testExample.com,password123,LoadTestUser109,LoadTestName109 +compassLoadTest110@testExample.com,password123,LoadTestUser110,LoadTestName110 +compassLoadTest111@testExample.com,password123,LoadTestUser111,LoadTestName111 +compassLoadTest112@testExample.com,password123,LoadTestUser112,LoadTestName112 +compassLoadTest113@testExample.com,password123,LoadTestUser113,LoadTestName113 +compassLoadTest114@testExample.com,password123,LoadTestUser114,LoadTestName114 +compassLoadTest115@testExample.com,password123,LoadTestUser115,LoadTestName115 +compassLoadTest116@testExample.com,password123,LoadTestUser116,LoadTestName116 +compassLoadTest117@testExample.com,password123,LoadTestUser117,LoadTestName117 +compassLoadTest118@testExample.com,password123,LoadTestUser118,LoadTestName118 +compassLoadTest119@testExample.com,password123,LoadTestUser119,LoadTestName119 +compassLoadTest120@testExample.com,password123,LoadTestUser120,LoadTestName120 +compassLoadTest121@testExample.com,password123,LoadTestUser121,LoadTestName121 +compassLoadTest122@testExample.com,password123,LoadTestUser122,LoadTestName122 +compassLoadTest123@testExample.com,password123,LoadTestUser123,LoadTestName123 +compassLoadTest124@testExample.com,password123,LoadTestUser124,LoadTestName124 +compassLoadTest125@testExample.com,password123,LoadTestUser125,LoadTestName125 +compassLoadTest126@testExample.com,password123,LoadTestUser126,LoadTestName126 +compassLoadTest127@testExample.com,password123,LoadTestUser127,LoadTestName127 +compassLoadTest128@testExample.com,password123,LoadTestUser128,LoadTestName128 +compassLoadTest129@testExample.com,password123,LoadTestUser129,LoadTestName129 +compassLoadTest130@testExample.com,password123,LoadTestUser130,LoadTestName130 +compassLoadTest131@testExample.com,password123,LoadTestUser131,LoadTestName131 +compassLoadTest132@testExample.com,password123,LoadTestUser132,LoadTestName132 +compassLoadTest133@testExample.com,password123,LoadTestUser133,LoadTestName133 +compassLoadTest134@testExample.com,password123,LoadTestUser134,LoadTestName134 +compassLoadTest135@testExample.com,password123,LoadTestUser135,LoadTestName135 +compassLoadTest136@testExample.com,password123,LoadTestUser136,LoadTestName136 +compassLoadTest137@testExample.com,password123,LoadTestUser137,LoadTestName137 +compassLoadTest138@testExample.com,password123,LoadTestUser138,LoadTestName138 +compassLoadTest139@testExample.com,password123,LoadTestUser139,LoadTestName139 +compassLoadTest140@testExample.com,password123,LoadTestUser140,LoadTestName140 +compassLoadTest141@testExample.com,password123,LoadTestUser141,LoadTestName141 +compassLoadTest142@testExample.com,password123,LoadTestUser142,LoadTestName142 +compassLoadTest143@testExample.com,password123,LoadTestUser143,LoadTestName143 +compassLoadTest144@testExample.com,password123,LoadTestUser144,LoadTestName144 +compassLoadTest145@testExample.com,password123,LoadTestUser145,LoadTestName145 +compassLoadTest146@testExample.com,password123,LoadTestUser146,LoadTestName146 +compassLoadTest147@testExample.com,password123,LoadTestUser147,LoadTestName147 +compassLoadTest148@testExample.com,password123,LoadTestUser148,LoadTestName148 +compassLoadTest149@testExample.com,password123,LoadTestUser149,LoadTestName149 +compassLoadTest150@testExample.com,password123,LoadTestUser150,LoadTestName150 +compassLoadTest151@testExample.com,password123,LoadTestUser151,LoadTestName151 +compassLoadTest152@testExample.com,password123,LoadTestUser152,LoadTestName152 +compassLoadTest153@testExample.com,password123,LoadTestUser153,LoadTestName153 +compassLoadTest154@testExample.com,password123,LoadTestUser154,LoadTestName154 +compassLoadTest155@testExample.com,password123,LoadTestUser155,LoadTestName155 +compassLoadTest156@testExample.com,password123,LoadTestUser156,LoadTestName156 +compassLoadTest157@testExample.com,password123,LoadTestUser157,LoadTestName157 +compassLoadTest158@testExample.com,password123,LoadTestUser158,LoadTestName158 +compassLoadTest159@testExample.com,password123,LoadTestUser159,LoadTestName159 +compassLoadTest160@testExample.com,password123,LoadTestUser160,LoadTestName160 +compassLoadTest161@testExample.com,password123,LoadTestUser161,LoadTestName161 +compassLoadTest162@testExample.com,password123,LoadTestUser162,LoadTestName162 +compassLoadTest163@testExample.com,password123,LoadTestUser163,LoadTestName163 +compassLoadTest164@testExample.com,password123,LoadTestUser164,LoadTestName164 +compassLoadTest165@testExample.com,password123,LoadTestUser165,LoadTestName165 +compassLoadTest166@testExample.com,password123,LoadTestUser166,LoadTestName166 +compassLoadTest167@testExample.com,password123,LoadTestUser167,LoadTestName167 +compassLoadTest168@testExample.com,password123,LoadTestUser168,LoadTestName168 +compassLoadTest169@testExample.com,password123,LoadTestUser169,LoadTestName169 +compassLoadTest170@testExample.com,password123,LoadTestUser170,LoadTestName170 +compassLoadTest171@testExample.com,password123,LoadTestUser171,LoadTestName171 +compassLoadTest172@testExample.com,password123,LoadTestUser172,LoadTestName172 +compassLoadTest173@testExample.com,password123,LoadTestUser173,LoadTestName173 +compassLoadTest174@testExample.com,password123,LoadTestUser174,LoadTestName174 +compassLoadTest175@testExample.com,password123,LoadTestUser175,LoadTestName175 +compassLoadTest176@testExample.com,password123,LoadTestUser176,LoadTestName176 +compassLoadTest177@testExample.com,password123,LoadTestUser177,LoadTestName177 +compassLoadTest178@testExample.com,password123,LoadTestUser178,LoadTestName178 +compassLoadTest179@testExample.com,password123,LoadTestUser179,LoadTestName179 +compassLoadTest180@testExample.com,password123,LoadTestUser180,LoadTestName180 +compassLoadTest181@testExample.com,password123,LoadTestUser181,LoadTestName181 +compassLoadTest182@testExample.com,password123,LoadTestUser182,LoadTestName182 +compassLoadTest183@testExample.com,password123,LoadTestUser183,LoadTestName183 +compassLoadTest184@testExample.com,password123,LoadTestUser184,LoadTestName184 +compassLoadTest185@testExample.com,password123,LoadTestUser185,LoadTestName185 +compassLoadTest186@testExample.com,password123,LoadTestUser186,LoadTestName186 +compassLoadTest187@testExample.com,password123,LoadTestUser187,LoadTestName187 +compassLoadTest188@testExample.com,password123,LoadTestUser188,LoadTestName188 +compassLoadTest189@testExample.com,password123,LoadTestUser189,LoadTestName189 +compassLoadTest190@testExample.com,password123,LoadTestUser190,LoadTestName190 +compassLoadTest191@testExample.com,password123,LoadTestUser191,LoadTestName191 +compassLoadTest192@testExample.com,password123,LoadTestUser192,LoadTestName192 +compassLoadTest193@testExample.com,password123,LoadTestUser193,LoadTestName193 +compassLoadTest194@testExample.com,password123,LoadTestUser194,LoadTestName194 +compassLoadTest195@testExample.com,password123,LoadTestUser195,LoadTestName195 +compassLoadTest196@testExample.com,password123,LoadTestUser196,LoadTestName196 +compassLoadTest197@testExample.com,password123,LoadTestUser197,LoadTestName197 +compassLoadTest198@testExample.com,password123,LoadTestUser198,LoadTestName198 +compassLoadTest199@testExample.com,password123,LoadTestUser199,LoadTestName199 +compassLoadTest200@testExample.com,password123,LoadTestUser200,LoadTestName200 \ No newline at end of file diff --git a/tests/load/check_results.py b/tests/load/check_results.py new file mode 100644 index 000000000..480936a00 --- /dev/null +++ b/tests/load/check_results.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Fail the build when a JMeter results file (.jtl) breaks the limits. + +JMeter exits with 0 even when requests fail, so CI runs this afterwards. + +Usage: + python3 check_results.py results.jtl --max-error-rate 1 --max-p95 800 +""" +import argparse +import csv +import math +import sys + +# Requests in the plan are labelled "SETUP | ...", "TEST | ..." or "TEARDOWN | ...". +# The error limit applies to all of them: a failed SETUP request means less load than asked for, and a +# failed TEARDOWN request leaves test accounts behind on the server. The p95 limit only applies to TEST +# requests, because setup and cleanup times aren't what the test measures. +TEST_LABEL_PREFIX = "TEST |" + + +def load_samples(path): + with open(path, newline="", encoding="utf-8") as f: + return list(csv.DictReader(f)) + + +def percentile(values, pct): + # nearest-rank: the value that pct% of samples are at or below + ordered = sorted(values) + rank = math.ceil(pct / 100 * len(ordered)) + return ordered[max(rank - 1, 0)] + + +def summarise(samples): + """Group samples by label (request name) and work out the numbers for each.""" + stats = {} + for label in sorted({s["label"] for s in samples}): + group = [s for s in samples if s["label"] == label] + errors = sum(1 for s in group if s["success"] != "true") + stats[label] = { + "count": len(group), + "error_rate": errors / len(group) * 100, # percent + "p95": percentile([int(s["elapsed"]) for s in group], 95), # ms + } + return stats + + +def check_limits(stats, max_error_rate, max_p95): + """Return one message per broken limit. An empty list means the run passed.""" + problems = [] + for label, s in stats.items(): + if s["error_rate"] > max_error_rate: + problems.append(f"{label}: error rate {s['error_rate']:.1f}% is over {max_error_rate}%") + if label.startswith(TEST_LABEL_PREFIX) and s["count"] >= 20 and s["p95"] > max_p95: + problems.append(f"{label}: p95 {s['p95']} ms is over {max_p95} ms") + return problems + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("jtl", help="results file written by jmeter -l") + parser.add_argument("--max-error-rate", type=float, default=1.0, help="percent, default 1") + parser.add_argument("--max-p95", type=int, default=800, help="milliseconds, default 800") + args = parser.parse_args() + + samples = load_samples(args.jtl) + if not any(s["label"].startswith(TEST_LABEL_PREFIX) for s in samples): + print("FAIL: no samples from the main test. Did setUp create any accounts?") + return 1 + + stats = summarise(samples) + for label, s in stats.items(): + print(f"{label}: {s['count']} samples, {s['error_rate']:.1f}% errors, p95 {s['p95']} ms") + + problems = check_limits(stats, args.max_error_rate, args.max_p95) + for p in problems: + print("FAIL:", p) + if not problems: + print("PASS") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/load/compass-testPlan.jmx b/tests/load/compass-testPlan.jmx new file mode 100644 index 000000000..6ff5560f2 --- /dev/null +++ b/tests/load/compass-testPlan.jmx @@ -0,0 +1,519 @@ + + + + + true + true + + + + false + + + + ${__P(resetFile,1)} + 1 + true + continue + + 1 + false + + + + + true + + + import org.apache.jmeter.services.FileServer +new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt").text = "" + groovy + + + + + + + baseAuthUrl + identitytoolkit.googleapis.com + + = + + + apiHost + ${__P(apiHost,api.dev.compassmeet.com)} + = + + + firebaseApiKey + ${__P(firebaseApiKey,AIzaSyBspL9glBXWbMsjmtt36dgb2yU0YGGhzKo)} + = + + + signupUrl + /v1/accounts:signUp?key= + = + + + signinUrl + /v1/accounts:signInWithPassword?key= + = + + + deleteAccountUrl + /v1/accounts:delete?key= + = + + + getUserDataUrl + /v1/accounts:lookup?key= + = + + + + + + ${__P(setupThreads,${__P(threads,100)})} + ${__P(setupRamp,10)} + true + continue + + 1 + false + + + + + accounts.csv + UTF-8 + userEmail,userPassword,username,name + false + , + false + false + true + shareMode.all + + + + ${baseAuthUrl} + https + ${signupUrl}${firebaseApiKey} + true + POST + true + true + + + + false + { "email": "${userEmail}", "password": "${userPassword}", "returnSecureToken": true } + = + + + + + + + + + Content-Type + application/json + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + authToken + $.idToken + 1 + + + + + ${apiHost} + https + /create-user-and-profile + true + POST + true + true + + + + false + {"username": "${username}", "name": "${name}", "profile": {"visibility": "public"}} + = + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + groovy + + + true + import org.apache.jmeter.services.FileServer + +def token = vars.get("authToken") +if (!token) { + log.warn("No token for ${vars.get('userEmail')}, not saving") + return +} +if (!prev.isSuccessful()) { + log.warn("Profile creation failed for ${vars.get('userEmail')}, saving anyway so it gets cleaned up") +} + +def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") +synchronized (FileServer.class) { + file.append([vars.get("userEmail"), vars.get("userPassword"), token, vars.get("name")].join("|") + "\n", "UTF-8") +} + + + + + false + true + false + + + + + ${__P(meThreads,${__P(threads,100)})} + ${__P(meRamp,${__P(ramp,10)})} + true + continue + + 1 + false + + + + + created_accounts.txt + UTF-8 + userEmail,userPassword,authToken,name + false + | + false + false + true + shareMode.group + + + + ${__P(meDuration,${__P(duration,120)})} + + + + ${apiHost} + https + /me + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + $.name + ${name} + true + false + false + false + + + + + ${__P(thinkTimeMs,0)} + + + + + + 1 + 1 + true + continue + + -1 + false + + + + + created_accounts.txt + UTF-8 + userEmail,userPassword,authToken,name + false + | + false + false + true + shareMode.group + + + + ${baseAuthUrl} + https + ${signinUrl}${firebaseApiKey} + true + POST + true + true + + + + false + {"email": "${userEmail}", "password": "${userPassword}", "returnSecureToken": true} + = + + + + + + + + + Content-Type + application/json + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + authToken + $.idToken + 1 + + + + + ${apiHost} + https + /me/delete + true + POST + true + true + + + + false + {"reasonCategory": "jmeter-load-test"} + = + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + true + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + true + false + true + true + false + false + false + true + 0 + true + true + true + true + true + true + + + errors.xml + + + + + From d231237ee17a16a894774c4a42847cb6f6874019 Mon Sep 17 00:00:00 2001 From: MartinBraquet Date: Wed, 28 Oct 2026 12:05:33 +0000 Subject: [PATCH 4/7] Implement retry logic in `fetch-media` to handle transient connection errors and timeouts; add custom `HttpError` class and retry attempts with exponential backoff. --- web/scripts/fetch-media.mjs | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/web/scripts/fetch-media.mjs b/web/scripts/fetch-media.mjs index 2e7b03250..7f9283e72 100644 --- a/web/scripts/fetch-media.mjs +++ b/web/scripts/fetch-media.mjs @@ -93,16 +93,44 @@ const exists = async (p) => { } } +/** + * r2.dev occasionally drops a connection mid-body, which undici reports as a bare + * "TypeError: terminated". That is transient, so retry it; an HTTP error status is not, so it fails + * on the first attempt. The timeout covers a stalled stream, which would otherwise hang the build + * until Vercel kills it with no message at all. + */ +const ATTEMPTS = 3 +const TIMEOUT_MS = 5 * 60_000 + +class HttpError extends Error {} + +async function downloadOnce(url, target) { + const res = await fetch(url, {signal: AbortSignal.timeout(TIMEOUT_MS)}) + if (!res.ok || !res.body) { + throw new HttpError(`${res.status} ${res.statusText}`) + } + await pipeline(Readable.fromWeb(res.body), createWriteStream(target)) +} + async function download({key, dest}, base) { const url = `${base}/${key}` const target = join(PUBLIC_DIR, dest) await mkdir(dirname(target), {recursive: true}) - const res = await fetch(url) - if (!res.ok || !res.body) { - throw new Error(`${res.status} ${res.statusText} for ${url}`) + for (let attempt = 1; ; attempt++) { + try { + await downloadOnce(url, target) + break + } catch (err) { + // undici puts the useful part (e.g. "other side closed") in `cause` + const reason = err.cause ? `${err.message} (${err.cause.message ?? err.cause})` : err.message + if (err instanceof HttpError || attempt === ATTEMPTS) { + throw new Error(`${reason} for ${url} (attempt ${attempt}/${ATTEMPTS})`) + } + console.warn(`[media] ${dest}: ${reason}; retrying (${attempt}/${ATTEMPTS})`) + await new Promise((r) => setTimeout(r, 2000 * attempt)) + } } - await pipeline(Readable.fromWeb(res.body), createWriteStream(target)) const {size} = await stat(target) console.log(`[media] ${dest} (${(size / 1024).toFixed(0)} KB)`) } From 67758dfb98561440353ba4054aea3b85b608b325 Mon Sep 17 00:00:00 2001 From: MartinBraquet Date: Wed, 28 Oct 2026 12:58:46 +0000 Subject: [PATCH 5/7] Improvements --- .github/workflows/load-test.yml | 27 +- backend/email/emails/functions/send-email.ts | 20 + docs/load-testing.md | 1283 ++++-------------- scripts/e2e.sh | 4 +- scripts/seed.sh | 2 +- tests/load/accounts.csv | 200 --- tests/load/check_results.py | 8 +- tests/load/compass-testPlan.jmx | 632 +++++++-- 8 files changed, 858 insertions(+), 1318 deletions(-) delete mode 100644 tests/load/accounts.csv diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml index 1794f8140..32a2a3dea 100644 --- a/.github/workflows/load-test.yml +++ b/.github/workflows/load-test.yml @@ -40,10 +40,14 @@ jobs: load-test: runs-on: ubuntu-latest timeout-minutes: 20 - # Never run two load tests at once: they hit the same dev API, so each would skew the other's results. + # GitHub keeps only one pending run per concurrency group and cancels the one it replaces, so a single + # shared group would let a PR push silently cancel a queued run of another PR or the weekly full run. + # Instead each PR gets its own smoke group, where a newer push replaces the older run (its accounts are + # still deleted by the cleanup step, which runs on cancel), and the full run gets a group of its own. + # Smoke runs use only a few users, so one overlapping another run barely skews its results. concurrency: - group: load-test - cancel-in-progress: false + group: load-test-${{ inputs.report_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} defaults: run: working-directory: tests/load # folder holding the .jmx and check_results.py @@ -77,22 +81,13 @@ jobs: - name: Add JMeter to PATH run: echo "$HOME/jmeter/bin" >> "$GITHUB_PATH" - - name: Make test accounts for this run - run: | - # one row per user: email,password,username,name (no header, no commas inside values) - # The run ID and attempt number make the accounts unique, including when a failed job is re-run. - id="${GITHUB_RUN_ID: -5}${GITHUB_RUN_ATTEMPT}" - rm -f accounts.csv - for i in $(seq 1 "$USERS"); do - echo "loadTest${id}x${i}@example.com,$(openssl rand -hex 8),loadUser${id}x${i},load Name${id}x${i}" >> accounts.csv - done - echo "Made $(wc -l < accounts.csv) accounts" - - name: Run load test id: jmeter run: | mkdir -p results - jmeter -n -t "$PLAN" \ + # The plan puts the run ID in each account's email and username. The run ID and attempt number keep + # them unique, including when a failed job is re-run. + jmeter -n -t "$PLAN" -JrunId="${GITHUB_RUN_ID: -5}${GITHUB_RUN_ATTEMPT}" \ -JsetupThreads="$USERS" -Jthreads="$USERS" -Jduration="$DURATION" \ -l results/results.jtl -j jmeter.log -e -o report @@ -105,7 +100,7 @@ jobs: run: | [ -s created_accounts.txt ] || { echo "No accounts to clean up"; exit 0; } jmeter -n -t "$PLAN" \ - -JresetFile=0 -JsetupThreads=0 -Jthreads=0 \ + -JcheckLeftovers=0 -JsetupThreads=0 -Jthreads=0 \ -l results/cleanup.jtl -j cleanup.log - name: Save report diff --git a/backend/email/emails/functions/send-email.ts b/backend/email/emails/functions/send-email.ts index 13436c5cb..14f4fa31f 100644 --- a/backend/email/emails/functions/send-email.ts +++ b/backend/email/emails/functions/send-email.ts @@ -20,6 +20,11 @@ export const sendEmail = async ( return null } + if (sendsOnlyToTestDomains(payload.to)) { + debug(`Skipping email to reserved test domain: ${payload.to}`) + return null + } + if (!resend) return null const {data, error} = await resend.emails.send( @@ -44,6 +49,21 @@ export const sendEmail = async ( return data } +// Domains reserved for testing (RFC 2606) can never receive mail. The load tests sign up with them, so +// sending would only bounce and hurt the sender reputation. +const isTestDomainAddress = (address: string) => { + const domain = address.trim().replace(/>$/, '').split('@').pop()?.toLowerCase() ?? '' + return ( + ['example.com', 'example.net', 'example.org'].includes(domain) || + ['.example', '.test', '.invalid'].some((tld) => domain.endsWith(tld)) + ) +} + +const sendsOnlyToTestDomains = (to: string | string[]) => { + const addresses = Array.isArray(to) ? to : [to] + return addresses.length > 0 && addresses.every(isTestDomainAddress) +} + let resend: Resend | null = null const getResend = () => { if (resend) return resend diff --git a/docs/load-testing.md b/docs/load-testing.md index 7f5fb1f0f..0c0341f09 100644 --- a/docs/load-testing.md +++ b/docs/load-testing.md @@ -6,190 +6,77 @@ them. They don't check that features work correctly; unit, integration and E2E t [testing.md](testing.md)). Load tests use [Apache JMeter](https://jmeter.apache.org/). The test plan, `tests/load/compass-testPlan.jmx`, load -tests the `/me` endpoint with real, temporary accounts. Each run has three stages: +plays a typical browsing session against the API, with real, temporary accounts. Each run has three stages: 1. **Create** a set of temporary accounts. -2. **Load test:** all the accounts call the API at the same time, for a set amount of time. +2. **Load test:** all the accounts browse at the same time, for a set amount of time. 3. **Delete** the accounts again. **In this guide:** -- [Placeholders](#placeholders) and [Prerequisites](#prerequisites), including - [setups for Windows, WSL, Linux and macOS](#choose-your-setup) -- [JMeter basics](#jmeter-basics): the terms and rules the rest of the guide relies on -- [How the plan works](#how-the-plan-works): what happens, stage by stage -- [accounts.csv](#accountscsv): the accounts the plan creates +- [Prerequisites](#prerequisites), including [setups for Windows, WSL, Linux and macOS](#choose-your-setup) +- [JMeter basics](#jmeter-basics) +- [How the plan works](#how-the-plan-works) and [the test accounts](#the-test-accounts) - [Running the test](#running-the-test): commands, stopping early, flags and properties -- [Why it is structured this way](#why-it-is-structured-this-way) -- [Element reference](#element-reference): what each element does, why it's used and how to change it +- [Element reference](#element-reference) - [Adding a new test to this plan](#adding-a-new-test-to-this-plan) - [Reading the results](#reading-the-results) - [Automated runs](#automated-runs-github-actions): the smoke and weekly full tests in GitHub Actions - [Troubleshooting](#troubleshooting) ---- - -## Placeholders - -Values in angle brackets are placeholders. Replace them with the real values when you copy a command. - -| Placeholder | What it is | Where to find the real value | -| -------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------------- | -| `` | The API host the plan sends requests to | `apiHost` in the plan's User Defined Variables | -| `` | The Firebase key used to sign up and sign in | `firebaseApiKey` in the plan's User Defined Variables (the dev key is also in `common/src/envs/dev.ts`) | -| `` | The email domain of the test accounts | `tests/load/accounts.csv` | -| `` | The password of the test accounts | `tests/load/accounts.csv` | +Values in angle brackets are placeholders: `` and `` are `apiHost` and `firebaseApiKey` in +the plan's User Defined Variables (the dev key is also in `common/src/envs/dev.ts`). --- ## Prerequisites -- **Java 8 or newer** (17+ recommended), installed on the system where JMeter runs. JMeter runs on Java. -- **[Apache JMeter 5.6+](https://jmeter.apache.org/download_jmeter.cgi)**, set up for your system as described in +- **Java 8 or newer** (17+ recommended), installed on the system where JMeter runs. +- **[Apache JMeter 5.6+](https://jmeter.apache.org/download_jmeter.cgi)**, set up as described in [Choose your setup](#choose-your-setup). -The plan's files live in `tests/load/`: +The plan's files live in `tests/load/`: `compass-testPlan.jmx` (the test plan) and `check_results.py` (checks a +results file against limits, used by CI, see [check_results.py](#check_resultspy)). -| File | What it is | -| ---------------------- | ------------------------------------------------------------------------------------------- | -| `compass-testPlan.jmx` | The test plan | -| `accounts.csv` | The accounts to create (see [accounts.csv](#accountscsv)) | -| `check_results.py` | Checks a results file against limits, used by CI (see [check_results.py](#check_resultspy)) | +A run creates these files next to the plan. They are git-ignored: -A run creates these files. The [commands](#commands) are run from `tests/load/`, so they all land there. They are -generated output and are git-ignored: - -| File or folder | What it is | -| ---------------------- | ------------------------------------------------------------------- | -| `created_accounts.txt` | The accounts created in this run, passed between stages | -| `errors.xml` | Details of failed requests | -| `jmeter.log` | JMeter's own log, including the warnings the plan's scripts write | -| `results/` | The raw results of each run ([`.jtl` files](#the-results-file-jtl)) | -| `report/` | The HTML report | +| File or folder | What it is | +| ---------------------- | --------------------------------------------------------------------------------------- | +| `created_accounts.txt` | The accounts that exist right now, passed between stages (see below) | +| `errors.xml` | Details of failed requests | +| `jmeter.log` | JMeter's own log, including the warnings the plan's scripts write | +| `results/`, `report/` | The raw results of each run ([`.jtl` files](#reading-the-results)), and the HTML report | ### Choose your setup -The [commands](#commands) use short paths such as `compass-testPlan.jmx` and `results/smoke.jtl`, which only work when -JMeter runs **inside `tests/load/`** on the same system as the project. Find the row that matches your machine, then -follow its section below. - -| Your setup | Run the commands in | Open the GUI with | -| ------------------------------------------- | ---------------------------------------------------------- | ----------------------------------------------- | -| Project and JMeter on the same system | A terminal on that system (PowerShell on Windows) | JMeter on that system | -| Project in WSL, JMeter installed on Windows | A WSL terminal, using the Windows install's Linux launcher | JMeter on Windows, with the plan's Windows path | -| Project in WSL, JMeter installed in WSL | A WSL terminal | JMeter in WSL, shown as a window on Windows | - -Common to every setup: - -- **A WSL terminal** (needed for both WSL setups) is a terminal running Linux inside WSL. Open one with the **Ubuntu** - app from the Start menu, an Ubuntu tab in Windows Terminal, or `wsl` in PowerShell. If the project is open in VS Code - through WSL (the bottom-left corner shows **WSL: Ubuntu**), VS Code's own terminal (**Terminal → New Terminal**) is - already one. -- **Java** must be installed on the system that runs the commands. For both WSL setups that means **in WSL**, even if - Java is also installed on Windows. Check with `java -version`. To install it in WSL, use the full package (the - `-headless` one can't show the GUI): - - ```bash - sudo apt install openjdk-21-jre - ``` - -- **Check your setup** by running `jmeter --version` in the terminal you'll run the commands from. -- **Commands** are then run exactly as written under [Commands](#commands), from `tests/load/`. -- **To open the plan in the GUI:** start JMeter, choose **File → Open**, paste the full path of - `compass-testPlan.jmx` into the file name box and open it. When JMeter runs on the same system as the project, you - can instead run `jmeter -t compass-testPlan.jmx` from `tests/load/`, which opens the plan straight away. - -#### Project and JMeter on the same system - -For example, everything on Linux, on macOS, or on Windows without WSL. - -1. Install JMeter. -2. Add JMeter's `bin/` folder to your PATH, so the `jmeter` command works from any folder. - -To start the GUI, run `jmeter`. On Windows, you can also double-click `jmeter.bat` in JMeter's `bin/` folder. - -#### Project in WSL, JMeter installed on Windows - -The JMeter download is the same for every system, so its `bin/` folder has a launcher for each: `jmeter.bat` for -Windows and `jmeter` (a shell script) for Linux and macOS. WSL sees your Windows drive under `/mnt/c`, so it can run the -Linux launcher from your Windows install, using WSL's Java. +The [commands](#commands) use short paths such as `results/smoke.jtl`, which only work when JMeter runs **inside +`tests/load/`** on the same system as the project. Check your setup with `jmeter --version` in the terminal you'll run +the commands from. Java must be installed on that same system; in WSL that means **in WSL** (use the full package, as +the `-headless` one can't show the GUI: `sudo apt install openjdk-21-jre`). -Put JMeter's `bin/` folder on your WSL PATH in **one** of these two ways. Either way, JMeter's `shutdown.sh` also -becomes available, which you need to [stop a run early](#stopping-a-run-early). - -**Option A: add it to your Windows PATH.** WSL adds the Windows PATH to its own, so this makes `jmeter` work in WSL -**and** in PowerShell. - -1. In the Start menu, search for **Edit environment variables for your account** and open it. -2. Select **Path**, click **Edit**, then **New**, and paste the `bin/` folder's Windows path, e.g. - `C:\tools\apache-jmeter-5.6.3\bin`. Click **OK** in both windows. -3. Close and reopen your terminals and VS Code, because WSL only sees the new PATH in a fresh session. If `jmeter` still - isn't found in WSL, run `wsl --shutdown` in PowerShell and open WSL again. - -This doesn't work if WSL is set not to include the Windows PATH (`appendWindowsPath = false` in `/etc/wsl.conf`). In -that case, use option B. - -**Option B: add it to your WSL PATH only.** Run these in a WSL terminal, from any folder. - -1. Find the WSL path of your JMeter folder, using the Windows path where you unpacked it: - - ```bash - wslpath 'C:\tools\apache-jmeter-5.6.3' - ``` - - This prints e.g. `/mnt/c/tools/apache-jmeter-5.6.3`. - -2. Add its `bin/` folder to your WSL PATH, using the path from step 1: - - ```bash - echo 'export PATH="$PATH:/mnt/c/tools/apache-jmeter-5.6.3/bin"' >> ~/.bashrc && source ~/.bashrc - ``` - -In this setup, `jmeter` starts about a second slower than with JMeter installed in WSL (the very first start of the day -can take a few seconds), because its files are read from the Windows drive. Only start-up is affected: a running test -is just as fast. - -**GUI:** start JMeter on Windows (double-click `jmeter.bat` in its `bin/` folder, or type `jmeter` in PowerShell if you -used option A) and open the plan with its Windows path. It looks like `\\wsl.localhost\\home\\…\tests\load\compass-testPlan.jmx`. To get it, run this from -`tests/load/` in WSL and copy the output: +**Project and JMeter on the same system** (everything on Linux, on macOS, or on Windows without WSL): install JMeter +and add its `bin/` folder to your PATH. On Linux or macOS: ```bash -wslpath -w compass-testPlan.jmx +cd ~ && curl -LO https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.6.3.tgz && tar -xzf apache-jmeter-5.6.3.tgz +echo 'export PATH="$HOME/apache-jmeter-5.6.3/bin:$PATH"' >> ~/.bashrc && source ~/.bashrc ``` -You can also open the folder in File Explorer with `explorer.exe .` and copy the path from the address bar. - -> **Don't run the commands with `jmeter.bat`.** Batch files run in `cmd.exe`, which can't use a WSL folder (a -> `\\wsl.localhost\…` path) as its working folder. It switches to `C:\Windows` instead, so the short paths in the -> commands point to the wrong place and the run fails. - -#### Project in WSL, JMeter installed in WSL - -Everything runs inside WSL. JMeter starts faster than in the setup above, and its GUI opens as a normal window on -Windows through WSLg, which is built into current versions of WSL. - -Run these in a WSL terminal, from any folder (the first command moves to your home folder itself): - -1. Download and unpack JMeter into your WSL home folder: +**Project in WSL, JMeter installed in WSL** (recommended for WSL): run the same two commands in a WSL terminal. Putting +`bin/` at the **front** of the PATH means this JMeter wins over any on your Windows PATH; check with +`command -v jmeter`. The GUI opens as a normal Windows window through WSLg. - ```bash - cd ~ && curl -LO https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.6.3.tgz && tar -xzf apache-jmeter-5.6.3.tgz - ``` +**Project in WSL, JMeter installed on Windows:** the JMeter download has a Linux launcher (`bin/jmeter`) as well as +`jmeter.bat`, so WSL can run it from `/mnt/c` with WSL's Java. Add the `bin/` folder to your WSL PATH, e.g. +`echo 'export PATH="$PATH:/mnt/c/tools/apache-jmeter-5.6.3/bin"' >> ~/.bashrc` (find the path with +`wslpath 'C:\tools\apache-jmeter-5.6.3'`). Start-up is about a second slower; a running test is just as fast. Open +the GUI on Windows with the plan's `\\wsl.localhost\…` path (`wslpath -w compass-testPlan.jmx` prints it). -2. Add its `bin/` folder to the **front** of your PATH: +> **Don't run the commands with `jmeter.bat` against a WSL folder.** `cmd.exe` can't use a `\\wsl.localhost\…` path as +> its working folder and switches to `C:\Windows`, so the short paths point to the wrong place. - ```bash - echo 'export PATH="$HOME/apache-jmeter-5.6.3/bin:$PATH"' >> ~/.bashrc && source ~/.bashrc - ``` - - Putting it first means this JMeter is used even if another one is on your Windows PATH, because WSL places the - Windows PATH in the middle of its own. - -3. Check which JMeter runs with `command -v jmeter`. It should print a path in your home folder, e.g. - `/home//apache-jmeter-5.6.3/bin/jmeter`. - -**GUI:** run `jmeter`. For **File → Open**, use the plan's WSL path, e.g. -`/home//…/tests/load/compass-testPlan.jmx`. To print it, run `realpath compass-testPlan.jmx` in `tests/load/`. +**To open the plan in the GUI:** run `jmeter -t compass-testPlan.jmx` from `tests/load/`, or start JMeter and use +**File → Open** with the plan's full path. --- @@ -197,289 +84,159 @@ Run these in a WSL terminal, from any folder (the first command moves to your ho A JMeter test plan is a **tree** of elements. Each element type has one job: -| Term | What it means | Example in this plan | -| ---------------------- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -| **Test plan** | The whole file and its top-level settings | [`compass-testPlan.jmx`](#test-plan) | -| **Thread group** | A group of threads that all run the same steps | [`1 - Create accounts`](#thread-groups) | -| **Thread** | One simulated user (see the note below) | One of the 100 threads in a thread group | -| **Sampler** | Something a thread _does_, usually one request | [`SETUP \| Firebase Signup POST`](#http-request) | -| **Config element** | Settings or data used by samplers | [CSV Data Set Config](#csv-data-set-config), [HTTP Header Manager](#http-header-manager) | -| **Pre/Post-processor** | Runs before/after a sampler, e.g. to pull a value out of a response | [JSON Extractor](#json-extractor) | -| **Assertion** | Checks a response and marks the request as failed if it's wrong | [Response Assertion](#response-assertion) (code must be `200`) | -| **Timer** | Adds a pause before requests | [Think time](#constant-timer) | -| **Logic controller** | Controls _how often_ or _whether_ its children run | [Runtime Controller](#runtime-controller) | -| **Listener** | Collects or shows results | [Simple Data Writer](#listeners) | - -> **Words used in this guide:** -> -> - **Thread:** one simulated user that JMeter runs. JMeter's own screens sometimes call these "users", e.g. -> "Number of Threads (users)". -> - **Account:** a login (email, password, token) that a thread uses. -> - **User:** only ever means a real person using the app. - -### Variables and properties - -- **Variables** belong to **one thread**. You read them with `${name}`, e.g. `${authToken}`. Each thread has its own - copy, so 100 threads can each hold a different token. A variable that a thread sets while it runs (like - `authToken`) **doesn't carry over** to other thread groups. -- The exception is variables defined in the Test Plan or in [User Defined Variables](#user-defined-variables). JMeter - gives every thread its own copy of those when the thread starts, so threads in every group can read them. That's why - `${apiHost}` works everywhere. -- **Properties** are **global** to the whole run. You read them with `${__P(name,default)}` and set them from the - command line with `-Jname=value`. This plan uses properties for anything you might want to change per run (thread - count, duration, host). See [Properties](#properties) and [Functions](#functions). - -### Scope: where an element applies - -An element applies to its **parent and everything under its parent** in the tree: - -- A Header Manager placed **under one request** only affects that request. -- A timer placed under a controller applies before **every** request in that controller. -- A CSV Data Set placed under a thread group feeds **every thread** in that group. - -### Order of execution for each request - -For every sampler, JMeter runs the elements in scope in this order: - -```text -config elements → pre-processors → timers → the sampler (request) → post-processors → assertions → listeners -``` - -This matters when one element depends on another. For example, post-processors run **before** assertions, so a script -in a post-processor can't see whether an assertion passed. The [Save account script](#jsr223-scripts-groovy) is an -example. +| Term | What it means | Example in this plan | +| ---------------------- | ---------------------------------------------------------------- | ---------------------------------------- | +| **Thread group** | A group of threads that all run the same steps | `1 - Create accounts` | +| **Thread** | One simulated user. Each uses one test **account** | One of the 100 threads in a thread group | +| **Sampler** | Something a thread _does_, usually one request | `SETUP \| Firebase Signup POST` | +| **Config element** | Settings or data used by samplers | CSV Data Set Config, HTTP Header Manager | +| **Pre/Post-processor** | Runs before/after a sampler, e.g. to pull a value out of a reply | JSON Extractor | +| **Assertion** | Checks a response and marks the request failed if it's wrong | Response Assertion (code must be `200`) | +| **Timer** | Adds a pause before requests | Think time | +| **Logic controller** | Controls _how often_ or _whether_ its children run | Runtime Controller, If Controller | +| **Listener** | Collects or shows results | Simple Data Writer | + +- **Variables** belong to **one thread** and are read with `${name}`. A variable a thread sets while it runs (like + `authToken`) **doesn't carry over** to other thread groups. Variables in User Defined Variables are copied to every + thread, which is why `${apiHost}` works everywhere. +- **Properties** are **global** to the run. Read them with `${__P(name,default)}`, set them with `-Jname=value`. +- **Scope:** an element applies to its parent and everything under it. A Header Manager under one request only affects + that request. +- **Order for each request:** config elements → pre-processors → timers → the request → post-processors → assertions + → listeners. So a post-processor can't see whether an assertion passed. --- ## How the plan works -The plan has one thread group per stage: - ```text compass-testPlan.jmx -├── User Defined Variables hosts and endpoints shared by all requests -├── 0 - Reset shared file (setUp) empties created_accounts.txt -├── 1 - Create accounts (setUp) creates accounts, saves them to created_accounts.txt -├── 2 - /me load test (Thread Group) the load test, reads created_accounts.txt -├── 3 - Delete accounts (tearDown) reads created_accounts.txt, deletes each account -└── Listeners Summary Report, Simple Data Writer, View Results Tree +├── User Defined Variables hosts, endpoints and the run ID +├── 0 - Check for leftover accounts (setUp) stops the run from creating accounts if earlier ones remain +├── 1 - Create accounts (setUp) creates accounts, saves them to created_accounts.txt +├── 2 - Browsing session (Thread Group) the load test, reads created_accounts.txt +├── 3 - Delete accounts (tearDown) reads created_accounts.txt, deletes each account +├── 4 - Forget deleted accounts (tearDown) removes the deleted accounts from created_accounts.txt +└── Listeners Summary Report, Simple Data Writer, View Results Tree ``` -Account data moves through the plan like this: - -```text -accounts.csv ──► 1 - Create accounts ──► created_accounts.txt ──► 2 - /me load test - └────────► 3 - Delete accounts -``` - -### Stage by stage - -The stages run one after another (see [Test Plan](#test-plan)). The elements in brackets are explained in the -[Element reference](#element-reference). - -**0 - Reset shared file** (setUp, 1 thread) - -- `SETUP | Reset created_accounts.txt` ([JSR223 Sampler](#jsr223-scripts-groovy)) empties `created_accounts.txt`, so - only this run's accounts end up in it. The stage is skipped when `resetFile=0` (used by - [Cleanup only](#cleanup-only)). - -**1 - Create accounts** (setUp, `setupThreads` threads, one account each) - -Each thread takes the next row of `accounts.csv` ([CSV Data Set Config](#csv-data-set-config)), then: - -1. `SETUP | Firebase Signup POST` creates the Firebase login. A [JSON Extractor](#json-extractor) saves the sign-in - token from the response as `authToken`. -2. `SETUP | create-user-and-profile POST` creates the account's app user and profile, signed in with `authToken`. -3. **Save account for test and teardown** ([JSR223 PostProcessor](#jsr223-scripts-groovy)) adds the account to - `created_accounts.txt` as one line: `email|password|token|name`. - -A disabled [Debug Sampler](#debug-sampler) sits at the end of this stage, for checking the thread's variables while -debugging. - -**2 - /me load test** (Thread Group, `meThreads` threads) - -Each thread takes one line of `created_accounts.txt`, so it gets one account. A -[Runtime Controller](#runtime-controller) then repeats this request for `meDuration` seconds: - -- `TEST | /me GET` calls `/me` with that account's token. It passes if the response code is `200` - ([Response Assertion](#response-assertion)) **and** the returned `name` matches the account - ([JSON Assertion](#json-assertion)). - -**Think time** ([Constant Timer](#constant-timer)) can add a pause before each request. It's 0 by default. - -**3 - Delete accounts** (tearDown, 1 thread) - -The thread goes through `created_accounts.txt` line by line until the file runs out. For each account: - -1. `TEARDOWN | Firebase Sign in POST` signs in again to get a fresh token (saved by a JSON Extractor). Tokens expire - after 1 hour, so the one from setUp may be too old. -2. `TEARDOWN | Delete app user` calls `/me/delete`, which deletes the app user, their files and their Firebase login. - It sends the reason `jmeter-load-test`, which the API stores in the `deleted_users` table, so load-test deletions - can be told apart from real users' deletions. - ---- - -## accounts.csv - -### Why it's needed - -The API only accepts most requests from **signed-in accounts**, so the test needs real accounts. Each thread needs its -**own** account: - -- Each row holds one account, so each thread acts like a separate person using the app, as in real traffic. -- Sign-up rejects an email that's already in use, so every row needs its own email. - -### How the plan uses it - -- The file is read by the **CSV - Accounts** element in `1 - Create accounts`. -- Each thread in that group takes the **next unused row**. The sharing mode is _All threads_, so no two threads get - the same row. -- The four columns become the variables `${userEmail}`, `${userPassword}`, `${username}` and `${name}`. These are used - in the bodies of the sign-up and profile requests. -- If there are more threads than rows, the extra threads **stop** without doing anything ("Stop thread on EOF"). - -### Format - -Four columns, separated by commas, with **no header row**: - -```text -email,password,username,name -``` - -Example row: - -```text -loadtest001@,,loadtest001,Load Test 001 -``` - -Rules for each column: - -| Column | Rule | -| ---------- | ------------------------------------------------------------------------------------------------------------------------ | -| `email` | Unique, and not already used by an account in the environment you're testing | -| `password` | At least 6 characters (Firebase's minimum) | -| `username` | Unique, 3-25 letters, numbers or underscores. If it's already taken, the API quietly picks a similar one | -| `name` | At most 30 characters. Stick to letters, numbers and single spaces, and avoid staff words such as "admin" or "moderator" | - -**No commas or `|` inside any value.** A comma always starts a new column, and the accounts are saved to -`created_accounts.txt` with `|` between values. - -**Why the name rules matter:** the API cleans names before saving them. It squeezes repeated spaces, removes emoji and -symbols, cuts names to 30 characters, and rejects names that look like staff. The `/me` test checks that the returned -name matches `accounts.csv` exactly, so if the API changes a name, every `/me` request for that account fails. - -### How to change it - -- **Support more threads:** you need **at least as many rows as `setupThreads`** (100 by default). To make a new file - with more rows, run this from `tests/load/`, using the domain and password from the current file: - - ```bash - domain=''; password=''; count=200 - for i in $(seq -f '%03g' 1 "$count"); do echo "loadtest$i@$domain,$password,loadtest$i,Load Test $i"; done > accounts.csv - ``` - - ```powershell - $domain = ''; $password = ''; $count = 200 - 1..$count | ForEach-Object { $n = '{0:D3}' -f $_; "loadtest$n@$domain,$password,loadtest$n,Load Test $n" } | Set-Content -Encoding ascii accounts.csv - ``` - -- **Keep values unique:** no two rows can share an email or username. -- **Reusing rows is fine:** the accounts are deleted at the end of every run, so the same file works run after run. If - a run fails to clean up, see [Troubleshooting](#sign-up-fails-with-email_exists). -- **Adding, removing or reordering columns:** the columns map to variables **by position**. If you change them, also - update: - 1. **CSV - Accounts → Variable Names**, so the names are in the same order as the columns. - 2. Every request body that uses the variables (e.g. `SETUP | Firebase Signup POST`). - 3. The **Save account for test and teardown** script, if the new value should be passed on to later stages. +The stages run one after another (**Run Thread Groups consecutively** is on). setUp groups run before the main test, +tearDown groups after it, and tearDown also runs after a [graceful shutdown](#stopping-a-run-early). + +**0 - Check for leftover accounts** (setUp, 1 thread). After a run, `created_accounts.txt` only lists accounts that +couldn't be deleted. If it isn't empty, an earlier run was killed or failed to clean up. Emptying the file would lose +the only record of those accounts, so instead this stage fails with a message, and sets `setupThreads`, `threads` and +`sessionThreads` to 0: this run creates nothing and only deletes the leftovers. Run again afterwards. Skipped when +`checkLeftovers=0`. + +**1 - Create accounts** (setUp, `setupThreads` threads, one account each): + +1. **Make this thread's account** (JSR223 PreProcessor) sets the thread's email, random password, username and name + (see [The test accounts](#the-test-accounts)). +2. `SETUP | Firebase Signup POST` creates the Firebase login. A JSON Extractor saves the token as `authToken`. +3. `SETUP | create-user-and-profile POST` creates the app user and profile. +4. **Save account for test and teardown** (JSR223 PostProcessor) appends `email|password|token|name|profileCreated` + to `created_accounts.txt`. An account whose profile creation failed is still saved, since its Firebase login + exists; `profileCreated=false` tells teardown how to delete it. + +**2 - Browsing session** (`sessionThreads` threads). Each thread takes one line of `created_accounts.txt`, then a +Runtime Controller repeats a visit for `sessionDuration` seconds. A visit makes the calls the web app makes when someone +opens it and browses profiles: + +1. `TEST | /me GET`: the signed-in user. Passes if the code is `200` **and** the returned `name` matches the account. + Saves the account's `userId` and username for the requests below. +2. `TEST | get-profile GET (own)`, `TEST | set-last-online-time POST`, `TEST | get-notifications GET` (`limit=450`, + as `use-notifications.ts` asks) and `TEST | compatible-profiles GET`: what loads in the background on every page. +3. `TEST | get-profiles GET (feed)`: the first page of the profile grid, as `profiles-home.tsx` asks for it (20 cards, + ordered by compatibility with this account). Saves the usernames and the last profile's id. +4. `TEST | get-profile GET (feed profile)`, `profileViews` times (3 by default): opens a random profile from that page. +5. `TEST | get-profiles GET (next page)`, in half the visits (a Throughput Controller): scrolls to the next page with + `after`. + +Steps 2 to 5 are skipped when `/me` fails, and 4 and 5 when the feed is empty. **Think time** (a Uniform Random Timer) +pauses 2 to 5 seconds before each request, like a person reading. It also keeps each account under the API's rate +limit of 120 requests a minute per user (`API_RATE_LIMIT_PER_MIN_AUTHED`): with no pause, most rate-limited requests +would come back `429`. + +**3 - Delete accounts** (tearDown, 1 thread, one loop per line of `created_accounts.txt`): + +1. `TEARDOWN | Firebase Sign in POST` gets a fresh token (setUp's may have expired after 1 hour). If Firebase says the + account doesn't exist, it is counted as already deleted and passes. +2. If the profile was created: `TEARDOWN | Delete app user` calls `/me/delete`, which deletes the app user, their + files and their Firebase login. It sends the reason `jmeter-load-test`, stored in `deleted_users`. +3. If there is no profile, or `/me/delete` failed: `TEARDOWN | Firebase delete account` deletes the Firebase login + directly, so it doesn't stay behind. + +Each successful delete is remembered, and **4 - Forget deleted accounts** then rewrites `created_accounts.txt` with +only the accounts that couldn't be deleted. A warning in `jmeter.log` says how many remain. + +## The test accounts + +The API only accepts most requests from signed-in accounts, so each thread needs its own real account. The plan makes +them itself; there's no account file to maintain. + +| Value | Made as | Example | +| -------- | --------------------------------- | -------------------------------- | +| email | `loadtestx@example.com` | `loadtest28120940x1@example.com` | +| password | A random UUID | | +| username | `loadUserx` | `loadUser28120940x1` | +| name | `Load Test ` | `Load Test 1` | + +- `` is the thread number. `` is the `runId` property, by default the day, hour, minute and second the run + started (`ddHHmmss`). CI passes its own. A fresh ID every run means an overlapping run, or leftovers from an earlier + one, can never cause `EMAIL_EXISTS` or a taken username. +- `example.com` is reserved for testing (RFC 2606) and can't receive mail. The API's `sendEmail` skips such addresses, + so creating accounts sends no welcome emails (once the API with that check is deployed). +- Usernames are cut to 25 characters, and the API cleans names (no emoji or symbols, at most 30 characters, no staff + words like "admin"). The `/me` request checks the name comes back unchanged, so keep both short and plain if you change + the script. --- ## Running the test -### Why non-GUI mode - -Always run load tests from the command line (**non-GUI mode**, the `-n` flag). Use the GUI only to build and debug the -plan. - -- **It uses far less CPU and memory.** The GUI draws results on screen and keeps them in memory while the test runs. - That takes resources away from sending requests, so the machine running JMeter can become the bottleneck, and the - results get less accurate. -- **It works in CI and scripts,** since no screen is needed. - -JMeter's own docs give the same advice, and JMeter prints a warning about it whenever the GUI opens. +Always run load tests from the command line (**non-GUI mode**, `-n`). The GUI uses far more CPU and memory, which can +make JMeter itself the bottleneck. Use the GUI only to build and debug the plan. ### Commands -Run these from `tests/load/`, in the terminal for your [setup](#choose-your-setup) (a WSL terminal if the project is in -WSL): - -```bash -cd tests/load -``` - -The results (`results/…`) and report (`report`) paths are relative to that folder. - -#### Smoke test +Run these from `tests/load/`. The test runs against the **shared dev API and Firebase**, not your local stack. -1 thread for 10 seconds. Run this 1hr(`Firebase creation limit resets every hour`) before every full run to check that each step works. +**Smoke test** (1 thread, 10 seconds). Run it before every full run to check each step works. Firebase allows 100 new +accounts per hour from one IP address, so leave an hour between a smoke test and a full run if you're near that. ```bash -jmeter -n -f -t compass-testPlan.jmx -l results/smoke.jtl -e -o report -Jthreads=1 -JsetupRamp=1 -Jramp=1 -Jduration=10 +jmeter -n -f -t compass-testPlan.jmx -l results/smoke.jtl -e -o report -Jthreads=1 -JsetupRamp=1 -Jramp=1 -Jduration=10 ``` -#### Full run with HTML report - -Uses the defaults (100 threads, 120 seconds). +**Full run with HTML report** (the defaults: 100 threads, 120 seconds): ```bash jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report ``` -#### Report with only the load test requests - -Hides the account setup and cleanup requests from the report. The filter is a regular expression that keeps only the -labels starting with `TEST |`. +**Report with only the load test requests** (hides setup and cleanup; in PowerShell use double quotes): ```bash jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report '-Jjmeter.reportgenerator.sample_filter=^TEST \|.*' ``` -```powershell -jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report "-Jjmeter.reportgenerator.sample_filter=^TEST \|.*" -``` - -#### Run against a different environment - -The plan tests the dev environment by default. An API only accepts sign-in tokens from its own Firebase project, so -when you change the API host, also pass the Firebase key of the same environment: +**Another environment.** An API only accepts tokens from its own Firebase project, so pass both: ```bash jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -JapiHost= -JfirebaseApiKey= ``` -```powershell -jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl "-JapiHost=" -JfirebaseApiKey= -``` - -> **PowerShell note:** wrap any `-J` argument whose value contains a dot in double quotes, as above. Without the -> quotes, PowerShell can split the argument at the dot, and JMeter gets a broken value. - -#### Cleanup only +> **PowerShell:** wrap any `-J` argument whose value contains a dot in double quotes, or PowerShell may split it. -Deletes the accounts still listed in `created_accounts.txt`, without creating accounts or running the test. Use it -when a run stopped before cleanup finished. +**Cleanup only.** Deletes the accounts still listed in `created_accounts.txt`, without creating any. A normal run +already does this when it finds leftovers, so you only need it to clean up without starting a test: ```bash -jmeter -n -f -t compass-testPlan.jmx -l results/cleanup.jtl -JresetFile=0 -Jthreads=0 +jmeter -n -f -t compass-testPlan.jmx -l results/cleanup.jtl -JcheckLeftovers=0 -Jthreads=0 ``` -- `-JresetFile=0` gives `0 - Reset shared file` zero threads, so the file isn't emptied. -- `-Jthreads=0` gives account creation and the load test zero threads, so they're skipped. -- `3 - Delete accounts` always runs with 1 thread, so cleanup is all that runs. - -#### Rebuild a report from an existing results file - -Builds the HTML report again without running the test. The `report` folder must be empty or not exist yet. +**Rebuild a report** from an existing results file (the `report` folder must be empty or missing): ```bash jmeter -g results/results.jtl -o report @@ -487,680 +244,184 @@ jmeter -g results/results.jtl -o report ### Stopping a run early -To stop a run before it finishes **and still delete the accounts**, stop it gracefully. From another terminal of the -same kind as the one running the test (a WSL terminal if the project is in WSL), run JMeter's shutdown script. It's -in JMeter's `bin/` folder, so it's on your PATH: - -```bash -shutdown.sh -``` - -```powershell -shutdown.cmd -``` - -Type the full name: plain `shutdown` is your operating system's command to turn the computer off. +To stop a run **and still delete the accounts**, run `shutdown.sh` (`shutdown.cmd` on Windows) from another terminal of +the same kind. It's in JMeter's `bin/` folder. Type the full name: plain `shutdown` turns your computer off. In the GUI, +use **Run → Shutdown**. -JMeter lets the requests in progress finish, then runs `3 - Delete accounts` as usual. In the GUI, use -**Run → Shutdown** for the same effect. - -Pressing **Ctrl+C**, running `stoptest.sh`/`stoptest.cmd`, or using **Run → Stop** in the GUI stops JMeter straight -away, and cleanup **doesn't** run. Afterwards, run [Cleanup only](#cleanup-only) to delete the accounts that were -created. +**Ctrl+C**, `stoptest.sh` or **Run → Stop** stop straight away and cleanup **doesn't** run. The accounts stay listed in +`created_accounts.txt`, and the next run deletes them before doing anything else. ### Command flags -| Flag | Meaning | -| ------------- | --------------------------------------------------------------------------- | -| `-n` | Non-GUI mode | -| `-t ` | The test plan to run | -| `-l ` | Where to save the [raw results](#the-results-file-jtl) (`.jtl`, a CSV file) | -| `-e -o ` | After the run, build an HTML report into this folder | -| `-f` | Delete the old results file and report folder first | -| `-g ` | Build a report from an existing `.jtl` without running a test | -| `-J=` | Set a property the plan reads (see below) | - -Without `-f`, `-l` **adds** to an existing results file (mixing runs together), and `-e -o` **fails** if the report -folder isn't empty. +| Flag | Meaning | +| ------------- | --------------------------------------------------------------------------------- | +| `-n` | Non-GUI mode | +| `-t ` | The test plan to run | +| `-l ` | Where to save the raw results (`.jtl`). Without `-f`, it adds to an existing file | +| `-e -o ` | After the run, build an HTML report into this folder (must be empty without `-f`) | +| `-f` | Delete the old results file and report folder first | +| `-g ` | Build a report from an existing `.jtl` without running a test | +| `-J=` | Set a property the plan reads (see below) | ### Properties -The plan reads its settings from properties, so you can change a run without editing the `.jmx`. - -| Property | Default | What it controls | -| ---------------- | -------------------- | -------------------------------------------------------------------------------------- | -| `threads` | `100` | Threads for account creation and every load test, unless overridden | -| `ramp` | `10` | Seconds to start all threads in each load test, unless overridden | -| `duration` | `120` | Seconds each load test runs, unless overridden | -| `setupThreads` | `threads` | Number of accounts to create | -| `setupRamp` | `10` | Seconds to start all account-creation threads | -| `meThreads` | `threads` | Threads in the `/me` test | -| `meRamp` | `ramp` | Seconds to start all threads in the `/me` test | -| `meDuration` | `duration` | Seconds the `/me` test runs | -| `thinkTimeMs` | `0` | Pause before each load test request, per thread (0 = none) | -| `apiHost` | `` | API host to test against | -| `firebaseApiKey` | `` | Firebase key for sign-up and sign-in. Must belong to the same environment as `apiHost` | -| `resetFile` | `1` | `0` keeps `created_accounts.txt` instead of emptying it | - -**Specific settings override general ones.** `-Jthreads=50` sets account creation and every load test to 50 threads. -Adding `-JmeThreads=200` changes only the `/me` test. The reset and delete stages always use 1 thread. - -Each load test has its own property prefix (`me` for the `/me` test). -A new test gets its own `Threads`, `Ramp` and `Duration`. - -**Keep `setupThreads` at least as high as the largest load test.** The tests reuse the accounts created in setUp, and -a thread with no account stops. - -> **Ramp-up** is how long JMeter takes to start all the threads. With 100 threads and a 10-second ramp-up, a new -> thread starts every 0.1 seconds. Starting them all at once is less realistic and can overload sign-in. - ---- - -## Why it is structured this way - -- **Stages allow tests to be the main focus.** Creating accounts is slow and only happens once per account. Doing it in a - separate stage keeps it out of the load test results, so they only measure what's being tested. -- **setUp and tearDown thread groups run automatically.** JMeter runs setUp groups before the main test and tearDown - groups after it. tearDown also runs after a [graceful shutdown](#stopping-a-run-early), so test accounts don't pile - up when you stop a run early that way. -- **Stages run in order.** The test plan has **Run Thread Groups consecutively** turned on, so each group waits for the - one before it to finish (see [Test Plan](#test-plan)). -- **One file, one command.** All stages live in one `.jmx`, so the order is guaranteed and CI only needs one command. -- **A shared file passes accounts between stages.** Variables belong to one thread in one thread group, so later stages - can't see what setUp created. setUp writes each account to `created_accounts.txt`, and later stages read it. -- **Properties instead of fixed values.** The same plan works as a smoke test, a full run or a cleanup, just by - changing the command. No edits to the file needed. -- **Paths are relative to the `.jmx`.** The plan looks for `accounts.csv`, `created_accounts.txt` and `errors.xml` - next to the `.jmx`, so JMeter can be started from any folder, on any machine, and still find them. -- **Clear labels.** Every request name starts with `SETUP |`, `TEST |` or `TEARDOWN |`. This makes the stages easy to - tell apart in the results, and lets the report [filter](#report-with-only-the-load-test-requests) to `TEST |` only. +| Property | Default | What it controls | +| ----------------- | -------------------- | ------------------------------------------------------------------------------- | +| `threads` | `100` | Threads for account creation and every load test, unless overridden | +| `ramp` | `10` | Seconds to start all threads in each load test, unless overridden | +| `duration` | `120` | Seconds each load test runs, unless overridden | +| `setupThreads` | `threads` | Number of accounts to create | +| `setupRamp` | `10` | Seconds to start all account-creation threads | +| `sessionThreads` | `threads` | Threads in the browsing session | +| `sessionRamp` | `ramp` | Seconds to start all browsing session threads | +| `sessionDuration` | `duration` | Seconds the browsing session runs | +| `profileViews` | `3` | Feed profiles opened per visit | +| `thinkTimeMs` | `2000` | Shortest pause before each load test request, per thread | +| `thinkRandomMs` | `3000` | Random extra pause on top of `thinkTimeMs` (0 to this) | +| `apiHost` | `` | API host to test against | +| `firebaseApiKey` | `` | Firebase key for sign-up and sign-in. Must match `apiHost`'s environment | +| `runId` | start time | Added to every email and username (see [The test accounts](#the-test-accounts)) | +| `checkLeftovers` | `1` | `0` skips the leftover check | + +**Specific settings override general ones:** `-Jthreads=50` sets everything to 50 threads; adding `-JsessionThreads=200` +changes only the browsing session. **Keep `setupThreads` at least as high as the largest load test**, since each load test +thread needs an account. **Ramp-up** is how long JMeter takes to start all threads; starting them all at once can +overload sign-in. **Lowering the think time** sends more load per account, but keep the average pause above +about 0.5 seconds, or the rate limit turns requests into `429`s. --- ## Element reference -Each element type used in the plan, with: - -- **What it does:** how the element works. -- **Why it's used here:** what the plan needs it for. -- **How it's set up here:** its key settings in this plan, and why. -- **How to change it:** where to edit it, what else has to change with it, and how to add a new one. - -**GUI basics:** to **add** an element, right-click the element it should go under, choose **Add**, then pick the -category and the element. To **change** an element, click it in the tree and edit the fields on the right. To turn an -element off without deleting it, right-click it → **Disable**. Save with **Ctrl+S**. - -### Test Plan - -**What it does:** The top of the tree. It holds settings that apply to the whole run. - -**Why it's used here:** Its settings make the stages run in order, and make cleanup run after a graceful early stop. - -**How it's set up here:** - -- **Run Thread Groups consecutively (i.e. one at a time): on.** Each stage waits for the one before it to finish. - Without it, groups of the same type start together: `0 - Reset shared file` could empty `created_accounts.txt` while - `1 - Create accounts` is saving accounts to it. -- **Run tearDown Thread Groups after shutdown of main threads: on.** Accounts are still deleted after a - [graceful shutdown](#stopping-a-run-early). It doesn't help after Ctrl+C or **Stop**. - -**How to change it:** Click the top element of the tree and tick or untick the boxes. - -### Thread groups - -**What they do:** A thread group is a set of threads that all run the steps under it. There are three types: - -- **setUp Thread Group:** runs **before** all normal thread groups. -- **Thread Group:** the normal kind, used for the load tests. -- **tearDown Thread Group:** runs **after** all normal thread groups. - -**Why they're used here:** Each stage is its own thread group. That keeps the slow, one-off account creation out of the -measured load, and lets JMeter run setUp and cleanup at the right time without any extra logic. - -**How they're set up here:** - -| Thread group | Type | Threads | Loops | -| ----------------------- | ------------ | --------------------------------------- | ------------------------------------------------------ | -| `0 - Reset shared file` | setUp | `resetFile` (1, or 0 to skip the group) | 1 | -| `1 - Create accounts` | setUp | `setupThreads` | 1 (one account per thread) | -| `2 - /me load test` | Thread Group | `meThreads` | 1 (the Runtime Controller inside repeats the requests) | -| `3 - Delete accounts` | tearDown | 1 | Infinite, stops when `created_accounts.txt` runs out | - -- **Number of Threads (users):** how many threads to run. Here it's a property with a fallback, e.g. - `${__P(setupThreads,${__P(threads,100)})}` (see [Functions](#functions)). A group with 0 threads is skipped, which - is how `resetFile=0` and `threads=0` turn stages off. -- **Ramp-up period (seconds):** how long it takes to start all the threads (see the ramp-up note under - [Properties](#properties)). -- **Loop Count:** how many times each thread runs the steps. "Infinite" means until something stops it (here, the end - of `created_accounts.txt`). -- **Action to be taken after a Sampler error:** **Continue** in all groups, so one failed request doesn't stop a - thread. - -**How to change it:** Click the thread group and edit the fields. To change the numbers for one run, use -[properties](#properties) instead of editing the fields. To add a group, right-click the Test Plan → **Add → Threads -(Users)** → pick the type. Within each type, groups run in their order in the tree. - -### User Defined Variables - -**What it does:** Defines named values once, so every element can use them with `${name}`. Every thread gets its own -copy when it starts (see [Variables and properties](#variables-and-properties)). - -**Why it's used here:** Hosts, endpoint paths and the Firebase key are used by many requests. Keeping them in one place -means you change them once instead of in every request. - -**How it's set up here:** - -| Name | Value | -| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | -| `baseAuthUrl` | Firebase Auth host | -| `apiHost` | `${__P(apiHost,)}` (can be overridden with `-JapiHost`) | -| `firebaseApiKey` | `${__P(firebaseApiKey,)}` (can be overridden with `-JfirebaseApiKey`) | -| `signupUrl`, `signinUrl`, `deleteAccountUrl`, `getUserDataUrl` | Firebase Auth endpoint paths. `deleteAccountUrl` and `getUserDataUrl` aren't used by any request at the moment | - -**How to change it:** Click **User Defined Variables** and edit the table, or add a row for a new value. To make a -value changeable from the command line, wrap it in `__P`, e.g. `${__P(myValue,default)}`, and run with -`-JmyValue=...`. To add another set: right-click the Test Plan → **Add → Config Element → User Defined Variables**. - -### CSV Data Set Config - -**What it does:** Reads a text file line by line. Each line is split into columns, and the columns become variables -for the thread that read it. - -**Why it's used here:** Each thread needs its own account. The CSV configs hand every thread a different line, from -`accounts.csv` in setUp and from `created_accounts.txt` (which passes the accounts on) in the later stages. - -**How it's set up here:** - -| Element | Thread group | File | Variables | Delimiter | Sharing mode | -| ------------------------ | --------------------- | ---------------------- | --------------------------------------- | --------- | -------------------- | -| `CSV - Accounts` | `1 - Create accounts` | `accounts.csv` | `userEmail,userPassword,username,name` | `,` | All threads | -| `CSV - Created accounts` | `2 - /me load test` | `created_accounts.txt` | `userEmail,userPassword,authToken,name` | `\|` | Current thread group | -| `CSV - created accounts` | `3 - Delete accounts` | `created_accounts.txt` | `userEmail,userPassword,authToken,name` | `\|` | Current thread group | - -- **Filename:** a plain file name is looked up **next to the `.jmx`**. -- **Variable Names:** one name per column, in column order, separated by commas. -- **Delimiter:** the character between columns. `created_accounts.txt` uses `|`, because that's how the - [Save account script](#jsr223-scripts-groovy) writes it. -- **Recycle on EOF?:** **False**. When the file runs out, don't start again from the top, because each account is - used once per group. -- **Stop thread on EOF?:** **True**. A thread with no line left stops, instead of running with empty values. -- **Sharing mode:** which threads share one position in the file. - - **All threads:** one shared reader for the whole run, so every thread in every group gets a different line. Used - for `accounts.csv`. - - **Current thread group:** each thread group gets its own reader, starting from line 1. Used for - `created_accounts.txt`, so every load test **and** the cleanup each go through all the accounts. - -**When threads read a line:** each thread reads the next line at the **start of every loop** of its thread group. In -`2 - /me load test`, each thread loops once, so it keeps one account for the whole test. In `3 - Delete accounts`, the -single thread loops until the file runs out, taking the next account each time. - -**How to change it:** Click the CSV config and edit the fields. If you change the columns of either file, update the -Variable Names to match (see [accounts.csv → How to change it](#how-to-change-it), and the -[Save account script](#jsr223-scripts-groovy) for `created_accounts.txt`). To add one: right-click the thread group → -**Add → Config Element → CSV Data Set Config**. - -### HTTP Request - -**What it does:** Sends one HTTP request and records how long the response took. - -**Why it's used here:** Every step of the plan is an HTTP request: signing up, creating the profile, the load test -itself and the cleanup. The plan's requests, in run order, are listed in [Stage by stage](#stage-by-stage). - -**How it's set up here** (Basic tab), using `SETUP | Firebase Signup POST` as the example: - -- **Name** (at the top): the label shown in results. It starts with `SETUP |`, `TEST |` or `TEARDOWN |` (see - [Clear labels](#why-it-is-structured-this-way)). -- **Protocol:** `https`. -- **Server Name or IP:** a variable, here `${baseAuthUrl}` (requests to the API use `${apiHost}`). Don't include - `https://` here. -- **HTTP Request (method):** `POST`. -- **Path:** `${signupUrl}${firebaseApiKey}`. Variables can be combined. -- **Body Data tab:** the JSON body, with variables inside: - `{ "email": "${userEmail}", "password": "${userPassword}", "returnSecureToken": true }`. - -**How to change it:** Click the request and edit the Basic tab. To add one: right-click the thread group or controller -→ **Add → Sampler → HTTP Request**. Give it a name with the right prefix, plus a -[Header Manager](#http-header-manager) and a [Response Assertion](#response-assertion), like the existing requests. - -### HTTP Header Manager - -**What it does:** Adds HTTP headers to the requests in its scope. - -**Why it's used here:** The API and Firebase expect JSON bodies, and signed-in requests must send a token to prove -which account they come from. - -**How it's set up here:** Each request has its own Header Manager underneath it: - -- `Content-Type: application/json` on every request. -- `Authorization: Bearer ${authToken}` on requests that need a signed-in account. - -**How to change it:** Click the Header Manager and edit its table of header **Name** and **Value**. Values can use -variables. To add one: right-click the **request** → **Add → Config Element → HTTP Header Manager**. Placing it under -the request means it only applies to that request. Placing it under a thread group would apply it to **every** request -in the group (see [Scope](#scope-where-an-element-applies)). - -### JSON Extractor - -**What it does:** Reads a value out of a JSON response and saves it as a variable. - -**Why it's used here:** The sign-in token only exists once Firebase sends it back. The extractor catches it, so later -requests can send it in the `Authorization` header. - -**How it's set up here:** There's one under `SETUP | Firebase Signup POST` and one under -`TEARDOWN | Firebase Sign in POST`. Both take `idToken` from the response and save it as `authToken`. - -| Field | Value | Meaning | -| -------------------------- | ----------- | ------------------------------------------------------- | -| Names of created variables | `authToken` | The variable to save into | -| JSON Path expressions | `$.idToken` | Where to find the value. `$` is the top of the response | -| Match No. (0 for Random) | `1` | Use the first match | -| Default Values | (empty) | What to save if nothing is found | - -**How to change it:** Click the extractor and edit the fields. To capture more values, list more variable names and -JSON paths, separated by `;`. To add one: right-click the **request** → **Add → Post Processors → JSON Extractor**. - -### JSR223 scripts (Groovy) - -**What they do:** Run a short script. A **JSR223 Sampler** is a step of its own and shows up in the results. A -**JSR223 PostProcessor** runs right after the request it sits under. This plan uses **Groovy**, the recommended (and -fastest) language for JMeter scripts. - -Objects a script can use: - -| Object | What it is | -| ------- | ------------------------------------------------------------------------ | -| `vars` | This thread's variables: `vars.get("name")`, `vars.put("name", "value")` | -| `props` | Global properties | -| `prev` | The result of the previous request, e.g. `prev.isSuccessful()` | -| `log` | Writes to `jmeter.log`, e.g. `log.warn("...")` | - -**Why they're used here:** Emptying `created_accounts.txt` and adding accounts to it are easiest with a few lines of -script. - -**How they're set up here:** - -- **Language:** `groovy`. -- **Cache compiled script if available:** **on**. The script is compiled once, which matters when it runs thousands - of times. - -`SETUP | Reset created_accounts.txt` (JSR223 Sampler in `0 - Reset shared file`): - -```groovy -import org.apache.jmeter.services.FileServer -new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt").text = "" -``` - -- `FileServer.getFileServer().getBaseDir()` is the folder containing the `.jmx`. Using it means the file is always - found next to the plan, wherever JMeter was started from. -- `.text = ""` empties the file. - -**Save account for test and teardown** (JSR223 PostProcessor under `SETUP | create-user-and-profile POST`): - -```groovy -import org.apache.jmeter.services.FileServer - -def token = vars.get("authToken") -if (!token) { - log.warn("No token for ${vars.get('userEmail')}, not saving") - return -} -if (!prev.isSuccessful()) { - log.warn("Profile creation failed for ${vars.get('userEmail')}, saving anyway so it gets cleaned up") -} - -def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") -synchronized (FileServer.class) { - file.append([vars.get("userEmail"), vars.get("userPassword"), token, vars.get("name")].join("|") + "\n", "UTF-8") -} -``` - -- **No token:** sign-up failed, so there's no account to save. It logs a warning and stops. -- **Profile failed:** `prev` is the profile request. `prev.isSuccessful()` is false when the response code is 4xx or - 5xx. The script runs before the Response Assertion, so the assertion's result isn't included (see - [Order of execution](#order-of-execution-for-each-request)). The Firebase login exists either way, so the account - is **saved anyway**, and cleanup deletes it. -- **`synchronized (FileServer.class)`:** many threads write to the same file at the same time. This lets only one - thread write at a time, so lines never get mixed together. -- **`join("|")`:** joins the values into one line in the format `created_accounts.txt` expects. - -**How to change them:** Click the element and edit the **Script** box. If you add a value to `created_accounts.txt`, -add it to the `join` list here **and** to the Variable Names of both CSV configs that read the file, in the same -position. To add one: right-click a thread group → **Add → Sampler → JSR223 Sampler**, or right-click a request → -**Add → Post Processors → JSR223 PostProcessor**. Check that the language is `groovy`. - -### Response Assertion - -**What it does:** Checks part of a response against a pattern. If it doesn't match, the request is marked as -**failed** in the results. - -**Why it's used here:** On its own, JMeter only marks a request as failed when the response code is 4xx or 5xx. The -assertion makes the check strict: anything other than exactly `200` counts as a failure. - -**How it's set up here:** Every request has one: - -- **Field to Test:** `Response Code` -- **Pattern Matching Rules:** `Equals` -- **Patterns to Test:** `200` - -**How to change it:** Click the assertion and edit **Patterns to Test**, e.g. if a new request should return `201`. -To add one: right-click the **request** → **Add → Assertions → Response Assertion**. - -### JSON Assertion - -**What it does:** Checks that a value exists in a JSON response, and optionally that it equals an expected value. - -**Why it's used here:** A `200` only says the request worked. Checking the returned `name` proves the API sent back -**this** account's data, not someone else's. - -**How it's set up here:** On `TEST | /me GET`: - -- **Assert JSON Path exists:** `$.name` -- **Additionally assert value:** on -- **Expected Value:** `${name}`, the account's name from `created_accounts.txt` -- **Match as regular expression:** **off**. When it's on, characters like `.` or `(` in a name are treated as pattern - symbols, and valid names can fail. - -Because of this check, names in `accounts.csv` must follow the [name rules](#format). - -**How to change it:** Click the assertion and change the JSON path and expected value to something that proves the -response is right. To add one: right-click the **request** → **Add → Assertions → JSON Assertion**. - -### Runtime Controller - -**What it does:** Repeats the elements under it, over and over, until a set number of seconds has passed. - -**Why it's used here:** It makes the load test run for a fixed time, so a test takes the same time however fast or slow -the API responds. - -**How it's set up here:** It wraps `TEST | /me GET`, so each thread keeps sending requests for `meDuration` seconds. - -- **Runtime (seconds):** `${__P(meDuration,${__P(duration,120)})}` - -**How to change it:** Change the time per run with `-JmeDuration` or `-Jduration` (see [Properties](#properties)) -rather than editing the field. To test a sequence of requests, put more requests under the controller. To add one: -right-click the thread group → **Add → Logic Controller → Runtime Controller**, then drag the requests onto it. - -### Constant Timer - -**What it does:** Pauses each thread for a fixed time **before every request in its scope**. - -**Why it's used here:** As **think time**: an optional pause between requests, so threads can act more like real -people. It defaults to `0` (no pause, maximum load). - -**How it's set up here:** It's called **Think time** and sits under the Runtime Controller. - -- **Thread Delay (in milliseconds):** `${__P(thinkTimeMs,0)}` - -**How to change it:** Set the pause per run with `-JthinkTimeMs=`. To add one: right-click the -controller or thread group → **Add → Timer → Constant Timer**. - -### Listeners - -**What they do:** Collect and show results. They apply to every request in their scope; here they sit at the top -level, so they see everything. - -**Why they're used here:** To keep the details of failed requests without slowing the run, and to show results live -when debugging in the GUI. - -**How they're set up here:** - -| Listener | Status | What it does | -| ---------------------- | -------- | ---------------------------------------------------------------------------------------- | -| **Summary Report** | Enabled | A live results table, only useful in the GUI | -| **Simple Data Writer** | Enabled | Saves **failed requests only** to `errors.xml`, with the response body showing why | -| **View Results Tree** | Disabled | Shows every request and response in detail. Great for debugging, too heavy for load runs | - -Simple Data Writer settings: - -- **Filename:** `errors.xml` (relative to the `.jmx`). -- **Log/Display Only: Errors:** ticked, so only failed requests are saved. -- **Configure** button: - - **Save Response Data (XML):** on, to see why a request failed. - - **Save Sampler Data (XML):** **off**. Request bodies contain passwords and tokens. - -The results file from `-l` is separate from these listeners. It always records every request. - -**How to change them:** To debug a smoke test in the GUI, enable **View Results Tree** (right-click → **Enable**), and -disable it again before a load run. To add one: right-click the Test Plan → **Add → Listener**. - -### Debug Sampler - -**What it does:** A sampler that doesn't send a request. Instead, it records the thread's variables at that point in -the steps, so you can read them in **View Results Tree**. - -**Why it's used here:** To check what a thread holds while building or fixing the plan, e.g. whether `authToken` was -extracted, or which row of `accounts.csv` the thread read. It's **disabled** normally, because it adds a row per thread -to the results and isn't needed during a load run. - -**How it's set up here:** It's the last step of `1 - Create accounts`, after `SETUP | create-user-and-profile POST`, so -it shows each thread's variables once its account exists. It shows **JMeter variables** only; JMeter properties and -system properties are turned off. - -**How to change it:** Right-click it → **Enable**, enable **View Results Tree** as well, and run a smoke test in the -GUI. Select the **Debug Sampler** row and open the **Response data** tab to see the variables, which include the -account's password and token. Disable both again before a load run. To check variables somewhere else, add one there: -right-click the thread group → **Add → Sampler → Debug Sampler**, and drag it to the step you want to check. - -### Functions - -**What they do:** Special `${__name(...)}` values that JMeter works out at run time. - -**Why they're used here:** `__P` lets the command line change the plan's settings without editing the `.jmx`. - -**How they're set up here:** The plan only uses `__P`, which reads a property: - -- `${__P(duration,120)}` means the `duration` property, or `120` if it isn't set. -- `${__P(setupThreads,${__P(threads,100)})}` means `setupThreads`; if that isn't set, `threads`; if that isn't set - either, `100`. This is how specific settings override general ones. - -**How to change them:** Edit the function where it's used, e.g. to change a default value. **Tools → Function Helper -Dialog** in the GUI lists every function and builds the syntax for you. +To **add** an element in the GUI, right-click the element it should go under → **Add**. To turn one off, right-click → +**Disable**. Save with **Ctrl+S**. + +| Element | How it's used here | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Test Plan** | **Run Thread Groups consecutively** on, so stages run in order. **Run tearDown Thread Groups after shutdown** on, so a graceful stop still cleans up. | +| **Thread groups** | Thread counts are properties with fallbacks, e.g. `${__P(setupThreads,${__P(threads,100)})}`, read when the group starts. A group with 0 threads is skipped. All groups **Continue** after a sampler error. | +| **User Defined Variables** | `baseAuthUrl`, `apiHost`, `firebaseApiKey`, `runId`, and the Firebase endpoint paths (`signupUrl`, `signinUrl`, `deleteAccountUrl`; `getUserDataUrl` is unused). Wrap a value in `__P` to make it settable with `-J`. | +| **CSV Data Set Config** | Two, both reading `created_accounts.txt` with `\|` as the delimiter and the sharing mode _Current thread group_, so the load test and teardown each go through every account. **Stop thread on EOF** is on, **Recycle** off. | +| **HTTP Request** | One per step. Names start with `SETUP \|`, `TEST \|` or `TEARDOWN \|`, so stages are easy to tell apart and the report can filter. Server is `${baseAuthUrl}` or `${apiHost}`, without `https://`. | +| **HTTP Header Manager** | Under each request: `Content-Type: application/json`, plus `Authorization: Bearer ${authToken}` where signed in. | +| **JSON Extractor** | Saves `$.idToken` from sign-up and teardown sign-in as `authToken`. | +| **JSR223 (Groovy)** | The leftover check, account generation, saving accounts, the teardown sign-in check, remembering deletes and rewriting the file. Scripts use `vars`, `props`, `prev`, `log`, and find files next to the `.jmx` with `FileServer.getFileServer().getBaseDir()`. **Cache compiled script** is on. | +| **Response Assertion** | On every request: response code must equal `200`. | +| **JSON Assertion** | On `TEST \| /me GET`: `$.name` must equal `${name}`, **not** as a regex (a `.` in a name would break it). | +| **If Controller** | In the session, skip requests whose input is missing (`/me` failed, empty feed). In teardown, choose between `/me/delete` and the direct Firebase delete, using `signedIn` and `profileCreated`. | +| **Loop, Throughput Controller** | Open `profileViews` feed profiles per visit; load the next page in 50% of visits. A random feed profile is `${__V(feedUsername_${__Random(1,${feedUsername_matchNr})})}`. | +| **Runtime Controller** | Repeats a visit for `${__P(sessionDuration,${__P(duration,120)})}` seconds. | +| **Uniform Random Timer** | **Think time**: `${__P(thinkTimeMs,2000)}` ms plus a random 0 to `${__P(thinkRandomMs,3000)}` ms before each session request. | +| **Listeners** | **Simple Data Writer** saves failed requests only to `errors.xml`, with response bodies but not request bodies (they hold passwords and tokens). **Summary Report** is GUI only. **View Results Tree** is disabled: great for debugging, too heavy for load runs. | +| **Debug Sampler** | Disabled, at the end of `1 - Create accounts`. Enable it with View Results Tree to see a thread's variables in the GUI; disable both before a load run. | + +If you add a column to `created_accounts.txt`, add it to the `join` in **Save account for test and teardown** and to +the Variable Names of both CSV configs, in the same position. --- ## Adding a new test to this plan -New load tests **reuse the accounts** created by setUp and deleted by tearDown, so you only need to build the test -itself. +New load tests reuse the accounts created by setUp and deleted by tearDown, so you only build the test itself. -1. **Duplicate the existing test group.** Right-click `2 - /me load test` → **Duplicate**. Drag the copy so it sits - **after the last load test and before `3 - Delete accounts`**. -2. **Rename it** `N - `, where `N` is its position in the run order. -3. **Choose a property prefix** for the new test, e.g. ``, and set these fields: +1. Right-click `2 - Browsing session` → **Duplicate**, and drag the copy after the last load test. (To add a + request to the session instead, put it inside the existing Runtime Controller with a `TEST |` name.) +2. Rename it `N - ` and pick a property prefix, e.g. ``: - Number of Threads: `${__P(Threads,${__P(threads,100)})}` - Ramp-up period: `${__P(Ramp,${__P(ramp,10)})}` - Runtime Controller → Runtime: `${__P(Duration,${__P(duration,120)})}` -4. **Keep the `CSV - Created accounts` element** as it is. Its sharing mode, _Current thread group_, gives this test - its own pass through all the accounts. -5. **Change the HTTP Request:** - - Name: `TEST | ` - - Method, Path and Body Data for the new request - - Keep the Header Manager with `Authorization: Bearer ${authToken}` if the request needs a signed-in account -6. **Update the assertions:** - - Keep the Response Assertion if `200` is the expected code, or change it. - - Change or remove the JSON Assertion so it checks something in **this** response. -7. **Optional:** adjust or remove the think time timer, or add more requests inside the Runtime Controller to test a - sequence of requests. -8. **Check account numbers:** `setupThreads` must cover the new test's thread count (see [Properties](#properties)). -9. **Save, then run a [smoke test](#smoke-test).** Check that the new `TEST |` rows show up and pass. - -**Things to keep in mind:** - -- **Tests run one after another and share the same accounts.** Total run time is the sum of all test durations. -- **Long runs need fresh tokens.** All tests use the tokens created in setUp, which expire after 1 hour. If the tests - together run longer than that, later tests start failing with `401`. In that case, add a sign-in request (like - `TEARDOWN | Firebase Sign in POST` and its JSON Extractor) at the start of the new test group, before the Runtime - Controller. -- **Tests that change account data affect later tests.** For example, a test that edits the profile name makes a later - test that checks the name fail. Put such tests last, or make later checks not depend on that data. +3. Add `Threads` to the list the leftover check sets to 0, in **SETUP | Check created_accounts.txt is empty**. +4. Keep `CSV - Created accounts` as it is. +5. Change the HTTP Request (name `TEST | `, method, path, body) and update the assertions so they check + **this** response. +6. Make sure `setupThreads` covers the new test's thread count, then save and run a [smoke test](#commands). + +Keep in mind: tests run one after another and share the same accounts, so total time is the sum of their durations. +Tokens expire after 1 hour; if tests together run longer, add a sign-in request (like the teardown one) at the start of +the new group. A test that changes account data (e.g. the name) affects later tests, so put it last. --- ## Reading the results -A run gives you three outputs: the results file (`.jtl`), the HTML report built from it, and `errors.xml`. - -### The results file (`.jtl`) - -The file you name with `-l` (e.g. `results/results.jtl`) holds the **raw results** of a run. JMeter writes one row for -every request it sends, in every stage, with how long it took and whether it passed. The HTML report and its statistics -are all calculated from this file. - -For example, a smoke test gives about 50 rows: one for each setup and cleanup request, plus one for every `/me` call -the thread made in its 10 seconds. - -It's a plain CSV file with a header row, so it opens in any spreadsheet. The most useful columns: - -| Column | Meaning | -| ------------------------------------------- | ------------------------------------------------------------------------------ | -| `timeStamp` | When the request was sent (milliseconds since 1970) | -| `elapsed` | Response time in ms, the main number in a load test | -| `label` | Which request it was, e.g. `TEST \| /me GET` | -| `responseCode`, `success`, `failureMessage` | Whether it passed, and why not if it failed | -| `Latency` | Time until the first byte of the response arrived | -| `Connect` | Time spent opening the connection | -| `allThreads` | How many threads were running at that moment, which shows the load at the time | -| `threadName` | Which thread sent it, e.g. `1 - Create accounts 2-1` | - -What it's used for: - -- **Building the HTML report.** `-e -o report` builds it at the end of a run, and - [Rebuild a report](#rebuild-a-report-from-an-existing-results-file) builds it again later from the same file. -- **Looking at single requests.** Sort by `elapsed` to find the slowest ones, or filter on `success` = `false` to see - what failed and when. -- **Comparing runs.** Keep the file from each run, e.g. before and after a change to the API. Because the commands use - `-f`, a run overwrites the file with the same name, so copy or rename the ones you want to keep. - -It doesn't contain passwords or tokens. The `URL` column does include the Firebase API key, but that key isn't secret -(the web app sends it to every browser), and `results/` is git-ignored anyway. - -### The HTML report +**The results file (`.jtl`)**, named with `-l`, has one CSV row per request, in every stage. The HTML report is built +from it. Useful columns: `elapsed` (response time in ms), `label`, `responseCode`, `success`, `failureMessage`, +`Latency`, `Connect` and `allThreads` (the load at that moment). Sort by `elapsed` for the slowest requests, or keep +files from several runs to compare them (`-f` overwrites a file with the same name). It holds no passwords or tokens. -- **Check the run matches what you asked for.** In the **Statistics** table, `#Samples` should fit the threads and - duration you used, e.g. 100 sign-up samples for 100 threads. -- **Error %** shows the share of failed requests per label. Any failures in `SETUP |` rows mean fewer accounts, and so - less load, than intended. -- **Response times:** _Average_ can hide slow outliers. The **90th/95th/99th pct** columns show the time that - 90/95/99% of requests finished within. +**The HTML report:** in **Statistics**, check `#Samples` fits what you asked for (e.g. 100 sign-ups for 100 threads). +Failures in `SETUP |` rows mean less load than intended. _Average_ hides slow outliers, so look at the **90th/95th/99th +pct** columns. -### errors.xml - -Holds only the failed requests, each with the response body that shows why it failed. Unlike the `.jtl`, which `-f` replaces, it -**adds to itself on every run**, so delete it before a run if you only want that run's errors. +**errors.xml** holds only failed requests, with the response body that shows why. It **adds to itself on every run**, +so delete it first if you only want one run's errors. --- ## Automated runs (GitHub Actions) -Two GitHub Actions workflows run the plan automatically. Both run the same steps, kept in -`.github/workflows/load-test.yml`, with different settings: - -| | Smoke test | Full test | -| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | -| Workflow | `load-smoke-test.yml` | `load-full-test.yml` | -| Runs when | A pull request into `main` changes API-related files: `backend/api/`, `backend/shared/`, `common/src/api/`, `tests/load/` or the load test workflows | Every Monday at 06:00 UTC | -| Size | 3 threads for 30 seconds | 100 threads for 120 seconds | -| Passes when | No request fails, and the p95 of `TEST \|` requests is at most 2000 ms | At most 1% of each request fails, and the p95 of `TEST \|` requests is at most 800 ms | -| Report artifact | `load-smoke-report` | `load-full-report` | +Two workflows run the plan, using the shared steps in `.github/workflows/load-test.yml`: -You can also start either one by hand: open the **Actions** tab, pick the workflow and click **Run workflow**. +| | Smoke test (`load-smoke-test.yml`) | Full test (`load-full-test.yml`) | +| ----------- | ------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------- | +| Runs when | A PR into `main` changes `backend/api/`, `backend/shared/`, `common/src/api/`, `tests/load/` or the load workflows | Every Monday at 06:00 UTC | +| Size | 3 threads for 30 seconds | 100 threads for 120 seconds | +| Passes when | No request fails, and the p95 of `TEST \|` requests is at most 2000 ms | At most 1% of each request fails, and `TEST \|` p95 is at most 800 ms | +| Artifact | `load-smoke-report` | `load-full-report` | -> **They test the API deployed to dev, not the pull request's code.** Nothing deploys to dev automatically: dev -> only changes when someone runs `./deploy-api.sh dev` in `backend/api/`. To load test a change, deploy it to dev -> first, then start the smoke or full test by hand. +Either can also be started by hand from the **Actions** tab → **Run workflow**. -### What a run does +> **They test the API deployed to dev, not the pull request's code.** Dev only changes when someone runs +> `./deploy-api.sh dev` in `backend/api/`. To load test a change, deploy it to dev first. -1. Installs JMeter (cached between runs) and checks the download against Apache's SHA-512 checksum. -2. Makes a fresh `accounts.csv` with one account per thread, each with a random password. The run's ID and attempt - number make every account unique, including when a failed run is re-run. The committed `accounts.csv` isn't used. -3. Runs the plan with the workflow's size, and builds the HTML report. -4. Checks the results with [check_results.py](#check_resultspy). This step fails the run when a limit is broken. -5. If JMeter didn't finish (it crashed, or the run was cancelled or timed out), runs [Cleanup only](#cleanup-only), - so the accounts don't stay on dev. -6. Uploads the report, the results, `errors.xml` and the logs. To get them, open the run in the **Actions** tab and - download the artifact under **Artifacts**. +A run installs JMeter (cached, checked against Apache's SHA-512), runs the plan with `runId` set from the workflow run +ID and attempt number, checks the results with [check_results.py](#check_resultspy), runs [Cleanup only](#commands) if +JMeter didn't finish, and uploads the report, results, `errors.xml` and logs. -Only one load test runs at a time, because two tests on the same dev API would skew each other's results. A run that -starts while another is going waits for it to finish; if several are waiting, only the newest one runs. +**Concurrency:** each PR has its own smoke group, where a newer push cancels the older run (its cleanup step still +runs), and the full test has its own group. A shared group would let one PR's push silently cancel another PR's queued +run or the weekly full run, because GitHub keeps only one pending run per group. -The full test's 100 threads match Firebase's limit of 100 new accounts per hour from one IP address. To use more, -raise the sign-up quota in the Firebase console first. - -To change a workflow's size or limits, edit the values under `with:` in `load-smoke-test.yml` or -`load-full-test.yml`. +To change a workflow's size or limits, edit the values under `with:`. Going above 100 accounts needs a higher sign-up +quota in the Firebase console. ### check_results.py -JMeter exits as if all went well even when requests fail, so CI needs something else to decide pass or fail. -`tests/load/check_results.py` reads a results file and checks two limits for each request label: - -- **Error rate**, for every request: `SETUP |`, `TEST |` and `TEARDOWN |`. A failed setup request means less load than - asked for, and a failed cleanup request leaves accounts behind on dev. -- **p95 response time**, for `TEST |` requests only, because setup and cleanup times aren't what the test measures. - It's only checked when a label has at least 20 samples, since fewer is too little to judge. +JMeter exits with `0` even when requests fail, so CI uses `tests/load/check_results.py` to pass or fail a run. It +checks, per request label: -It also fails when there are no `TEST |` results at all, which means no accounts were created. +- **Error rate**, for every request. A failed setup request means less load; a failed cleanup request leaves accounts + behind. +- **p95 response time**, for `TEST |` requests with at least 20 samples. -You can run it on a local run's results too, from `tests/load/` (on Windows, use `python` instead of `python3`): +It also fails when there are no `TEST |` results at all. Run it locally from `tests/load/` (use `python` on Windows): ```bash python3 check_results.py results/results.jtl --max-error-rate 1 --max-p95 800 ``` -It prints each request's sample count, error rate and p95, then either `PASS` or one `FAIL:` line per broken limit. -It exits with `0` for a pass and `1` for a fail. Without the options, the limits are 1% errors and 800 ms. +It prints each label's samples, error rate and p95, then `PASS` or one `FAIL:` line per broken limit, and exits with +`0` or `1`. The defaults are 1% and 800 ms. --- ## Troubleshooting -### Sign-up fails with `EMAIL_EXISTS` - -A previous run didn't finish cleaning up, so accounts from `accounts.csv` still exist. - -- If `created_accounts.txt` still lists them, run [Cleanup only](#cleanup-only). -- If it doesn't (every run empties it at the start), rebuild it from `accounts.csv`, then run - [Cleanup only](#cleanup-only). Cleanup signs in again with each email and password, so the token column can be any - text: - - ```bash - awk -F, '{print $1 "|" $2 "|x|" $4}' accounts.csv > created_accounts.txt - ``` - - ```powershell - Import-Csv accounts.csv -Header email,password,username,name | ForEach-Object { "$($_.email)|$($_.password)|x|$($_.name)" } | Set-Content -Encoding ascii created_accounts.txt - ``` - - Accounts that no longer exist just show up as failed `TEARDOWN |` requests. - -Don't delete the accounts in the Firebase console instead. That only removes the Firebase login. The app user stays in -the database, and nothing warns you: the next run still works, because the API quietly gives the new accounts -different usernames, so the leftover users pile up. - -### Fewer load test threads than expected - -`created_accounts.txt` has fewer lines than the test has threads, so some threads stop right away. Check the `SETUP |` -rows in the results for failures. Also make sure `setupThreads` is high enough (see [Properties](#properties)), and -that `accounts.csv` has enough rows. - -### `/me` requests fail the JSON Assertion - -The `name` the API returned doesn't match the one in `accounts.csv`. The API cleans names when it saves them, so a -name with emoji, symbols, double spaces or more than 30 characters comes back changed. Fix the names (see the -[name rules](#format)) and run again. - -### `${something}` appears literally in a request +**The run says `created_accounts.txt lists N accounts from an earlier run`.** An earlier run didn't finish cleaning up. +This run deleted those accounts instead of testing; check the `TEARDOWN |` rows, then run again. Accounts that still +couldn't be deleted stay in the file, and `jmeter.log` says how many. Don't delete test accounts in the Firebase console +instead: that only removes the login, and the app user stays in the database. -The variable was never set. Common causes: +**Requests fail with `429`.** The think time is too short for the rate limit; raise `thinkTimeMs`. -- The name is spelled differently (names are case-sensitive). -- The element that sets it is out of scope (e.g. it's in another thread group). -- The extractor found nothing. +**Fewer load test threads than expected.** `created_accounts.txt` has fewer lines than the test has threads. Check the +`SETUP |` rows for failures, and that `setupThreads` is high enough. -Enable **View Results Tree** and run a smoke test in the GUI to see the actual requests sent. To see every variable a -thread holds, also enable the [Debug Sampler](#debug-sampler). +**`TEST | /me GET` fails the JSON Assertion.** The API changed the name when saving it. Keep names plain (see +[The test accounts](#the-test-accounts)). -### Report folder error +**`${something}` appears literally in a request.** The variable was never set: it's misspelled (names are +case-sensitive), set in another thread group, or the extractor found nothing. Enable **View Results Tree** (and the +Debug Sampler) and run a smoke test in the GUI. -JMeter won't write a report into a folder that isn't empty. Add `-f`, or delete the report folder first. +**Report folder error.** JMeter won't write a report into a folder that isn't empty. Add `-f`, or delete the folder. diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 45080095b..7866ebe1e 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -60,7 +60,7 @@ fi # ✅ Kill stale processes from previous runs print_status "Killing any stale processes..." ./scripts/firebase_stop.sh -supabase stop --no-backup 2>/dev/null || true +npx supabase stop --no-backup 2>/dev/null || true sleep 2 # Give ports time to free up # Install Playwright browsers @@ -81,7 +81,7 @@ npx wait-on \ yarn test:db:reset # Get connection details -STATUS_JSON=$(supabase status --output json) +STATUS_JSON=$(npx supabase status --output json) export NEXT_PUBLIC_SUPABASE_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL') export NEXT_PUBLIC_SUPABASE_ANON_KEY=$(echo "$STATUS_JSON" | jq -r '.ANON_KEY') export DATABASE_URL=$(echo "$STATUS_JSON" | jq -r '.DB_URL') diff --git a/scripts/seed.sh b/scripts/seed.sh index 4a9750205..40feb1a9e 100755 --- a/scripts/seed.sh +++ b/scripts/seed.sh @@ -8,7 +8,7 @@ cd "$(dirname "$0")"/.. export $(cat .env.test | grep -v '^#' | xargs) # Get connection details -STATUS_JSON=$(supabase status --output json) +STATUS_JSON=$(npx supabase status --output json) export NEXT_PUBLIC_SUPABASE_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL') export NEXT_PUBLIC_SUPABASE_ANON_KEY=$(echo "$STATUS_JSON" | jq -r '.ANON_KEY') export DATABASE_URL=$(echo "$STATUS_JSON" | jq -r '.DB_URL') diff --git a/tests/load/accounts.csv b/tests/load/accounts.csv deleted file mode 100644 index 283536503..000000000 --- a/tests/load/accounts.csv +++ /dev/null @@ -1,200 +0,0 @@ -compassLoadTest001@testExample.com,password123,LoadTestUser001,LoadTestName001 -compassLoadTest002@testExample.com,password123,LoadTestUser002,LoadTestName002 -compassLoadTest003@testExample.com,password123,LoadTestUser003,LoadTestName003 -compassLoadTest004@testExample.com,password123,LoadTestUser004,LoadTestName004 -compassLoadTest005@testExample.com,password123,LoadTestUser005,LoadTestName005 -compassLoadTest006@testExample.com,password123,LoadTestUser006,LoadTestName006 -compassLoadTest007@testExample.com,password123,LoadTestUser007,LoadTestName007 -compassLoadTest008@testExample.com,password123,LoadTestUser008,LoadTestName008 -compassLoadTest009@testExample.com,password123,LoadTestUser009,LoadTestName009 -compassLoadTest010@testExample.com,password123,LoadTestUser010,LoadTestName010 -compassLoadTest011@testExample.com,password123,LoadTestUser011,LoadTestName011 -compassLoadTest012@testExample.com,password123,LoadTestUser012,LoadTestName012 -compassLoadTest013@testExample.com,password123,LoadTestUser013,LoadTestName013 -compassLoadTest014@testExample.com,password123,LoadTestUser014,LoadTestName014 -compassLoadTest015@testExample.com,password123,LoadTestUser015,LoadTestName015 -compassLoadTest016@testExample.com,password123,LoadTestUser016,LoadTestName016 -compassLoadTest017@testExample.com,password123,LoadTestUser017,LoadTestName017 -compassLoadTest018@testExample.com,password123,LoadTestUser018,LoadTestName018 -compassLoadTest019@testExample.com,password123,LoadTestUser019,LoadTestName019 -compassLoadTest020@testExample.com,password123,LoadTestUser020,LoadTestName020 -compassLoadTest021@testExample.com,password123,LoadTestUser021,LoadTestName021 -compassLoadTest022@testExample.com,password123,LoadTestUser022,LoadTestName022 -compassLoadTest023@testExample.com,password123,LoadTestUser023,LoadTestName023 -compassLoadTest024@testExample.com,password123,LoadTestUser024,LoadTestName024 -compassLoadTest025@testExample.com,password123,LoadTestUser025,LoadTestName025 -compassLoadTest026@testExample.com,password123,LoadTestUser026,LoadTestName026 -compassLoadTest027@testExample.com,password123,LoadTestUser027,LoadTestName027 -compassLoadTest028@testExample.com,password123,LoadTestUser028,LoadTestName028 -compassLoadTest029@testExample.com,password123,LoadTestUser029,LoadTestName029 -compassLoadTest030@testExample.com,password123,LoadTestUser030,LoadTestName030 -compassLoadTest031@testExample.com,password123,LoadTestUser031,LoadTestName031 -compassLoadTest032@testExample.com,password123,LoadTestUser032,LoadTestName032 -compassLoadTest033@testExample.com,password123,LoadTestUser033,LoadTestName033 -compassLoadTest034@testExample.com,password123,LoadTestUser034,LoadTestName034 -compassLoadTest035@testExample.com,password123,LoadTestUser035,LoadTestName035 -compassLoadTest036@testExample.com,password123,LoadTestUser036,LoadTestName036 -compassLoadTest037@testExample.com,password123,LoadTestUser037,LoadTestName037 -compassLoadTest038@testExample.com,password123,LoadTestUser038,LoadTestName038 -compassLoadTest039@testExample.com,password123,LoadTestUser039,LoadTestName039 -compassLoadTest040@testExample.com,password123,LoadTestUser040,LoadTestName040 -compassLoadTest041@testExample.com,password123,LoadTestUser041,LoadTestName041 -compassLoadTest042@testExample.com,password123,LoadTestUser042,LoadTestName042 -compassLoadTest043@testExample.com,password123,LoadTestUser043,LoadTestName043 -compassLoadTest044@testExample.com,password123,LoadTestUser044,LoadTestName044 -compassLoadTest045@testExample.com,password123,LoadTestUser045,LoadTestName045 -compassLoadTest046@testExample.com,password123,LoadTestUser046,LoadTestName046 -compassLoadTest047@testExample.com,password123,LoadTestUser047,LoadTestName047 -compassLoadTest048@testExample.com,password123,LoadTestUser048,LoadTestName048 -compassLoadTest049@testExample.com,password123,LoadTestUser049,LoadTestName049 -compassLoadTest050@testExample.com,password123,LoadTestUser050,LoadTestName050 -compassLoadTest051@testExample.com,password123,LoadTestUser051,LoadTestName051 -compassLoadTest052@testExample.com,password123,LoadTestUser052,LoadTestName052 -compassLoadTest053@testExample.com,password123,LoadTestUser053,LoadTestName053 -compassLoadTest054@testExample.com,password123,LoadTestUser054,LoadTestName054 -compassLoadTest055@testExample.com,password123,LoadTestUser055,LoadTestName055 -compassLoadTest056@testExample.com,password123,LoadTestUser056,LoadTestName056 -compassLoadTest057@testExample.com,password123,LoadTestUser057,LoadTestName057 -compassLoadTest058@testExample.com,password123,LoadTestUser058,LoadTestName058 -compassLoadTest059@testExample.com,password123,LoadTestUser059,LoadTestName059 -compassLoadTest060@testExample.com,password123,LoadTestUser060,LoadTestName060 -compassLoadTest061@testExample.com,password123,LoadTestUser061,LoadTestName061 -compassLoadTest062@testExample.com,password123,LoadTestUser062,LoadTestName062 -compassLoadTest063@testExample.com,password123,LoadTestUser063,LoadTestName063 -compassLoadTest064@testExample.com,password123,LoadTestUser064,LoadTestName064 -compassLoadTest065@testExample.com,password123,LoadTestUser065,LoadTestName065 -compassLoadTest066@testExample.com,password123,LoadTestUser066,LoadTestName066 -compassLoadTest067@testExample.com,password123,LoadTestUser067,LoadTestName067 -compassLoadTest068@testExample.com,password123,LoadTestUser068,LoadTestName068 -compassLoadTest069@testExample.com,password123,LoadTestUser069,LoadTestName069 -compassLoadTest070@testExample.com,password123,LoadTestUser070,LoadTestName070 -compassLoadTest071@testExample.com,password123,LoadTestUser071,LoadTestName071 -compassLoadTest072@testExample.com,password123,LoadTestUser072,LoadTestName072 -compassLoadTest073@testExample.com,password123,LoadTestUser073,LoadTestName073 -compassLoadTest074@testExample.com,password123,LoadTestUser074,LoadTestName074 -compassLoadTest075@testExample.com,password123,LoadTestUser075,LoadTestName075 -compassLoadTest076@testExample.com,password123,LoadTestUser076,LoadTestName076 -compassLoadTest077@testExample.com,password123,LoadTestUser077,LoadTestName077 -compassLoadTest078@testExample.com,password123,LoadTestUser078,LoadTestName078 -compassLoadTest079@testExample.com,password123,LoadTestUser079,LoadTestName079 -compassLoadTest080@testExample.com,password123,LoadTestUser080,LoadTestName080 -compassLoadTest081@testExample.com,password123,LoadTestUser081,LoadTestName081 -compassLoadTest082@testExample.com,password123,LoadTestUser082,LoadTestName082 -compassLoadTest083@testExample.com,password123,LoadTestUser083,LoadTestName083 -compassLoadTest084@testExample.com,password123,LoadTestUser084,LoadTestName084 -compassLoadTest085@testExample.com,password123,LoadTestUser085,LoadTestName085 -compassLoadTest086@testExample.com,password123,LoadTestUser086,LoadTestName086 -compassLoadTest087@testExample.com,password123,LoadTestUser087,LoadTestName087 -compassLoadTest088@testExample.com,password123,LoadTestUser088,LoadTestName088 -compassLoadTest089@testExample.com,password123,LoadTestUser089,LoadTestName089 -compassLoadTest090@testExample.com,password123,LoadTestUser090,LoadTestName090 -compassLoadTest091@testExample.com,password123,LoadTestUser091,LoadTestName091 -compassLoadTest092@testExample.com,password123,LoadTestUser092,LoadTestName092 -compassLoadTest093@testExample.com,password123,LoadTestUser093,LoadTestName093 -compassLoadTest094@testExample.com,password123,LoadTestUser094,LoadTestName094 -compassLoadTest095@testExample.com,password123,LoadTestUser095,LoadTestName095 -compassLoadTest096@testExample.com,password123,LoadTestUser096,LoadTestName096 -compassLoadTest097@testExample.com,password123,LoadTestUser097,LoadTestName097 -compassLoadTest098@testExample.com,password123,LoadTestUser098,LoadTestName098 -compassLoadTest099@testExample.com,password123,LoadTestUser099,LoadTestName099 -compassLoadTest100@testExample.com,password123,LoadTestUser100,LoadTestName100 -compassLoadTest101@testExample.com,password123,LoadTestUser101,LoadTestName101 -compassLoadTest102@testExample.com,password123,LoadTestUser102,LoadTestName102 -compassLoadTest103@testExample.com,password123,LoadTestUser103,LoadTestName103 -compassLoadTest104@testExample.com,password123,LoadTestUser104,LoadTestName104 -compassLoadTest105@testExample.com,password123,LoadTestUser105,LoadTestName105 -compassLoadTest106@testExample.com,password123,LoadTestUser106,LoadTestName106 -compassLoadTest107@testExample.com,password123,LoadTestUser107,LoadTestName107 -compassLoadTest108@testExample.com,password123,LoadTestUser108,LoadTestName108 -compassLoadTest109@testExample.com,password123,LoadTestUser109,LoadTestName109 -compassLoadTest110@testExample.com,password123,LoadTestUser110,LoadTestName110 -compassLoadTest111@testExample.com,password123,LoadTestUser111,LoadTestName111 -compassLoadTest112@testExample.com,password123,LoadTestUser112,LoadTestName112 -compassLoadTest113@testExample.com,password123,LoadTestUser113,LoadTestName113 -compassLoadTest114@testExample.com,password123,LoadTestUser114,LoadTestName114 -compassLoadTest115@testExample.com,password123,LoadTestUser115,LoadTestName115 -compassLoadTest116@testExample.com,password123,LoadTestUser116,LoadTestName116 -compassLoadTest117@testExample.com,password123,LoadTestUser117,LoadTestName117 -compassLoadTest118@testExample.com,password123,LoadTestUser118,LoadTestName118 -compassLoadTest119@testExample.com,password123,LoadTestUser119,LoadTestName119 -compassLoadTest120@testExample.com,password123,LoadTestUser120,LoadTestName120 -compassLoadTest121@testExample.com,password123,LoadTestUser121,LoadTestName121 -compassLoadTest122@testExample.com,password123,LoadTestUser122,LoadTestName122 -compassLoadTest123@testExample.com,password123,LoadTestUser123,LoadTestName123 -compassLoadTest124@testExample.com,password123,LoadTestUser124,LoadTestName124 -compassLoadTest125@testExample.com,password123,LoadTestUser125,LoadTestName125 -compassLoadTest126@testExample.com,password123,LoadTestUser126,LoadTestName126 -compassLoadTest127@testExample.com,password123,LoadTestUser127,LoadTestName127 -compassLoadTest128@testExample.com,password123,LoadTestUser128,LoadTestName128 -compassLoadTest129@testExample.com,password123,LoadTestUser129,LoadTestName129 -compassLoadTest130@testExample.com,password123,LoadTestUser130,LoadTestName130 -compassLoadTest131@testExample.com,password123,LoadTestUser131,LoadTestName131 -compassLoadTest132@testExample.com,password123,LoadTestUser132,LoadTestName132 -compassLoadTest133@testExample.com,password123,LoadTestUser133,LoadTestName133 -compassLoadTest134@testExample.com,password123,LoadTestUser134,LoadTestName134 -compassLoadTest135@testExample.com,password123,LoadTestUser135,LoadTestName135 -compassLoadTest136@testExample.com,password123,LoadTestUser136,LoadTestName136 -compassLoadTest137@testExample.com,password123,LoadTestUser137,LoadTestName137 -compassLoadTest138@testExample.com,password123,LoadTestUser138,LoadTestName138 -compassLoadTest139@testExample.com,password123,LoadTestUser139,LoadTestName139 -compassLoadTest140@testExample.com,password123,LoadTestUser140,LoadTestName140 -compassLoadTest141@testExample.com,password123,LoadTestUser141,LoadTestName141 -compassLoadTest142@testExample.com,password123,LoadTestUser142,LoadTestName142 -compassLoadTest143@testExample.com,password123,LoadTestUser143,LoadTestName143 -compassLoadTest144@testExample.com,password123,LoadTestUser144,LoadTestName144 -compassLoadTest145@testExample.com,password123,LoadTestUser145,LoadTestName145 -compassLoadTest146@testExample.com,password123,LoadTestUser146,LoadTestName146 -compassLoadTest147@testExample.com,password123,LoadTestUser147,LoadTestName147 -compassLoadTest148@testExample.com,password123,LoadTestUser148,LoadTestName148 -compassLoadTest149@testExample.com,password123,LoadTestUser149,LoadTestName149 -compassLoadTest150@testExample.com,password123,LoadTestUser150,LoadTestName150 -compassLoadTest151@testExample.com,password123,LoadTestUser151,LoadTestName151 -compassLoadTest152@testExample.com,password123,LoadTestUser152,LoadTestName152 -compassLoadTest153@testExample.com,password123,LoadTestUser153,LoadTestName153 -compassLoadTest154@testExample.com,password123,LoadTestUser154,LoadTestName154 -compassLoadTest155@testExample.com,password123,LoadTestUser155,LoadTestName155 -compassLoadTest156@testExample.com,password123,LoadTestUser156,LoadTestName156 -compassLoadTest157@testExample.com,password123,LoadTestUser157,LoadTestName157 -compassLoadTest158@testExample.com,password123,LoadTestUser158,LoadTestName158 -compassLoadTest159@testExample.com,password123,LoadTestUser159,LoadTestName159 -compassLoadTest160@testExample.com,password123,LoadTestUser160,LoadTestName160 -compassLoadTest161@testExample.com,password123,LoadTestUser161,LoadTestName161 -compassLoadTest162@testExample.com,password123,LoadTestUser162,LoadTestName162 -compassLoadTest163@testExample.com,password123,LoadTestUser163,LoadTestName163 -compassLoadTest164@testExample.com,password123,LoadTestUser164,LoadTestName164 -compassLoadTest165@testExample.com,password123,LoadTestUser165,LoadTestName165 -compassLoadTest166@testExample.com,password123,LoadTestUser166,LoadTestName166 -compassLoadTest167@testExample.com,password123,LoadTestUser167,LoadTestName167 -compassLoadTest168@testExample.com,password123,LoadTestUser168,LoadTestName168 -compassLoadTest169@testExample.com,password123,LoadTestUser169,LoadTestName169 -compassLoadTest170@testExample.com,password123,LoadTestUser170,LoadTestName170 -compassLoadTest171@testExample.com,password123,LoadTestUser171,LoadTestName171 -compassLoadTest172@testExample.com,password123,LoadTestUser172,LoadTestName172 -compassLoadTest173@testExample.com,password123,LoadTestUser173,LoadTestName173 -compassLoadTest174@testExample.com,password123,LoadTestUser174,LoadTestName174 -compassLoadTest175@testExample.com,password123,LoadTestUser175,LoadTestName175 -compassLoadTest176@testExample.com,password123,LoadTestUser176,LoadTestName176 -compassLoadTest177@testExample.com,password123,LoadTestUser177,LoadTestName177 -compassLoadTest178@testExample.com,password123,LoadTestUser178,LoadTestName178 -compassLoadTest179@testExample.com,password123,LoadTestUser179,LoadTestName179 -compassLoadTest180@testExample.com,password123,LoadTestUser180,LoadTestName180 -compassLoadTest181@testExample.com,password123,LoadTestUser181,LoadTestName181 -compassLoadTest182@testExample.com,password123,LoadTestUser182,LoadTestName182 -compassLoadTest183@testExample.com,password123,LoadTestUser183,LoadTestName183 -compassLoadTest184@testExample.com,password123,LoadTestUser184,LoadTestName184 -compassLoadTest185@testExample.com,password123,LoadTestUser185,LoadTestName185 -compassLoadTest186@testExample.com,password123,LoadTestUser186,LoadTestName186 -compassLoadTest187@testExample.com,password123,LoadTestUser187,LoadTestName187 -compassLoadTest188@testExample.com,password123,LoadTestUser188,LoadTestName188 -compassLoadTest189@testExample.com,password123,LoadTestUser189,LoadTestName189 -compassLoadTest190@testExample.com,password123,LoadTestUser190,LoadTestName190 -compassLoadTest191@testExample.com,password123,LoadTestUser191,LoadTestName191 -compassLoadTest192@testExample.com,password123,LoadTestUser192,LoadTestName192 -compassLoadTest193@testExample.com,password123,LoadTestUser193,LoadTestName193 -compassLoadTest194@testExample.com,password123,LoadTestUser194,LoadTestName194 -compassLoadTest195@testExample.com,password123,LoadTestUser195,LoadTestName195 -compassLoadTest196@testExample.com,password123,LoadTestUser196,LoadTestName196 -compassLoadTest197@testExample.com,password123,LoadTestUser197,LoadTestName197 -compassLoadTest198@testExample.com,password123,LoadTestUser198,LoadTestName198 -compassLoadTest199@testExample.com,password123,LoadTestUser199,LoadTestName199 -compassLoadTest200@testExample.com,password123,LoadTestUser200,LoadTestName200 \ No newline at end of file diff --git a/tests/load/check_results.py b/tests/load/check_results.py index 480936a00..1e86e1ff6 100644 --- a/tests/load/check_results.py +++ b/tests/load/check_results.py @@ -10,6 +10,7 @@ import csv import math import sys +from collections import defaultdict # Requests in the plan are labelled "SETUP | ...", "TEST | ..." or "TEARDOWN | ...". # The error limit applies to all of them: a failed SETUP request means less load than asked for, and a @@ -32,9 +33,12 @@ def percentile(values, pct): def summarise(samples): """Group samples by label (request name) and work out the numbers for each.""" + groups = defaultdict(list) + for s in samples: # one pass, so large full-run files stay fast + groups[s["label"]].append(s) + stats = {} - for label in sorted({s["label"] for s in samples}): - group = [s for s in samples if s["label"] == label] + for label, group in sorted(groups.items()): errors = sum(1 for s in group if s["success"] != "true") stats[label] = { "count": len(group), diff --git a/tests/load/compass-testPlan.jmx b/tests/load/compass-testPlan.jmx index 6ff5560f2..e1ee70435 100644 --- a/tests/load/compass-testPlan.jmx +++ b/tests/load/compass-testPlan.jmx @@ -10,8 +10,8 @@ false - - ${__P(resetFile,1)} + + ${__P(checkLeftovers,1)} 1 true continue @@ -21,12 +21,27 @@ - + true import org.apache.jmeter.services.FileServer -new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt").text = "" + +// Teardown removes each account from the file once it is deleted, so anything still listed was left +// behind by an earlier run that was killed or failed to delete it. Emptying the file would lose the only +// record of those accounts. Instead, give account creation and every load test zero threads (thread +// counts are read when each group starts), so this run only deletes the leftovers. Then run again. +// A new load test must add its own <prefix>Threads here. +def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") +if (!file.exists()) file.text = "" // the later stages read it even when no account gets saved +def leftovers = file.readLines().findAll { it.trim() } +if (leftovers) { + def message = "created_accounts.txt lists ${leftovers.size()} accounts from an earlier run. Deleting them instead of running the test; run again afterwards." + log.error(message) + SampleResult.setSuccessful(false) + SampleResult.setResponseMessage(message) + ["setupThreads", "threads", "sessionThreads"].each { props.put(it, "0") } +} groovy @@ -49,6 +64,12 @@ new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt&quo ${__P(firebaseApiKey,AIzaSyBspL9glBXWbMsjmtt36dgb2yU0YGGhzKo)} = + + runId + ${__P(runId,${__time(ddHHmmss,)})} + Added to every email and username so each run's accounts are new + = + signupUrl /v1/accounts:signUp?key= @@ -83,18 +104,6 @@ new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt&quo - - accounts.csv - UTF-8 - userEmail,userPassword,username,name - false - , - false - false - true - shareMode.all - - ${baseAuthUrl} https @@ -114,6 +123,22 @@ new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt&quo + + groovy + + + true + // One new account per thread. The run ID makes it unique: reusing fixed accounts would fail every +// sign-up with EMAIL_EXISTS or a taken username when a run overlaps another, or follows one that leaked +// accounts. example.com is reserved for testing, so the API sends no welcome email to it. +// Usernames are cut to 25 characters, so keep them short. +def id = "${vars.get('runId')}x${ctx.getThreadNum() + 1}" +vars.put("userEmail", "loadtest${id}@example.com") +vars.put("userPassword", UUID.randomUUID().toString()) +vars.put("username", "loadUser${id}") +vars.put("name", "Load Test ${ctx.getThreadNum() + 1}") + + @@ -194,13 +219,15 @@ if (!token) { log.warn("No token for ${vars.get('userEmail')}, not saving") return } -if (!prev.isSuccessful()) { +def profileCreated = prev.isSuccessful() +if (!profileCreated) { log.warn("Profile creation failed for ${vars.get('userEmail')}, saving anyway so it gets cleaned up") } +// profileCreated tells teardown how to delete it: through the API, or only from Firebase def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") synchronized (FileServer.class) { - file.append([vars.get("userEmail"), vars.get("userPassword"), token, vars.get("name")].join("|") + "\n", "UTF-8") + file.append([vars.get("userEmail"), vars.get("userPassword"), token, vars.get("name"), profileCreated].join("|") + "\n", "UTF-8") } @@ -212,9 +239,9 @@ synchronized (FileServer.class) { - - ${__P(meThreads,${__P(threads,100)})} - ${__P(meRamp,${__P(ramp,10)})} + + ${__P(sessionThreads,${__P(threads,100)})} + ${__P(sessionRamp,${__P(ramp,10)})} true continue @@ -226,7 +253,7 @@ synchronized (FileServer.class) { created_accounts.txt UTF-8 - userEmail,userPassword,authToken,name + userEmail,userPassword,authToken,name,profileCreated false | false @@ -236,7 +263,7 @@ synchronized (FileServer.class) { - ${__P(meDuration,${__P(duration,120)})} + ${__P(sessionDuration,${__P(duration,120)})} @@ -265,15 +292,12 @@ synchronized (FileServer.class) { - - - 200 - - - Assertion.response_code - false - 8 - + + userId;ownUsername + $.id;$.username + 1;1 + NOT_FOUND;NOT_FOUND + $.name @@ -284,10 +308,325 @@ synchronized (FileServer.class) { false + + + 200 + + + Assertion.response_code + false + 8 + + - - ${__P(thinkTimeMs,0)} - + + ${__groovy(vars.get("userId") != "NOT_FOUND")} + false + true + + + + ${apiHost} + https + /get-profile?username=${ownUsername} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${apiHost} + https + /set-last-online-time + true + POST + true + true + + + + false + {} + = + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${apiHost} + https + /get-notifications?limit=450 + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${apiHost} + https + /compatible-profiles?userId=${userId} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${apiHost} + https + /get-profiles?limit=20&projection=card&compatibleWithUserId=${userId} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + feedUsername;feedAfter + $.profiles[*].user.username;$.profiles[-1:].id + -1;1 + NOT_FOUND;NOT_FOUND + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${__groovy((vars.get("feedUsername_matchNr") ?: "0") as int > 0)} + false + true + + + + ${__P(profileViews,3)} + true + + + + ${apiHost} + https + /get-profile?username=${__V(feedUsername_${__Random(1,${feedUsername_matchNr})})} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + + 1 + false + 1 + + ThroughputController.percentThroughput + 50.0 + 0.0 + + + + + ${apiHost} + https + /get-profiles?limit=20&projection=card&compatibleWithUserId=${userId}&after=${feedAfter} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + + + + ${__P(thinkTimeMs,2000)} + ${__P(thinkRandomMs,3000)} + @@ -305,7 +644,7 @@ synchronized (FileServer.class) { created_accounts.txt UTF-8 - userEmail,userPassword,authToken,name + userEmail,userPassword,authToken,name,profileCreated false | false @@ -342,15 +681,29 @@ synchronized (FileServer.class) { - - - 200 - - - Assertion.response_code - false - 8 - + + groovy + + + true + // An account that no longer exists is already deleted, e.g. a row an older version of this plan left +// in created_accounts.txt after a successful run. Count it as deleted instead of failing it forever. +// Firebase answers INVALID_LOGIN_CREDENTIALS instead of EMAIL_NOT_FOUND when email enumeration +// protection is on, so this also matches a wrong password; the password comes from the file, so it +// is only wrong if the file was edited. +def signedIn = prev.getResponseCode() == "200" +vars.put("signedIn", String.valueOf(signedIn)) +if (signedIn) return +if (prev.getResponseDataAsString() =~ /EMAIL_NOT_FOUND|INVALID_LOGIN_CREDENTIALS/) { + prev.setSuccessful(true) + prev.setResponseMessage("Account already deleted") + props.putIfAbsent("deletedAccounts", java.util.concurrent.ConcurrentHashMap.newKeySet()) + props.get("deletedAccounts").add(vars.get("userEmail")) + return +} +AssertionResult.setFailure(true) +AssertionResult.setFailureMessage("Sign-in failed with ${prev.getResponseCode()}") + authToken @@ -359,50 +712,157 @@ synchronized (FileServer.class) { - - ${apiHost} - https - /me/delete - true - POST - true - true - - - - false - {"reasonCategory": "jmeter-load-test"} - = - - - - + + ${__groovy(vars.get("signedIn") == "true" && vars.get("profileCreated") != "false")} + false + true + - - - - Content-Type - application/json - - - Authorization - Bearer ${authToken} - - - - - - - 200 - - - Assertion.response_code - false - 8 - - + + ${apiHost} + https + /me/delete + true + POST + true + true + + + + false + {"reasonCategory": "jmeter-load-test"} + = + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + groovy + + + true + // "4 - Forget deleted accounts" removes these from created_accounts.txt. Failed ones stay listed. +if (prev.isSuccessful()) { + props.putIfAbsent("deletedAccounts", java.util.concurrent.ConcurrentHashMap.newKeySet()) + props.get("deletedAccounts").add(vars.get("userEmail")) +} + + + + + + ${__groovy(vars.get("signedIn") == "true" && (vars.get("profileCreated") == "false" || !ctx.getPreviousResult().isSuccessful()))} + false + true + + + + ${baseAuthUrl} + https + ${deleteAccountUrl}${firebaseApiKey} + true + POST + true + true + + + + false + {"idToken": "${authToken}"} + = + + + + + + + + + Content-Type + application/json + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + groovy + + + true + // "4 - Forget deleted accounts" removes these from created_accounts.txt. Failed ones stay listed. +if (prev.isSuccessful()) { + props.putIfAbsent("deletedAccounts", java.util.concurrent.ConcurrentHashMap.newKeySet()) + props.get("deletedAccounts").add(vars.get("userEmail")) +} + + + + + 1 + 1 + true + continue + + 1 + false + + + + + true + + + import org.apache.jmeter.services.FileServer + +// Keep only the accounts teardown couldn't delete, so a later run (or Cleanup only) can still find them +def deleted = props.remove("deletedAccounts") ?: [] as Set +def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") +if (!file.exists()) return +def remaining = file.readLines().findAll { it.trim() && !deleted.contains(it.split(/\|/)[0]) } +file.setText(remaining ? remaining.join("\n") + "\n" : "", "UTF-8") +if (remaining) { + log.warn("${remaining.size()} accounts could not be deleted and are still listed in created_accounts.txt") +} + groovy + + + false From 11e42459ffc025a2955f5b15be24647cec302a5a Mon Sep 17 00:00:00 2001 From: MartinBraquet Date: Wed, 28 Oct 2026 13:06:56 +0000 Subject: [PATCH 6/7] Increase max acceptable p95 latency threshold in test script from 800ms to 1200ms. --- tests/load/check_results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/load/check_results.py b/tests/load/check_results.py index 1e86e1ff6..53d8c170b 100644 --- a/tests/load/check_results.py +++ b/tests/load/check_results.py @@ -63,7 +63,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("jtl", help="results file written by jmeter -l") parser.add_argument("--max-error-rate", type=float, default=1.0, help="percent, default 1") - parser.add_argument("--max-p95", type=int, default=800, help="milliseconds, default 800") + parser.add_argument("--max-p95", type=int, default=1200, help="milliseconds, default 800") args = parser.parse_args() samples = load_samples(args.jtl) @@ -84,4 +84,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) From 4fffc2d37df9635e06cebc33f8a9fa2634314c73 Mon Sep 17 00:00:00 2001 From: MartinBraquet Date: Wed, 28 Oct 2026 13:08:20 +0000 Subject: [PATCH 7/7] Add `AbortSignal` to ESLint globals; increase max p95 latency threshold to 1200ms --- .github/workflows/load-full-test.yml | 2 +- web/eslint.config.mjs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/load-full-test.yml b/.github/workflows/load-full-test.yml index 7448cd90e..cd34023ee 100644 --- a/.github/workflows/load-full-test.yml +++ b/.github/workflows/load-full-test.yml @@ -18,5 +18,5 @@ jobs: users: 100 duration: 120 max_error_rate: 1 - max_p95: 800 + max_p95: 1200 report_name: load-full-report diff --git a/web/eslint.config.mjs b/web/eslint.config.mjs index 120fc829f..ff4322b6f 100644 --- a/web/eslint.config.mjs +++ b/web/eslint.config.mjs @@ -51,6 +51,7 @@ export default tseslint.config( Request: 'readonly', Response: 'readonly', AbortController: 'readonly', + AbortSignal: 'readonly', localStorage: 'readonly', sessionStorage: 'readonly', IntersectionObserver: 'readonly',