diff --git a/.github/workflows/load-full-test.yml b/.github/workflows/load-full-test.yml new file mode 100644 index 00000000..cd34023e --- /dev/null +++ b/.github/workflows/load-full-test.yml @@ -0,0 +1,22 @@ +name: Load test (full) + +# The full load test against the dev API, once a week: 100 users for 120 seconds. +# 100 users is Firebase's limit of 100 new accounts per hour from one IP address. Raising it needs a +# higher sign-up quota in the Firebase console first. +on: + schedule: + - cron: '0 6 * * 1' # Mondays at 06:00 UTC + workflow_dispatch: # adds a "Run workflow" button in the Actions tab + +permissions: + contents: read + +jobs: + full: + uses: ./.github/workflows/load-test.yml + with: + users: 100 + duration: 120 + max_error_rate: 1 + max_p95: 1200 + report_name: load-full-report diff --git a/.github/workflows/load-smoke-test.yml b/.github/workflows/load-smoke-test.yml new file mode 100644 index 00000000..d50f59d3 --- /dev/null +++ b/.github/workflows/load-smoke-test.yml @@ -0,0 +1,28 @@ +name: Load test (smoke) + +# A small JMeter run when a pull request changes API-related files, to check the test plan still works. +# It tests the API deployed to dev, not the pull request's code: dev is only updated by running +# backend/api/deploy-api.sh dev by hand. The weekly full run is in load-full-test.yml. +on: + pull_request: + branches: [main] + paths: + - 'backend/api/**' + - 'backend/shared/**' + - 'common/src/api/**' + - 'tests/load/**' + - '.github/workflows/load-*.yml' + workflow_dispatch: # adds a "Run workflow" button in the Actions tab + +permissions: + contents: read + +jobs: + smoke: + uses: ./.github/workflows/load-test.yml + with: + users: 3 + duration: 30 + max_error_rate: 0 + max_p95: 2000 + report_name: load-smoke-report diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml new file mode 100644 index 00000000..32a2a3de --- /dev/null +++ b/.github/workflows/load-test.yml @@ -0,0 +1,117 @@ +name: Load test + +# The steps shared by the smoke and full load tests. It doesn't run on its own: load-smoke-test.yml and +# load-full-test.yml call it with their own settings. See docs/load-testing.md. +# +# It tests the API deployed to dev (the plan's default apiHost), not the code in this checkout. +on: + workflow_call: + inputs: + users: + description: Threads for account creation and the load test (one account each) + type: number + required: true + duration: + description: Seconds the load test runs + type: number + required: true + max_error_rate: + description: Highest error rate allowed for any request, in percent + type: number + required: true + max_p95: + description: Highest p95 response time allowed for TEST requests, in milliseconds + type: number + required: true + report_name: + description: Name of the uploaded report artifact + type: string + required: true + +env: + JMETER_VERSION: '5.6.3' + PLAN: compass-testPlan.jmx + USERS: ${{ inputs.users }} + DURATION: ${{ inputs.duration }} + MAX_ERROR_RATE: ${{ inputs.max_error_rate }} + MAX_P95: ${{ inputs.max_p95 }} + +jobs: + load-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + # GitHub keeps only one pending run per concurrency group and cancels the one it replaces, so a single + # shared group would let a PR push silently cancel a queued run of another PR or the weekly full run. + # Instead each PR gets its own smoke group, where a newer push replaces the older run (its accounts are + # still deleted by the cleanup step, which runs on cancel), and the full run gets a group of its own. + # Smoke runs use only a few users, so one overlapping another run barely skews its results. + concurrency: + group: load-test-${{ inputs.report_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + defaults: + run: + working-directory: tests/load # folder holding the .jmx and check_results.py + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + + - name: Cache JMeter + id: jmeter-cache + uses: actions/cache@v4 + with: + path: ~/jmeter + key: jmeter-${{ env.JMETER_VERSION }} + + - name: Install JMeter + if: steps.jmeter-cache.outputs.cache-hit != 'true' + run: | + cd "$HOME" + base="https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-${JMETER_VERSION}.tgz" + curl -fsSLO "$base" + # make sure the download wasn't corrupted or tampered with + echo "$(curl -fsSL "$base.sha512" | cut -d' ' -f1) apache-jmeter-${JMETER_VERSION}.tgz" | sha512sum -c - + mkdir -p jmeter + tar -xzf "apache-jmeter-${JMETER_VERSION}.tgz" -C jmeter --strip-components=1 + + - name: Add JMeter to PATH + run: echo "$HOME/jmeter/bin" >> "$GITHUB_PATH" + + - name: Run load test + id: jmeter + run: | + mkdir -p results + # The plan puts the run ID in each account's email and username. The run ID and attempt number keep + # them unique, including when a failed job is re-run. + jmeter -n -t "$PLAN" -JrunId="${GITHUB_RUN_ID: -5}${GITHUB_RUN_ATTEMPT}" \ + -JsetupThreads="$USERS" -Jthreads="$USERS" -Jduration="$DURATION" \ + -l results/results.jtl -j jmeter.log -e -o report + + - name: Check results + run: python3 check_results.py results/results.jtl --max-error-rate "$MAX_ERROR_RATE" --max-p95 "$MAX_P95" + + # tearDown normally deletes the accounts. This only runs if JMeter crashed or the job was cancelled. + - name: Clean up accounts if the test didn't finish + if: always() && steps.jmeter.outcome != 'success' + run: | + [ -s created_accounts.txt ] || { echo "No accounts to clean up"; exit 0; } + jmeter -n -t "$PLAN" \ + -JcheckLeftovers=0 -JsetupThreads=0 -Jthreads=0 \ + -l results/cleanup.jtl -j cleanup.log + + - name: Save report + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.report_name }} + path: | + tests/load/report + tests/load/results + tests/load/errors.xml + tests/load/jmeter.log + tests/load/cleanup.log + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 34ff2b9f..b9497a10 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,13 @@ /tests/reports/playwright-report /tests/e2e/web/.auth/ +# JMeter load tests (generated by each run, see docs/load-testing.md) +/tests/load/created_accounts.txt +/tests/load/errors.xml +/tests/load/jmeter.log +/tests/load/results/ +/tests/load/report/ + # next.js /.next/ /out/ diff --git a/CLAUDE.md b/CLAUDE.md index 1c8f4ee2..14da4eea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,6 +132,7 @@ Cross-cutting: [`docs/compatibility-questions.md`](docs/compatibility-questions. compatibility prompts), [`docs/feed.md`](docs/feed.md), [`docs/internationalization.md`](docs/internationalization.md), [`docs/profile-fields.md`](docs/profile-fields.md), [`docs/testing.md`](docs/testing.md), +[`docs/load-testing.md`](docs/load-testing.md) (JMeter load tests), [`docs/logging-monitoring.md`](docs/logging-monitoring.md), [`docs/troubleshooting.md`](docs/troubleshooting.md). Mobile: [`android/README.md`](android/README.md), [`ios/README.md`](ios/README.md), diff --git a/README.md b/README.md index 45cff17e..f353e731 100644 --- a/README.md +++ b/README.md @@ -196,45 +196,33 @@ Then, install the dependencies for this project: yarn install --frozen-lockfile ``` -### Tests +### Running the app -Make sure the Jest tests pass: +There are two ways to run Compass locally: -```bash -yarn test -``` +| | Quick start (`yarn dev`) | Full isolation (`yarn dev:isolated`) | +| --------------- | ------------------------------------- | ---------------------------------------------------- | +| Database & auth | Shared remote dev environment | Local Supabase + Firebase emulators | +| Extra setup | None | Docker, Java 21+, Firebase CLI | +| Best for | UI changes, wording fixes, first look | Database, auth, or anything that writes lots of data | -If they don't and you can't find out why, simply raise an issue! Sometimes it's something on our end that we overlooked. +If you're unsure, start with the quick start. You can switch to full isolation later. -### Running the Development Server +After pulling new changes, run `yarn install --frozen-lockfile` before launching either one. -Start the development server: +#### Option A: Quick start (shared dev database) ```bash yarn dev ``` -[//]: # 'Note: the dev remote server is not available until further notice (to save money). The above command will start a local' -[//]: # "server instead (same as `yarn dev:isolated` below). You'll have to set up Firebase and Supabase first (see below)." -[//]: # 'Contact us if you need help; we may spin up the dev remote server for you.' - -Once the server is running, visit http://localhost:3000 to start using the app. You can sign up and visit the profiles; -you should see a few synthetic profiles. - -Note: it's normal if page loading locally is much slower than the deployed version. It can take up to 10 seconds, it -would be great to improve that though! +> ⚠️ The remote dev database is shared by all contributors. Don't reset or wipe it. If you need a clean or disposable +> database, use full isolation. -If you update the database schema (in the SQL files) or the database is not super clean or too large (e.g., someone -clogged it up), you can reset it to a clean state by running: +#### Option B: Full isolation (local database) -```bash -yarn dev:db:reset -``` - -#### Full isolation - -Running `yarn dev:isolated` spins up a local Supabase and Firebase emulator instead of pointing at the shared remote -database. This is strongly recommended for day-to-day development: +`yarn dev:isolated` runs a local Supabase database and Firebase emulators instead of using the shared remote +environment: - **Freedom** — Reset, wipe, and reseed your local database as many times as you want without affecting other contributors. @@ -242,7 +230,7 @@ database. This is strongly recommended for day-to-day development: - **Works offline** — No internet required once services are started locally. - **Faster** — No network latency on every database query. -However, running in full isolation requires installing several heavy dependencies: +The trade-off is heavier setup: - **Docker** (~500MB) — runs the Supabase Postgres container - **Supabase CLI** — manages the local Supabase stack (10+ Docker containers, ~1-2GB RAM when running) @@ -252,11 +240,7 @@ However, running in full isolation requires installing several heavy dependencie First startup is slow (30-60s) and the stack uses significant memory. If your machine has less than 8GB RAM, you may notice slowdowns. -If this feels like too much, you can skip isolation entirely — `yarn dev` works out of the box against the shared remote -and is perfectly fine for most contributions, especially UI changes, wording fixes, or anything that doesn't touch the -database or authentication. - -###### Setup instructions +##### Prerequisites As always, don't hesitate to raise an issue if you run into any problems! @@ -305,14 +289,34 @@ npm install -g firebase-tools firebase --version ``` -Run in isolation +##### Run + +Start each session with a fresh local database that has all the latest migrations: ```bash +yarn test:db:reset yarn dev:isolated ``` -Visit http://localhost:3000 as usual. Your local database comes preloaded with synthetic test profiles so the app -looks and feels like the real thing. +`yarn test:db:reset` only affects your local database and never touches the shared dev one. The Firebase emulator +doesn't keep users between runs, so resetting the database each time keeps the two in sync. + +#### What to expect + +Visit http://localhost:3000. You can sign up and browse a few synthetic profiles. + +Local page loads can be much slower than the deployed site, up to 10 seconds. That's expected (though improvements are +welcome!). + +### Tests + +Before opening a pull request, make sure the Jest tests pass: + +```bash +yarn test +``` + +If they fail and you can't work out why, raise an issue. Sometimes it's something on our end that we overlooked. ### Contributing diff --git a/backend/email/emails/functions/send-email.ts b/backend/email/emails/functions/send-email.ts index 13436c5c..14f4fa31 100644 --- a/backend/email/emails/functions/send-email.ts +++ b/backend/email/emails/functions/send-email.ts @@ -20,6 +20,11 @@ export const sendEmail = async ( return null } + if (sendsOnlyToTestDomains(payload.to)) { + debug(`Skipping email to reserved test domain: ${payload.to}`) + return null + } + if (!resend) return null const {data, error} = await resend.emails.send( @@ -44,6 +49,21 @@ export const sendEmail = async ( return data } +// Domains reserved for testing (RFC 2606) can never receive mail. The load tests sign up with them, so +// sending would only bounce and hurt the sender reputation. +const isTestDomainAddress = (address: string) => { + const domain = address.trim().replace(/>$/, '').split('@').pop()?.toLowerCase() ?? '' + return ( + ['example.com', 'example.net', 'example.org'].includes(domain) || + ['.example', '.test', '.invalid'].some((tld) => domain.endsWith(tld)) + ) +} + +const sendsOnlyToTestDomains = (to: string | string[]) => { + const addresses = Array.isArray(to) ? to : [to] + return addresses.length > 0 && addresses.every(isTestDomainAddress) +} + let resend: Resend | null = null const getResend = () => { if (resend) return resend diff --git a/docs/load-testing.md b/docs/load-testing.md new file mode 100644 index 00000000..0c0341f0 --- /dev/null +++ b/docs/load-testing.md @@ -0,0 +1,427 @@ +# JMeter (Load) Testing Guide + +Load tests check how the API holds up when many people use it at the same time. They measure response times, +throughput (requests per second) and error rate, so slowdowns and failures under load show up before real users find +them. They don't check that features work correctly; unit, integration and E2E tests do that (see +[testing.md](testing.md)). + +Load tests use [Apache JMeter](https://jmeter.apache.org/). The test plan, `tests/load/compass-testPlan.jmx`, load +plays a typical browsing session against the API, with real, temporary accounts. Each run has three stages: + +1. **Create** a set of temporary accounts. +2. **Load test:** all the accounts browse at the same time, for a set amount of time. +3. **Delete** the accounts again. + +**In this guide:** + +- [Prerequisites](#prerequisites), including [setups for Windows, WSL, Linux and macOS](#choose-your-setup) +- [JMeter basics](#jmeter-basics) +- [How the plan works](#how-the-plan-works) and [the test accounts](#the-test-accounts) +- [Running the test](#running-the-test): commands, stopping early, flags and properties +- [Element reference](#element-reference) +- [Adding a new test to this plan](#adding-a-new-test-to-this-plan) +- [Reading the results](#reading-the-results) +- [Automated runs](#automated-runs-github-actions): the smoke and weekly full tests in GitHub Actions +- [Troubleshooting](#troubleshooting) + +Values in angle brackets are placeholders: `` and `` are `apiHost` and `firebaseApiKey` in +the plan's User Defined Variables (the dev key is also in `common/src/envs/dev.ts`). + +--- + +## Prerequisites + +- **Java 8 or newer** (17+ recommended), installed on the system where JMeter runs. +- **[Apache JMeter 5.6+](https://jmeter.apache.org/download_jmeter.cgi)**, set up as described in + [Choose your setup](#choose-your-setup). + +The plan's files live in `tests/load/`: `compass-testPlan.jmx` (the test plan) and `check_results.py` (checks a +results file against limits, used by CI, see [check_results.py](#check_resultspy)). + +A run creates these files next to the plan. They are git-ignored: + +| File or folder | What it is | +| ---------------------- | --------------------------------------------------------------------------------------- | +| `created_accounts.txt` | The accounts that exist right now, passed between stages (see below) | +| `errors.xml` | Details of failed requests | +| `jmeter.log` | JMeter's own log, including the warnings the plan's scripts write | +| `results/`, `report/` | The raw results of each run ([`.jtl` files](#reading-the-results)), and the HTML report | + +### Choose your setup + +The [commands](#commands) use short paths such as `results/smoke.jtl`, which only work when JMeter runs **inside +`tests/load/`** on the same system as the project. Check your setup with `jmeter --version` in the terminal you'll run +the commands from. Java must be installed on that same system; in WSL that means **in WSL** (use the full package, as +the `-headless` one can't show the GUI: `sudo apt install openjdk-21-jre`). + +**Project and JMeter on the same system** (everything on Linux, on macOS, or on Windows without WSL): install JMeter +and add its `bin/` folder to your PATH. On Linux or macOS: + +```bash +cd ~ && curl -LO https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.6.3.tgz && tar -xzf apache-jmeter-5.6.3.tgz +echo 'export PATH="$HOME/apache-jmeter-5.6.3/bin:$PATH"' >> ~/.bashrc && source ~/.bashrc +``` + +**Project in WSL, JMeter installed in WSL** (recommended for WSL): run the same two commands in a WSL terminal. Putting +`bin/` at the **front** of the PATH means this JMeter wins over any on your Windows PATH; check with +`command -v jmeter`. The GUI opens as a normal Windows window through WSLg. + +**Project in WSL, JMeter installed on Windows:** the JMeter download has a Linux launcher (`bin/jmeter`) as well as +`jmeter.bat`, so WSL can run it from `/mnt/c` with WSL's Java. Add the `bin/` folder to your WSL PATH, e.g. +`echo 'export PATH="$PATH:/mnt/c/tools/apache-jmeter-5.6.3/bin"' >> ~/.bashrc` (find the path with +`wslpath 'C:\tools\apache-jmeter-5.6.3'`). Start-up is about a second slower; a running test is just as fast. Open +the GUI on Windows with the plan's `\\wsl.localhost\…` path (`wslpath -w compass-testPlan.jmx` prints it). + +> **Don't run the commands with `jmeter.bat` against a WSL folder.** `cmd.exe` can't use a `\\wsl.localhost\…` path as +> its working folder and switches to `C:\Windows`, so the short paths point to the wrong place. + +**To open the plan in the GUI:** run `jmeter -t compass-testPlan.jmx` from `tests/load/`, or start JMeter and use +**File → Open** with the plan's full path. + +--- + +## JMeter basics + +A JMeter test plan is a **tree** of elements. Each element type has one job: + +| Term | What it means | Example in this plan | +| ---------------------- | ---------------------------------------------------------------- | ---------------------------------------- | +| **Thread group** | A group of threads that all run the same steps | `1 - Create accounts` | +| **Thread** | One simulated user. Each uses one test **account** | One of the 100 threads in a thread group | +| **Sampler** | Something a thread _does_, usually one request | `SETUP \| Firebase Signup POST` | +| **Config element** | Settings or data used by samplers | CSV Data Set Config, HTTP Header Manager | +| **Pre/Post-processor** | Runs before/after a sampler, e.g. to pull a value out of a reply | JSON Extractor | +| **Assertion** | Checks a response and marks the request failed if it's wrong | Response Assertion (code must be `200`) | +| **Timer** | Adds a pause before requests | Think time | +| **Logic controller** | Controls _how often_ or _whether_ its children run | Runtime Controller, If Controller | +| **Listener** | Collects or shows results | Simple Data Writer | + +- **Variables** belong to **one thread** and are read with `${name}`. A variable a thread sets while it runs (like + `authToken`) **doesn't carry over** to other thread groups. Variables in User Defined Variables are copied to every + thread, which is why `${apiHost}` works everywhere. +- **Properties** are **global** to the run. Read them with `${__P(name,default)}`, set them with `-Jname=value`. +- **Scope:** an element applies to its parent and everything under it. A Header Manager under one request only affects + that request. +- **Order for each request:** config elements → pre-processors → timers → the request → post-processors → assertions + → listeners. So a post-processor can't see whether an assertion passed. + +--- + +## How the plan works + +```text +compass-testPlan.jmx +├── User Defined Variables hosts, endpoints and the run ID +├── 0 - Check for leftover accounts (setUp) stops the run from creating accounts if earlier ones remain +├── 1 - Create accounts (setUp) creates accounts, saves them to created_accounts.txt +├── 2 - Browsing session (Thread Group) the load test, reads created_accounts.txt +├── 3 - Delete accounts (tearDown) reads created_accounts.txt, deletes each account +├── 4 - Forget deleted accounts (tearDown) removes the deleted accounts from created_accounts.txt +└── Listeners Summary Report, Simple Data Writer, View Results Tree +``` + +The stages run one after another (**Run Thread Groups consecutively** is on). setUp groups run before the main test, +tearDown groups after it, and tearDown also runs after a [graceful shutdown](#stopping-a-run-early). + +**0 - Check for leftover accounts** (setUp, 1 thread). After a run, `created_accounts.txt` only lists accounts that +couldn't be deleted. If it isn't empty, an earlier run was killed or failed to clean up. Emptying the file would lose +the only record of those accounts, so instead this stage fails with a message, and sets `setupThreads`, `threads` and +`sessionThreads` to 0: this run creates nothing and only deletes the leftovers. Run again afterwards. Skipped when +`checkLeftovers=0`. + +**1 - Create accounts** (setUp, `setupThreads` threads, one account each): + +1. **Make this thread's account** (JSR223 PreProcessor) sets the thread's email, random password, username and name + (see [The test accounts](#the-test-accounts)). +2. `SETUP | Firebase Signup POST` creates the Firebase login. A JSON Extractor saves the token as `authToken`. +3. `SETUP | create-user-and-profile POST` creates the app user and profile. +4. **Save account for test and teardown** (JSR223 PostProcessor) appends `email|password|token|name|profileCreated` + to `created_accounts.txt`. An account whose profile creation failed is still saved, since its Firebase login + exists; `profileCreated=false` tells teardown how to delete it. + +**2 - Browsing session** (`sessionThreads` threads). Each thread takes one line of `created_accounts.txt`, then a +Runtime Controller repeats a visit for `sessionDuration` seconds. A visit makes the calls the web app makes when someone +opens it and browses profiles: + +1. `TEST | /me GET`: the signed-in user. Passes if the code is `200` **and** the returned `name` matches the account. + Saves the account's `userId` and username for the requests below. +2. `TEST | get-profile GET (own)`, `TEST | set-last-online-time POST`, `TEST | get-notifications GET` (`limit=450`, + as `use-notifications.ts` asks) and `TEST | compatible-profiles GET`: what loads in the background on every page. +3. `TEST | get-profiles GET (feed)`: the first page of the profile grid, as `profiles-home.tsx` asks for it (20 cards, + ordered by compatibility with this account). Saves the usernames and the last profile's id. +4. `TEST | get-profile GET (feed profile)`, `profileViews` times (3 by default): opens a random profile from that page. +5. `TEST | get-profiles GET (next page)`, in half the visits (a Throughput Controller): scrolls to the next page with + `after`. + +Steps 2 to 5 are skipped when `/me` fails, and 4 and 5 when the feed is empty. **Think time** (a Uniform Random Timer) +pauses 2 to 5 seconds before each request, like a person reading. It also keeps each account under the API's rate +limit of 120 requests a minute per user (`API_RATE_LIMIT_PER_MIN_AUTHED`): with no pause, most rate-limited requests +would come back `429`. + +**3 - Delete accounts** (tearDown, 1 thread, one loop per line of `created_accounts.txt`): + +1. `TEARDOWN | Firebase Sign in POST` gets a fresh token (setUp's may have expired after 1 hour). If Firebase says the + account doesn't exist, it is counted as already deleted and passes. +2. If the profile was created: `TEARDOWN | Delete app user` calls `/me/delete`, which deletes the app user, their + files and their Firebase login. It sends the reason `jmeter-load-test`, stored in `deleted_users`. +3. If there is no profile, or `/me/delete` failed: `TEARDOWN | Firebase delete account` deletes the Firebase login + directly, so it doesn't stay behind. + +Each successful delete is remembered, and **4 - Forget deleted accounts** then rewrites `created_accounts.txt` with +only the accounts that couldn't be deleted. A warning in `jmeter.log` says how many remain. + +## The test accounts + +The API only accepts most requests from signed-in accounts, so each thread needs its own real account. The plan makes +them itself; there's no account file to maintain. + +| Value | Made as | Example | +| -------- | --------------------------------- | -------------------------------- | +| email | `loadtestx@example.com` | `loadtest28120940x1@example.com` | +| password | A random UUID | | +| username | `loadUserx` | `loadUser28120940x1` | +| name | `Load Test ` | `Load Test 1` | + +- `` is the thread number. `` is the `runId` property, by default the day, hour, minute and second the run + started (`ddHHmmss`). CI passes its own. A fresh ID every run means an overlapping run, or leftovers from an earlier + one, can never cause `EMAIL_EXISTS` or a taken username. +- `example.com` is reserved for testing (RFC 2606) and can't receive mail. The API's `sendEmail` skips such addresses, + so creating accounts sends no welcome emails (once the API with that check is deployed). +- Usernames are cut to 25 characters, and the API cleans names (no emoji or symbols, at most 30 characters, no staff + words like "admin"). The `/me` request checks the name comes back unchanged, so keep both short and plain if you change + the script. + +--- + +## Running the test + +Always run load tests from the command line (**non-GUI mode**, `-n`). The GUI uses far more CPU and memory, which can +make JMeter itself the bottleneck. Use the GUI only to build and debug the plan. + +### Commands + +Run these from `tests/load/`. The test runs against the **shared dev API and Firebase**, not your local stack. + +**Smoke test** (1 thread, 10 seconds). Run it before every full run to check each step works. Firebase allows 100 new +accounts per hour from one IP address, so leave an hour between a smoke test and a full run if you're near that. + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/smoke.jtl -e -o report -Jthreads=1 -JsetupRamp=1 -Jramp=1 -Jduration=10 +``` + +**Full run with HTML report** (the defaults: 100 threads, 120 seconds): + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report +``` + +**Report with only the load test requests** (hides setup and cleanup; in PowerShell use double quotes): + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report '-Jjmeter.reportgenerator.sample_filter=^TEST \|.*' +``` + +**Another environment.** An API only accepts tokens from its own Firebase project, so pass both: + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -JapiHost= -JfirebaseApiKey= +``` + +> **PowerShell:** wrap any `-J` argument whose value contains a dot in double quotes, or PowerShell may split it. + +**Cleanup only.** Deletes the accounts still listed in `created_accounts.txt`, without creating any. A normal run +already does this when it finds leftovers, so you only need it to clean up without starting a test: + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/cleanup.jtl -JcheckLeftovers=0 -Jthreads=0 +``` + +**Rebuild a report** from an existing results file (the `report` folder must be empty or missing): + +```bash +jmeter -g results/results.jtl -o report +``` + +### Stopping a run early + +To stop a run **and still delete the accounts**, run `shutdown.sh` (`shutdown.cmd` on Windows) from another terminal of +the same kind. It's in JMeter's `bin/` folder. Type the full name: plain `shutdown` turns your computer off. In the GUI, +use **Run → Shutdown**. + +**Ctrl+C**, `stoptest.sh` or **Run → Stop** stop straight away and cleanup **doesn't** run. The accounts stay listed in +`created_accounts.txt`, and the next run deletes them before doing anything else. + +### Command flags + +| Flag | Meaning | +| ------------- | --------------------------------------------------------------------------------- | +| `-n` | Non-GUI mode | +| `-t ` | The test plan to run | +| `-l ` | Where to save the raw results (`.jtl`). Without `-f`, it adds to an existing file | +| `-e -o ` | After the run, build an HTML report into this folder (must be empty without `-f`) | +| `-f` | Delete the old results file and report folder first | +| `-g ` | Build a report from an existing `.jtl` without running a test | +| `-J=` | Set a property the plan reads (see below) | + +### Properties + +| Property | Default | What it controls | +| ----------------- | -------------------- | ------------------------------------------------------------------------------- | +| `threads` | `100` | Threads for account creation and every load test, unless overridden | +| `ramp` | `10` | Seconds to start all threads in each load test, unless overridden | +| `duration` | `120` | Seconds each load test runs, unless overridden | +| `setupThreads` | `threads` | Number of accounts to create | +| `setupRamp` | `10` | Seconds to start all account-creation threads | +| `sessionThreads` | `threads` | Threads in the browsing session | +| `sessionRamp` | `ramp` | Seconds to start all browsing session threads | +| `sessionDuration` | `duration` | Seconds the browsing session runs | +| `profileViews` | `3` | Feed profiles opened per visit | +| `thinkTimeMs` | `2000` | Shortest pause before each load test request, per thread | +| `thinkRandomMs` | `3000` | Random extra pause on top of `thinkTimeMs` (0 to this) | +| `apiHost` | `` | API host to test against | +| `firebaseApiKey` | `` | Firebase key for sign-up and sign-in. Must match `apiHost`'s environment | +| `runId` | start time | Added to every email and username (see [The test accounts](#the-test-accounts)) | +| `checkLeftovers` | `1` | `0` skips the leftover check | + +**Specific settings override general ones:** `-Jthreads=50` sets everything to 50 threads; adding `-JsessionThreads=200` +changes only the browsing session. **Keep `setupThreads` at least as high as the largest load test**, since each load test +thread needs an account. **Ramp-up** is how long JMeter takes to start all threads; starting them all at once can +overload sign-in. **Lowering the think time** sends more load per account, but keep the average pause above +about 0.5 seconds, or the rate limit turns requests into `429`s. + +--- + +## Element reference + +To **add** an element in the GUI, right-click the element it should go under → **Add**. To turn one off, right-click → +**Disable**. Save with **Ctrl+S**. + +| Element | How it's used here | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Test Plan** | **Run Thread Groups consecutively** on, so stages run in order. **Run tearDown Thread Groups after shutdown** on, so a graceful stop still cleans up. | +| **Thread groups** | Thread counts are properties with fallbacks, e.g. `${__P(setupThreads,${__P(threads,100)})}`, read when the group starts. A group with 0 threads is skipped. All groups **Continue** after a sampler error. | +| **User Defined Variables** | `baseAuthUrl`, `apiHost`, `firebaseApiKey`, `runId`, and the Firebase endpoint paths (`signupUrl`, `signinUrl`, `deleteAccountUrl`; `getUserDataUrl` is unused). Wrap a value in `__P` to make it settable with `-J`. | +| **CSV Data Set Config** | Two, both reading `created_accounts.txt` with `\|` as the delimiter and the sharing mode _Current thread group_, so the load test and teardown each go through every account. **Stop thread on EOF** is on, **Recycle** off. | +| **HTTP Request** | One per step. Names start with `SETUP \|`, `TEST \|` or `TEARDOWN \|`, so stages are easy to tell apart and the report can filter. Server is `${baseAuthUrl}` or `${apiHost}`, without `https://`. | +| **HTTP Header Manager** | Under each request: `Content-Type: application/json`, plus `Authorization: Bearer ${authToken}` where signed in. | +| **JSON Extractor** | Saves `$.idToken` from sign-up and teardown sign-in as `authToken`. | +| **JSR223 (Groovy)** | The leftover check, account generation, saving accounts, the teardown sign-in check, remembering deletes and rewriting the file. Scripts use `vars`, `props`, `prev`, `log`, and find files next to the `.jmx` with `FileServer.getFileServer().getBaseDir()`. **Cache compiled script** is on. | +| **Response Assertion** | On every request: response code must equal `200`. | +| **JSON Assertion** | On `TEST \| /me GET`: `$.name` must equal `${name}`, **not** as a regex (a `.` in a name would break it). | +| **If Controller** | In the session, skip requests whose input is missing (`/me` failed, empty feed). In teardown, choose between `/me/delete` and the direct Firebase delete, using `signedIn` and `profileCreated`. | +| **Loop, Throughput Controller** | Open `profileViews` feed profiles per visit; load the next page in 50% of visits. A random feed profile is `${__V(feedUsername_${__Random(1,${feedUsername_matchNr})})}`. | +| **Runtime Controller** | Repeats a visit for `${__P(sessionDuration,${__P(duration,120)})}` seconds. | +| **Uniform Random Timer** | **Think time**: `${__P(thinkTimeMs,2000)}` ms plus a random 0 to `${__P(thinkRandomMs,3000)}` ms before each session request. | +| **Listeners** | **Simple Data Writer** saves failed requests only to `errors.xml`, with response bodies but not request bodies (they hold passwords and tokens). **Summary Report** is GUI only. **View Results Tree** is disabled: great for debugging, too heavy for load runs. | +| **Debug Sampler** | Disabled, at the end of `1 - Create accounts`. Enable it with View Results Tree to see a thread's variables in the GUI; disable both before a load run. | + +If you add a column to `created_accounts.txt`, add it to the `join` in **Save account for test and teardown** and to +the Variable Names of both CSV configs, in the same position. + +--- + +## Adding a new test to this plan + +New load tests reuse the accounts created by setUp and deleted by tearDown, so you only build the test itself. + +1. Right-click `2 - Browsing session` → **Duplicate**, and drag the copy after the last load test. (To add a + request to the session instead, put it inside the existing Runtime Controller with a `TEST |` name.) +2. Rename it `N - ` and pick a property prefix, e.g. ``: + - Number of Threads: `${__P(Threads,${__P(threads,100)})}` + - Ramp-up period: `${__P(Ramp,${__P(ramp,10)})}` + - Runtime Controller → Runtime: `${__P(Duration,${__P(duration,120)})}` +3. Add `Threads` to the list the leftover check sets to 0, in **SETUP | Check created_accounts.txt is empty**. +4. Keep `CSV - Created accounts` as it is. +5. Change the HTTP Request (name `TEST | `, method, path, body) and update the assertions so they check + **this** response. +6. Make sure `setupThreads` covers the new test's thread count, then save and run a [smoke test](#commands). + +Keep in mind: tests run one after another and share the same accounts, so total time is the sum of their durations. +Tokens expire after 1 hour; if tests together run longer, add a sign-in request (like the teardown one) at the start of +the new group. A test that changes account data (e.g. the name) affects later tests, so put it last. + +--- + +## Reading the results + +**The results file (`.jtl`)**, named with `-l`, has one CSV row per request, in every stage. The HTML report is built +from it. Useful columns: `elapsed` (response time in ms), `label`, `responseCode`, `success`, `failureMessage`, +`Latency`, `Connect` and `allThreads` (the load at that moment). Sort by `elapsed` for the slowest requests, or keep +files from several runs to compare them (`-f` overwrites a file with the same name). It holds no passwords or tokens. + +**The HTML report:** in **Statistics**, check `#Samples` fits what you asked for (e.g. 100 sign-ups for 100 threads). +Failures in `SETUP |` rows mean less load than intended. _Average_ hides slow outliers, so look at the **90th/95th/99th +pct** columns. + +**errors.xml** holds only failed requests, with the response body that shows why. It **adds to itself on every run**, +so delete it first if you only want one run's errors. + +--- + +## Automated runs (GitHub Actions) + +Two workflows run the plan, using the shared steps in `.github/workflows/load-test.yml`: + +| | Smoke test (`load-smoke-test.yml`) | Full test (`load-full-test.yml`) | +| ----------- | ------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------- | +| Runs when | A PR into `main` changes `backend/api/`, `backend/shared/`, `common/src/api/`, `tests/load/` or the load workflows | Every Monday at 06:00 UTC | +| Size | 3 threads for 30 seconds | 100 threads for 120 seconds | +| Passes when | No request fails, and the p95 of `TEST \|` requests is at most 2000 ms | At most 1% of each request fails, and `TEST \|` p95 is at most 800 ms | +| Artifact | `load-smoke-report` | `load-full-report` | + +Either can also be started by hand from the **Actions** tab → **Run workflow**. + +> **They test the API deployed to dev, not the pull request's code.** Dev only changes when someone runs +> `./deploy-api.sh dev` in `backend/api/`. To load test a change, deploy it to dev first. + +A run installs JMeter (cached, checked against Apache's SHA-512), runs the plan with `runId` set from the workflow run +ID and attempt number, checks the results with [check_results.py](#check_resultspy), runs [Cleanup only](#commands) if +JMeter didn't finish, and uploads the report, results, `errors.xml` and logs. + +**Concurrency:** each PR has its own smoke group, where a newer push cancels the older run (its cleanup step still +runs), and the full test has its own group. A shared group would let one PR's push silently cancel another PR's queued +run or the weekly full run, because GitHub keeps only one pending run per group. + +To change a workflow's size or limits, edit the values under `with:`. Going above 100 accounts needs a higher sign-up +quota in the Firebase console. + +### check_results.py + +JMeter exits with `0` even when requests fail, so CI uses `tests/load/check_results.py` to pass or fail a run. It +checks, per request label: + +- **Error rate**, for every request. A failed setup request means less load; a failed cleanup request leaves accounts + behind. +- **p95 response time**, for `TEST |` requests with at least 20 samples. + +It also fails when there are no `TEST |` results at all. Run it locally from `tests/load/` (use `python` on Windows): + +```bash +python3 check_results.py results/results.jtl --max-error-rate 1 --max-p95 800 +``` + +It prints each label's samples, error rate and p95, then `PASS` or one `FAIL:` line per broken limit, and exits with +`0` or `1`. The defaults are 1% and 800 ms. + +--- + +## Troubleshooting + +**The run says `created_accounts.txt lists N accounts from an earlier run`.** An earlier run didn't finish cleaning up. +This run deleted those accounts instead of testing; check the `TEARDOWN |` rows, then run again. Accounts that still +couldn't be deleted stay in the file, and `jmeter.log` says how many. Don't delete test accounts in the Firebase console +instead: that only removes the login, and the app user stays in the database. + +**Requests fail with `429`.** The think time is too short for the rate limit; raise `thinkTimeMs`. + +**Fewer load test threads than expected.** `created_accounts.txt` has fewer lines than the test has threads. Check the +`SETUP |` rows for failures, and that `setupThreads` is high enough. + +**`TEST | /me GET` fails the JSON Assertion.** The API changed the name when saving it. Keep names plain (see +[The test accounts](#the-test-accounts)). + +**`${something}` appears literally in a request.** The variable was never set: it's misspelled (names are +case-sensitive), set in another thread group, or the extractor found nothing. Enable **View Results Tree** (and the +Debug Sampler) and run a smoke test in the GUI. + +**Report folder error.** JMeter won't write a report into a folder that isn't empty. Add `-f`, or delete the folder. diff --git a/docs/testing.md b/docs/testing.md index db10ae48..47f601b6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -22,7 +22,7 @@ How we apply it here ### Test types at a glance -This project uses three complementary test types. Use the right level for the job: +This project uses four complementary test types. Use the right level for the job: - Unit tests - Purpose: Verify a single function/module in isolation; fast, deterministic. @@ -48,6 +48,14 @@ This project uses three complementary test types. Use the right level for the jo - Naming: `*.e2e.spec.ts`. - When to use: Critical journeys (signup, login, checkout), cross-service interactions, smoke tests for deployments. +- Load tests + - Purpose: Check how the API behaves when many users hit it at once (response times, throughput, error rate). + - Where: `tests/load/` (`compass-testPlan.jmx`). + - Runner: Apache JMeter 5.6+. + - When to use: Before releases or after changes to frequently used endpoints. Not for checking correctness; + unit, integration and E2E tests cover that. + - Guide: [Load testing guide](load-testing.md). + Quick commands ```bash diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 45080095..7866ebe1 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -60,7 +60,7 @@ fi # ✅ Kill stale processes from previous runs print_status "Killing any stale processes..." ./scripts/firebase_stop.sh -supabase stop --no-backup 2>/dev/null || true +npx supabase stop --no-backup 2>/dev/null || true sleep 2 # Give ports time to free up # Install Playwright browsers @@ -81,7 +81,7 @@ npx wait-on \ yarn test:db:reset # Get connection details -STATUS_JSON=$(supabase status --output json) +STATUS_JSON=$(npx supabase status --output json) export NEXT_PUBLIC_SUPABASE_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL') export NEXT_PUBLIC_SUPABASE_ANON_KEY=$(echo "$STATUS_JSON" | jq -r '.ANON_KEY') export DATABASE_URL=$(echo "$STATUS_JSON" | jq -r '.DB_URL') diff --git a/scripts/run_local_isolated.sh b/scripts/run_local_isolated.sh index c803a885..6a0b825a 100755 --- a/scripts/run_local_isolated.sh +++ b/scripts/run_local_isolated.sh @@ -35,13 +35,13 @@ export $(cat .env.local | grep -v '^#' | xargs) export $(cat .env.test | grep -v '^#' | xargs) # Ensure Supabase local stack is running; if not, reset/start it -STATUS_JSON=$(supabase status --output json 2>/dev/null || echo '') +STATUS_JSON=$(npx supabase status --output json 2>/dev/null || echo '') API_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL // empty') if [ -z "$API_URL" ]; then echo "Supabase is not running. Bootstrapping local stack with: yarn test:db:reset" yarn test:db:reset - STATUS_JSON=$(supabase status --output json) + STATUS_JSON=$(npx supabase status --output json) fi export NEXT_PUBLIC_SUPABASE_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL') diff --git a/scripts/seed.sh b/scripts/seed.sh index 4a975020..40feb1a9 100755 --- a/scripts/seed.sh +++ b/scripts/seed.sh @@ -8,7 +8,7 @@ cd "$(dirname "$0")"/.. export $(cat .env.test | grep -v '^#' | xargs) # Get connection details -STATUS_JSON=$(supabase status --output json) +STATUS_JSON=$(npx supabase status --output json) export NEXT_PUBLIC_SUPABASE_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL') export NEXT_PUBLIC_SUPABASE_ANON_KEY=$(echo "$STATUS_JSON" | jq -r '.ANON_KEY') export DATABASE_URL=$(echo "$STATUS_JSON" | jq -r '.DB_URL') diff --git a/tests/load/check_results.py b/tests/load/check_results.py new file mode 100644 index 00000000..53d8c170 --- /dev/null +++ b/tests/load/check_results.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Fail the build when a JMeter results file (.jtl) breaks the limits. + +JMeter exits with 0 even when requests fail, so CI runs this afterwards. + +Usage: + python3 check_results.py results.jtl --max-error-rate 1 --max-p95 800 +""" +import argparse +import csv +import math +import sys +from collections import defaultdict + +# Requests in the plan are labelled "SETUP | ...", "TEST | ..." or "TEARDOWN | ...". +# The error limit applies to all of them: a failed SETUP request means less load than asked for, and a +# failed TEARDOWN request leaves test accounts behind on the server. The p95 limit only applies to TEST +# requests, because setup and cleanup times aren't what the test measures. +TEST_LABEL_PREFIX = "TEST |" + + +def load_samples(path): + with open(path, newline="", encoding="utf-8") as f: + return list(csv.DictReader(f)) + + +def percentile(values, pct): + # nearest-rank: the value that pct% of samples are at or below + ordered = sorted(values) + rank = math.ceil(pct / 100 * len(ordered)) + return ordered[max(rank - 1, 0)] + + +def summarise(samples): + """Group samples by label (request name) and work out the numbers for each.""" + groups = defaultdict(list) + for s in samples: # one pass, so large full-run files stay fast + groups[s["label"]].append(s) + + stats = {} + for label, group in sorted(groups.items()): + errors = sum(1 for s in group if s["success"] != "true") + stats[label] = { + "count": len(group), + "error_rate": errors / len(group) * 100, # percent + "p95": percentile([int(s["elapsed"]) for s in group], 95), # ms + } + return stats + + +def check_limits(stats, max_error_rate, max_p95): + """Return one message per broken limit. An empty list means the run passed.""" + problems = [] + for label, s in stats.items(): + if s["error_rate"] > max_error_rate: + problems.append(f"{label}: error rate {s['error_rate']:.1f}% is over {max_error_rate}%") + if label.startswith(TEST_LABEL_PREFIX) and s["count"] >= 20 and s["p95"] > max_p95: + problems.append(f"{label}: p95 {s['p95']} ms is over {max_p95} ms") + return problems + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("jtl", help="results file written by jmeter -l") + parser.add_argument("--max-error-rate", type=float, default=1.0, help="percent, default 1") + parser.add_argument("--max-p95", type=int, default=1200, help="milliseconds, default 800") + args = parser.parse_args() + + samples = load_samples(args.jtl) + if not any(s["label"].startswith(TEST_LABEL_PREFIX) for s in samples): + print("FAIL: no samples from the main test. Did setUp create any accounts?") + return 1 + + stats = summarise(samples) + for label, s in stats.items(): + print(f"{label}: {s['count']} samples, {s['error_rate']:.1f}% errors, p95 {s['p95']} ms") + + problems = check_limits(stats, args.max_error_rate, args.max_p95) + for p in problems: + print("FAIL:", p) + if not problems: + print("PASS") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/load/compass-testPlan.jmx b/tests/load/compass-testPlan.jmx new file mode 100644 index 00000000..e1ee7043 --- /dev/null +++ b/tests/load/compass-testPlan.jmx @@ -0,0 +1,979 @@ + + + + + true + true + + + + false + + + + ${__P(checkLeftovers,1)} + 1 + true + continue + + 1 + false + + + + + true + + + import org.apache.jmeter.services.FileServer + +// Teardown removes each account from the file once it is deleted, so anything still listed was left +// behind by an earlier run that was killed or failed to delete it. Emptying the file would lose the only +// record of those accounts. Instead, give account creation and every load test zero threads (thread +// counts are read when each group starts), so this run only deletes the leftovers. Then run again. +// A new load test must add its own <prefix>Threads here. +def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") +if (!file.exists()) file.text = "" // the later stages read it even when no account gets saved +def leftovers = file.readLines().findAll { it.trim() } +if (leftovers) { + def message = "created_accounts.txt lists ${leftovers.size()} accounts from an earlier run. Deleting them instead of running the test; run again afterwards." + log.error(message) + SampleResult.setSuccessful(false) + SampleResult.setResponseMessage(message) + ["setupThreads", "threads", "sessionThreads"].each { props.put(it, "0") } +} + groovy + + + + + + + baseAuthUrl + identitytoolkit.googleapis.com + + = + + + apiHost + ${__P(apiHost,api.dev.compassmeet.com)} + = + + + firebaseApiKey + ${__P(firebaseApiKey,AIzaSyBspL9glBXWbMsjmtt36dgb2yU0YGGhzKo)} + = + + + runId + ${__P(runId,${__time(ddHHmmss,)})} + Added to every email and username so each run's accounts are new + = + + + signupUrl + /v1/accounts:signUp?key= + = + + + signinUrl + /v1/accounts:signInWithPassword?key= + = + + + deleteAccountUrl + /v1/accounts:delete?key= + = + + + getUserDataUrl + /v1/accounts:lookup?key= + = + + + + + + ${__P(setupThreads,${__P(threads,100)})} + ${__P(setupRamp,10)} + true + continue + + 1 + false + + + + + ${baseAuthUrl} + https + ${signupUrl}${firebaseApiKey} + true + POST + true + true + + + + false + { "email": "${userEmail}", "password": "${userPassword}", "returnSecureToken": true } + = + + + + + + + groovy + + + true + // One new account per thread. The run ID makes it unique: reusing fixed accounts would fail every +// sign-up with EMAIL_EXISTS or a taken username when a run overlaps another, or follows one that leaked +// accounts. example.com is reserved for testing, so the API sends no welcome email to it. +// Usernames are cut to 25 characters, so keep them short. +def id = "${vars.get('runId')}x${ctx.getThreadNum() + 1}" +vars.put("userEmail", "loadtest${id}@example.com") +vars.put("userPassword", UUID.randomUUID().toString()) +vars.put("username", "loadUser${id}") +vars.put("name", "Load Test ${ctx.getThreadNum() + 1}") + + + + + + Content-Type + application/json + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + authToken + $.idToken + 1 + + + + + ${apiHost} + https + /create-user-and-profile + true + POST + true + true + + + + false + {"username": "${username}", "name": "${name}", "profile": {"visibility": "public"}} + = + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + groovy + + + true + import org.apache.jmeter.services.FileServer + +def token = vars.get("authToken") +if (!token) { + log.warn("No token for ${vars.get('userEmail')}, not saving") + return +} +def profileCreated = prev.isSuccessful() +if (!profileCreated) { + log.warn("Profile creation failed for ${vars.get('userEmail')}, saving anyway so it gets cleaned up") +} + +// profileCreated tells teardown how to delete it: through the API, or only from Firebase +def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") +synchronized (FileServer.class) { + file.append([vars.get("userEmail"), vars.get("userPassword"), token, vars.get("name"), profileCreated].join("|") + "\n", "UTF-8") +} + + + + + false + true + false + + + + + ${__P(sessionThreads,${__P(threads,100)})} + ${__P(sessionRamp,${__P(ramp,10)})} + true + continue + + 1 + false + + + + + created_accounts.txt + UTF-8 + userEmail,userPassword,authToken,name,profileCreated + false + | + false + false + true + shareMode.group + + + + ${__P(sessionDuration,${__P(duration,120)})} + + + + ${apiHost} + https + /me + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + userId;ownUsername + $.id;$.username + 1;1 + NOT_FOUND;NOT_FOUND + + + + $.name + ${name} + true + false + false + false + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${__groovy(vars.get("userId") != "NOT_FOUND")} + false + true + + + + ${apiHost} + https + /get-profile?username=${ownUsername} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${apiHost} + https + /set-last-online-time + true + POST + true + true + + + + false + {} + = + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${apiHost} + https + /get-notifications?limit=450 + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${apiHost} + https + /compatible-profiles?userId=${userId} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${apiHost} + https + /get-profiles?limit=20&projection=card&compatibleWithUserId=${userId} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + feedUsername;feedAfter + $.profiles[*].user.username;$.profiles[-1:].id + -1;1 + NOT_FOUND;NOT_FOUND + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + ${__groovy((vars.get("feedUsername_matchNr") ?: "0") as int > 0)} + false + true + + + + ${__P(profileViews,3)} + true + + + + ${apiHost} + https + /get-profile?username=${__V(feedUsername_${__Random(1,${feedUsername_matchNr})})} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + + 1 + false + 1 + + ThroughputController.percentThroughput + 50.0 + 0.0 + + + + + ${apiHost} + https + /get-profiles?limit=20&projection=card&compatibleWithUserId=${userId}&after=${feedAfter} + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + + + + ${__P(thinkTimeMs,2000)} + ${__P(thinkRandomMs,3000)} + + + + + + 1 + 1 + true + continue + + -1 + false + + + + + created_accounts.txt + UTF-8 + userEmail,userPassword,authToken,name,profileCreated + false + | + false + false + true + shareMode.group + + + + ${baseAuthUrl} + https + ${signinUrl}${firebaseApiKey} + true + POST + true + true + + + + false + {"email": "${userEmail}", "password": "${userPassword}", "returnSecureToken": true} + = + + + + + + + + + Content-Type + application/json + + + + + + groovy + + + true + // An account that no longer exists is already deleted, e.g. a row an older version of this plan left +// in created_accounts.txt after a successful run. Count it as deleted instead of failing it forever. +// Firebase answers INVALID_LOGIN_CREDENTIALS instead of EMAIL_NOT_FOUND when email enumeration +// protection is on, so this also matches a wrong password; the password comes from the file, so it +// is only wrong if the file was edited. +def signedIn = prev.getResponseCode() == "200" +vars.put("signedIn", String.valueOf(signedIn)) +if (signedIn) return +if (prev.getResponseDataAsString() =~ /EMAIL_NOT_FOUND|INVALID_LOGIN_CREDENTIALS/) { + prev.setSuccessful(true) + prev.setResponseMessage("Account already deleted") + props.putIfAbsent("deletedAccounts", java.util.concurrent.ConcurrentHashMap.newKeySet()) + props.get("deletedAccounts").add(vars.get("userEmail")) + return +} +AssertionResult.setFailure(true) +AssertionResult.setFailureMessage("Sign-in failed with ${prev.getResponseCode()}") + + + + authToken + $.idToken + 1 + + + + + ${__groovy(vars.get("signedIn") == "true" && vars.get("profileCreated") != "false")} + false + true + + + + ${apiHost} + https + /me/delete + true + POST + true + true + + + + false + {"reasonCategory": "jmeter-load-test"} + = + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + groovy + + + true + // "4 - Forget deleted accounts" removes these from created_accounts.txt. Failed ones stay listed. +if (prev.isSuccessful()) { + props.putIfAbsent("deletedAccounts", java.util.concurrent.ConcurrentHashMap.newKeySet()) + props.get("deletedAccounts").add(vars.get("userEmail")) +} + + + + + + ${__groovy(vars.get("signedIn") == "true" && (vars.get("profileCreated") == "false" || !ctx.getPreviousResult().isSuccessful()))} + false + true + + + + ${baseAuthUrl} + https + ${deleteAccountUrl}${firebaseApiKey} + true + POST + true + true + + + + false + {"idToken": "${authToken}"} + = + + + + + + + + + Content-Type + application/json + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + groovy + + + true + // "4 - Forget deleted accounts" removes these from created_accounts.txt. Failed ones stay listed. +if (prev.isSuccessful()) { + props.putIfAbsent("deletedAccounts", java.util.concurrent.ConcurrentHashMap.newKeySet()) + props.get("deletedAccounts").add(vars.get("userEmail")) +} + + + + + + + 1 + 1 + true + continue + + 1 + false + + + + + true + + + import org.apache.jmeter.services.FileServer + +// Keep only the accounts teardown couldn't delete, so a later run (or Cleanup only) can still find them +def deleted = props.remove("deletedAccounts") ?: [] as Set +def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") +if (!file.exists()) return +def remaining = file.readLines().findAll { it.trim() && !deleted.contains(it.split(/\|/)[0]) } +file.setText(remaining ? remaining.join("\n") + "\n" : "", "UTF-8") +if (remaining) { + log.warn("${remaining.size()} accounts could not be deleted and are still listed in created_accounts.txt") +} + groovy + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + true + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + true + false + true + true + false + false + false + true + 0 + true + true + true + true + true + true + + + errors.xml + + + + + diff --git a/web/eslint.config.mjs b/web/eslint.config.mjs index 120fc829..ff4322b6 100644 --- a/web/eslint.config.mjs +++ b/web/eslint.config.mjs @@ -51,6 +51,7 @@ export default tseslint.config( Request: 'readonly', Response: 'readonly', AbortController: 'readonly', + AbortSignal: 'readonly', localStorage: 'readonly', sessionStorage: 'readonly', IntersectionObserver: 'readonly', diff --git a/web/scripts/fetch-media.mjs b/web/scripts/fetch-media.mjs index 2e7b0325..7f9283e7 100644 --- a/web/scripts/fetch-media.mjs +++ b/web/scripts/fetch-media.mjs @@ -93,16 +93,44 @@ const exists = async (p) => { } } +/** + * r2.dev occasionally drops a connection mid-body, which undici reports as a bare + * "TypeError: terminated". That is transient, so retry it; an HTTP error status is not, so it fails + * on the first attempt. The timeout covers a stalled stream, which would otherwise hang the build + * until Vercel kills it with no message at all. + */ +const ATTEMPTS = 3 +const TIMEOUT_MS = 5 * 60_000 + +class HttpError extends Error {} + +async function downloadOnce(url, target) { + const res = await fetch(url, {signal: AbortSignal.timeout(TIMEOUT_MS)}) + if (!res.ok || !res.body) { + throw new HttpError(`${res.status} ${res.statusText}`) + } + await pipeline(Readable.fromWeb(res.body), createWriteStream(target)) +} + async function download({key, dest}, base) { const url = `${base}/${key}` const target = join(PUBLIC_DIR, dest) await mkdir(dirname(target), {recursive: true}) - const res = await fetch(url) - if (!res.ok || !res.body) { - throw new Error(`${res.status} ${res.statusText} for ${url}`) + for (let attempt = 1; ; attempt++) { + try { + await downloadOnce(url, target) + break + } catch (err) { + // undici puts the useful part (e.g. "other side closed") in `cause` + const reason = err.cause ? `${err.message} (${err.cause.message ?? err.cause})` : err.message + if (err instanceof HttpError || attempt === ATTEMPTS) { + throw new Error(`${reason} for ${url} (attempt ${attempt}/${ATTEMPTS})`) + } + console.warn(`[media] ${dest}: ${reason}; retrying (${attempt}/${ATTEMPTS})`) + await new Promise((r) => setTimeout(r, 2000 * attempt)) + } } - await pipeline(Readable.fromWeb(res.body), createWriteStream(target)) const {size} = await stat(target) console.log(`[media] ${dest} (${(size / 1024).toFixed(0)} KB)`) }