diff --git a/.github/workflows/location-storage.yml b/.github/workflows/location-storage.yml new file mode 100644 index 00000000..796b49bd --- /dev/null +++ b/.github/workflows/location-storage.yml @@ -0,0 +1,39 @@ +name: Location storage regression + +on: + pull_request: + paths: + - 'backend/**' + - '.github/workflows/location-storage.yml' + +permissions: + contents: read + +jobs: + location-storage: + runs-on: ubuntu-latest + services: + dynamodb: + image: amazon/dynamodb-local@sha256:6ada631ec7aa5fc83446855eb17233e3dad184fec47477e8140c6c569cfe0c86 + ports: + - 18001:8000 + defaults: + run: + working-directory: backend + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: backend/package-lock.json + - run: npm ci + - run: npm test + - name: Wait for DynamoDB Local + run: | + for attempt in $(seq 1 30); do + if curl --silent --output /dev/null http://127.0.0.1:18001; then exit 0; fi + sleep 1 + done + exit 1 + - run: node --test scripts/check-locations.js diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 16a08b84..7e62a006 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -44,6 +44,10 @@ Adopts the covenant scale (LBTAS, v2 floor) and carries its own record instance. The reference economy. Its operator/federation model is the named reference for the substrate's frontend-as-operator design target. +### Operator-local location data + +Location persistence uses the separate `Locations` table, never the commons record or generic Users federation payloads. `backend/controllers/locationController.js` requires a verified account JWT for owner writes/reloads and explicit boolean consent for discovery. Public discovery emits an allowlisted location DTO and encrypted pagination cursors. `backend/scripts/check-locations.js` binds these claims to real DynamoDB storage checks. See [deployment, privacy semantics and residual limits](docs/location-api.md). This does not promote any in-review ledger binding or fix legacy user/federation account-security gaps. + The application message API supports optional retry receipts, scoped to the authenticated caller and conversation. These are private operational metadata, not sealed dialog or commons entries; they store hashes and message references, diff --git a/backend/.env.example b/backend/.env.example index 11edf7df..44f33084 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -22,3 +22,7 @@ STRIPE_KEY= UI_STRING=Fruitful_app NODE_ID=backend-node-01 NODE_TYPE=backend + +# Operator-local location table (string partition key: id). +# Provision before enabling location persistence; never include in federation exports. +LOCATION_TABLE_NAME=Locations diff --git a/backend/README.md b/backend/README.md index ef3c8d5f..a2a192c5 100644 --- a/backend/README.md +++ b/backend/README.md @@ -151,3 +151,7 @@ See `backend/package.json` for Jest config and tests. Add route/model tests unde --- Contributions welcome! Please keep `server.js` slim by adding routes and logic in domain folders. + +## Location persistence + +See [the location API guide](../docs/location-api.md) for table provisioning, owner JWTs, explicit publication consent, precision, kilometre radius semantics, and bounded discovery pagination. New location data stays in a separate operator-local table, outside generic user exports. diff --git a/backend/__tests__/locationRateLimit.test.js b/backend/__tests__/locationRateLimit.test.js new file mode 100644 index 00000000..39c950ff --- /dev/null +++ b/backend/__tests__/locationRateLimit.test.js @@ -0,0 +1,74 @@ +const request = require('supertest'); +const http = require('node:http'); +const jwt = require('jsonwebtoken'); +const { rateLimit } = require('express-rate-limit'); + +jest.mock('../lib/dynamodbClient', () => ({ + get: ({ Key }) => ({ promise: async () => ({ Item: { id: Key.id, coordinates: { lat: 0, long: 0 }, privacy: false } }) }), +})); + +describe('location request budgets', () => { + let app, server, agent; + const send = (method, url) => request(server)[method](url).agent(agent); + const env = { ...process.env }; + const authorization = userId => `Bearer ${jwt.sign({ userId }, process.env.JWT_SECRET, { expiresIn: '2h' })}`; + + beforeAll(() => { + process.env.JWT_SECRET = 'location-budget-test'; + process.env.API_KEY = 'location-budget-key'; + }); + beforeEach(async () => { + jest.resetModules(); + const express = require('express'); + app = express(); + // Bound this test server too, while allowing the route's 1,000-request + // budget and independent-account behavior to be exercised first. + app.use(express.json(), rateLimit({ windowMs: 60 * 1000, limit: 10000 }), require('../middleware/authMiddleware')); + app.use('/api/location', require('../routes/locationRoutes')); + agent = new http.Agent({ keepAlive: true }); + server = app.listen(0, '127.0.0.1'); + await new Promise(resolve => server.once('listening', resolve)); + }); + afterEach(async () => { + agent.destroy(); + await new Promise(resolve => server.close(resolve)); + }); + afterAll(() => { process.env = env; }); + + test('owner reads and writes share a budget without changing ownership denials', async () => { + const token = authorization('alice'); + for (let i = 0; i < 1000; i++) { + const method = i % 2 ? 'post' : 'get'; + const response = await send(method, '/api/location/foreign').set('Authorization', token); + expect(response.status).toBe(403); + } + const blocked = await send('get', '/api/location/foreign').set('Authorization', authorization('alice')); + expect(blocked.status).toBe(429); + expect(Number(blocked.headers['retry-after'])).toBeGreaterThan(0); + expect((await send('post', '/api/location/foreign').set('Authorization', token) + .set('X-Forwarded-For', '203.0.113.9')).status).toBe(429); + expect((await send('get', '/api/location/foreign').set('Authorization', authorization('bob'))).status).toBe(403); + }, 20000); + + test('API-key requests use an IP budget that cannot be reset by forwarded headers', async () => { + for (let i = 0; i < 1000; i++) { + expect((await send('get', '/api/location/foreign').set('x-api-key', process.env.API_KEY)).status).toBe(401); + } + const blocked = await send('get', '/api/location/foreign').set('x-api-key', process.env.API_KEY) + .set('X-Forwarded-For', '203.0.113.10'); + expect(blocked.status).toBe(429); + // Owner authentication accepts any nonempty string ID, including spaces. + expect((await send('get', '/api/location/%20').set('Authorization', authorization(' '))).status).toBe(200); + }, 20000); + + test('adding an API key cannot give an owner another account budget', async () => { + const token = authorization('credential-owner'); + for (let i = 0; i < 1000; i++) { + expect((await send('get', '/api/location/credential-owner').set('Authorization', token)).status).toBe(200); + } + for (const key of [process.env.API_KEY, 'invalid-key']) { + expect((await send('get', '/api/location/credential-owner').set('Authorization', token) + .set('x-api-key', key)).status).toBe(429); + } + }, 20000); +}); diff --git a/backend/__tests__/message_retries.test.js b/backend/__tests__/message_retries.test.js index 58a1ca07..be9418b7 100644 --- a/backend/__tests__/message_retries.test.js +++ b/backend/__tests__/message_retries.test.js @@ -291,6 +291,19 @@ test('deleting a conversation never lets an old retry resurrect its messages or assert.equal(aiCalls, 1); }); + +test('a missing persisted assistant reply cannot replay a partial success', async () => { + const first = await send(); + assert.equal(first.status, 201); + const [replyKey] = [...db.tables.get('Messages')].find(([, row]) => row.id === first.body.reply.id); + db.tables.get('Messages').delete(replyKey); + const retry = await send(); + assert.equal(retry.status, 410); + assert.equal(retry.body.code, 'message_send_removed'); + assert.equal(messages().length, 1); + assert.equal(aiCalls, 1); +}); + test('a missing receipt table refuses keyed sends before any message or AI work', async () => { const put = docClient.put; docClient.put = params => params.TableName === 'MessageSends' diff --git a/backend/controllers/locationController.js b/backend/controllers/locationController.js index 12d01d48..0a58c939 100644 --- a/backend/controllers/locationController.js +++ b/backend/controllers/locationController.js @@ -1,58 +1,82 @@ -const User = require('../models/user'); +const jwt = require('jsonwebtoken'); +const docClient = require('../lib/dynamodbClient'); +const { LOCATION_TABLE_NAME, publicLocation, encodeCursor, decodeCursor } = require('../models/location'); const { parseAgriNet, validateCoords, isWithinGeofence } = require('../utils/geo'); +const allowedTypes = ['farm', 'production', 'delivery']; -// Store coordinates for a user using AgriNet format -exports.setLocation = async (req, res) => { - try { - const { coordinates, locationType, privacy } = req.body; - const { lat, long } = parseAgriNet(coordinates); - validateCoords(lat, long); +function invalid(message) { return Object.assign(new Error(message), { status: 400 }); } +function fail(res, err) { + return res.status(err.status || 500).json({ error: err.status ? err.message : 'Location storage unavailable' }); +} - const allowedTypes = ['farm', 'production', 'delivery']; - if (locationType && !allowedTypes.includes(locationType)) { - throw new Error('Invalid location type'); - } +// General API-key authentication does not establish an account owner. +exports.requireOwner = (req, res, next) => { + try { + const header = req.headers.authorization; + if (!header || !header.startsWith('Bearer ')) throw new Error('Missing token'); + const owner = jwt.verify(header.slice(7), process.env.JWT_SECRET, { algorithms: ['HS256'] }); + if (typeof owner.userId !== 'string' || !owner.userId) throw new Error('Missing account'); + if (owner.userId !== req.params.id) return res.status(403).json({ error: 'Location belongs to another account' }); + req.user = owner; + next(); + } catch { + return res.status(401).json({ error: 'A valid account token is required' }); + } +}; - const update = { - coordinates: { lat, long }, - locationType, - locationPrivacy: !!privacy - }; +exports.setLocation = async (req, res) => { + try { + const { coordinates, locationType, privacy } = req.body || {}; + if (typeof coordinates !== 'string') throw invalid('Coordinates must use AgriNet format'); + if (typeof privacy !== 'boolean') throw invalid('privacy must explicitly be true (shared) or false (private)'); + if (locationType !== undefined && !allowedTypes.includes(locationType)) throw invalid('Invalid location type'); + let parsed; + try { parsed = parseAgriNet(coordinates); validateCoords(parsed.lat, parsed.long); } + catch (err) { throw invalid(err.message); } + const item = { id: req.params.id, coordinates: parsed, privacy, + ...(locationType === undefined ? {} : { locationType }) }; + await docClient.put({ TableName: LOCATION_TABLE_NAME, Item: item }).promise(); + res.json({ message: 'Location saved', ...item }); + } catch (err) { fail(res, err); } +}; - // Update user record; implementation depends on DB layer - if (User.findByIdAndUpdate) { - await User.findByIdAndUpdate(req.params.id, update, { new: true }); - } +exports.getLocation = async (req, res) => { + try { + const { Item } = await docClient.get({ TableName: LOCATION_TABLE_NAME, Key: { id: req.params.id }, ConsistentRead: true }).promise(); + if (!Item) return res.status(404).json({ error: 'Location not found' }); + res.json({ ...publicLocation(Item), privacy: Item.privacy === true }); + } catch (err) { fail(res, err); } +}; - res.json({ message: 'Location saved', coordinates: update.coordinates }); - } catch (err) { - res.status(400).json({ error: err.message }); +function number(value) { + if (typeof value !== 'string' || !/^-?(?:\d+(?:\.\d*)?|\.\d+)$/.test(value) || !Number.isFinite(Number(value))) { + throw invalid('Invalid geofence parameters'); } -}; + return Number(value); +} -// Retrieve public locations within a geofence for marketplace discovery exports.getPublicLocations = async (req, res) => { try { - const { lat, long, radius, type } = req.query; - const geofence = { - lat: parseFloat(lat), - long: parseFloat(long), - radius: parseFloat(radius) - }; - if (isNaN(geofence.lat) || isNaN(geofence.long) || isNaN(geofence.radius)) { - throw new Error('Invalid geofence parameters'); + const geofence = { lat: number(req.query.lat), long: number(req.query.long), radius: number(req.query.radius) }; + try { validateCoords(geofence.lat, geofence.long); } catch (err) { throw invalid(err.message); } + if (geofence.radius < 0) throw invalid('Radius must be non-negative kilometres'); + const type = req.query.type; + if (type !== undefined && !allowedTypes.includes(type)) throw invalid('Invalid location type'); + const scope = JSON.stringify({ ...geofence, type: type || null }); + let start; + if (req.query.cursor !== undefined) { + try { start = decodeCursor(req.query.cursor, scope); } catch { throw invalid('Invalid location cursor'); } } - - let users = []; - if (User.find) { - const query = { locationPrivacy: true }; - if (type) query.locationType = type; - users = await User.find(query, '-password'); - } - - const filtered = users.filter(u => u.coordinates && isWithinGeofence(u.coordinates, geofence)); - res.json(filtered); - } catch (err) { - res.status(400).json({ error: err.message }); - } + const page = await docClient.scan({ TableName: LOCATION_TABLE_NAME, Limit: 100, ConsistentRead: true, + ProjectionExpression: 'id, coordinates, locationType, privacy', + ...(start ? { ExclusiveStartKey: start } : {}) }).promise(); + // Do not rely on client-supplied filters or include complete user records. + const found = (page.Items || []).filter(item => { + if (item.privacy !== true || (type && item.locationType !== type)) return false; + try { validateCoords(item.coordinates?.lat, item.coordinates?.long); } catch { return false; } + return isWithinGeofence(item.coordinates, geofence); + }).map(publicLocation); + if (page.LastEvaluatedKey) res.set('X-Next-Cursor', encodeCursor(page.LastEvaluatedKey, scope)); + res.json(found); + } catch (err) { fail(res, err); } }; diff --git a/backend/middleware/requestRateLimit.js b/backend/middleware/requestRateLimit.js new file mode 100644 index 00000000..76e7f315 --- /dev/null +++ b/backend/middleware/requestRateLimit.js @@ -0,0 +1,12 @@ +const { rateLimit, ipKeyGenerator } = require('express-rate-limit'); + +// req.user is populated by authentication middleware, never by request data. +module.exports = () => rateLimit({ + windowMs: 60 * 1000, + limit: 1000, + keyGenerator: req => typeof req.user?.userId === 'string' && req.user.userId + ? `account:${req.user.userId}` : `ip:${ipKeyGenerator(req.ip)}`, + standardHeaders: 'draft-6', + legacyHeaders: false, + message: { error: 'Too many requests, please try again later' }, +}); diff --git a/backend/models/location.js b/backend/models/location.js new file mode 100644 index 00000000..f5ad162d --- /dev/null +++ b/backend/models/location.js @@ -0,0 +1,31 @@ +const { createHash, randomBytes, createCipheriv, createDecipheriv } = require('node:crypto'); + +// Separate storage keeps local coordinates out of raw Users exports/imports. +const LOCATION_TABLE_NAME = process.env.LOCATION_TABLE_NAME || 'Locations'; +function publicLocation(item) { + return { id: item.id, coordinates: { lat: item.coordinates.lat, long: item.coordinates.long }, + ...(item.locationType === undefined ? {} : { locationType: item.locationType }) }; +} +function key() { + if (!process.env.JWT_SECRET) throw new Error('JWT_SECRET is required'); + return createHash('sha256').update('agrinet-location-cursor-v1\0').update(process.env.JWT_SECRET).digest(); +} +// Encrypt the scan key: it may belong to a private location, even on an empty page. +function encodeCursor(start, scope) { + const iv = randomBytes(12); + const cipher = createCipheriv('aes-256-gcm', key(), iv); + cipher.setAAD(Buffer.from(scope)); + const data = Buffer.concat([cipher.update(JSON.stringify({ start, expires: Date.now() + 3600000 })), cipher.final()]); + return Buffer.concat([iv, cipher.getAuthTag(), data]).toString('base64url'); +} +function decodeCursor(token, scope) { + if (typeof token !== 'string' || token.length > 2048 || !/^[\w-]+$/.test(token)) throw new Error('Invalid cursor'); + const bytes = Buffer.from(token, 'base64url'); + const cipher = createDecipheriv('aes-256-gcm', key(), bytes.subarray(0, 12)); + cipher.setAAD(Buffer.from(scope)); + cipher.setAuthTag(bytes.subarray(12, 28)); + const result = JSON.parse(Buffer.concat([cipher.update(bytes.subarray(28)), cipher.final()]).toString()); + if (!Number.isFinite(result.expires) || result.expires < Date.now() || typeof result.start?.id !== 'string') throw new Error('Expired cursor'); + return { id: result.start.id }; +} +module.exports = { LOCATION_TABLE_NAME, publicLocation, encodeCursor, decodeCursor }; diff --git a/backend/package-lock.json b/backend/package-lock.json index f066c735..71cdb57a 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -17,6 +17,7 @@ "cors": "^2.8.5", "dotenv": "^16.4.7", "express": "^4.21.2", + "express-rate-limit": "^8.7.2", "franc": "^6.1.0", "jsonwebtoken": "^9.0.2", "openai": "^4.76.0", @@ -3662,6 +3663,48 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.7.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.2.tgz", + "integrity": "sha512-U1ScocxDD9WI09LaBVypvNkOtNR5yWAmZLkuKJdAzIchA4YKFYF6GT5IKER7epG58oXtjvGlV0/6ii0F83EyZg==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/express-rate-limit/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/express-rate-limit/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -4211,6 +4254,15 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/ip-address": { + "version": "10.7.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.5.tgz", + "integrity": "sha512-t2v0rkILYv9WXmSL7TpAaU3HXgaFYOPUcH3QHgwQ++gpJuH3bqyMy4Fvt/DJCTKwzH3Cf+bAp5W/mFL39KPLzA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", diff --git a/backend/package.json b/backend/package.json index 0e1551e8..ba7c7356 100644 --- a/backend/package.json +++ b/backend/package.json @@ -16,6 +16,7 @@ "cors": "^2.8.5", "dotenv": "^16.4.7", "express": "^4.21.2", + "express-rate-limit": "^8.7.2", "franc": "^6.1.0", "jsonwebtoken": "^9.0.2", "openai": "^4.76.0", diff --git a/backend/routes/locationRoutes.js b/backend/routes/locationRoutes.js index b1283be8..74aa46a5 100644 --- a/backend/routes/locationRoutes.js +++ b/backend/routes/locationRoutes.js @@ -1,9 +1,19 @@ const express = require('express'); const router = express.Router(); const locationController = require('../controllers/locationController'); +const requestRateLimit = require('../middleware/requestRateLimit'); + +router.use((_req, res, next) => { + res.set('Cache-Control', 'no-store'); + res.set('Access-Control-Expose-Headers', 'X-Next-Cursor'); + next(); +}); // Update coordinates for a user -router.post('/:id', locationController.setLocation); +const ownerLimit = requestRateLimit(); +const accountLimit = requestRateLimit(); +router.post('/:id', ownerLimit, locationController.requireOwner, accountLimit, locationController.setLocation); +router.get('/:id', ownerLimit, locationController.requireOwner, accountLimit, locationController.getLocation); // Retrieve public locations filtered by geofence router.get('/', locationController.getPublicLocations); diff --git a/backend/scripts/check-locations.js b/backend/scripts/check-locations.js new file mode 100644 index 00000000..4441ede7 --- /dev/null +++ b/backend/scripts/check-locations.js @@ -0,0 +1,161 @@ +// Explicit integration check: node --test scripts/check-locations.js +// Requires a disposable DynamoDB Local endpoint; never accepts a remote host. +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { randomUUID } = require('node:crypto'); +const endpoint = new URL(process.env.DYNAMODB_ENDPOINT || 'http://127.0.0.1:18001'); +if (!['127.0.0.1', 'localhost', '[::1]'].includes(endpoint.hostname)) { + throw new Error('Location check requires loopback DynamoDB Local'); +} +process.env.DYNAMODB_ENDPOINT = endpoint.href; +process.env.AWS_ACCESS_KEY_ID = 'localcheck'; +process.env.AWS_SECRET_ACCESS_KEY = 'localcheck'; +process.env.AWS_REGION = 'us-east-1'; +process.env.JWT_SECRET = 'location-integration-only'; +process.env.API_KEY = 'location-integration-key'; +process.env.LOCATION_TABLE_NAME = `LocationCheck-${randomUUID()}`; +const { DynamoDBClient, CreateTableCommand, DeleteTableCommand } = require('@aws-sdk/client-dynamodb'); +const admin = new DynamoDBClient({ region: 'us-east-1', endpoint: endpoint.href }); +const db = require('../lib/dynamodbClient'); +const jwt = require('jsonwebtoken'); +const express = require('express'); +const auth = require('../middleware/authMiddleware'); +const app = express(); +app.use(express.json(), require('../middleware/requestRateLimit')(), auth); +app.use('/api/location', require('../routes/locationRoutes')); +const body = (privacy = false, coordinates = 'geoLat_0.0,geoLong_0.0') => ({ coordinates, locationType: 'farm', privacy }); + +test('location persistence and discovery against DynamoDB Local', async t => { + await admin.send(new CreateTableCommand({ + TableName: process.env.LOCATION_TABLE_NAME, + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + BillingMode: 'PAY_PER_REQUEST' + })); + const server = app.listen(0, '127.0.0.1'); + await new Promise(resolve => server.once('listening', resolve)); + const base = `http://127.0.0.1:${server.address().port}/api/location`; + async function request(path, { owner, payload, token, key = false } = {}) { + const headers = { 'Content-Type': 'application/json' }; + if (owner) headers.Authorization = `Bearer ${jwt.sign({ userId: owner }, process.env.JWT_SECRET, { expiresIn: '1h' })}`; + if (token) headers.Authorization = `Bearer ${token}`; + if (key || !headers.Authorization) headers['x-api-key'] = process.env.API_KEY; + const res = await fetch(base + path, { method: payload === undefined ? 'GET' : 'POST', headers, + ...(payload === undefined ? {} : { body: JSON.stringify(payload) }) }); + return { status: res.status, body: await res.json(), cursor: res.headers.get('x-next-cursor'), cache: res.headers.get('cache-control'), exposed: res.headers.get('access-control-expose-headers') }; + } + async function check(name, fn) { await fn(); t.diagnostic(name); } + const search = '?lat=0&long=0&radius=1'; + try { + await check('save writes durable data and owner can reload it', async () => { + assert.equal((await request('/private', { owner: 'private', payload: body() })).status, 200); + const item = await db.get({ TableName: process.env.LOCATION_TABLE_NAME, Key: { id: 'private' }, ConsistentRead: true }).promise(); + assert.deepEqual(item.Item?.coordinates, { lat: 0, long: 0 }); + const loaded = await request('/private', { owner: 'private' }); + assert.equal(loaded.status, 200); + assert.equal(loaded.cache, 'no-store'); + assert.equal(loaded.body.privacy, false); + assert.deepEqual(loaded.body.coordinates, { lat: 0, long: 0 }); + }); + await check('API keys and other accounts cannot read or mutate private locations', async () => { + for (const options of [{}, { owner: 'other' }, { token: jwt.sign({ userId: 'private' }, 'wrong-secret') }, + { token: jwt.sign({ userId: 'private' }, process.env.JWT_SECRET, { expiresIn: -1 }) }]) { + for (const payload of [undefined, body(true)]) { + const res = await request('/private', { ...options, payload, key: true }); + assert.ok([401, 403].includes(res.status), JSON.stringify(res)); + } + } + assert.equal((await request('/private', { owner: 'private' })).body.privacy, false); + }); + await check('explicit sharing discovers safe DTOs inside radius only; unsharing removes them', async () => { + await request('/near', { owner: 'near', payload: body(true, 'geoLat_0.005,geoLong_0.0') }); + await request('/far', { owner: 'far', payload: body(true, 'geoLat_1.0,geoLong_0.0') }); + let found = await request('/' + search); + assert.equal(found.status, 200); + assert.deepEqual(found.body.map(x => x.id), ['near']); + assert.deepEqual(Object.keys(found.body[0]).sort(), ['coordinates', 'id', 'locationType']); + await request('/near', { owner: 'near', payload: body(false) }); + found = await request('/' + search); + assert.deepEqual(found.body, []); + }); + await check('invalid input cannot alter a saved location', async () => { + for (const payload of [body('true'), { ...body(), privacy: undefined }, body(true, 'geoLat_91.0,geoLong_0.0'), + body(true, 'geoLat_0.0,geoLong_181.0'), { ...body(), locationType: 'unknown' }, { ...body(), coordinates: 7 }]) { + assert.equal((await request('/private', { owner: 'private', payload })).status, 400); + } + assert.equal((await request('/private', { owner: 'private' })).body.privacy, false); + for (const query of ['lat=0junk&long=0&radius=1', 'lat=91&long=0&radius=1', 'lat=0&long=181&radius=1', + 'lat=0&long=0&radius=-1', 'lat=0&long=0&radius=Infinity', 'lat=&long=0&radius=1', + 'lat=0&long=0&radius=1&type=unknown', 'lat=0&lat=1&long=0&radius=1']) { + assert.equal((await request('/?' + query)).status, 400, query); + } + }); + await check('coordinate boundaries and zero radius are supported', async () => { + for (const [id, coordinates] of [['north', 'geoLat_90.0,geoLong_180.0'], ['south', 'geoLat_-90.0,geoLong_-180.0']]) { + assert.equal((await request('/' + id, { owner: id, payload: body(true, coordinates) })).status, 200); + } + await request('/zero', { owner: 'zero', payload: body(true) }); + assert.deepEqual((await request('/?lat=0&long=0&radius=0&type=farm')).body.map(x => x.id), ['zero']); + assert.equal((await request('/missing', { owner: 'missing' })).status, 404); + }); + await check('bounded pages continue past private rows and cursors reject tampering or query changes', async () => { + for (let i = 0; i < 120; i++) { + await request(`/page-${i}`, { owner: `page-${i}`, payload: body(i % 2 === 0) }); + } + let cursor, pages = 0; + const ids = []; + do { + const result = await request('/' + search + (cursor ? '&cursor=' + encodeURIComponent(cursor) : '')); + assert.equal(result.status, 200); + assert.ok(result.body.length <= 100); + assert.equal(result.exposed, 'X-Next-Cursor'); + ids.push(...result.body.map(x => x.id)); + cursor = result.cursor; + if (cursor) { + assert.equal((await request('/' + search + '&cursor=' + cursor.slice(0, -4) + 'xxxx')).status, 400); + assert.equal((await request('/?lat=0&long=0&radius=2&cursor=' + cursor)).status, 400); + } + assert.ok(++pages < 10); + } while (cursor); + assert.ok(pages >= 2); + assert.equal(new Set(ids).size, 61); + assert.equal(ids.length, 61); + }); + await check('empty private pages retain continuation without exposing private row data', async () => { + const first = await db.scan({ TableName: process.env.LOCATION_TABLE_NAME, Limit: 100, ConsistentRead: true }).promise(); + assert.ok(first.LastEvaluatedKey); + for (const item of first.Items) { + await db.put({ TableName: process.env.LOCATION_TABLE_NAME, Item: { ...item, privacy: false } }).promise(); + } + const empty = await request('/' + search); + assert.deepEqual(empty.body, []); + assert.ok(empty.cursor); + assert.equal((await request('/' + search + '&cursor=' + empty.cursor)).status, 200); + }); + await check('storage failure never reports saved or returns a successful empty discovery', async () => { + await admin.send(new DeleteTableCommand({ TableName: process.env.LOCATION_TABLE_NAME })); + assert.equal((await request('/private', { owner: 'private', payload: body() })).status, 500); + assert.equal((await request('/' + search)).status, 500); + }); + await check('the verification server bounds authentication attempts', async () => { + let limited = false; + for (let i = 0; i < 1001; i++) { + const response = await fetch(base, { headers: { Authorization: 'Bearer invalid' } }); + await response.text(); + if (response.status === 429) { + assert.ok(Number(response.headers.get('retry-after')) > 0); + limited = true; + break; + } + assert.equal(response.status, 401); + } + assert.ok(limited, 'authentication attempts must be rate limited'); + }); + } finally { + await new Promise(resolve => server.close(resolve)); + await admin.send(new DeleteTableCommand({ TableName: process.env.LOCATION_TABLE_NAME })).catch(err => { + if (err.name !== 'ResourceNotFoundException') throw err; + }); + admin.destroy(); + } +}); diff --git a/backend/scripts/check-message-retries.js b/backend/scripts/check-message-retries.js index bedef976..1ca9e60b 100644 --- a/backend/scripts/check-message-retries.js +++ b/backend/scripts/check-message-retries.js @@ -6,6 +6,24 @@ const { randomUUID } = require('node:crypto'); const { DynamoDBClient, CreateTableCommand, DeleteTableCommand, ScanCommand } = require('@aws-sdk/client-dynamodb'); const jwt = require('../utils/jwt'); +async function createTables(client, definitions, created) { + for (const [name, composite] of definitions) { + await client.send(new CreateTableCommand({ TableName: name, BillingMode: 'PAY_PER_REQUEST', + AttributeDefinitions: composite ? [{ AttributeName: 'conversationId', AttributeType: 'S' }, { AttributeName: 'id', AttributeType: 'S' }] + : [{ AttributeName: 'id', AttributeType: 'S' }], + KeySchema: composite ? [{ AttributeName: 'conversationId', KeyType: 'HASH' }, { AttributeName: 'id', KeyType: 'RANGE' }] + : [{ AttributeName: 'id', KeyType: 'HASH' }], + })); + created.push(name); + } +} + +async function stopWorkers(workers, stop) { + for (const target of workers) { + if (target.child.exitCode === null && target.child.signalCode === null) await stop(target); + } +} + async function main() { const endpoint = process.env.DYNAMODB_TEST_ENDPOINT; if (!endpoint || !['localhost', '127.0.0.1', '[::1]'].includes(new URL(endpoint).hostname)) { @@ -71,15 +89,7 @@ async function main() { } try { - for (const [name, composite] of [[messages, true], [receipts, false]]) { - await client.send(new CreateTableCommand({ TableName: name, BillingMode: 'PAY_PER_REQUEST', - AttributeDefinitions: composite ? [{ AttributeName: 'conversationId', AttributeType: 'S' }, { AttributeName: 'id', AttributeType: 'S' }] - : [{ AttributeName: 'id', AttributeType: 'S' }], - KeySchema: composite ? [{ AttributeName: 'conversationId', KeyType: 'HASH' }, { AttributeName: 'id', KeyType: 'RANGE' }] - : [{ AttributeName: 'id', KeyType: 'HASH' }], - })); - created.push(name); - } + await createTables(client, [[messages, true], [receipts, false]], created); const firstWorker = await worker(true); const secondWorker = await worker(false); const first = send(firstWorker); @@ -113,9 +123,7 @@ async function main() { assert.equal(afterCrash.Count, 3); console.log('PASS: process death leaves a pending receipt and does not duplicate a message or AI invocation.'); } finally { - for (const target of workers) { - if (target.child.exitCode === null && target.child.signalCode === null) await stop(target); - } + await stopWorkers(workers, stop); for (const name of created.reverse()) await client.send(new DeleteTableCommand({ TableName: name })); client.destroy(); } diff --git a/backend/services/retrySafeMessage.js b/backend/services/retrySafeMessage.js index 1657a837..3f742526 100644 --- a/backend/services/retrySafeMessage.js +++ b/backend/services/retrySafeMessage.js @@ -20,16 +20,41 @@ function problem(status, code, error) { return { status, body: { code, error } } const failed = () => problem(500, 'message_send_failed', 'This send failed and will not be repeated automatically. Inspect the conversation before starting a new send.'); -async function send(req, deliver) { +function readInput(req) { const key = req.headers['idempotency-key']; if (typeof key !== 'string' || !/^[\x21-\x7e]{1,128}$/.test(key)) { - return problem(400, 'invalid_idempotency_key', 'Idempotency-Key must be 1–128 printable ASCII characters without spaces.'); + return { error: problem(400, 'invalid_idempotency_key', 'Idempotency-Key must be 1–128 printable ASCII characters without spaces.') }; } // Auth middleware has already verified the JWT or accepted the configured // API key. Never use body.from (a display/role label) as the caller identity. const userId = typeof req.user?.userId === 'string' && req.user.userId ? req.user.userId : null; const apiKey = req.user ? null : req.headers['x-api-key']; - if (!userId && !apiKey) return problem(400, 'idempotency_identity_required', 'Retry-safe sends require a stable authenticated user ID or API key.'); + if (!userId && !apiKey) { + return { error: problem(400, 'idempotency_identity_required', 'Retry-safe sends require a stable authenticated user ID or API key.') }; + } + return { key, userId, apiKey }; +} + +async function replay(receipt, fingerprint, conversationId) { + if (receipt.fingerprint !== fingerprint) { + return problem(409, 'idempotency_key_conflict', 'This key was already used with a different message payload.'); + } + if (receipt.state !== 'complete') { + return problem(409, 'message_send_pending', 'This send is still running or was interrupted. Retry this key later; inspect the conversation before starting a new send.'); + } + if (receipt.outcome.status === 500) return failed(); + const message = await Message.getMessage(conversationId, receipt.outcome.messageId); + const reply = receipt.outcome.replyId ? await Message.getMessage(conversationId, receipt.outcome.replyId) : null; + if (!message || (receipt.outcome.replyId && !reply)) { + return problem(410, 'message_send_removed', 'The original message or reply is no longer available. It will not be recreated.'); + } + return { status: 201, body: reply ? { message, ai: reply, reply } : { message } }; +} + +async function send(req, deliver) { + const input = readInput(req); + if (input.error) return input.error; + const { key, userId, apiKey } = input; const conversationId = req.params.conversationId; const { from, to, content, type = 'text', file } = req.body; @@ -49,22 +74,7 @@ async function send(req, deliver) { } catch { return problem(503, 'message_send_unavailable', 'Retry protection is unavailable. No send was started by this request. Retry with the same key.'); } - if (!claim.claimed) { - const { receipt } = claim; - if (receipt.fingerprint !== fingerprint) { - return problem(409, 'idempotency_key_conflict', 'This key was already used with a different message payload.'); - } - if (receipt.state !== 'complete') { - return problem(409, 'message_send_pending', 'This send is still running or was interrupted. Retry this key later; inspect the conversation before starting a new send.'); - } - if (receipt.outcome.status === 500) return failed(); - const message = await Message.getMessage(conversationId, receipt.outcome.messageId); - const reply = receipt.outcome.replyId ? await Message.getMessage(conversationId, receipt.outcome.replyId) : null; - if (!message || (receipt.outcome.replyId && !reply)) { - return problem(410, 'message_send_removed', 'The original message or reply is no longer available. It will not be recreated.'); - } - return { status: 201, body: reply ? { message, ai: reply, reply } : { message } }; - } + if (!claim.claimed) return replay(claim.receipt, fingerprint, conversationId); let result, outcome; try { diff --git a/docs/location-api.md b/docs/location-api.md new file mode 100644 index 00000000..8a2c79eb --- /dev/null +++ b/docs/location-api.md @@ -0,0 +1,61 @@ +# Local location persistence and discovery + +Locations are mutable application data, kept in a separate operator-local DynamoDB table. They are not ledger entries or federation payloads. `Users` is deliberately not the storage target: existing user/admin/federation endpoints return whole user records, and federation imports can replace them. Those existing account-security gaps remain separate work; this feature still trusts account JWTs issued by the backend. + +## Deployment + +Create a table named `Locations` (or set `LOCATION_TABLE_NAME`), with string partition key `id`, no sort key, and on-demand billing. Grant the backend `dynamodb:GetItem`, `dynamodb:PutItem`, and `dynamodb:Scan` on this table. For example, using the operator's configured AWS region: + +```sh +aws dynamodb create-table --table-name Locations \ + --attribute-definitions AttributeName=id,AttributeType=S \ + --key-schema AttributeName=id,KeyType=HASH --billing-mode PAY_PER_REQUEST +``` + +Use the normal AWS region/credentials configuration and a strong `JWT_SECRET`. Missing storage returns HTTP 500; it never reports a successful save. Do not add this table to federation exports. Restrict database access and backups as private member data. + +Existing `Users.coordinates`, `locationType` and `locationPrivacy` are not copied or consulted. Members must explicitly save and opt in again. This avoids silently publishing legacy coordinates; it does not repair any pre-existing leakage of legacy user records. + +## Save and reload + +`POST /api/location/:id` and `GET /api/location/:id` require an unexpired HS256 backend bearer token whose `userId` exactly equals `:id`. An API key, role, body ID, or another account token cannot establish ownership, even when an API key passes the outer middleware. The existing server's authentication middleware still applies to discovery as well. + +Example POST body: + +```json +{"coordinates":"geoLat_17.18113,geoLong_48.11553","locationType":"farm","privacy":false} +``` + +`privacy` must be an explicit JSON boolean on every save. For compatibility with the original discovery code, **true means shared; false means private**. The label is historical and counterintuitive. Any future form must label this as “Share exact coordinates in discovery”, unchecked initially, and disclose the precision before opting in. No location form currently exists in this repository. + +Coordinates use the existing AgriNet decimal-string format. Latitude is inclusive [-90, 90], longitude [-180, 180]. Shared coordinates retain the supplied numeric precision; no rounding or approximate-location claim is made. `locationType` is optional; when present it is `farm`, `production`, or `delivery`. Each save replaces the owner's complete location record; omitting the type removes the previous type. Unknown body properties are ignored. Saving with `privacy: false` withdraws it from subsequent discovery reads; it cannot recall copies already downloaded. + +The save response includes `message`, `id`, `coordinates`, `privacy`, and optional `locationType`. Owner reload returns the same location fields without `message`; an unsaved location returns 404. Responses use `Cache-Control: no-store`. Invalid values return 400, missing/invalid account tokens 401, other owners 403, and storage failures 500. + +Owner reads and writes share a limit of 1,000 requests per minute. A guard before owner authentication uses the account already verified by outer middleware, or an IP fallback. A separate budget after owner authentication enforces 1,000 requests per account even when an API key accompanies the bearer token. Excess requests return HTTP 429 with `Retry-After`. Ownership, privacy and response fields remain unchanged below the limit; discovery is unchanged. + +Budgets are local to each server process and reset on restart. Multiple instances need a shared limiter store for a deployment-wide budget. Proxy trust settings are unchanged; arbitrary forwarded headers cannot select another budget. + +## Bounded discovery + +`GET /api/location?lat=17.18113&long=48.11553&radius=10&type=farm` + +Radius is a finite, non-negative number in **kilometres**, including the boundary (`distance <= radius`). Zero selects exact matching coordinates. Latitude/longitude use the same bounds as saved coordinates. Numeric query parameters must be complete decimal values, not trailing text, arrays, blank strings, NaN or infinity. `type` is optional and uses the same three values. + +The body remains a JSON array. Each entry contains only `id`, `coordinates`, and optional `locationType`. Only rows explicitly saved with `privacy: true` are eligible. No contact, password, verification, role, reputation or other user attributes are returned. + +Each request evaluates at most 100 table records (and DynamoDB's 1 MiB page limit), then filters by sharing, type and distance. If response header `X-Next-Cursor` is present, repeat the same query with `cursor=` until it is absent. An empty page can have a continuation. The header is exposed for browser clients. Cursors encrypt and authenticate the scan key, so a page ending on a private row does not reveal its account ID. They are bound to the parsed query, expire after one hour, and become invalid on `JWT_SECRET` rotation. Invalid/tampered/query-mismatched cursors return 400. Do not construct or interpret them. + +This is bounded DynamoDB scan pagination, not a spatial index or snapshot. Concurrent changes can affect later pages. Strong reads reduce stale publication after withdrawal; there is no cross-request snapshot or recall guarantee. A spatial index is separate work if table size makes scans unsuitable. + +## Verification + +Run the normal backend suite with `npm test`. For real storage checks, start a disposable DynamoDB Local instance bound to loopback port 18001, then run from `backend`: + +```sh +node --test scripts/check-locations.js +``` + +`DYNAMODB_ENDPOINT` may select another loopback endpoint. The check refuses remote hosts, supplies dummy credentials, creates a uniquely named table, and deletes it on completion. It verifies durable writes, owner reload/access denial, explicit publication/withdrawal, exact safe response fields, coordinate/radius validation, multiple pages, cursor tampering/query binding, and honest storage failures. CI runs this check against a digest-pinned DynamoDB Local image. + +The disposable HTTP server also limits authentication attempts to 1,000 per minute per IP before verifying credentials. The check verifies budget exhaustion without bypassing authentication or throttling in test mode.