diff --git a/.github/workflows/collection-pagination.yml b/.github/workflows/collection-pagination.yml new file mode 100644 index 00000000..50883197 --- /dev/null +++ b/.github/workflows/collection-pagination.yml @@ -0,0 +1,48 @@ +name: Collection pagination regression + +on: + pull_request: + paths: + - 'backend/**' + - 'frontend/chat-ui/**' + - '.github/workflows/collection-pagination.yml' + +permissions: + contents: read + +jobs: + collection-pagination: + runs-on: ubuntu-latest + services: + dynamodb: + image: amazon/dynamodb-local@sha256:6ada631ec7aa5fc83446855eb17233e3dad184fec47477e8140c6c569cfe0c86 + ports: + - 18001:8000 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: backend/package-lock.json + - run: npm ci + working-directory: backend + - run: npm test + working-directory: backend + - run: npm ci + working-directory: frontend/chat-ui + - run: npm test + working-directory: frontend/chat-ui + - run: npm run build + working-directory: frontend/chat-ui + - name: Wait for DynamoDB Local + run: | + for attempt in $(seq 1 30); do + if curl --silent --output /dev/null http://127.0.0.1:18001; then exit 0; fi + sleep 1 + done + exit 1 + - run: node scripts/check-collection-pagination.js + working-directory: backend + env: + DYNAMODB_ENDPOINT: http://127.0.0.1:18001 diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 16a08b84..4207d723 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -42,8 +42,39 @@ Adopts the covenant scale (LBTAS, v2 floor) and carries its own record instance. ## Product-specific notes (last, per the ordering rule) +### Bounded application reads + +Conversation pages retain the account/owner boundary documented in +`backend/docs/conversation-ownership.md`. The user directory allowlists local +public fields, location-type discovery and sharing-opted-in coarse location; it is not an account +export. Scalar reputation fields stay excluded per the covenant’s full-distribution rule. Cursors encrypt private scan positions and bind the account/query. +`backend/__tests__/collectionPagination.test.js` and the real DynamoDB check +`backend/scripts/check-collection-pagination.js` exercise these boundaries. + +Federation export is an explicitly read-only application view, using +per-peer HMAC authentication with signed responses. Account rows, local dialog narrative and +scalar ratings are omitted. Scheduled paths inspect pages without importing +summaries, rewriting canonical records or advancing synchronization time. +Source provenance/conflict policy and safe manual import remain open work; +no witnessed-federation or full-stack privacy conformance is asserted. See +`backend/docs/collection-pagination.md` for exact omissions, compatibility +changes, configuration, rollout dependencies and residuals. Existing record, +covenant and economy status claims above remain unchanged. + The reference economy. Its operator/federation model is the named reference for the substrate's frontend-as-operator design target. +Private application chat uses the verified backend account `userId` as an +immutable conversation owner. Conversation/message routes and attachment +downloads enforce that boundary; shared integration keys supply no participant +identity. Legacy ownerless rows remain inaccessible pending explicit operator +assignment. Bindings: `backend/middleware/chatAuth.js`, +`conversationOwner.js`, `attachmentOwner.js`, the conditional conversation +model writes, and `backend/__tests__/conversationAccess.test.js`. This does not +promote private chat to a sealed commons dialog or change the in-review ledger +claims above. SSE requires the companion authenticated-stream change before +rollout. See `backend/docs/conversation-ownership.md` for migration and client +integration limits. + The application message API supports optional retry receipts, scoped to the authenticated caller and conversation. These are private operational metadata, not sealed dialog or commons entries; they store hashes and message references, diff --git a/backend/__tests__/collectionPagination.test.js b/backend/__tests__/collectionPagination.test.js new file mode 100644 index 00000000..01d5e28a --- /dev/null +++ b/backend/__tests__/collectionPagination.test.js @@ -0,0 +1,246 @@ +const request = require('supertest'); +const jwt = require('jsonwebtoken'); +const { createPeerHeaders, verifyPeerResponse } = require('../middleware/federationPeerAuth'); +const peerSecret = '0123456789abcdef0123456789abcdef'; +const peerHeaders = () => createPeerHeaders({ nodeId: 'peer-1', secret: peerSecret, method: 'GET', path: '/federation/export' }); + +const mockTables = {}; +let mockScans = []; +jest.mock('../lib/dynamodbClient', () => ({ + scan: params => ({ promise: async () => { + mockScans.push(params); + const rows = mockTables[params.TableName] || []; + const start = params.ExclusiveStartKey ? rows.findIndex(row => row.id === params.ExclusiveStartKey.id) + 1 : 0; + const evaluated = rows.slice(start, start + (params.Limit || rows.length)); + let Items = evaluated; + const values = params.ExpressionAttributeValues || {}; + if (values[':ownerId']) Items = Items.filter(row => row.ownerId === values[':ownerId']); + if (values[':locationType']) Items = Items.filter(row => row.locationType === values[':locationType']); + if (values[':role']) Items = Items.filter(row => row.role === values[':role']); + if (values[':location']) Items = Items.filter(row => row.location === values[':location'] && row.locationPrivacy === true); + // Return complete records deliberately: response DTOs must also protect + // against unexpected fields, independently of the database projection. + return { Items, ...(start + evaluated.length < rows.length && { + LastEvaluatedKey: { id: evaluated.at(-1).id }, + }) }; + } }), +})); +jest.mock('../bull/pingJobs', () => ({ addPingJob: jest.fn() })); +jest.mock('../bull/smsQueue', () => ({ enqueueSms: jest.fn() })); +jest.mock('../federation/federationSyncJob', () => jest.fn()); + +describe('bounded collection HTTP responses', () => { + let app; + const env = { ...process.env }; + const authorization = userId => `Bearer ${jwt.sign({ userId }, process.env.JWT_SECRET, { expiresIn: '2h' })}`; + beforeAll(() => { + process.env.MINIMAL_SERVER = 'false'; + process.env.JWT_SECRET = 'collection-test-secret'; + process.env.API_KEY = 'collection-operator-key'; + process.env.FEDERATION_LOCAL_NODE_ID = 'local-node'; + process.env.FEDERATION_PEERS_JSON = JSON.stringify({ 'peer-1': peerSecret }); + ({ app } = require('../server')); + }); + afterAll(() => { process.env = env; }); + beforeEach(() => { + mockScans = []; + mockTables.Conversations = [ + { id: 'bobs-secret-id', ownerId: 'bob', title: 'private' }, + { id: 'mine-1', ownerId: 'alice', title: 'first' }, + { id: 'legacy', title: 'unowned' }, + { id: 'mine-2', ownerId: 'alice', title: 'second' }, + ]; + mockTables.Users = [ + { id: 'u1', username: 'grower', role: 'producer', locationType: 'farm', reputationScore: 99, reputationWeight: 1000, location: 'Town', locationPrivacy: false, password: 'hash', verificationCode: 'code', email: 'private@example.invalid', phone: 'private', coordinates: { lat: 1, long: 2 }, futureSecret: 'secret' }, + { id: 'u2', username: 'buyer', role: 'consumer', locationType: 'delivery', location: 'Town', locationPrivacy: true }, + { id: 'u3', username: 'seller', role: 'producer', locationType: 'production', location: 'Town', locationPrivacy: true }, + ]; + mockTables.Listings = [{ id: 'l1', userId: 'u1', title: 'Public listing', price: 2, futureSecret: 'not a listing field' }, { id: 'l2', userId: 'u2', title: 'Second' }]; + mockTables.Transactions = [{ id: 't1', consumerId: 'u1', producerId: 'u2', status: 'pending', dialogNotes: 'private narrative', futureSecret: 'secret' }]; + }); + + async function collect(path, header, value) { + const pages = []; + let cursor; + do { + const result = await request(app).get(path + (cursor ? `&cursor=${encodeURIComponent(cursor)}` : '')).set(header === 'peer' ? peerHeaders() : { [header]: value }); + expect(result.status).toBe(200); + expect(result.headers['x-pagination']).toBe('cursor-v1'); + expect(result.headers['cache-control']).toContain('no-store'); + if (header === 'peer') expect(verifyPeerResponse({ headers: result.headers, expectedNodeId: 'local-node', secret: peerSecret, method: 'GET', path: '/federation/export', body: JSON.stringify(result.body) })).toBe(true); + pages.push(result.body); + cursor = result.headers['x-next-cursor']; + expect(pages.length).toBeLessThan(10); + } while (cursor); + return pages; + } + + test('conversation pages include empty filtered pages but never foreign or ownerless rows', async () => { + const pages = await collect('/conversations?limit=1', 'Authorization', authorization('alice')); + expect(pages).toEqual([[], [{ id: 'mine-1', ownerId: 'alice', title: 'first' }], [], [{ id: 'mine-2', ownerId: 'alice', title: 'second' }]]); + expect(mockScans).toHaveLength(4); + expect(mockScans.every(scan => scan.Limit === 1 && scan.ExpressionAttributeValues[':ownerId'] === 'alice')).toBe(true); + }); + + test('user pages allowlist public fields and do not expose private location by filtering', async () => { + const pages = await collect('/users?limit=1', 'Authorization', authorization('alice')); + expect(pages.flat()).toEqual([ + { id: 'u1', username: 'grower', role: 'producer', locationType: 'farm' }, + { id: 'u2', username: 'buyer', role: 'consumer', locationType: 'delivery', location: 'Town' }, + { id: 'u3', username: 'seller', role: 'producer', locationType: 'production', location: 'Town' }, + ]); + mockScans = []; + const filtered = await collect('/users?limit=1&role=producer&location=Town', 'Authorization', authorization('alice')); + expect(filtered.flat().map(row => row.id)).toEqual(['u3']); + expect(filtered[0]).toEqual([]); + }); + + test('location-type discovery preserves empty pages and excludes scalar reputation and private location', async () => { + const pages = await collect('/users?limit=1&locationType=production', 'Authorization', authorization('alice')); + expect(pages).toEqual([[], [], [{ id: 'u3', username: 'seller', role: 'producer', locationType: 'production', location: 'Town' }]]); + const privatePage = await collect('/users?limit=1&locationType=farm', 'Authorization', authorization('alice')); + expect(privatePage.flat()).toEqual([{ id: 'u1', username: 'grower', role: 'producer', locationType: 'farm' }]); + }); + + test('location-type cursors cannot be reused with another or omitted filter', async () => { + const token = authorization('alice'); + const first = await request(app).get('/users?limit=1&locationType=farm').set('Authorization', token); + expect(first.status).toBe(200); + const cursor = first.headers['x-next-cursor']; + expect(typeof cursor).toBe('string'); + mockScans = []; + for (const path of ['/users?limit=1&locationType=delivery', '/users?limit=1']) { + const result = await request(app).get(path).query({ cursor }).set('Authorization', token); + expect(result.status).toBe(400); + } + expect(mockScans).toHaveLength(0); + }); + + test('repeated and structured location-type filters fail before storage access', async () => { + for (const suffix of ['locationType=farm&locationType=delivery', 'locationType[kind]=farm']) { + const result = await request(app).get('/users?' + suffix).set('Authorization', authorization('alice')); + expect(result.status).toBe(400); + } + expect(mockScans).toHaveLength(0); + }); + + test('federation pages finish collections independently without exporting accounts or narrative', async () => { + const pages = await collect('/federation/export?limit=1', 'peer', null); + expect(pages).toEqual([ + { listings: [{ id: 'l1', userId: 'u1', title: 'Public listing', price: 2 }], transactions: [{ id: 't1', consumerId: 'u1', producerId: 'u2', status: 'pending' }], users: [] }, + { listings: [{ id: 'l2', userId: 'u2', title: 'Second' }], transactions: [], users: [] }, + ]); + expect(mockScans.map(scan => scan.TableName)).toEqual(['Listings', 'Transactions', 'Listings']); + }); + + test('a federation continuation is bound to its authenticated peer', async () => { + const first = await request(app).get('/federation/export?limit=1').set(peerHeaders()); + expect(first.status).toBe(200); + const cursor = first.headers['x-next-cursor']; + expect(typeof cursor).toBe('string'); + process.env.FEDERATION_PEERS_JSON = JSON.stringify({ 'peer-1': peerSecret, 'peer-2': peerSecret }); + try { + const other = createPeerHeaders({ nodeId: 'peer-2', secret: peerSecret, method: 'GET', path: '/federation/export' }); + const response = await request(app).get('/federation/export?limit=1').query({ cursor }).set(other); + expect(response.status).toBe(400); + } finally { process.env.FEDERATION_PEERS_JSON = JSON.stringify({ 'peer-1': peerSecret }); } + }); + + test('missing response signing identity fails before scanning or publishing a cursor', async () => { + const local = process.env.FEDERATION_LOCAL_NODE_ID; + const node = process.env.NODE_ID; + delete process.env.FEDERATION_LOCAL_NODE_ID; + delete process.env.NODE_ID; + try { + const result = await request(app).get('/federation/export?limit=1').set(peerHeaders()); + expect(result.status).toBe(503); + expect(result.headers['x-next-cursor']).toBeUndefined(); + expect(mockScans).toHaveLength(0); + } finally { + process.env.FEDERATION_LOCAL_NODE_ID = local; + if (node === undefined) delete process.env.NODE_ID; else process.env.NODE_ID = node; + } + }); + + test('ordinary account tokens cannot export federation data and shared keys cannot list private chats', async () => { + expect((await request(app).get('/federation/export').set('Authorization', authorization('alice'))).status).toBe(401); + expect((await request(app).get('/federation/export').set('x-api-key', process.env.API_KEY)).status).toBe(401); + expect((await request(app).get('/conversations').set('x-api-key', process.env.API_KEY)).status).toBe(401); + expect((await request(app).get('/users').set('x-api-key', process.env.API_KEY)).status).toBe(401); + expect(mockScans).toHaveLength(0); + }); + + test('obsolete export keys cannot bypass per-peer HMAC authentication', async () => { + const result = await request(app).get('/federation/export').set('x-federation-key', 'old-read-only-key'); + expect(result.status).toBe(401); + expect(mockScans).toHaveLength(0); + }); + + test('approved browser origins can read pagination headers without exposing private keys', async () => { + const result = await request(app).get('/users?limit=1').set('Authorization', authorization('alice')).set('Origin', 'https://www.ntari.org'); + expect(result.status).toBe(200); + expect(result.headers['access-control-expose-headers']).toContain('X-Next-Cursor'); + expect(result.headers['access-control-expose-headers']).toContain('X-Pagination'); + expect(JSON.stringify(result.body)).not.toContain('private@example.invalid'); + }); + + test.each(['0', '-1', '101', '1junk', '1.5', '', '1&limit=2'])('invalid limit %s fails before a scan', async limit => { + const result = await request(app).get(`/conversations?limit=${limit}`).set('Authorization', authorization('alice')); + expect(result.status).toBe(400); + expect(mockScans).toHaveLength(0); + }); + + test('defaults are bounded and malformed cursors fail before scanning', async () => { + const first = await request(app).get('/users').set('Authorization', authorization('alice')); + expect(first.status).toBe(200); + expect(mockScans[0].Limit).toBe(50); + mockScans = []; + for (const cursor of ['invalid', '%%%','']) { + const result = await request(app).get('/users').query({ cursor }).set('Authorization', authorization('alice')); + expect(result.status).toBe(400); + } + expect(mockScans).toHaveLength(0); + }); + + test('encrypted cursors bind account, collection, filters, and limit', async () => { + const token = authorization('alice'); + mockTables.Users[0].id = 'private-account-continuation-key'; + const first = await request(app).get('/users?limit=1&role=producer').set('Authorization', token); + const cursor = first.headers['x-next-cursor']; + expect(typeof cursor).toBe('string'); + expect(Buffer.from(cursor, 'base64url').toString()).not.toContain('private-account-continuation-key'); + mockScans = []; + for (const [path, auth] of [ + ['/users?limit=1&role=consumer', token], + ['/users?limit=2&role=producer', token], + ['/users?limit=1&role=producer', authorization('bob')], + ['/conversations?limit=1&role=producer', token], + ]) { + const result = await request(app).get(path).query({ cursor }).set('Authorization', auth); + expect(result.status).toBe(400); + } + const changed = cursor.slice(0, 12) + (cursor[12] === 'A' ? 'B' : 'A') + cursor.slice(13); + expect((await request(app).get('/users?limit=1&role=producer').query({ cursor: changed }).set('Authorization', token)).status).toBe(400); + expect(mockScans).toHaveLength(0); + }); + + test('cursors expire and database failures never return a successful partial page', async () => { + const token = authorization('alice'); + const first = await request(app).get('/users?limit=1').set('Authorization', token); + const cursor = first.headers['x-next-cursor']; + const clock = jest.spyOn(Date, 'now').mockReturnValue(Date.now() + 3600001); + try { + const expired = await request(app).get('/users?limit=1').query({ cursor }).set('Authorization', token); + expect(expired.status).toBe(400); + } finally { clock.mockRestore(); } + mockTables.Listings = null; + const db = require('../lib/dynamodbClient'); + const originalScan = db.scan; + db.scan = () => ({ promise: async () => { throw new Error('database unavailable'); } }); + try { + const result = await request(app).get('/federation/export').set(peerHeaders()); + expect(result.status).toBe(500); + expect(result.headers['x-next-cursor']).toBeUndefined(); + } finally { db.scan = originalScan; } + }); +}); diff --git a/backend/__tests__/collectionRateLimit.test.js b/backend/__tests__/collectionRateLimit.test.js new file mode 100644 index 00000000..b7360ccf --- /dev/null +++ b/backend/__tests__/collectionRateLimit.test.js @@ -0,0 +1,91 @@ +const request = require('supertest'); +const http = require('node:http'); +const jwt = require('jsonwebtoken'); + +jest.mock('../bull/pingJobs', () => ({ addPingJob: jest.fn() })); +jest.mock('../bull/smsQueue', () => ({ enqueueSms: jest.fn() })); +jest.mock('../federation/federationSyncJob', () => jest.fn()); +jest.mock('../lib/dynamodbClient', () => ({ + get: ({ Key }) => ({ promise: async () => ({ + Item: Key.id === 'owned' ? { id: 'owned', ownerId: 'alice' } : undefined, + }) }), + query: () => ({ promise: async () => ({ Items: [] }) }), + put: () => ({ promise: async () => ({}) }), +})); + +describe('account collection request budgets', () => { + let app, server, agent; + const send = (method, url) => request(server)[method](url).agent(agent); + const env = { ...process.env }; + const authorization = userId => `Bearer ${jwt.sign({ userId }, process.env.JWT_SECRET, { expiresIn: '2h' })}`; + beforeAll(() => { + process.env.MINIMAL_SERVER = 'false'; + process.env.JWT_SECRET = 'collection-budget-test'; + process.env.API_KEY = 'collection-budget-key'; + }); + beforeEach(async () => { + jest.resetModules(); + ({ app } = require('../server')); + agent = new http.Agent({ keepAlive: true }); + server = app.listen(0, '127.0.0.1'); + await new Promise(resolve => server.once('listening', resolve)); + }); + afterEach(async () => { + agent.destroy(); + await new Promise(resolve => server.close(resolve)); + }); + afterAll(() => { process.env = env; }); + + test.each([ + ['/users?limit=0', 400], + ['/uploads/bad%00name', 404], + ['/messages/missing', 404], + ])('%s preserves normal responses and throttles each account independently', async (url, status) => { + const token = authorization('alice'); + for (let i = 0; i < 120; i++) { + expect((await send('get', url).set('Authorization', token)).status).toBe(status); + } + const blocked = await send('get', url).set('Authorization', authorization('alice')) + .set('X-Forwarded-For', '203.0.113.9'); + expect(blocked.status).toBe(429); + expect(Number(blocked.headers['retry-after'])).toBeGreaterThan(0); + expect((await send('get', url).set('Authorization', authorization('bob'))).status).toBe(status); + }, 20000); + + test.each(['/users', '/uploads/bad%00name', '/messages/missing'])('%s limits API-key-only callers without granting account access', async url => { + for (let i = 0; i < 120; i++) { + expect((await send('get', url).set('x-api-key', process.env.API_KEY)).status).toBe(401); + } + const blocked = await send('get', url).set('x-api-key', process.env.API_KEY) + .set('X-Forwarded-For', '203.0.113.10'); + expect(blocked.status).toBe(429); + }, 20000); + + test.each([ + ['/users?limit=0', 400], + ['/uploads/bad%00name', 404], + ['/messages/missing', 404], + ])('%s cannot gain another account budget by adding an API key', async (url, status) => { + const token = authorization('credential-owner'); + for (let i = 0; i < 120; i++) { + expect((await send('get', url).set('Authorization', token)).status).toBe(status); + } + for (const key of [process.env.API_KEY, 'invalid-key']) { + expect((await send('get', url).set('Authorization', token).set('x-api-key', key)).status).toBe(429); + } + }, 20000); + + test('message reads and sends share an account budget across conversation IDs and encoded paths', async () => { + const token = authorization('alice'); + for (let i = 0; i < 120; i++) { + expect((await send('get', '/messages/missing').set('Authorization', token)).status).toBe(404); + } + for (const method of ['get', 'post']) { + const blocked = await send(method, '/messages/%6fwned') + .set('Authorization', token).send({ from: 'bob', content: 'hello' }); + expect(blocked.status).toBe(429); + expect(Number(blocked.headers['retry-after'])).toBeGreaterThan(0); + } + expect((await send('get', '/messages/owned').set('Authorization', authorization('bob'))).status).toBe(404); + }, 20000); +}); diff --git a/backend/__tests__/conversation.test.js b/backend/__tests__/conversation.test.js index da0bcaa7..2e168e0d 100644 --- a/backend/__tests__/conversation.test.js +++ b/backend/__tests__/conversation.test.js @@ -36,10 +36,10 @@ test('removing a conversation deletes its messages', async () => { Conversation.setDocClient(mockClient); Message.setDocClient(mockClient); - await mockClient.put({ TableName: 'Conversations', Item: { id: '1', title: 'seed', pinned: false, createdAt: new Date().toISOString() } }).promise(); + await mockClient.put({ TableName: 'Conversations', Item: { id: '1', ownerId: 'owner', title: 'seed', pinned: false, createdAt: new Date().toISOString() } }).promise(); await mockClient.put({ TableName: 'Messages', Item: { id: 'm1', conversationId: '1', content: 'hi' } }).promise(); - await Conversation.remove('1'); + await Conversation.remove('1', 'owner'); const msgs = await Message.listMessages('1'); assert.equal(msgs.length, 0); }); diff --git a/backend/__tests__/conversationAccess.test.js b/backend/__tests__/conversationAccess.test.js new file mode 100644 index 00000000..580d6a5c --- /dev/null +++ b/backend/__tests__/conversationAccess.test.js @@ -0,0 +1,183 @@ +const request = require('supertest'); +const jwt = require('jsonwebtoken'); +const fs = require('fs'); +const path = require('path'); + +const mockTables = { Conversations: new Map(), Messages: new Map() }; +let mockMessageWrites = 0; +let mockMessageReads = 0; +function mockConditionalFailure() { + return Object.assign(new Error('condition failed'), { code: 'ConditionalCheckFailedException' }); +} +jest.mock('../lib/dynamodbClient', () => ({ + put: ({ TableName, Item }) => ({ promise: async () => { + if (TableName === 'Messages') mockMessageWrites++; + mockTables[TableName].set(Item.id, { ...Item }); + return {}; + } }), + get: ({ TableName, Key }) => ({ promise: async () => ({ Item: mockTables[TableName].get(Key.id) }) }), + scan: ({ TableName, ExpressionAttributeValues, ExclusiveStartKey }) => ({ promise: async () => { + let Items = [...mockTables[TableName].values()]; + // Deliberately include a continuation page with no matching items first. + if (ExclusiveStartKey) Items = Items.slice(1); + else if (Items.length > 1) Items = Items.slice(0, 1); + const LastEvaluatedKey = !ExclusiveStartKey && mockTables[TableName].size > 1 ? { id: Items[0].id } : undefined; + if (ExpressionAttributeValues?.[':ownerId']) { + Items = Items.filter(item => item.ownerId === ExpressionAttributeValues[':ownerId']); + } + if (ExpressionAttributeValues?.[':filePath']) { + Items = Items.filter(item => item.file?.path === ExpressionAttributeValues[':filePath']); + } + return { Items, LastEvaluatedKey }; + } }), + update: ({ TableName, Key, ExpressionAttributeNames, ExpressionAttributeValues, ConditionExpression }) => ({ promise: async () => { + const item = mockTables[TableName].get(Key.id); + if (!item || (ConditionExpression.includes('ownerId') && item.ownerId !== ExpressionAttributeValues[':ownerId'])) throw mockConditionalFailure(); + item[ExpressionAttributeNames['#field']] = ExpressionAttributeValues[':value']; + return { Attributes: { ...item } }; + } }), + delete: ({ TableName, Key, ConditionExpression, ExpressionAttributeValues }) => ({ promise: async () => { + const item = mockTables[TableName].get(Key.id); + if (ConditionExpression && (!item || item.ownerId !== ExpressionAttributeValues[':ownerId'])) throw mockConditionalFailure(); + mockTables[TableName].delete(Key.id); + return { Attributes: item }; + } }), + query: ({ ExpressionAttributeValues }) => ({ promise: async () => { + mockMessageReads++; + return { Items: [...mockTables.Messages.values()].filter(item => item.conversationId === ExpressionAttributeValues[':cid']) }; + } }), + batchWrite: ({ RequestItems }) => ({ promise: async () => { + for (const { DeleteRequest: { Key } } of RequestItems.Messages || []) mockTables.Messages.delete(Key.id); + return {}; + } }), +})); +jest.mock('../bull/pingJobs', () => ({ addPingJob: jest.fn() })); +jest.mock('../bull/smsQueue', () => ({ enqueueSms: jest.fn() })); +jest.mock('../federation/federationSyncJob', () => jest.fn()); +jest.mock('../services/openAIResponder', () => ({ generateResponse: async () => ({ from: 'ai', content: 'fixture reply' }) })); + +describe('private conversation HTTP boundary', () => { + let app; + const originalEnv = { ...process.env }; + const globals = Object.fromEntries(['broadcast', 'emitToken', 'emitMessage'].map(key => [key, global[key]])); + const filename = 'ownership-fixture.txt'; + const attachment = `/uploads/${filename}`; + const authorization = (userId, extra = {}) => `Bearer ${jwt.sign({ userId, ...extra }, process.env.JWT_SECRET, { expiresIn: '1h' })}`; + + beforeAll(() => { + process.env.MINIMAL_SERVER = 'false'; + process.env.API_KEY = 'conversation-test-key'; + process.env.JWT_SECRET = 'conversation-test-secret'; + ({ app } = require('../server')); + fs.writeFileSync(path.join(__dirname, '../uploads', filename), 'private attachment'); + }); + beforeEach(() => { + mockTables.Conversations.clear(); + mockTables.Messages.clear(); + mockMessageWrites = 0; + mockMessageReads = 0; + mockTables.Conversations.set('foreign', { id: 'foreign', ownerId: 'bob', title: 'Bobs private chat' }); + mockTables.Conversations.set('mine', { id: 'mine', ownerId: 'alice', title: 'My chat', pinned: false }); + mockTables.Conversations.set('legacy', { id: 'legacy', title: 'Unknown owner' }); + mockTables.Messages.set('secret', { id: 'secret', conversationId: 'foreign', content: 'private', file: { path: attachment } }); + }); + afterAll(() => { + fs.unlinkSync(path.join(__dirname, '../uploads', filename)); + process.env = originalEnv; + for (const [key, value] of Object.entries(globals)) { + if (value === undefined) delete global[key]; else global[key] = value; + } + }); + + test('creation derives immutable ownership from the verified account, ignoring supplied owner fields', async () => { + const result = await request(app).post('/conversations').set('Authorization', authorization('alice')) + .send({ title: 'New chat', ownerId: 'bob', userId: 'bob' }); + expect(result.status).toBe(201); + expect(result.body.ownerId).toBe('alice'); + expect(mockTables.Conversations.get(result.body.id).ownerId).toBe('alice'); + }); + async function allConversations(userId, apiKey) { + const rows = []; + let cursor; + do { + let call = request(app).get('/conversations').set('Authorization', authorization(userId)); + if (apiKey) call = call.set('x-api-key', apiKey); + if (cursor) call = call.query({ cursor }); + const result = await call; + expect(result.status).toBe(200); + rows.push(...result.body); + cursor = result.headers['x-next-cursor']; + } while (cursor); + return rows; + } + test('listing traverses empty filtered pages without exposing foreign or legacy chats', async () => { + expect((await allConversations('alice')).map(item => item.id)).toEqual(['mine']); + }); + test.each(['foreign', 'legacy', 'missing'])('all ID routes conceal %s and have no message side effects', async id => { + for (const [method, url, body] of [ + ['get', `/messages/${id}`], ['get', `/conversations/${id}`], ['put', `/conversations/${id}`, { title: 'stolen' }], + ['post', `/conversations/${id}/pin`, { pinned: true }], ['delete', `/conversations/${id}`], + ['get', `/messages/${id}`], ['post', `/messages/${id}`, { from: 'user', content: 'private AI trigger' }], + ]) { + const result = await request(app)[method](url).set('Authorization', authorization('alice')).send(body); + expect(result.status).toBe(404); + expect(result.body).toEqual({ error: 'Not found' }); + } + expect(mockMessageReads).toBe(0); + expect(mockMessageWrites).toBe(0); + expect(mockTables.Conversations.get('foreign').title).toBe('Bobs private chat'); + expect(mockTables.Messages.get('secret').content).toBe('private'); + }); + test.each([undefined, '', ' ', 7, ['alice']])('rejects a signed token without a valid userId (%p)', async userId => { + const result = await request(app).get('/conversations').set('Authorization', authorization(userId)); + expect(result.status).toBe(401); + }); + test('a shared API key cannot replace or bypass individual bearer authentication', async () => { + for (const bearer of [undefined, 'Bearer invalid']) { + let call = request(app).get('/conversations').set('x-api-key', process.env.API_KEY); + if (bearer) call = call.set('Authorization', bearer); + expect((await call).status).toBe(401); + } + const result = await request(app).get('/conversations').set('x-api-key', process.env.API_KEY) + .set('Authorization', authorization('alice')); + expect(result.status).toBe(200); + expect((await allConversations('alice', process.env.API_KEY)).map(item => item.id)).toEqual(['mine']); + }); + test('the rightful owner can read, rename, pin, send, list and delete', async () => { + const auth = authorization('alice'); + expect((await request(app).get('/conversations/mine').set('Authorization', auth)).status).toBe(200); + expect((await request(app).put('/conversations/mine').set('Authorization', auth).send({ title: 'Renamed', ownerId: 'bob' })).body.title).toBe('Renamed'); + expect((await request(app).post('/conversations/mine/pin').set('Authorization', auth).send({ pinned: true })).body.pinned).toBe(true); + const sent = await request(app).post('/messages/mine').set('Authorization', auth).send({ from: 'alice', content: 'hello', type: 'text' }); + expect(sent.status).toBe(201); + const messages = await request(app).get('/messages/mine').set('Authorization', auth); + expect(messages.body.map(item => item.content)).toEqual(['hello']); + expect((await request(app).delete('/conversations/mine').set('Authorization', auth)).status).toBe(204); + expect(mockTables.Conversations.has('mine')).toBe(false); + expect([...mockTables.Messages.values()].some(item => item.conversationId === 'mine')).toBe(false); + expect(mockTables.Conversations.get('foreign').ownerId).toBe('bob'); + }); + test('attachment paths enforce the same owner boundary, including encoded paths', async () => { + for (const url of [attachment, '/uploads/%6fwnership-fixture.txt']) { + const foreign = await request(app).get(url).set('Authorization', authorization('alice')); + expect(foreign.status).toBe(404); + const owner = await request(app).get(url).set('Authorization', authorization('bob')); + expect(owner.status).toBe(200); + expect(owner.body.toString()).toBe('private attachment'); + expect(owner.headers['cache-control']).toContain('no-store'); + expect(owner.headers['content-disposition']).toBe('attachment'); + } + expect((await request(app).get(attachment).set('x-api-key', process.env.API_KEY)).status).toBe(401); + }); + + test('conditional model writes cannot rename, pin or delete another owner or a legacy row', async () => { + const Conversation = require('../models/conversation'); + for (const id of ['foreign', 'legacy', 'missing']) { + expect(await Conversation.rename(id, 'stolen', 'alice')).toBeNull(); + expect(await Conversation.pin(id, true, 'alice')).toBeNull(); + expect(await Conversation.remove(id, 'alice')).toBe(false); + } + expect(mockTables.Messages.get('secret').content).toBe('private'); + expect(mockTables.Conversations.get('foreign').title).toBe('Bobs private chat'); + }); +}); diff --git a/backend/__tests__/message_retries.test.js b/backend/__tests__/message_retries.test.js index be67bc7c..8c50738b 100644 --- a/backend/__tests__/message_retries.test.js +++ b/backend/__tests__/message_retries.test.js @@ -49,18 +49,26 @@ function memoryDatabase() { }; } -let db, app, originalClient, originalEnv, aiCalls, emitted; +let db, app, serviceApp, originalClient, originalEnv, aiCalls, emitted; const payload = { from: 'user', to: 'ai', content: 'When should I plant tomatoes?', type: 'text' }; -function makeApp() { +function makeApp(service = false) { const instance = express(); instance.use(express.json()); - instance.use('/messages', require('../routes/communicationRoutes')); + if (service) { + // Exercise the upstream retry service's credential scopes separately from + // private chat, whose production route requires an account and ownership. + instance.use(require('../middleware/requestRateLimit')(), require('../middleware/authMiddleware')); + instance.post('/messages/:conversationId', require('../utils/asyncHandler')( + require('../controllers/communication_controller').sendMessage)); + } else { + instance.use('/messages', require('../routes/communicationRoutes')); + } return instance; } -function send({ key = 'send-1', body = payload, user = 'grower-1', conversation = 'conversation-1', apiKey } = {}) { - let pending = request(app).post(`/messages/${conversation}`); +function send({ key = 'send-1', body = payload, user = 'grower-1', conversation = 'conversation-1', apiKey, service = false } = {}) { + let pending = request(service ? serviceApp : app).post(`/messages/${conversation}`); if (apiKey) pending = pending.set('X-API-Key', apiKey); else pending = pending.set('Authorization', `Bearer ${jwt.sign({ userId: user }, process.env.JWT_SECRET)}`); if (key !== undefined && key !== null) pending = pending.set('Idempotency-Key', key); @@ -69,7 +77,7 @@ function send({ key = 'send-1', body = payload, user = 'grower-1', conversation function messages() { return [...(db.tables.get('Messages')?.values() || [])]; } -beforeEach(() => { +beforeEach(async () => { originalEnv = { JWT_SECRET: process.env.JWT_SECRET, API_KEY: process.env.API_KEY }; process.env.JWT_SECRET = 'test-only-message-retry-secret'; process.env.API_KEY = 'test-only-api-key'; @@ -77,6 +85,9 @@ beforeEach(() => { originalClient = { ...docClient }; Object.assign(docClient, db); Message.setDocClient(docClient); + for (const [id, ownerId] of [['conversation-1', 'grower-1'], ['conversation-2', 'grower-1'], ['foreign', 'grower-2']]) { + await db.put({ TableName: 'Conversations', Item: { id, ownerId } }).promise(); + } aiCalls = 0; emitted = []; global.emitMessage = (conversation, message) => emitted.push({ conversation, message }); @@ -85,6 +96,7 @@ beforeEach(() => { return (async function* () { yield { choices: [{ delta: { content: 'Plant after the last frost.' } }] }; })(); } } } }); app = makeApp(); + serviceApp = makeApp(true); }); afterEach(() => { @@ -118,9 +130,10 @@ test('reusing a key with different content conflicts without creating another me assert.equal(aiCalls, 1); }); -test('keys are isolated by authenticated caller and conversation, never by body.from', async () => { +test('keys are isolated by owned conversation and cannot be replayed by another account', async () => { const first = await send(); - const otherUser = await send({ user: 'grower-2' }); + assert.equal((await send({ user: 'grower-2' })).status, 404); + const otherUser = await send({ user: 'grower-2', conversation: 'foreign' }); const otherConversation = await send({ conversation: 'conversation-2' }); assert.equal(otherUser.status, 201); assert.equal(otherConversation.status, 201); @@ -148,35 +161,30 @@ test('invalid keys are rejected before persistence or AI work', async () => { assert.equal(aiCalls, 0); }); -test('parallel JWT retries cannot acquire a second send while the first AI response is running', async () => { - let started, finish; - const running = new Promise(resolve => { started = resolve; }); - const release = new Promise(resolve => { finish = resolve; }); - setClient({ chat: { completions: { create: async () => { - aiCalls++; - started(); - await release; - return (async function* () { yield { choices: [{ delta: { content: 'Ready' } }] }; })(); - } } } }); - const first = send().then(response => response); - await running; - const racing = await Promise.all(Array.from({ length: 8 }, () => send())); - finish(); - const done = await first; - assert.ok(racing.every(response => response.status === 409 && response.body.code === 'message_send_pending')); - assert.deepEqual((await send()).body, done.body); - assert.equal(messages().length, 2); - assert.equal(aiCalls, 1); - assert.equal(emitted.length, 2); -}); - -test('shared API-key-only callers are rejected before message or AI work', async () => { - const response = await send({ apiKey: process.env.API_KEY }); - assert.equal(response.status, 401); - assert.equal(response.body.error, 'Account bearer token required'); - assert.equal(messages().length, 0); - assert.equal(aiCalls, 0); -}); +for (const apiCaller of [false, true]) { + test(`parallel ${apiCaller ? 'API-key service' : 'JWT route'} retries cannot acquire a second send while the first AI response is running`, async () => { + const auth = apiCaller ? { service: true, apiKey: process.env.API_KEY } : {}; + let started, finish; + const running = new Promise(resolve => { started = resolve; }); + const release = new Promise(resolve => { finish = resolve; }); + setClient({ chat: { completions: { create: async () => { + aiCalls++; + started(); + await release; + return (async function* () { yield { choices: [{ delta: { content: 'Ready' } }] }; })(); + } } } }); + const first = send(auth).then(response => response); + await running; + const racing = await Promise.all(Array.from({ length: 8 }, () => send(auth))); + finish(); + const done = await first; + assert.ok(racing.every(response => response.status === 409 && response.body.code === 'message_send_pending')); + assert.deepEqual((await send(auth)).body, done.body); + assert.equal(messages().length, 2); + assert.equal(aiCalls, 1); + assert.equal(emitted.length, 2); + }); +} test('completed receipts survive reloading the request handler', async () => { const first = await send(); @@ -208,11 +216,77 @@ test('changing sender label, recipient, type or attachment conflicts, without re assert.equal(aiCalls, 1); }); -test('an invalid API-key header cannot alter a verified JWT retry scope', async () => { +test('API-key callers replay within their credential scope and do not share JWT receipts', async () => { + const first = await send({ service: true, apiKey: process.env.API_KEY }); + assert.deepEqual((await send({ service: true, apiKey: process.env.API_KEY })).body, first.body); + const user = await send(); + assert.equal(user.status, 201); + assert.notEqual(user.body.message.id, first.body.message.id); + assert.equal(aiCalls, 2); +}); + +test('API-key-only callers cannot send or replay private chat messages', async () => { + const denied = await send({ service: false, apiKey: process.env.API_KEY }); + assert.equal(denied.status, 401); + assert.equal(messages().length, 0); + assert.equal(db.tables.has('MessageSends'), false); + const first = await send(); + assert.equal(first.status, 201); + const deniedReplay = await send({ service: false, apiKey: process.env.API_KEY }); + assert.equal(deniedReplay.status, 401); + assert.equal(messages().length, 2); + assert.equal(aiCalls, 1); +}); + +test('API-key receipts use the stable scrypt scope instead of a fast credential hash', async () => { + const first = await send({ service: true, apiKey: process.env.API_KEY }); + assert.equal(first.status, 201); + const [receipt] = db.tables.get('MessageSends').values(); + // Independent Python hashlib.scrypt fixture: password=test-only-api-key, + // salt=SHA256(JSON([message-send-api-key-v2, conversation-1, send-1])).hex, + // N=16384, r=8, p=5, dklen=32. Locks the durable replay namespace. + assert.equal(receipt.id, 'e54be044e9e970d309de6ebe2b3763f1cbb49c546d04c7379102ba52709867e6'); + assert.ok(!JSON.stringify(receipt).includes(process.env.API_KEY)); + delete require.cache[require.resolve('../services/retrySafeMessage')]; + delete require.cache[require.resolve('../controllers/communication_controller')]; + delete require.cache[require.resolve('../routes/communicationRoutes')]; + app = makeApp(); + assert.deepEqual((await send({ service: true, apiKey: process.env.API_KEY })).body, first.body); + assert.equal(aiCalls, 1); +}); + +test('API-key rotation, conversation and send keys have separate retry scopes', async () => { + const first = await send({ service: true, apiKey: process.env.API_KEY }); + const otherConversation = await send({ service: true, apiKey: process.env.API_KEY, conversation: 'conversation-2' }); + const otherSend = await send({ service: true, apiKey: process.env.API_KEY, key: 'send-2' }); + process.env.API_KEY = 'rotated-test-api-key'; + const rotated = await send({ service: true, apiKey: process.env.API_KEY }); + for (const response of [first, otherConversation, otherSend, rotated]) assert.equal(response.status, 201); + assert.equal(new Set([first, otherConversation, otherSend, rotated].map(response => response.body.message.id)).size, 4); + assert.deepEqual((await send({ service: true, apiKey: process.env.API_KEY })).body, rotated.body); + assert.equal(aiCalls, 4); +}); + +test('WIX API-key retries work without a JWT secret', async () => { + const priorWixKey = process.env.WIX_API_KEY; + delete process.env.API_KEY; + delete process.env.JWT_SECRET; + process.env.WIX_API_KEY = 'test-only-wix-api-key'; + try { + const first = await send({ service: true, apiKey: process.env.WIX_API_KEY }); + assert.equal(first.status, 201); + assert.deepEqual((await send({ service: true, apiKey: process.env.WIX_API_KEY })).body, first.body); + assert.equal(aiCalls, 1); + } finally { + if (priorWixKey === undefined) delete process.env.WIX_API_KEY; else process.env.WIX_API_KEY = priorWixKey; + } +}); + +for (const validKey of [false, true]) test(`the ${validKey ? 'valid' : 'invalid'} API-key header cannot alter a verified JWT retry scope`, async () => { const first = await send(); const retry = await request(app).post('/messages/conversation-1') .set('Authorization', `Bearer ${jwt.sign({ userId: 'grower-1' }, process.env.JWT_SECRET)}`) - .set('X-API-Key', 'invalid-api-key') + .set('X-API-Key', validKey ? process.env.API_KEY : 'invalid-api-key') .set('Idempotency-Key', 'send-1').send(payload); assert.equal(retry.status, 201); assert.deepEqual(retry.body, first.body); @@ -225,10 +299,11 @@ test('authentication failures and JWTs without a stable user ID create no receip .set('Authorization', `Bearer ${jwt.sign({ role: 'member' }, process.env.JWT_SECRET)}`) .set('Idempotency-Key', 'key').send(payload); assert.equal(noIdentity.status, 401); - assert.equal(db.tables.size, 0); + assert.equal(db.tables.has('MessageSends'), false); + assert.equal(messages().length, 0); }); -test('deleting a conversation never lets an old retry resurrect its messages or retained narrative', async () => { +test('removing original messages never lets a retry resurrect them or retained narrative', async () => { await send(); await Message.removeByConversation('conversation-1'); const retry = await send(); @@ -243,6 +318,19 @@ test('deleting a conversation never lets an old retry resurrect its messages or assert.equal(aiCalls, 1); }); + +test('a missing persisted assistant reply cannot replay a partial success', async () => { + const first = await send(); + assert.equal(first.status, 201); + const [replyKey] = [...db.tables.get('Messages')].find(([, row]) => row.id === first.body.reply.id); + db.tables.get('Messages').delete(replyKey); + const retry = await send(); + assert.equal(retry.status, 410); + assert.equal(retry.body.code, 'message_send_removed'); + assert.equal(messages().length, 1); + assert.equal(aiCalls, 1); +}); + test('a missing receipt table refuses keyed sends before any message or AI work', async () => { const put = docClient.put; docClient.put = params => params.TableName === 'MessageSends' @@ -343,3 +431,12 @@ test('browser preflight permits the idempotency header on the existing allowed o if (priorMinimal === undefined) delete process.env.MINIMAL_SERVER; else process.env.MINIMAL_SERVER = priorMinimal; } }); + + +test('shared API-key-only callers are rejected before message or AI work', async () => { + const response = await send({ apiKey: process.env.API_KEY }); + assert.equal(response.status, 401); + assert.equal(response.body.error, 'Account bearer token required'); + assert.equal(messages().length, 0); + assert.equal(aiCalls, 0); +}); diff --git a/backend/controllers/conversation_controller.js b/backend/controllers/conversation_controller.js index e4b0d94f..6db6089c 100644 --- a/backend/controllers/conversation_controller.js +++ b/backend/controllers/conversation_controller.js @@ -1,34 +1,42 @@ const Conversation = require('../models/conversation'); +const { collectionPage, pageError } = require('../lib/collectionPage'); exports.create = async (req, res) => { const { title } = req.body; - const convo = await Conversation.create(title); + const convo = await Conversation.create(title, req.user.userId); res.status(201).json(convo); }; -exports.list = async (_req, res) => { - const convos = await Conversation.list(); - res.json(convos); +exports.list = async (req, res) => { + try { + const page = collectionPage(req.query, ['conversations', req.user.userId]); + const { items, next } = await Conversation.listPage(req.user.userId, page.limit, page.state?.start); + page.finish(res, next && { start: next }); + res.json(items); + } catch (error) { pageError(res, error, 'Failed to fetch conversations'); } }; +exports.get = (req, res) => res.json(req.conversation); + exports.rename = async (req, res) => { const { id } = req.params; const { title } = req.body; - const convo = await Conversation.rename(id, title); + const convo = await Conversation.rename(id, title, req.user.userId); if (!convo) return res.status(404).json({ error: 'Not found' }); res.json(convo); }; exports.remove = async (req, res) => { const { id } = req.params; - await Conversation.remove(id); + const removed = await Conversation.remove(id, req.user.userId); + if (!removed) return res.status(404).json({ error: 'Not found' }); res.status(204).end(); }; exports.pin = async (req, res) => { const { id } = req.params; const { pinned } = req.body; - const convo = await Conversation.pin(id, pinned); + const convo = await Conversation.pin(id, pinned, req.user.userId); if (!convo) return res.status(404).json({ error: 'Not found' }); res.json(convo); }; diff --git a/backend/docs/collection-pagination.md b/backend/docs/collection-pagination.md new file mode 100644 index 00000000..803b54a2 --- /dev/null +++ b/backend/docs/collection-pagination.md @@ -0,0 +1,182 @@ +# Collection pages and read-only federation exports + +## Purpose and scope + +`GET /conversations`, `GET /users`, and `GET /federation/export` now return +bounded DynamoDB scan pages with explicit continuation. Previously these +paths either stopped at one DynamoDB page or assembled an unbounded array. +This change requires the conversation ownership work in Agrinet #191. Its +private-chat rollout still requires the authenticated-stream companion #192. + +Pagination must not make more private account data available. The directory +uses an allowlist, account records are omitted from federation exports, and +federation uses independently authenticated per-peer HMAC credentials. +These are deliberate compatibility changes, not a claim that all existing +account or federation security gaps are closed. + +## HTTP contract + +`GET /users` and attachment downloads each have an independent limit of +120 requests per minute. A guard before account authentication uses the +account already verified by outer middleware, or an IP fallback. A separate +budget after account authentication enforces 120 requests per account even +when an API key accompanies the bearer token. Excess requests return HTTP 429 +with `Retry-After`. Existing ownership, response fields and pagination remain +unchanged below the limit. Message reads and sends (`GET` and `POST` +`/messages/:conversationId`) use the same two-guard policy with a separate +120-request budget shared across methods and conversation IDs for each +account. The guards run before ownership lookups, message storage and AI +responses. Adding an API key or refreshing a token does not reset an account's +budget. Conversation routes also retain the authentication-attempt and account limits from the security hardening change. Federation retains its separate peer limits. + +The additional collection guards are process-local and reset on restart. The outer API and federation guards retain the existing Redis-backed store when configured. Configure Redis as described in `security-hardening.md` for multi-instance deployment; the collection guards do not replace that shared protection. Proxy trust settings are unchanged. The disposable storage-check HTTP server also +limits authentication attempts to 120 per minute per IP before verifying +credentials, and its acceptance check verifies exhaustion of that budget. + +JSON response shapes remain arrays for conversations/users and +`{listings: [], transactions: [], users: []}` for federation export. +Pagination metadata is carried in response headers: + +- `X-Pagination: cursor-v1` declares the contract. +- `X-Page-Limit` is the maximum number of records **evaluated**, per table, + in that request. `limit` defaults to 50 and accepts decimal integers 1–100. + Federation export can evaluate that many listings and transactions each. +- `X-Next-Cursor` is present only when there is another scan position. + Pass its exact opaque value as `cursor`, with the original filters and + limit, to the **same endpoint**. An empty result can still have a cursor: + DynamoDB applies filters after evaluating records. Stop only when the + continuation header is absent. +- `Cache-Control: private, no-store` prevents shared caching. + +The approved browser origin can read these headers through CORS. Existing +clients outside this repository must be updated to follow the header; +otherwise they see only the first bounded page. The bundled chat account +client follows array pages, including empty pages, preserves filters and +credentials, and rejects account changes or later-page errors instead of +returning a partial old-account list. It still accumulates the full list in +browser memory, as before; this PR bounds individual server responses and +does not introduce a lazy-loading sidebar. + +Cursors encrypt and authenticate DynamoDB continuation keys using AES-GCM +with a key derived from the existing `JWT_SECRET`. They expire after one +hour and bind endpoint, limit, directory filters and account identity where +applicable. Raw scan keys can identify private rows even on empty pages, so +they are not base64-encoded plaintext. Changing `JWT_SECRET` invalidates +cursors. Missing secret configuration returns 503 before scanning; invalid, +expired, tampered or mismatched parameters return 400. A storage failure +returns 500 without a successful partial page or continuation header. + +Scans do not provide a snapshot or stable sort. For unchanged tables, following +all continuations returns the matching records. Concurrent additions, +updates and deletions can change what is observed; the API does not promise +exactly-once snapshot enumeration or claim an ordering DynamoDB does not give. +Restart from the first page if a cursor expires. + +## Private conversation and directory boundaries + +Conversations retain #191's account-only authentication and owner filter on +every scan. Other owners' and legacy ownerless rows remain inaccessible. +The existing all-results model helper remains for programmatic compatibility; +the HTTP route uses its bounded page counterpart. + +`GET /users` now requires a verified account token; a shared integration key +alone is insufficient. Returned fields are `id`, `username`, legacy `role`, and +`locationType` when present, plus +the existing coarse `location` only when `locationPrivacy === true`. That +boolean is the existing sharing opt-in convention, not an inverted default. +Location filtering also requires that opt-in, preventing the filter itself +from revealing a private location. Missing/false privacy never publishes a +location. Coordinates, email, phone, password, verification code, scalar +reputation values, and unknown future fields are excluded. The database +projection and the response allowlist both enforce this boundary. Existing +role filtering remains available; `role`, `location`, and `locationType` each +accept one string and are bound into continuation cursors. `locationType` retains +the structural discovery field/filter from the account hardening change. Scalar +`reputationScore` and `reputationWeight` remain excluded: the repository covenant +requires full distributions rather than a single ranking score. This directory +introduces no replacement reputation schema. + +The merged account hardening remains intact: user provisioning is admin-only, account IDs are server-generated, writes refuse collisions, and login/admin responses use their existing sanitized DTOs. Pagination does not replace or broaden those authorization policies. + +## Federation export is read-only inspection + +Federation uses the existing per-peer HMAC contract from +`security-hardening.md`. Both export and manual import require a configured +peer identity, a fresh signed request and the existing peer request limits. +Ordinary account JWTs, browser API keys and the obsolete `x-federation-key` +header cannot authorize a federation request. The former export-key flow is +removed; `FEDERATION_EXPORT_KEY` and `FEDERATION_PEER_KEYS` are not used. + +Each export page has a response signature checked against the expected peer +identity before its contents reach an inspection consumer. Encrypted export +cursors also bind the authenticated peer. Manual imports retain authenticated +source provenance, field sanitization and conditional no-overwrite writes. + +Export pages contain only explicitly selected advertised listing fields and +structural transaction fields. Existing participant/contract/listing IDs and +lifecycle flags are retained; `dialogNotes`, scalar ratings and all Users +records are excluded. `users` is always an empty compatibility array, not a +claim that the operator has zero members. `X-Federation-Omissions` states +`users, dialogNotes, ratings`. Advertised listing descriptions and locations +are still included; this is not an automatic PII scrubber for public content. +The duplicate legacy router delegates to the mounted router's same policy. + +For outbound inspection, keep the operator-managed `NodeRegistry` records, +including each peer's `nodeId` and `url` (or legacy `nodeUrl`). Set +`FEDERATION_LOCAL_NODE_ID` (or `NODE_ID`) and +`FEDERATION_OUTBOUND_PEERS_JSON`, mapping the remote node ID to a pairwise +secret of at least 32 characters. The peer's `FEDERATION_PEERS_JSON` maps the +local sender ID to that same secret. No credential is sent for an unconfigured +node ID. There is no default credential; neither browser API keys nor HMAC +secrets are forwarded in plaintext. + +Peer URLs reject userinfo, query strings and fragments. HTTPS is required +except for loopback development. Redirects are refused; continuation is a token, +never a destination URL. Keep clocks synchronized for the existing five-minute +signature window. +Requests have a five-second timeout, a 16 MiB response limit and a 10,000-page +safety ceiling. Exceeding a limit is an explicit incomplete inspection, not +successful truncation. Older peers without `X-Pagination: cursor-v1`, +malformed pages and any peer page containing Users records are rejected. + +The server-mounted worker and both legacy scheduled paths inspect all pages +and report counts. **Automatic import is disabled, and `lastSyncAt` is not +advanced.** The old importers overwrite canonical records without a source +namespace or auditable conflict policy; importing redacted summaries could +erase local narrative or account data. This pagination change does not define those reconciliation rules. +Source-aware reconciliation remains separate design work. No temporary unscoped +cache or new authority store is added. The legacy status router (not mounted by the default server) totals all pages +and reports `exportMode: read-only` and `usersExported: false`; its `lastSync` +value is historical data, not proof this inspection synchronized anything. + +The manual `/federation/import` endpoint retains the merged security controls, but is not called by scheduled inspection. Source-aware reconciliation of listing and transaction records, witnessed checkpoints and independent peers remain separate work; authenticated imports do not establish those guarantees. + +These exports are interim application views, not commons artifacts, portable +reputation, signed exchange receipts, witnessed federation, or a coordination +wire extension. No ledger entry is edited, deleted or promoted to authority. +Escrow, credit configuration and the existing in-review conformance claims +remain outside the change. + +## Rollout and verification + +Review #191 before this pagination change; deploy private chat with #192. No table +schema migration or new index is required. The application retains existing +Scan permissions. Configure per-peer HMAC credentials +before enabling inspection; unset configuration deliberately fails closed. +Update external clients to consume pagination headers before rollout, and +communicate that automated merging and account replication are disabled. + +`cd backend && npm test` covers production-mounted HTTP boundaries, empty +filtered pages, owner and query binding, cursor tampering/expiry, export +credential isolation, partial failures and non-mutating scheduled inspection. +`cd frontend/chat-ui && npm test && npm run build` checks the account adapter +and production bundle. + +With DynamoDB Local on a loopback endpoint, run +`DYNAMODB_ENDPOINT=http://127.0.0.1:18001 node scripts/check-collection-pagination.js` +from `backend`. It creates uniquely named disposable tables and removes them +afterward. Records exceed DynamoDB's 1 MiB scan boundary, so continuation, +privacy projections, filtering, the actual account adapter and read-only +export traversal are checked against real storage. The check confirms local +credentials and narrative remain unchanged. CI runs the same check with a +digest-pinned local database image; no live peer, payment or AI call is made. diff --git a/backend/docs/conversation-ownership.md b/backend/docs/conversation-ownership.md new file mode 100644 index 00000000..bdb7cf5b --- /dev/null +++ b/backend/docs/conversation-ownership.md @@ -0,0 +1,90 @@ +# Private conversation ownership + +## Boundary + +Private chat uses the existing backend login JWT's `userId`, verified as an +HS256 account token. A shared `API_KEY`/`WIX_API_KEY` identifies an integration, +not a participant, and grants no conversation, message or attachment access. +Supplying a valid integration key alongside a bearer never bypasses bearer +verification. No Cognito `sub`, caller-supplied owner, message sender label or +administrative role is treated as a substitute account identity. + +New Conversations rows carry immutable `ownerId`. Listing filters on that +owner across DynamoDB scan pages. Detail reads, rename, pin, delete, message +reads and sends check ownership. Rename/pin/delete also carry a conditional +owner check in DynamoDB. Foreign, missing and ownerless IDs return the same +404. A denied send reaches neither file writes, messages, responders nor +broadcasts. Attachments require a message reference belonging to an owned +conversation; responses disable caching and download as inert attachments. + +This is application-local private chat. It does not seal a JFA dialog or +modify Mycelium commitments, exchange records, reputation, escrow or federation. +The separate status of those mechanisms in CONFORMANCE.md remains accurate. + +## Existing data and deployment + +Back up Conversations and Messages before rollout. Existing rows without an +owner remain stored but inaccessible; the application never grants them to +the first caller or guesses an owner from `from`. An operator must establish +the rightful existing account through independent evidence before making an +explicit, conditional administrative backfill. This PR supplies no public +claim/transfer endpoint and performs no automatic data migration or deletion. +If ownership cannot be established, preserve the inaccessible row for the +operator's data-retention process. Ownership transfer is not supported here. + +The existing Messages deletion helper still has its previously documented +large-history batching/retry limitations. This change establishes who may +invoke deletion; it does not claim complete erasure of large histories or +physical attachment bytes. Stale attachment files become inaccessible when +their owned message/conversation reference no longer exists. + +The deployment needs DynamoDB GetItem/Scan/PutItem/UpdateItem/DeleteItem access +to the configured conversation table and the existing message access, now +including Scan for attachment-reference lookup. Scans require no new index +or table. They prioritize correct isolation; an indexed attachment lookup is +a possible measured follow-up, not supplied by this change. + +**Roll out together with the authenticated-stream companion PR (A2).** A1 +closes REST and attachment access; unauthenticated SSE remains a separate +existing path until A2 lands. Other open hardening work, including admin, +user/federation authorization and attachment write containment, remains +separate and must be assessed before exposing public signup. + +## Browser integration + +The chat UI previously sent only a shared build-time API key and has no +working account-login screen. The host application's account flow must obtain +the existing backend `/api/auth/login` response and pass its token to the +exported `setAccountToken(token)` in `frontend/chat-ui/src/api.js`. Call +`setAccountToken(null)` on logout. Tokens stay in memory; the adapter uses +Authorization headers, never query strings or browser storage. Account changes +clear chat state and discard in-flight responses from the old account. + +Do not substitute an Amplify/Cognito token without an approved backend identity +binding. PR #183 separately makes backend login/registration publicly +reachable; until it lands, login still needs the server's existing integration +authentication. This PR does not claim completed browser onboarding. + +Attachment buttons download through the authenticated adapter rather than +opening uncredentialed URLs. Malformed, external and non-upload download paths +are rejected before forwarding credentials. Redirects are rejected. + +## Verification + +`cd backend && npm test` includes production-server HTTP tests for two users, +legacy records, missing IDs, conditional mutation ownership, malformed subjects, +shared-key bypass attempts and protected attachments. AI provider calls and +background queues are isolated; account verification, mounted routes and the +authorization/model logic remain real. `cd frontend/chat-ui && npm test` +checks credential placement, account changes, attachment destinations and the +existing chat reducer. Database-backed verification uses isolated DynamoDB +Local tables, never live records. + +Run the storage check with `DYNAMODB_ENDPOINT=http://127.0.0.1:18001 node +scripts/check-conversation-ownership.js` from `backend`, after starting +DynamoDB Local on that loopback port. It creates uniquely named disposable +tables and removes them afterward. The check refuses non-loopback endpoints. + +Scope follows the [Fruitful communications overview](https://ntari.atlassian.net/wiki/spaces/Fruitful/pages/44761089) +and [Slack's instruction to use Janus and bylaws](https://ntari.slack.com/archives/C0AENHR5KG8/p1791982812690669). +Current repository invariants take precedence over older architecture summaries. diff --git a/backend/docs/retry-safe-messages.md b/backend/docs/retry-safe-messages.md index 5f5f09b6..fd0add29 100644 --- a/backend/docs/retry-safe-messages.md +++ b/backend/docs/retry-safe-messages.md @@ -28,6 +28,12 @@ Clients that consume both HTTP and SSE should reconcile using message IDs. ## Contract +On this branch, private chat routes require the owning account's bearer token. +An API key alone cannot send or replay a private message. The retry service +retains its API-key receipt format, but its credential-scope tests and the +`--api-key` storage check exercise a separate service harness, not permission +to access private chats. JWT storage checks use the production ownership path. + Keys contain 1–128 printable ASCII characters, without spaces. Scope is the conversation plus the authenticated JWT `userId`, or the accepted API credential. It is never the caller-supplied `from` label. All clients sharing an API credential diff --git a/backend/docs/security-hardening.md b/backend/docs/security-hardening.md index 9e6669ec..738186ff 100644 --- a/backend/docs/security-hardening.md +++ b/backend/docs/security-hardening.md @@ -26,7 +26,7 @@ Requests sign the timestamp, HTTP method, path and SHA-256 digest of the exact r Federation user imports accept an explicit user-field allowlist, reject existing IDs atomically, strip passwords, contact fields, roles and verification state, and record sourceNodeId from the authenticated peer plus remoteUserId. Claimed origin identifiers are retained as provenance metadata from the signed peer payload; they never determine the authenticated sender. -Exports use explicit field allowlists. The location feature's exact owner coordinates remain in its separate local Locations table and are not part of the user export. +Exports use bounded cursor pages and explicit listing/transaction field allowlists; account records, local dialog narrative and scalar ratings are omitted. Scheduled federation paths verify signed pages and report inspection counts without importing or advancing lastSyncAt. Manual imports retain the source-bound security controls above. See collection-pagination.md for the complete inspection contract. ## Distributed request and connection limits @@ -48,9 +48,9 @@ The concurrency middleware releases a reservation on response close/finish and R Until the owner-checked chat changes are deployed: - /events and /stream/:conversationId return 410 instead of exposing the legacy unauthenticated feed. -- /uploads returns 410 instead of serving files to any caller holding the shared browser key. +- /uploads serves attachments only after independent account authentication, request throttling and attachment-owner authorization; shared browser keys alone grant no download access. -After #191 is rebased and approved, restore attachment serving only behind chatAuth, the shared chat limiter, and attachmentOwner. After #192 is rebased and approved, replace the temporary stream 410 with the ordered authenticated handler above. Do not restore the old handlers before those protections are present. +This branch carries the owner-checked attachment path from #191; retain it when integrating subsequent chat changes. After #192 is rebased and approved, replace the temporary stream 410 with the ordered authenticated handler above. Do not restore the old handlers before those protections are present. No upload-creation endpoint exists in the current main tree; when one is introduced, place upload throttling on that actual write operation, not only on static download paths. diff --git a/backend/federation/__tests__/federationRoutes.test.js b/backend/federation/__tests__/federationRoutes.test.js index 80ac72d5..00d71206 100644 --- a/backend/federation/__tests__/federationRoutes.test.js +++ b/backend/federation/__tests__/federationRoutes.test.js @@ -45,7 +45,7 @@ describe('federation route security', () => { expect(docClient.scan).not.toHaveBeenCalled(); }); - test('exports only allowlisted user, listing and transaction fields and signs response', async () => { + test('exports bounded redacted listing and transaction fields and signs response', async () => { docClient.scan .mockReturnValueOnce({ promise: async () => ({ Items: [{ id: 'l1', title: 'Apples', password: 'secret', internal: 'x' }] }) }) .mockReturnValueOnce({ promise: async () => ({ Items: [{ id: 't1', status: 'pending', cardToken: 'secret' }] }) }) @@ -58,12 +58,11 @@ describe('federation route security', () => { const headers = createPeerHeaders({ nodeId: 'peer-1', secret, method: 'GET', path: '/federation/export' }); + process.env.JWT_SECRET = 'federation-cursor-test-secret'; const response = await request(app).get('/federation/export').set(headers); expect(response.status).toBe(200); - expect(response.body.users).toEqual([{ - id: 'u1', username: 'alice', reputationScore: 3, - originNodeId: 'local-node', originUserId: 'u1' - }]); + expect(response.body.users).toEqual([]); + expect(response.headers['x-pagination']).toBe('cursor-v1'); expect(response.body.listings).toEqual([{ id: 'l1', title: 'Apples' }]); expect(response.body.transactions).toEqual([{ id: 't1', status: 'pending' }]); expect(verifyPeerResponse({ diff --git a/backend/federation/__tests__/federationSyncJob.test.js b/backend/federation/__tests__/federationSyncJob.test.js index 57999c6c..d78119ea 100644 --- a/backend/federation/__tests__/federationSyncJob.test.js +++ b/backend/federation/__tests__/federationSyncJob.test.js @@ -1,95 +1,66 @@ +const { configurePeers, signedResponse } = require('../../test-support/federation-pages'); jest.useFakeTimers(); jest.mock('axios'); -jest.mock('../../lib/dynamodbClient', () => ({ - scan: jest.fn(), - update: jest.fn() -})); - +jest.mock('../../lib/dynamodbClient', () => ({ scan: jest.fn(), update: jest.fn(), put: jest.fn() })); const axios = require('axios'); -const dynamodbClient = require('../../lib/dynamodbClient'); -const { createPeerResponseHeaders } = require('../../middleware/federationPeerAuth'); -const runFederationSync = require('../federationSyncJob'); - -describe('runFederationSync', () => { - const originalEnv = { - nodeId: process.env.NODE_ID, - localNodeId: process.env.FEDERATION_LOCAL_NODE_ID, - outbound: process.env.FEDERATION_OUTBOUND_PEERS_JSON - }; - const peerSecret = '0123456789abcdef0123456789abcdef'; +const db = require('../../lib/dynamodbClient'); +const run = require('../federationSyncJob'); +describe('scheduled federation inspection', () => { + const env = { ...process.env }; + let info; + let errors; beforeEach(() => { - jest.clearAllMocks(); - process.env.NODE_ID = 'local-node'; - process.env.FEDERATION_LOCAL_NODE_ID = 'local-node'; - process.env.FEDERATION_OUTBOUND_PEERS_JSON = JSON.stringify({ 'peer-1': peerSecret }); + jest.resetAllMocks(); + configurePeers(); + db.scan.mockReturnValue({ promise: async () => ({ Items: [{ nodeId: 'peer-1', url: 'https://peer.example' }] }) }); + info = jest.spyOn(console, 'info').mockImplementation(() => {}); + errors = jest.spyOn(console, 'error').mockImplementation(() => {}); }); - - afterAll(() => { - if (originalEnv.nodeId === undefined) delete process.env.NODE_ID; - else process.env.NODE_ID = originalEnv.nodeId; - if (originalEnv.localNodeId === undefined) delete process.env.FEDERATION_LOCAL_NODE_ID; - else process.env.FEDERATION_LOCAL_NODE_ID = originalEnv.localNodeId; - if (originalEnv.outbound === undefined) delete process.env.FEDERATION_OUTBOUND_PEERS_JSON; - else process.env.FEDERATION_OUTBOUND_PEERS_JSON = originalEnv.outbound; + afterEach(() => { info.mockRestore(); errors.mockRestore(); }); + afterAll(() => { process.env = env; jest.useRealTimers(); }); + const page = (listings, next) => ({ data: { listings, transactions: [], users: [] }, headers: { 'x-pagination': 'cursor-v1', ...(next && { 'x-next-cursor': next }) } }); + test('reads every page and reports inspection without canonical writes or a sync timestamp', async () => { + axios.get.mockResolvedValueOnce(signedResponse(page([{ id: 'l1' }], 'next'))).mockResolvedValueOnce(signedResponse(page([{ id: 'l2' }]))); + await run(); + expect(info).toHaveBeenCalledWith('Federation export inspected; automatic import disabled', { peer: 'https://peer.example', counts: { listings: 2, transactions: 0, users: 0 } }); + expect(db.put).not.toHaveBeenCalled(); + expect(db.update).not.toHaveBeenCalled(); + expect(axios.post).not.toHaveBeenCalled(); }); - - it('verifies peer response signatures before importing and updates lastSyncAt', async () => { - dynamodbClient.scan.mockReturnValue({ - promise: async () => ({ Items: [{ nodeId: 'peer-1', url: 'https://node1.example' }] }) - }); - dynamodbClient.update.mockReturnValue({ promise: async () => ({}) }); - const data = { users: [] }; - const body = JSON.stringify(data); - axios.get.mockResolvedValue({ - data, - headers: createPeerResponseHeaders({ - nodeId: 'peer-1', secret: peerSecret, method: 'GET', path: '/federation/export', body - }) - }); - - await runFederationSync(); - - expect(axios.get).toHaveBeenCalledWith('https://node1.example/federation/export', expect.objectContaining({ - headers: expect.objectContaining({ - 'x-federation-node-id': 'local-node', - 'x-federation-signature': expect.any(String) - }), - timeout: 15000 - })); - expect(dynamodbClient.update).toHaveBeenCalledTimes(1); - const params = dynamodbClient.update.mock.calls[0][0]; - expect(params).toMatchObject({ - Key: { url: 'https://node1.example' }, - UpdateExpression: 'set lastSyncAt = :time', - ExpressionAttributeValues: { ':time': expect.any(String) } - }); - const timestamp = params.ExpressionAttributeValues[':time']; - expect(new Date(timestamp).toISOString()).toBe(timestamp); + test('a failed later page never reports a successful inspection or sync', async () => { + axios.get.mockResolvedValueOnce(signedResponse(page([{ id: 'l1' }], 'next'))).mockRejectedValueOnce(new Error('later page failed')); + await run(); + expect(info).not.toHaveBeenCalled(); + expect(errors).toHaveBeenCalled(); + expect(db.put).not.toHaveBeenCalled(); + expect(db.update).not.toHaveBeenCalled(); }); - - it('does not import or update metadata when the peer response signature is invalid', async () => { - dynamodbClient.scan.mockReturnValue({ - promise: async () => ({ Items: [{ nodeId: 'peer-1', url: 'https://node1.example' }] }) - }); - axios.get.mockResolvedValue({ - data: { users: [] }, - headers: { - 'x-federation-response-node-id': 'attacker', - 'x-federation-response-timestamp': String(Math.floor(Date.now() / 1000)), - 'x-federation-response-signature': '0'.repeat(64) - } - }); - await runFederationSync(); - expect(dynamodbClient.update).not.toHaveBeenCalled(); + test('continues through registry pages even when an earlier peer fails', async () => { + configurePeers(); + db.scan.mockReturnValueOnce({ promise: async () => ({ Items: [{ nodeId: 'peer-1', url: 'https://peer.example' }], LastEvaluatedKey: { url: 'https://peer.example' } }) }) + .mockReturnValueOnce({ promise: async () => ({ Items: [{ nodeId: 'peer-1', url: 'https://other.example' }] }) }); + axios.get.mockRejectedValueOnce(new Error('unavailable')).mockResolvedValueOnce(signedResponse(page([]))); + await run(); + expect(axios.get.mock.calls.map(call => call[0])).toEqual(['https://peer.example/federation/export', 'https://other.example/federation/export']); + expect(db.scan.mock.calls[1][0].ExclusiveStartKey).toEqual({ url: 'https://peer.example' }); + expect(info).toHaveBeenCalledTimes(1); + expect(db.update).not.toHaveBeenCalled(); }); - - it('does not send credentials to non-HTTPS production peers', async () => { - dynamodbClient.scan.mockReturnValue({ - promise: async () => ({ Items: [{ nodeId: 'peer-1', url: 'http://node1.example' }] }) - }); - await runFederationSync(); + test('a forged peer response is never reported as a successful inspection', async () => { + const response = signedResponse(page([])); + response.headers['x-federation-response-signature'] = '0'.repeat(64); + axios.get.mockResolvedValueOnce(response); + await run(); + expect(info).not.toHaveBeenCalled(); + expect(errors).toHaveBeenCalled(); + expect(db.put).not.toHaveBeenCalled(); + expect(db.update).not.toHaveBeenCalled(); + }); + test('HTTP production peers receive no credentials', async () => { + db.scan.mockReturnValue({ promise: async () => ({ Items: [{ nodeId: 'peer-1', url: 'http://peer.example' }] }) }); + await run(); expect(axios.get).not.toHaveBeenCalled(); - expect(dynamodbClient.update).not.toHaveBeenCalled(); + expect(db.update).not.toHaveBeenCalled(); }); }); diff --git a/backend/federation/exportFields.js b/backend/federation/exportFields.js new file mode 100644 index 00000000..21369554 --- /dev/null +++ b/backend/federation/exportFields.js @@ -0,0 +1,18 @@ +// Interim application export, not a commons record or a reputation transfer. +// Only existing structural transaction fields and advertised listing fields +// cross this boundary. Never spread an entire DynamoDB record into a response. +function fields(item, names) { + return Object.fromEntries(names.filter(name => Object.hasOwn(item, name)).map(name => [name, item[name]])); +} + +const listingFields = ['id', 'userId', 'type', 'title', 'category', 'description', 'price', 'location', + 'media', 'logisticsRange', 'processingCategory', 'compostingService', 'availability', 'tags', 'createdAt']; +const transactionFields = ['id', 'transactionId', 'contractId', 'consumerId', 'producerId', 'buyerId', + 'sellerId', 'listingId', 'status', 'createdAt', 'lastPing', 'pingCount', 'buyerRated', 'sellerRated', + 'ratingGiven', 'dialogConfirmed', 'flaggedForReview', 'escrowLocked', 'escrowReleasedAt']; + +module.exports = { + listingFields, transactionFields, + listing: item => fields(item, listingFields), + transaction: item => fields(item, transactionFields), +}; diff --git a/backend/federation/federationRoutes.js b/backend/federation/federationRoutes.js index 2afe333e..34d94916 100644 --- a/backend/federation/federationRoutes.js +++ b/backend/federation/federationRoutes.js @@ -3,58 +3,45 @@ const router = express.Router(); const docClient = require('../lib/dynamodbClient'); const { LISTING_TABLE_NAME } = require('../models/listing'); const { TRANSACTION_TABLE_NAME } = require('../models/transaction'); -const { USER_TABLE_NAME } = require('../models/user'); const { importFederatedData } = require('../sync/importHelper'); +const { collectionPage, scanPage, pageError } = require('../lib/collectionPage'); +const exportFields = require('./exportFields'); const { requireFederationPeer, createPeerResponseHeaders } = require('../middleware/federationPeerAuth'); const { federationAttemptRateLimit, federationRateLimit } = require('../middleware/rateLimit'); -const pick = (item, fields) => Object.fromEntries( - fields.filter(field => item[field] !== undefined).map(field => [field, item[field]]) -); - -// Browser API keys and ordinary member tokens are never federation credentials. router.use(federationAttemptRateLimit, requireFederationPeer, federationRateLimit); -router.get('/export', async (_req, res) => { +// Export route for peer sync +router.get('/export', async (req, res) => { try { - const [listingsRes, transactionsRes, usersRes] = await Promise.all([ - docClient.scan({ TableName: LISTING_TABLE_NAME }).promise(), - docClient.scan({ TableName: TRANSACTION_TABLE_NAME }).promise(), - docClient.scan({ - TableName: USER_TABLE_NAME, - ProjectionExpression: '#id, username, reputationScore, reputationWeight, originNodeId, originUserId', - ExpressionAttributeNames: { '#id': 'id' } - }).promise() - ]); - - const configuredNodeId = process.env.FEDERATION_LOCAL_NODE_ID || process.env.NODE_ID; - const users = (usersRes.Items || []).map(user => ({ - ...pick(user, ['id', 'username', 'reputationScore', 'reputationWeight']), - originNodeId: typeof user.originNodeId === 'string' ? user.originNodeId : configuredNodeId, - originUserId: typeof user.originUserId === 'string' ? user.originUserId : user.id - })); - - // Export only fields required for federation; never serialize full DynamoDB rows. - const listings = (listingsRes.Items || []).map(item => pick(item, [ - 'id', 'userId', 'type', 'title', 'description', 'price', 'location', 'createdAt' - ])); - const transactions = (transactionsRes.Items || []).map(item => pick(item, [ - 'id', 'contractId', 'consumerId', 'producerId', 'rating', 'status' - ])); - - const payload = JSON.stringify({ listings, transactions, users }); + const nodeId = process.env.FEDERATION_LOCAL_NODE_ID || process.env.NODE_ID; const peers = JSON.parse(process.env.FEDERATION_PEERS_JSON || '{}'); - const secret = peers[_req.federationPeer.nodeId]; - const responseNodeId = process.env.FEDERATION_LOCAL_NODE_ID || process.env.NODE_ID; - if (!responseNodeId || typeof secret !== 'string' || secret.length < 32) { + const secret = peers[req.federationPeer.nodeId]; + if (!nodeId || typeof secret !== 'string' || secret.length < 32) { return res.status(503).json({ error: 'Federation response signing is not configured' }); } - res.set(createPeerResponseHeaders({ - nodeId: responseNodeId, secret, method: 'GET', path: '/federation/export', body: payload - })); - return res.type('application/json').send(payload); - } catch { - return res.status(500).json({ error: 'Export failed' }); + const page = collectionPage(req.query, ['federation-export', req.federationPeer.nodeId]); + const result = { listings: [], transactions: [], users: [] }; + const next = {}; + for (const [name, table, allowed, present] of [ + ['listings', LISTING_TABLE_NAME, exportFields.listingFields, exportFields.listing], + ['transactions', TRANSACTION_TABLE_NAME, exportFields.transactionFields, exportFields.transaction], + ]) { + if (page.state && page.state[name] === null) { next[name] = null; continue; } + const names = Object.fromEntries(allowed.map((field, i) => [`#f${i}`, field])); + const data = await scanPage(docClient, { + TableName: table, ProjectionExpression: Object.keys(names).join(', '), ExpressionAttributeNames: names, + }, page.limit, page.state?.[name]); + result[name] = data.items.map(present); + next[name] = data.next; + } + page.finish(res, Object.values(next).some(Boolean) ? next : null); + res.set('X-Federation-Omissions', 'users, dialogNotes, ratings'); + const payload = JSON.stringify(result); + res.set(createPeerResponseHeaders({ nodeId, secret, method: 'GET', path: '/federation/export', body: payload })); + res.type('application/json').send(payload); + } catch (err) { + pageError(res, err, 'Export failed'); } }); diff --git a/backend/federation/federationSyncJob.js b/backend/federation/federationSyncJob.js index 2863cf83..1c9f31ea 100644 --- a/backend/federation/federationSyncJob.js +++ b/backend/federation/federationSyncJob.js @@ -1,78 +1,21 @@ -const axios = require('axios'); const dynamodbClient = require('../lib/dynamodbClient'); const { NODE_REGISTRY_TABLE_NAME } = require('./models/nodeRegistry'); -const { createPeerHeaders, verifyPeerResponse } = require('../middleware/federationPeerAuth'); -const { importFederatedData } = require('../sync/importHelper'); +const { scanItems } = require('../lib/collectionPage'); +const { inspectExport } = require('../sync/exportPages'); -const runFederationSync = async () => { - const localNodeId = process.env.FEDERATION_LOCAL_NODE_ID || process.env.NODE_ID; - if (!localNodeId) { - console.error('Federation sync skipped: FEDERATION_LOCAL_NODE_ID or NODE_ID is not configured'); - return; - } - let outboundSecrets; - try { - outboundSecrets = JSON.parse(process.env.FEDERATION_OUTBOUND_PEERS_JSON || '{}'); - } catch { - console.error('Federation sync skipped: FEDERATION_OUTBOUND_PEERS_JSON is invalid'); - return; - } - - const { Items: nodes = [] } = await dynamodbClient - .scan({ TableName: NODE_REGISTRY_TABLE_NAME }) - .promise(); - - for (const node of nodes) { +async function runFederationSync() { + for await (const node of scanItems(dynamodbClient, { TableName: NODE_REGISTRY_TABLE_NAME })) { + const url = node.url || node.nodeUrl; try { - // A peer must be present in the operator's authenticated peer registry. - const peerSecret = node.nodeId ? outboundSecrets[node.nodeId] : null; - if (!node.nodeId || typeof peerSecret !== 'string' || peerSecret.length < 32) { - console.error('Federation sync skipped for an unconfigured peer'); - continue; - } - - const exportPath = '/federation/export'; - const headers = createPeerHeaders({ - nodeId: localNodeId, secret: peerSecret, method: 'GET', path: exportPath - }); - const peerUrl = new URL(node.url); - if (peerUrl.protocol !== 'https:' && !['localhost', '127.0.0.1', '[::1]'].includes(peerUrl.hostname)) { - throw new Error('Federation peers must use HTTPS'); - } - const response = await axios.get(new URL(exportPath, peerUrl).toString(), { - headers, timeout: 15000 - }); - const responseBody = JSON.stringify(response.data); - if (!verifyPeerResponse({ - headers: response.headers, - expectedNodeId: node.nodeId, - secret: peerSecret, - method: 'GET', - path: exportPath, - body: responseBody - })) { - throw new Error('Federation peer response signature is invalid'); - } - const data = response.data; - - // Source identity comes from the locally managed NodeRegistry record whose - // peer credential is configured above, never from fields in the response body. - await importFederatedData(data, { sourceNodeId: node.nodeId }); - - await dynamodbClient.update({ - TableName: NODE_REGISTRY_TABLE_NAME, - Key: { url: node.url }, - UpdateExpression: 'set lastSyncAt = :time', - ExpressionAttributeValues: { ':time': new Date().toISOString() } - }).promise(); - console.log('Federation sync completed with a registered peer'); + const counts = await inspectExport(node); + // Redacted summaries cannot replace canonical records or local narrative. + console.info('Federation export inspected; automatic import disabled', { peer: url, counts }); } catch (error) { - console.error('Federation sync failed for a registered peer:', error.message); + console.error('Federation export inspection failed; no sync recorded', { peer: url, error: error.message }); } } -}; +} -const timer = setInterval(runFederationSync, 6 * 60 * 60 * 1000); +const timer = setInterval(() => runFederationSync().catch(() => console.error('Federation registry scan failed')), 6 * 60 * 60 * 1000); if (typeof timer.unref === 'function') timer.unref(); - module.exports = runFederationSync; diff --git a/backend/federation/models/nodeRegistry.js b/backend/federation/models/nodeRegistry.js index 27f2b609..03368bdb 100644 --- a/backend/federation/models/nodeRegistry.js +++ b/backend/federation/models/nodeRegistry.js @@ -1,4 +1,5 @@ const docClient = require("../../lib/dynamodbClient"); +const { scanItems } = require('../../lib/collectionPage'); const { applyNodeTemplate } = require("./nodeTemplate"); const NODE_REGISTRY_TABLE_NAME = "NodeRegistry"; @@ -58,8 +59,9 @@ module.exports = { * @returns {Promise>} */ async function getAllNodes() { - const data = await docClient.scan({ TableName: NODE_REGISTRY_TABLE_NAME }).promise(); - return data.Items || []; + const nodes = []; + for await (const node of scanItems(docClient, { TableName: NODE_REGISTRY_TABLE_NAME })) nodes.push(node); + return nodes; } /** diff --git a/backend/lib/collectionPage.js b/backend/lib/collectionPage.js new file mode 100644 index 00000000..ff0878c5 --- /dev/null +++ b/backend/lib/collectionPage.js @@ -0,0 +1,80 @@ +const { createHash, randomBytes, createCipheriv, createDecipheriv } = require('node:crypto'); + +function badPage() { + return Object.assign(new Error('Invalid or expired pagination parameters'), { status: 400 }); +} + +function encryptionKey() { + if (!process.env.JWT_SECRET) { + throw Object.assign(new Error('Pagination requires JWT_SECRET configuration'), { status: 503 }); + } + return createHash('sha256').update('agrinet-collection-page-v1\0').update(process.env.JWT_SECRET).digest(); +} + +function encode(state, scope) { + const iv = randomBytes(12); + const cipher = createCipheriv('aes-256-gcm', encryptionKey(), iv); + cipher.setAAD(Buffer.from(scope)); + const bytes = Buffer.concat([ + cipher.update(JSON.stringify({ state, expires: Date.now() + 3600000 })), cipher.final(), + ]); + return Buffer.concat([iv, cipher.getAuthTag(), bytes]).toString('base64url'); +} + +function decode(token, scope) { + const key = encryptionKey(); + try { + if (typeof token !== 'string' || token.length > 16384 || !/^[\w-]+$/.test(token)) throw badPage(); + const bytes = Buffer.from(token, 'base64url'); + if (bytes.length < 29 || bytes.toString('base64url') !== token) throw badPage(); + const decipher = createDecipheriv('aes-256-gcm', key, bytes.subarray(0, 12)); + decipher.setAAD(Buffer.from(scope)); + decipher.setAuthTag(bytes.subarray(12, 28)); + const decoded = JSON.parse(Buffer.concat([decipher.update(bytes.subarray(28)), decipher.final()]).toString()); + if (!Number.isFinite(decoded.expires) || decoded.expires <= Date.now() || !decoded.state || typeof decoded.state !== 'object' || Array.isArray(decoded.state)) throw badPage(); + return decoded.state; + } catch { throw badPage(); } +} + +function collectionPage(query, scope) { + const rawLimit = query.limit === undefined ? '50' : query.limit; + if (typeof rawLimit !== 'string' || !/^[1-9]\d*$/.test(rawLimit)) throw badPage(); + const limit = Number(rawLimit); + if (!Number.isSafeInteger(limit) || limit > 100) throw badPage(); + const boundScope = JSON.stringify([scope, limit]); + // Fail configuration before scanning, including first/last pages with no cursor. + encryptionKey(); + return { + limit, + state: query.cursor === undefined ? undefined : decode(query.cursor, boundScope), + finish(res, state) { + const next = state ? encode(state, boundScope) : null; + res.set('Cache-Control', 'private, no-store'); + res.set('X-Pagination', 'cursor-v1'); + res.set('X-Page-Limit', String(limit)); + if (next) res.set('X-Next-Cursor', next); + }, + }; +} + +async function scanPage(client, params, limit, start) { + const result = await client.scan({ ...params, Limit: limit, ...(start && { ExclusiveStartKey: start }) }).promise(); + const next = result.LastEvaluatedKey; + return { items: result.Items || [], next: next && Object.keys(next).length ? next : null }; +} + +function pageError(res, error, fallback) { + const status = error.status === 400 || error.status === 503 ? error.status : 500; + return res.status(status).json({ error: status === 500 ? fallback : error.message }); +} + +async function* scanItems(client, params) { + let start; + do { + const page = await scanPage(client, params, 50, start); + yield* page.items; + start = page.next; + } while (start); +} + +module.exports = { collectionPage, scanPage, scanItems, pageError }; diff --git a/backend/middleware/attachmentOwner.js b/backend/middleware/attachmentOwner.js new file mode 100644 index 00000000..bde65905 --- /dev/null +++ b/backend/middleware/attachmentOwner.js @@ -0,0 +1,19 @@ +const Message = require('../models/message'); +const Conversation = require('../models/conversation'); + +module.exports = async (req, res, next) => { + let filename; + try { filename = decodeURIComponent(req.path).slice(1); } catch { /* invalid URL */ } + if (!filename || filename.includes('/') || filename.includes('\\') || filename.includes('\0') || filename === '.' || filename === '..') { + return res.status(404).json({ error: 'Not found' }); + } + const messages = await Message.findByAttachmentPath(`/uploads/${filename}`); + for (const message of messages) { + if (await Conversation.getOwned(message.conversationId, req.user.userId)) { + // Private response must not be cached across accounts after authorization. + res.set('Cache-Control', 'private, no-store'); + return next(); + } + } + return res.status(404).json({ error: 'Not found' }); +}; diff --git a/backend/middleware/conversationOwner.js b/backend/middleware/conversationOwner.js new file mode 100644 index 00000000..5338198b --- /dev/null +++ b/backend/middleware/conversationOwner.js @@ -0,0 +1,8 @@ +const Conversation = require('../models/conversation'); + +module.exports = async (req, res, next) => { + const conversation = await Conversation.getOwned(req.params.conversationId || req.params.id, req.user.userId); + if (!conversation) return res.status(404).json({ error: 'Not found' }); + req.conversation = conversation; + return next(); +}; diff --git a/backend/middleware/requestRateLimit.js b/backend/middleware/requestRateLimit.js new file mode 100644 index 00000000..b402447e --- /dev/null +++ b/backend/middleware/requestRateLimit.js @@ -0,0 +1,12 @@ +const { rateLimit, ipKeyGenerator } = require('express-rate-limit'); + +// req.user is populated by authentication middleware, never by request data. +module.exports = () => rateLimit({ + windowMs: 60 * 1000, + limit: 120, + keyGenerator: req => typeof req.user?.userId === 'string' && req.user.userId + ? `account:${req.user.userId}` : `ip:${ipKeyGenerator(req.ip)}`, + standardHeaders: 'draft-6', + legacyHeaders: false, + message: { error: 'Too many requests, please try again later' }, +}); diff --git a/backend/models/conversation.js b/backend/models/conversation.js index 79d5080c..ffbf965e 100644 --- a/backend/models/conversation.js +++ b/backend/models/conversation.js @@ -1,12 +1,15 @@ const { randomUUID } = require('crypto'); let docClient = require('../lib/dynamodbClient'); const Message = require('./message'); +const { scanPage } = require('../lib/collectionPage'); const CONVERSATION_TABLE_NAME = process.env.CONVERSATION_TABLE_NAME || 'Conversations'; -async function create(title = 'New Chat') { +async function create(title = 'New Chat', ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) throw new Error('Conversation owner required'); const convo = { id: randomUUID(), + ownerId, title, pinned: false, createdAt: new Date().toISOString(), @@ -15,12 +18,40 @@ async function create(title = 'New Chat') { return convo; } -async function list() { - const res = await docClient.scan({ TableName: CONVERSATION_TABLE_NAME }).promise(); - return res.Items || []; +async function list(ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) return []; + const items = []; + let cursor; + do { + const res = await docClient.scan({ + TableName: CONVERSATION_TABLE_NAME, + FilterExpression: 'ownerId = :ownerId', + ExpressionAttributeValues: { ':ownerId': ownerId }, + ...(cursor && { ExclusiveStartKey: cursor }), + }).promise(); + items.push(...(res.Items || [])); + cursor = res.LastEvaluatedKey; + } while (cursor); + return items; } -async function update(id, field, value) { +async function listPage(ownerId, limit, start) { + if (typeof ownerId !== 'string' || !ownerId.trim()) throw new Error('Conversation owner required'); + return scanPage(docClient, { + TableName: CONVERSATION_TABLE_NAME, + FilterExpression: 'ownerId = :ownerId', + ExpressionAttributeValues: { ':ownerId': ownerId }, + }, limit, start); +} + +async function getOwned(id, ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) return null; + const { Item } = await docClient.get({ TableName: CONVERSATION_TABLE_NAME, Key: { id }, ConsistentRead: true }).promise(); + return Item && Item.ownerId === ownerId ? Item : null; +} + +async function update(id, field, value, ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) return null; try { const res = await docClient .update({ @@ -28,9 +59,9 @@ async function update(id, field, value) { Key: { id }, UpdateExpression: 'SET #field = :value', ExpressionAttributeNames: { '#field': field }, - ExpressionAttributeValues: { ':value': value }, + ExpressionAttributeValues: { ':value': value, ':ownerId': ownerId }, ReturnValues: 'ALL_NEW', - ConditionExpression: 'attribute_exists(id)', + ConditionExpression: 'attribute_exists(id) AND ownerId = :ownerId', }) .promise(); return res.Attributes; @@ -40,17 +71,28 @@ async function update(id, field, value) { } } -function rename(id, title) { return update(id, 'title', title); } +function rename(id, title, ownerId) { return update(id, 'title', title, ownerId); } -async function remove(id) { - await docClient.delete({ TableName: CONVERSATION_TABLE_NAME, Key: { id } }).promise(); +async function remove(id, ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) return false; + try { + await docClient.delete({ + TableName: CONVERSATION_TABLE_NAME, Key: { id }, + ConditionExpression: 'attribute_exists(id) AND ownerId = :ownerId', + ExpressionAttributeValues: { ':ownerId': ownerId }, + }).promise(); + } catch (err) { + if (err.code === 'ConditionalCheckFailedException') return false; + throw err; + } await Message.removeByConversation(id); + return true; } -function pin(id, pinned) { return update(id, 'pinned', pinned); } +function pin(id, pinned, ownerId) { return update(id, 'pinned', pinned, ownerId); } function setDocClient(client) { docClient = client; } -module.exports = { create, list, rename, remove, pin, setDocClient }; +module.exports = { create, list, listPage, getOwned, rename, remove, pin, setDocClient }; diff --git a/backend/models/message.js b/backend/models/message.js index 9bd12ae9..77e9ea2c 100644 --- a/backend/models/message.js +++ b/backend/models/message.js @@ -73,4 +73,21 @@ function setDocClient(client) { docClient = client; } -module.exports = { sendMessage, listMessages, getMessage, removeByConversation, setDocClient }; +async function findByAttachmentPath(filePath) { + const items = []; + let cursor; + do { + const page = await docClient.scan({ + TableName: MESSAGE_TABLE_NAME, + FilterExpression: '#file.#path = :filePath', + ExpressionAttributeNames: { '#file': 'file', '#path': 'path' }, + ExpressionAttributeValues: { ':filePath': filePath }, + ...(cursor && { ExclusiveStartKey: cursor }), + }).promise(); + items.push(...(page.Items || [])); + cursor = page.LastEvaluatedKey; + } while (cursor); + return items; +} + +module.exports = { sendMessage, listMessages, getMessage, removeByConversation, findByAttachmentPath, setDocClient }; diff --git a/backend/routes/__tests__/federationStatus.test.js b/backend/routes/__tests__/federationStatus.test.js index ab33ff3a..7198b8f5 100644 --- a/backend/routes/__tests__/federationStatus.test.js +++ b/backend/routes/__tests__/federationStatus.test.js @@ -1,3 +1,4 @@ +const { configurePeers, signedResponse } = require('../../test-support/federation-pages'); const request = require('supertest'); const express = require('express'); @@ -16,27 +17,29 @@ app.use(federationStatusRouter); describe('GET /federation/status', () => { beforeEach(() => { jest.clearAllMocks(); + configurePeers(); }); it('returns federation health with last sync times', async () => { dynamodbClient.scan.mockReturnValue({ promise: () => Promise.resolve({ Items: [ - { nodeUrl: 'http://node1.com', lastSyncAt: '2024-01-01T00:00:00Z' }, - { nodeUrl: 'http://node2.com', lastSyncAt: '2024-01-02T00:00:00Z' } + { nodeId: 'peer-1', nodeUrl: 'https://node1.com', lastSyncAt: '2024-01-01T00:00:00Z' }, + { nodeId: 'peer-1', nodeUrl: 'https://node2.com', lastSyncAt: '2024-01-02T00:00:00Z' } ] }) }); axios.get.mockImplementation((url) => { if (url.includes('node1')) { - return Promise.resolve({ + return Promise.resolve(signedResponse({ data: { listings: [1, 2], transactions: [1], - users: [1] - } - }); + users: [] + }, + headers: { 'x-pagination': 'cursor-v1' } + })); } return Promise.reject(new Error('Network Error')); }); @@ -45,17 +48,18 @@ describe('GET /federation/status', () => { expect(res.status).toBe(200); expect(res.body).toEqual({ + exportMode: 'read-only', usersExported: false, federationHealth: [ { - node: 'http://node1.com', + node: 'https://node1.com', status: 'ONLINE', listings: 2, transactions: 1, - users: 1, + users: 0, lastSync: '2024-01-01T00:00:00Z' }, { - node: 'http://node2.com', + node: 'https://node2.com', status: 'OFFLINE', error: 'Network Error', listings: 0, diff --git a/backend/routes/__tests__/userRoutes.test.js b/backend/routes/__tests__/userRoutes.test.js index c1de31aa..159ad33b 100644 --- a/backend/routes/__tests__/userRoutes.test.js +++ b/backend/routes/__tests__/userRoutes.test.js @@ -1,6 +1,7 @@ const request = require('supertest'); const express = require('express'); const bcrypt = require('bcryptjs'); +const jwt = require('jsonwebtoken'); jest.mock('../../lib/dynamodbClient', () => ({ scan: jest.fn(), @@ -10,41 +11,48 @@ jest.mock('../../lib/dynamodbClient', () => ({ const docClient = require('../../lib/dynamodbClient'); const userRoutes = require('../userRoutes'); -describe('userRoutes security', () => { +describe('userRoutes', () => { let app; beforeEach(() => { app = express(); app.use(express.json()); - app.use((req, _res, next) => { - req.user = { userId: 'admin-1', role: 'admin' }; - next(); - }); + process.env.JWT_SECRET = 'directory-test-key'; + app.use((req, _res, next) => { req.user = { userId: 'admin-1', role: 'admin' }; next(); }); app.use('/users', userRoutes); docClient.scan.mockReset(); docClient.put.mockReset(); }); - test('public directory returns only an explicit public DTO', async () => { - docClient.scan.mockReturnValue({ - promise: async () => ({ - Items: [{ - id: 'u1', username: 'alice', role: 'admin', email: 'alice@example.test', - phone: '555-0100', password: 'hash', verificationCode: 'secret', - verified: true, location: 'private address', coordinates: { lat: 1, long: 2 }, - locationType: 'farm', reputationScore: 5, reputationWeight: 2 - }] - }) + describe('GET /users', () => { + it('returns users without passwords', async () => { + docClient.scan.mockReturnValue({ + promise: () => Promise.resolve({ + Items: [ + { id: '1', username: 'john', password: 'hash', role: 'consumer', location: 'NY' } + ] + }) + }); + + const res = await request(app).get('/users').set('Authorization', `Bearer ${jwt.sign({ userId: 'test' }, process.env.JWT_SECRET)}`); + expect(res.status).toBe(200); + expect(res.body).toEqual([ + { id: '1', username: 'john', role: 'consumer' } + ]); + expect(res.body[0].password).toBeUndefined(); }); - const response = await request(app).get('/users'); - expect(response.status).toBe(200); - expect(response.body).toEqual([{ - id: 'u1', username: 'alice', locationType: 'farm', - reputationScore: 5, reputationWeight: 2 - }]); - expect(docClient.scan).toHaveBeenCalledWith(expect.objectContaining({ - ProjectionExpression: expect.stringContaining('username') - })); + it('applies role and location filters', async () => { + docClient.scan.mockReturnValue({ promise: () => Promise.resolve({ Items: [] }) }); + + await request(app).get('/users?role=farmer&location=Texas').set('Authorization', `Bearer ${jwt.sign({ userId: 'test' }, process.env.JWT_SECRET)}`); + + expect(docClient.scan).toHaveBeenCalledWith( + expect.objectContaining({ + FilterExpression: '#role = :role AND #location = :location AND #privacy = :public', + ExpressionAttributeValues: { ':role': 'farmer', ':location': 'Texas', ':public': true } + }) + ); + }); }); test('rejects caller-supplied IDs before writing', async () => { diff --git a/backend/routes/communicationRoutes.js b/backend/routes/communicationRoutes.js index 8f058204..69b9b11a 100644 --- a/backend/routes/communicationRoutes.js +++ b/backend/routes/communicationRoutes.js @@ -2,9 +2,12 @@ const express = require('express'); const router = express.Router(); const commController = require('../controllers/communication_controller'); const auth = require('../middleware/chatAuth'); +const owner = require('../middleware/conversationOwner'); const asyncHandler = require('../utils/asyncHandler'); const { rateLimit, ipKeyGenerator } = require('express-rate-limit'); +router.use(require('../middleware/requestRateLimit')()); + router.post('/:conversationId', rateLimit({ windowMs: 60 * 1000, limit: 300, @@ -21,7 +24,7 @@ router.post('/:conversationId', rateLimit({ ? 'account:' + req.user.userId : 'ip:' + ipKeyGenerator(req.ip), message: { error: 'Too many requests, please try again later' } -}), asyncHandler(commController.sendMessage)); +}), require('../middleware/requestRateLimit')(), asyncHandler(owner), asyncHandler(commController.sendMessage)); router.get('/:conversationId', rateLimit({ windowMs: 60 * 1000, limit: 300, @@ -38,6 +41,6 @@ router.get('/:conversationId', rateLimit({ ? 'account:' + req.user.userId : 'ip:' + ipKeyGenerator(req.ip), message: { error: 'Too many requests, please try again later' } -}), asyncHandler(commController.listMessages)); +}), require('../middleware/requestRateLimit')(), asyncHandler(owner), asyncHandler(commController.listMessages)); module.exports = router; diff --git a/backend/routes/conversationRoutes.js b/backend/routes/conversationRoutes.js index a89e5e8a..138eae87 100644 --- a/backend/routes/conversationRoutes.js +++ b/backend/routes/conversationRoutes.js @@ -2,6 +2,7 @@ const express = require('express'); const router = express.Router(); const auth = require('../middleware/chatAuth'); const ctrl = require('../controllers/conversation_controller'); +const owner = require('../middleware/conversationOwner'); const asyncHandler = require('../utils/asyncHandler'); const { rateLimit, ipKeyGenerator } = require('express-rate-limit'); @@ -39,6 +40,23 @@ router.get('/', rateLimit({ : 'ip:' + ipKeyGenerator(req.ip), message: { error: 'Too many requests, please try again later' } }), asyncHandler(ctrl.list)); +router.get('/:id', rateLimit({ + windowMs: 60 * 1000, + limit: 300, + standardHeaders: 'draft-6', + legacyHeaders: false, + keyGenerator: req => 'ip:' + ipKeyGenerator(req.ip), + message: { error: 'Too many authentication attempts' } +}), auth, rateLimit({ + windowMs: 60 * 1000, + limit: 120, + standardHeaders: 'draft-6', + legacyHeaders: false, + keyGenerator: req => req.user?.userId + ? 'account:' + req.user.userId + : 'ip:' + ipKeyGenerator(req.ip), + message: { error: 'Too many requests, please try again later' } +}), asyncHandler(owner), ctrl.get); router.put('/:id', rateLimit({ windowMs: 60 * 1000, limit: 300, @@ -55,7 +73,7 @@ router.put('/:id', rateLimit({ ? 'account:' + req.user.userId : 'ip:' + ipKeyGenerator(req.ip), message: { error: 'Too many requests, please try again later' } -}), asyncHandler(ctrl.rename)); +}), asyncHandler(owner), asyncHandler(ctrl.rename)); router.delete('/:id', rateLimit({ windowMs: 60 * 1000, limit: 300, @@ -72,7 +90,7 @@ router.delete('/:id', rateLimit({ ? 'account:' + req.user.userId : 'ip:' + ipKeyGenerator(req.ip), message: { error: 'Too many requests, please try again later' } -}), asyncHandler(ctrl.remove)); +}), asyncHandler(owner), asyncHandler(ctrl.remove)); router.post('/:id/pin', rateLimit({ windowMs: 60 * 1000, limit: 300, @@ -89,6 +107,6 @@ router.post('/:id/pin', rateLimit({ ? 'account:' + req.user.userId : 'ip:' + ipKeyGenerator(req.ip), message: { error: 'Too many requests, please try again later' } -}), asyncHandler(ctrl.pin)); +}), asyncHandler(owner), asyncHandler(ctrl.pin)); module.exports = router; diff --git a/backend/routes/federationRoutes.js b/backend/routes/federationRoutes.js index f9a57436..60efd66d 100644 --- a/backend/routes/federationRoutes.js +++ b/backend/routes/federationRoutes.js @@ -1,42 +1,2 @@ -const express = require('express'); -const router = express.Router(); -const docClient = require('../lib/dynamodbClient'); -const { LISTING_TABLE_NAME } = require('../models/listing'); -const { TRANSACTION_TABLE_NAME } = require('../models/transaction'); -const { USER_TABLE_NAME } = require('../models/user'); -const { importFederatedData } = require('../sync/importHelper'); - -// Export core objects -router.get('/export', async (req, res) => { - try { - const listingsRes = await docClient.scan({ TableName: LISTING_TABLE_NAME }).promise(); - const transactionsRes = await docClient.scan({ TableName: TRANSACTION_TABLE_NAME }).promise(); - const usersRes = await docClient - .scan({ - TableName: USER_TABLE_NAME, - ProjectionExpression: 'id, email, reputationScore, role' - }) - .promise(); - - res.json({ - listings: listingsRes.Items || [], - transactions: transactionsRes.Items || [], - users: usersRes.Items || [] - }); - } catch (error) { - res.status(500).json({ error: 'Export error' }); - } -}); - -// Import core objects -router.post('/import', async (req, res) => { - try { - await importFederatedData(req.body); - - res.json({ message: 'Data imported successfully' }); - } catch (error) { - res.status(500).json({ error: 'Import error' }); - } -}); - -module.exports = router; +// Keep the legacy import path on the same export policy as the mounted router. +module.exports = require('../federation/federationRoutes'); diff --git a/backend/routes/federationStatus.js b/backend/routes/federationStatus.js index 7eb46b20..9b855644 100644 --- a/backend/routes/federationStatus.js +++ b/backend/routes/federationStatus.js @@ -1,43 +1,35 @@ const express = require("express"); const router = express.Router(); const dynamodbClient = require("../lib/dynamodbClient"); -const axios = require("axios"); +const { inspectExport } = require('../sync/exportPages'); +const { scanItems } = require('../lib/collectionPage'); router.get("/federation/status", async (req, res) => { try { - const { Items = [] } = await dynamodbClient - .scan({ TableName: "NodeRegistry" }) - .promise(); - - const checks = await Promise.all( - Items.map(async (node) => { - const url = node.nodeUrl; - try { - const { data } = await axios.get(`${url}/federation/export`, { - timeout: 5000, - }); - return { - node: url, - status: "ONLINE", - listings: data.listings?.length || 0, - transactions: data.transactions?.length || 0, - users: data.users?.length || 0, - lastSync: node.lastSyncAt, - }; - } catch (err) { - return { - node: url, - status: "OFFLINE", - error: err.message, - listings: 0, - transactions: 0, - users: 0, - lastSync: node.lastSyncAt, - }; - } - }) - ); - res.json({ federationHealth: checks }); + const checks = []; + for await (const node of scanItems(dynamodbClient, { TableName: 'NodeRegistry' })) { + const url = node.nodeUrl || node.url; + try { + const counts = await inspectExport(node); + checks.push({ + node: url, + status: "ONLINE", + ...counts, + lastSync: node.lastSyncAt, + }); + } catch (err) { + checks.push({ + node: url, + status: "OFFLINE", + error: err.message, + listings: 0, + transactions: 0, + users: 0, + lastSync: node.lastSyncAt, + }); + } + } + res.json({ federationHealth: checks, exportMode: 'read-only', usersExported: false }); } catch (err) { res.status(500).json({ error: "Failed to validate federation status" }); } diff --git a/backend/routes/userRoutes.js b/backend/routes/userRoutes.js index a532ab2e..38a5dfc1 100644 --- a/backend/routes/userRoutes.js +++ b/backend/routes/userRoutes.js @@ -7,22 +7,60 @@ const { USER_TABLE_NAME, createUserItem, sanitizeUserForPublic } = require('../m const requireAdmin = require('../middleware/requireAdmin'); const { userWriteRateLimit } = require('../middleware/rateLimit'); -// GET /users is a public directory, not an account export. -router.get('/', async (req, res) => { +const account = require('../middleware/chatAuth'); +const requestRateLimit = require('../middleware/requestRateLimit'); +const { collectionPage, scanPage, pageError } = require('../lib/collectionPage'); + +// GET /users - bounded directory, never account or private location records +router.get('/', requestRateLimit(), account, requestRateLimit(), async (req, res) => { try { + for (const name of ['role', 'location', 'locationType']) { + if (req.query[name] !== undefined && typeof req.query[name] !== 'string') { + return res.status(400).json({ error: 'Filters must be single strings' }); + } + } + const page = collectionPage(req.query, ['users', req.user.userId, req.query.role || '', req.query.location || '', req.query.locationType || '']); const params = { TableName: USER_TABLE_NAME, - ProjectionExpression: '#id, username, locationType, reputationScore, reputationWeight', - ExpressionAttributeNames: { '#id': 'id' } + ProjectionExpression: '#id, #username, #role, #location, #privacy, #locationType', + ExpressionAttributeNames: { '#id': 'id', '#username': 'username', '#role': 'role', '#location': 'location', '#privacy': 'locationPrivacy', '#locationType': 'locationType' }, }; + + const filters = []; + const values = {}; + if (req.query.role) { + filters.push('#role = :role'); + values[':role'] = req.query.role; + } if (req.query.locationType) { - params.FilterExpression = 'locationType = :locationType'; - params.ExpressionAttributeValues = { ':locationType': req.query.locationType }; + filters.push('#locationType = :locationType'); + values[':locationType'] = req.query.locationType; } - const data = await docClient.scan(params).promise(); - res.json((data.Items || []).map(sanitizeUserForPublic)); - } catch { - res.status(500).json({ error: 'Failed to fetch users' }); + if (req.query.location) { + filters.push('#location = :location AND #privacy = :public'); + values[':location'] = req.query.location; + values[':public'] = true; + } + if (filters.length) { + params.FilterExpression = filters.join(' AND '); + params.ExpressionAttributeValues = values; + } + + const { items, next } = await scanPage(docClient, params, page.limit, page.state?.start); + // Preserve structural discovery (including locationType), not scalar ranking: + // CLAUDE.md requires full reputation distributions, never a single score. + // Exact coordinates/contact fields stay private; coarse location is opt-in. + const users = items.map(item => ({ + ...(typeof item.id === 'string' && { id: item.id }), + ...(typeof item.username === 'string' && { username: item.username }), + ...(typeof item.locationType === 'string' && { locationType: item.locationType }), + ...(typeof item.role === 'string' && { role: item.role }), + ...(item.locationPrivacy === true && typeof item.location === 'string' && { location: item.location }), + })); + page.finish(res, next && { start: next }); + res.json(users); + } catch (err) { + pageError(res, err, 'Failed to fetch users'); } }); diff --git a/backend/scripts/check-collection-pagination.js b/backend/scripts/check-collection-pagination.js new file mode 100644 index 00000000..87e6146c --- /dev/null +++ b/backend/scripts/check-collection-pagination.js @@ -0,0 +1,140 @@ +// Disposable local-storage check. No production endpoint or data is accepted. +const assert = require('node:assert/strict'); +const { randomUUID } = require('node:crypto'); +const { DynamoDBClient, CreateTableCommand, DeleteTableCommand } = require('@aws-sdk/client-dynamodb'); +const { DynamoDBDocument, PutCommand, GetCommand } = require('@aws-sdk/lib-dynamodb'); +const express = require('express'); +const jwt = require('jsonwebtoken'); + +async function checkAuthenticationBudget(base) { + for (let i = 0; i < 1001; i++) { + const response = await fetch(`${base}/users`, { headers: { Authorization: 'Bearer invalid' } }); + await response.text(); + if (response.status === 429) { + assert.ok(Number(response.headers.get('retry-after')) > 0); + return; + } + assert.equal(response.status, 401); + } + assert.fail('verification server authentication attempts must be rate limited'); +} + +async function main() { + const endpoint = new URL(process.env.DYNAMODB_ENDPOINT); + if (!['127.0.0.1', 'localhost', '[::1]'].includes(endpoint.hostname)) throw new Error('DynamoDB Local loopback endpoint required'); + process.env.AWS_ACCESS_KEY_ID = 'localcheck'; + process.env.AWS_SECRET_ACCESS_KEY = 'localcheck'; + process.env.AWS_REGION = 'us-east-1'; + process.env.JWT_SECRET = randomUUID(); + process.env.API_KEY = randomUUID(); + const peerSecret = randomUUID(); + process.env.FEDERATION_LOCAL_NODE_ID = 'local-check'; + process.env.FEDERATION_PEERS_JSON = JSON.stringify({ 'local-check': peerSecret }); + process.env.FEDERATION_OUTBOUND_PEERS_JSON = JSON.stringify({ 'local-check': peerSecret }); + const admin = new DynamoDBClient({ endpoint: endpoint.href, region: 'us-east-1', credentials: { accessKeyId: 'localcheck', secretAccessKey: 'localcheck' } }); + const store = DynamoDBDocument.from(admin); + const prefix = `CollectionPages_${randomUUID().replaceAll('-', '')}`; + const tables = Object.fromEntries(['Conversations', 'Users', 'Listings', 'Transactions'].map(name => [name, `${prefix}_${name}`])); + process.env.CONVERSATION_TABLE_NAME = tables.Conversations; + const created = []; + let server; + try { + for (const TableName of Object.values(tables)) { + await admin.send(new CreateTableCommand({ TableName, BillingMode: 'PAY_PER_REQUEST', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + })); + created.push(TableName); + } + const put = (table, Item) => store.send(new PutCommand({ TableName: tables[table], Item })); + for (let i = 0; i < 8; i++) { + await put('Conversations', { id: `c${i}`, ownerId: i % 2 ? 'alice' : 'bob', title: 'private'.repeat(33000) }); + await put('Users', { id: `u${i}`, username: `member${i}`, role: i % 2 ? 'producer' : 'consumer', locationType: i % 2 ? 'farm' : 'delivery', reputationScore: 99, reputationWeight: 1000, location: 'Town', locationPrivacy: i % 2 === 1, + password: 'private hash', verificationCode: 'private code', email: 'private@example.invalid', coordinates: { lat: 1, long: 2 }, privatePadding: 'x'.repeat(230000) }); + await put('Listings', { id: `l${i}`, userId: `u${i}`, title: `Public listing ${i}`, description: 'advertised text'.repeat(16000), price: i + 1, privateExtra: 'not an advertised field' }); + await put('Transactions', { id: `t${i}`, buyerId: 'alice', sellerId: 'bob', status: 'pending', dialogNotes: 'local narrative'.repeat(15000) }); + } + await put('Conversations', { id: 'legacy', title: 'no owner' }); + + const db = require('../lib/dynamodbClient'); + const scan = db.scan; + const scans = []; + // Only substitute isolated table names. The production SDK adapter, scan + // limit, filters, projections and continuation keys all remain real. + db.scan = params => { + scans.push(params); + return scan({ ...params, TableName: tables[params.TableName] || params.TableName }); + }; + const app = express(); + app.use(express.json()); + app.use('/federation', require('../federation/federationRoutes')); + app.use(require('../middleware/requestRateLimit')(), require('../middleware/authMiddleware')); + app.use('/conversations', require('../routes/conversationRoutes')); + app.use('/users', require('../routes/userRoutes')); + server = await new Promise(resolve => { const listener = app.listen(0, '127.0.0.1', () => resolve(listener)); }); + const base = `http://127.0.0.1:${server.address().port}`; + const token = jwt.sign({ userId: 'alice' }, process.env.JWT_SECRET, { expiresIn: '2h' }); + const headers = { Authorization: `Bearer ${token}` }; + async function collect(path) { + const result = []; + let cursor; + let pages = 0; + do { + const response = await fetch(`${base}${path}${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`, { headers, redirect: 'error' }); + assert.equal(response.status, 200); + assert.equal(response.headers.get('X-Pagination'), 'cursor-v1'); + result.push(...await response.json()); + cursor = response.headers.get('X-Next-Cursor'); + assert.ok(++pages < 30); + } while (cursor); + return { result, pages }; + } + const directory = await collect('/users?limit=100'); + assert.ok(directory.pages > 1, 'the 1 MiB scan boundary must require continuation even with a projection'); + assert.equal(directory.result.length, 8); + for (const user of directory.result) { + assert.deepEqual(Object.keys(user).sort(), ['id', 'username', 'role', 'locationType', ...(Number(user.id.slice(1)) % 2 ? ['location'] : [])].sort()); + } + const byType = await collect('/users?limit=1&locationType=delivery'); + assert.deepEqual(byType.result.map(user => user.id).sort(), ['u0', 'u2', 'u4', 'u6']); + assert.ok(byType.result.every(user => !Object.hasOwn(user, 'location') && !Object.hasOwn(user, 'reputationScore') && !Object.hasOwn(user, 'reputationWeight'))); + const filtered = await collect('/users?limit=1&role=producer&location=Town'); + assert.deepEqual(filtered.result.map(user => user.id).sort(), ['u1', 'u3', 'u5', 'u7']); + const { createAccountClient } = await import('../../frontend/chat-ui/src/accountClient.mjs'); + const client = createAccountClient({ baseUrl: base }); + client.setToken(token); + const conversations = await client.request('GET', '/conversations?limit=100'); + assert.deepEqual(conversations.map(item => item.id).sort(), ['c1', 'c3', 'c5', 'c7']); + const first = await fetch(`${base}/conversations?limit=1`, { headers }); + const cursor = first.headers.get('X-Next-Cursor'); + assert.ok(cursor); + const foreign = await fetch(`${base}/conversations?limit=1&cursor=${cursor}`, { headers: { Authorization: `Bearer ${jwt.sign({ userId: 'bob' }, process.env.JWT_SECRET)}` } }); + assert.equal(foreign.status, 400); + + const { forEachExportPage, inspectExport } = require('../sync/exportPages'); + const peer = { nodeId: 'local-check', url: base }; + const exported = { listings: [], transactions: [], users: [] }; + let exportPages = 0; + await forEachExportPage(peer, page => { + exportPages++; + for (const name of Object.keys(exported)) exported[name].push(...page[name]); + }); + assert.ok(exportPages > 1); + assert.deepEqual(exported.listings.map(item => item.id).sort(), Array.from({ length: 8 }, (_, i) => `l${i}`)); + assert.deepEqual(exported.transactions.map(item => item.id).sort(), Array.from({ length: 8 }, (_, i) => `t${i}`)); + assert.deepEqual(exported.users, []); + assert.ok(exported.listings.every(item => !Object.hasOwn(item, 'privateExtra'))); + assert.ok(exported.transactions.every(item => !Object.hasOwn(item, 'dialogNotes'))); + assert.deepEqual(await inspectExport(peer), { listings: 8, transactions: 8, users: 0 }); + assert.equal((await store.send(new GetCommand({ TableName: tables.Users, Key: { id: 'u0' } }))).Item.password, 'private hash'); + assert.ok((await store.send(new GetCommand({ TableName: tables.Transactions, Key: { id: 't0' } }))).Item.dialogNotes.length > 100000); + assert.ok(scans.some(params => params.ExclusiveStartKey)); + assert.ok(scans.every(params => params.Limit > 0 && params.Limit <= 100)); + await checkAuthenticationBudget(base); + console.log('PASS: real DynamoDB 1 MiB continuation, ownership, private projections, filters, browser adapter, read-only exports and unchanged local records'); + } finally { + if (server) await new Promise(resolve => { server.close(resolve); server.closeAllConnections(); }); + for (const TableName of created) await admin.send(new DeleteTableCommand({ TableName })); + admin.destroy(); + } +} +main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/backend/scripts/check-conversation-ownership.js b/backend/scripts/check-conversation-ownership.js new file mode 100644 index 00000000..61ba60b9 --- /dev/null +++ b/backend/scripts/check-conversation-ownership.js @@ -0,0 +1,56 @@ +// Explicit integration check: creates and removes isolated DynamoDB Local tables. +const assert = require('node:assert/strict'); +const { randomUUID } = require('node:crypto'); +const { DynamoDBClient, CreateTableCommand, DeleteTableCommand } = require('@aws-sdk/client-dynamodb'); + +async function main() { + const endpoint = process.env.DYNAMODB_ENDPOINT; + const parsed = new URL(endpoint); + if (!['127.0.0.1', 'localhost', '[::1]'].includes(parsed.hostname)) throw new Error('DynamoDB Local loopback endpoint required'); + process.env.AWS_ACCESS_KEY_ID = 'localcheck'; + process.env.AWS_SECRET_ACCESS_KEY = 'localcheck'; + process.env.AWS_REGION = 'us-east-1'; + const suffix = randomUUID().replaceAll('-', ''); + process.env.CONVERSATION_TABLE_NAME = `ChatOwnership_${suffix}`; + process.env.MESSAGE_TABLE_NAME = `ChatMessages_${suffix}`; + const admin = new DynamoDBClient({ endpoint, region: 'us-east-1', credentials: { accessKeyId: 'localcheck', secretAccessKey: 'localcheck' } }); + const tables = []; + try { + for (const [TableName, keys] of [ + [process.env.CONVERSATION_TABLE_NAME, [{ AttributeName: 'id', KeyType: 'HASH' }]], + [process.env.MESSAGE_TABLE_NAME, [{ AttributeName: 'conversationId', KeyType: 'HASH' }, { AttributeName: 'id', KeyType: 'RANGE' }]], + ]) { + await admin.send(new CreateTableCommand({ + TableName, KeySchema: keys, AttributeDefinitions: keys.map(({ AttributeName }) => ({ AttributeName, AttributeType: 'S' })), BillingMode: 'PAY_PER_REQUEST', + })); + tables.push(TableName); + } + const Conversation = require('../models/conversation'); + const Message = require('../models/message'); + const db = require('../lib/dynamodbClient'); + const mine = await Conversation.create('private', 'alice'); + await Conversation.create('foreign', 'bob'); + await db.put({ TableName: process.env.CONVERSATION_TABLE_NAME, Item: { id: 'legacy', title: 'unknown owner' } }).promise(); + assert.equal(await Conversation.getOwned(mine.id, 'bob'), null); + assert.equal(await Conversation.getOwned('legacy', 'alice'), null); + assert.equal(await Conversation.rename(mine.id, 'stolen', 'bob'), null); + assert.equal(await Conversation.pin(mine.id, true, 'bob'), null); + assert.equal(await Conversation.remove(mine.id, 'bob'), false); + assert.equal((await Conversation.rename(mine.id, 'renamed', 'alice')).title, 'renamed'); + assert.equal((await Conversation.pin(mine.id, true, 'alice')).pinned, true); + for (let i = 0; i < 6; i++) await Conversation.create('x'.repeat(230000), i % 2 ? 'alice' : 'bob'); + assert.equal((await Conversation.list('alice')).length, 4); + const message = await Message.sendMessage(mine.id, { from: 'alice', content: 'private content' }); + assert.equal((await Message.listMessages(mine.id))[0].id, message.id); + await db.put({ TableName: process.env.MESSAGE_TABLE_NAME, Item: { id: 'attachment', conversationId: mine.id, file: { path: '/uploads/example.txt' } } }).promise(); + assert.equal((await Message.findByAttachmentPath('/uploads/example.txt'))[0].id, 'attachment'); + assert.equal(await Conversation.remove(mine.id, 'alice'), true); + assert.deepEqual(await Message.listMessages(mine.id), []); + assert.equal(await Conversation.getOwned(mine.id, 'alice'), null); + console.log('PASS: real DynamoDB owner conditions, legacy isolation, multi-page owner listing, message lookup and deletion'); + } finally { + for (const TableName of tables) await admin.send(new DeleteTableCommand({ TableName })); + admin.destroy(); + } +} +main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/backend/scripts/check-message-retries.js b/backend/scripts/check-message-retries.js index bedef976..06ce6672 100644 --- a/backend/scripts/check-message-retries.js +++ b/backend/scripts/check-message-retries.js @@ -3,9 +3,39 @@ const path = require('node:path'); const { fork } = require('node:child_process'); const { once } = require('node:events'); const { randomUUID } = require('node:crypto'); -const { DynamoDBClient, CreateTableCommand, DeleteTableCommand, ScanCommand } = require('@aws-sdk/client-dynamodb'); +const { DynamoDBClient, CreateTableCommand, DeleteTableCommand, ScanCommand, PutItemCommand } = require('@aws-sdk/client-dynamodb'); const jwt = require('../utils/jwt'); +async function seedConversations(client, table) { + for (const id of ['conversation-1', 'crash-conversation']) { + await client.send(new PutItemCommand({ TableName: table, Item: { id: { S: id }, ownerId: { S: 'local-grower' } } })); + } +} + +async function checkPrivateAccess(send, target, apiCaller) { + if (apiCaller) return; + assert.equal((await send(target, { apiKeyOnly: true })).status, 401); + assert.equal((await send(target, { user: 'another-grower' })).status, 404); +} + +async function createTables(client, definitions, created) { + for (const [name, composite] of definitions) { + await client.send(new CreateTableCommand({ TableName: name, BillingMode: 'PAY_PER_REQUEST', + AttributeDefinitions: composite ? [{ AttributeName: 'conversationId', AttributeType: 'S' }, { AttributeName: 'id', AttributeType: 'S' }] + : [{ AttributeName: 'id', AttributeType: 'S' }], + KeySchema: composite ? [{ AttributeName: 'conversationId', KeyType: 'HASH' }, { AttributeName: 'id', KeyType: 'RANGE' }] + : [{ AttributeName: 'id', KeyType: 'HASH' }], + })); + created.push(name); + } +} + +async function stopWorkers(workers, stop) { + for (const target of workers) { + if (target.child.exitCode === null && target.child.signalCode === null) await stop(target); + } +} + async function main() { const endpoint = process.env.DYNAMODB_TEST_ENDPOINT; if (!endpoint || !['localhost', '127.0.0.1', '[::1]'].includes(new URL(endpoint).hostname)) { @@ -16,6 +46,7 @@ async function main() { const suffix = randomUUID(); const messages = `RetryMessages-${suffix}`; const receipts = `RetrySends-${suffix}`; + const conversations = `RetryConversations-${suffix}`; const created = []; const workers = []; const secret = 'local-integration-secret'; @@ -27,7 +58,8 @@ async function main() { const child = fork(path.join(__dirname, '../test-support/message-retry-worker.js'), [], { env: { ...process.env, DYNAMODB_ENDPOINT: endpoint, AWS_REGION: 'us-east-1', AWS_ACCESS_KEY_ID: 'local', AWS_SECRET_ACCESS_KEY: 'local', JWT_SECRET: secret, API_KEY: apiKey, - MESSAGE_TABLE_NAME: messages, MESSAGE_SEND_TABLE_NAME: receipts, TEST_HOLD_AI: String(hold) }, + MESSAGE_TABLE_NAME: messages, MESSAGE_SEND_TABLE_NAME: receipts, CONVERSATION_TABLE_NAME: conversations, + TEST_RETRY_SERVICE: String(apiCaller), TEST_HOLD_AI: String(hold) }, stdio: ['ignore', 'inherit', 'inherit', 'ipc'], }); const state = { child, ai: 0 }; @@ -45,11 +77,11 @@ async function main() { return state; } - async function send(target, { key = 'send-1', conversation = 'conversation-1', body = payload } = {}) { + async function send(target, { key = 'send-1', conversation = 'conversation-1', body = payload, apiKeyOnly = apiCaller, user = 'local-grower' } = {}) { const response = await fetch(`${target.url}/messages/${conversation}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Idempotency-Key': key, - ...(apiCaller ? { 'X-API-Key': apiKey } - : { Authorization: `Bearer ${jwt.sign({ userId: 'local-grower' }, secret)}` }) }, + ...(apiKeyOnly ? { 'X-API-Key': apiKey } + : { Authorization: `Bearer ${jwt.sign({ userId: user }, secret)}` }) }, body: JSON.stringify(body), signal: AbortSignal.timeout(15000), }); return { status: response.status, body: await response.json() }; @@ -71,17 +103,11 @@ async function main() { } try { - for (const [name, composite] of [[messages, true], [receipts, false]]) { - await client.send(new CreateTableCommand({ TableName: name, BillingMode: 'PAY_PER_REQUEST', - AttributeDefinitions: composite ? [{ AttributeName: 'conversationId', AttributeType: 'S' }, { AttributeName: 'id', AttributeType: 'S' }] - : [{ AttributeName: 'id', AttributeType: 'S' }], - KeySchema: composite ? [{ AttributeName: 'conversationId', KeyType: 'HASH' }, { AttributeName: 'id', KeyType: 'RANGE' }] - : [{ AttributeName: 'id', KeyType: 'HASH' }], - })); - created.push(name); - } + await createTables(client, [[messages, true], [receipts, false], [conversations, false]], created); + await seedConversations(client, conversations); const firstWorker = await worker(true); const secondWorker = await worker(false); + await checkPrivateAccess(send, secondWorker, apiCaller); const first = send(firstWorker); await waitForAI(firstWorker); const racing = await Promise.all(Array.from({ length: 8 }, () => send(secondWorker))); @@ -113,9 +139,7 @@ async function main() { assert.equal(afterCrash.Count, 3); console.log('PASS: process death leaves a pending receipt and does not duplicate a message or AI invocation.'); } finally { - for (const target of workers) { - if (target.child.exitCode === null && target.child.signalCode === null) await stop(target); - } + await stopWorkers(workers, stop); for (const name of created.reverse()) await client.send(new DeleteTableCommand({ TableName: name })); client.destroy(); } diff --git a/backend/server.js b/backend/server.js index 9c1aeccc..82d871ef 100644 --- a/backend/server.js +++ b/backend/server.js @@ -1,5 +1,9 @@ const express = require("express"); const http = require('http'); +const path = require('path'); +const chatAuth = require('./middleware/chatAuth'); +const asyncHandler = require('./utils/asyncHandler'); +const requestRateLimit = require('./middleware/requestRateLimit'); let cors; try { cors = require('cors'); } catch { cors = () => (_req, _res, next) => next(); } let dotenv; @@ -10,7 +14,7 @@ const { apiRateLimit, streamRateLimit, locationRateLimit } = require('./middlewa const minimal = process.env.MINIMAL_SERVER === 'true'; const app = express(); const allowedOrigins = (process.env.CORS_ALLOWED_ORIGINS || 'https://www.ntari.org').split(',').map(origin => origin.trim()).filter(Boolean); -app.use(cors({ origin(origin, callback) { if (!origin || allowedOrigins.includes(origin)) return callback(null, true); return callback(new Error('CORS not allowed for this origin')); }, credentials: true, methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], allowedHeaders: ['Content-Type', 'Authorization', 'Idempotency-Key', 'X-API-Key', 'X-Federation-Node-Id', 'X-Federation-Timestamp', 'X-Federation-Signature'] })); +app.use(cors({ origin(origin, callback) { if (!origin || allowedOrigins.includes(origin)) return callback(null, true); return callback(new Error('CORS not allowed for this origin')); }, credentials: true, methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], allowedHeaders: ['Content-Type', 'Authorization', 'Idempotency-Key', 'X-API-Key', 'X-Federation-Node-Id', 'X-Federation-Timestamp', 'X-Federation-Signature'], exposedHeaders: ['X-Pagination', 'X-Next-Cursor', 'X-Page-Limit', 'X-Federation-Omissions'] })); app.use(express.json({ verify(req, _res, buffer) { req.rawBody = Buffer.from(buffer); } })); const tryMount = (route, modPath) => { try { app.use(route, require(modPath)); } catch (err) { console.warn('Skipping ' + modPath + ': ' + err.message); } }; if (!minimal) tryMount('/deposit', './routes/depositRoutes'); @@ -31,8 +35,15 @@ if (!minimal) tryMount('/federation', './federation/federationRoutes'); if (!minimal) tryMount('/api/auth', './routes/authRoutes'); app.use(authMiddleware); app.use(apiRateLimit); -// Private uploads are disabled until conversation-owner authorization is deployed (#191). -app.use('/uploads', (_req, res) => res.status(410).json({ error: 'Attachment downloads are temporarily unavailable pending owner authorization' })); +// Attachment content remains behind independent account authentication and ownership. +app.use('/uploads', requestRateLimit(), chatAuth, requestRateLimit(), asyncHandler(require('./middleware/attachmentOwner')), express.static(path.join(__dirname, 'uploads'), { + dotfiles: 'deny', index: false, + setHeaders(res) { + res.setHeader('Content-Disposition', 'attachment'); + res.setHeader('Content-Type', 'application/octet-stream'); + res.setHeader('X-Content-Type-Options', 'nosniff'); + }, +})); app.use('/api/location', locationRateLimit); let runFederationSync; if (!minimal) { diff --git a/backend/services/retrySafeMessage.js b/backend/services/retrySafeMessage.js index 1657a837..3f742526 100644 --- a/backend/services/retrySafeMessage.js +++ b/backend/services/retrySafeMessage.js @@ -20,16 +20,41 @@ function problem(status, code, error) { return { status, body: { code, error } } const failed = () => problem(500, 'message_send_failed', 'This send failed and will not be repeated automatically. Inspect the conversation before starting a new send.'); -async function send(req, deliver) { +function readInput(req) { const key = req.headers['idempotency-key']; if (typeof key !== 'string' || !/^[\x21-\x7e]{1,128}$/.test(key)) { - return problem(400, 'invalid_idempotency_key', 'Idempotency-Key must be 1–128 printable ASCII characters without spaces.'); + return { error: problem(400, 'invalid_idempotency_key', 'Idempotency-Key must be 1–128 printable ASCII characters without spaces.') }; } // Auth middleware has already verified the JWT or accepted the configured // API key. Never use body.from (a display/role label) as the caller identity. const userId = typeof req.user?.userId === 'string' && req.user.userId ? req.user.userId : null; const apiKey = req.user ? null : req.headers['x-api-key']; - if (!userId && !apiKey) return problem(400, 'idempotency_identity_required', 'Retry-safe sends require a stable authenticated user ID or API key.'); + if (!userId && !apiKey) { + return { error: problem(400, 'idempotency_identity_required', 'Retry-safe sends require a stable authenticated user ID or API key.') }; + } + return { key, userId, apiKey }; +} + +async function replay(receipt, fingerprint, conversationId) { + if (receipt.fingerprint !== fingerprint) { + return problem(409, 'idempotency_key_conflict', 'This key was already used with a different message payload.'); + } + if (receipt.state !== 'complete') { + return problem(409, 'message_send_pending', 'This send is still running or was interrupted. Retry this key later; inspect the conversation before starting a new send.'); + } + if (receipt.outcome.status === 500) return failed(); + const message = await Message.getMessage(conversationId, receipt.outcome.messageId); + const reply = receipt.outcome.replyId ? await Message.getMessage(conversationId, receipt.outcome.replyId) : null; + if (!message || (receipt.outcome.replyId && !reply)) { + return problem(410, 'message_send_removed', 'The original message or reply is no longer available. It will not be recreated.'); + } + return { status: 201, body: reply ? { message, ai: reply, reply } : { message } }; +} + +async function send(req, deliver) { + const input = readInput(req); + if (input.error) return input.error; + const { key, userId, apiKey } = input; const conversationId = req.params.conversationId; const { from, to, content, type = 'text', file } = req.body; @@ -49,22 +74,7 @@ async function send(req, deliver) { } catch { return problem(503, 'message_send_unavailable', 'Retry protection is unavailable. No send was started by this request. Retry with the same key.'); } - if (!claim.claimed) { - const { receipt } = claim; - if (receipt.fingerprint !== fingerprint) { - return problem(409, 'idempotency_key_conflict', 'This key was already used with a different message payload.'); - } - if (receipt.state !== 'complete') { - return problem(409, 'message_send_pending', 'This send is still running or was interrupted. Retry this key later; inspect the conversation before starting a new send.'); - } - if (receipt.outcome.status === 500) return failed(); - const message = await Message.getMessage(conversationId, receipt.outcome.messageId); - const reply = receipt.outcome.replyId ? await Message.getMessage(conversationId, receipt.outcome.replyId) : null; - if (!message || (receipt.outcome.replyId && !reply)) { - return problem(410, 'message_send_removed', 'The original message or reply is no longer available. It will not be recreated.'); - } - return { status: 201, body: reply ? { message, ai: reply, reply } : { message } }; - } + if (!claim.claimed) return replay(claim.receipt, fingerprint, conversationId); let result, outcome; try { diff --git a/backend/sync/__tests__/exportPages.test.js b/backend/sync/__tests__/exportPages.test.js new file mode 100644 index 00000000..a8d2037f --- /dev/null +++ b/backend/sync/__tests__/exportPages.test.js @@ -0,0 +1,80 @@ +jest.mock('axios'); +const axios = require('axios'); +const { createHmac, createHash } = require('node:crypto'); +const { createPeerResponseHeaders } = require('../../middleware/federationPeerAuth'); +const peer = { nodeId: 'peer-1', url: 'https://peer.example/' }; +const secret = '0123456789abcdef0123456789abcdef'; +const { forEachExportPage } = require('../exportPages'); + +describe('federation export traversal', () => { + const env = { ...process.env }; + beforeEach(() => { + jest.resetAllMocks(); + process.env.API_KEY = 'never-send-the-local-write-key'; + process.env.FEDERATION_PEER_KEYS = JSON.stringify({ 'https://peer.example': 'obsolete-read-key-must-not-be-used' }); + process.env.FEDERATION_LOCAL_NODE_ID = 'local-node'; + process.env.FEDERATION_OUTBOUND_PEERS_JSON = JSON.stringify({ 'peer-1': secret }); + }); + afterAll(() => { process.env = env; }); + const signed = data => ({ data, headers: createPeerResponseHeaders({ nodeId: 'peer-1', secret, method: 'GET', path: '/federation/export', body: JSON.stringify(data) }) }); + const page = (listings, cursor) => { + const data = { listings, transactions: [], users: [] }; + return { data, headers: { ...createPeerResponseHeaders({ nodeId: 'peer-1', secret, method: 'GET', path: '/federation/export', body: JSON.stringify(data) }), + 'x-pagination': 'cursor-v1', ...(cursor && { 'x-next-cursor': cursor }) } }; + }; + test('inspects every signed page including empty continuations, using per-peer HMAC credentials', async () => { + axios.get.mockResolvedValueOnce(page([{ id: 'l1' }], 'first')) + .mockResolvedValueOnce(page([], 'second')).mockResolvedValueOnce(page([{ id: 'l2' }])); + const seen = []; + await forEachExportPage(peer, data => { seen.push(data); }); + expect(seen.flatMap(data => data.listings)).toEqual([{ id: 'l1' }, { id: 'l2' }]); + expect(axios.get).toHaveBeenCalledTimes(3); + expect(axios.get.mock.calls[0]).toEqual(['https://peer.example/federation/export', { + headers: expect.objectContaining({ 'x-federation-node-id': 'local-node', 'x-federation-signature': expect.any(String) }), params: { limit: 50 }, + timeout: 5000, maxRedirects: 0, maxContentLength: 16 * 1024 * 1024, + }]); + const sent = axios.get.mock.calls[0][1].headers; + const digest = createHash('sha256').update('').digest('hex'); + expect(sent['x-federation-signature']).toBe(createHmac('sha256', secret).update([sent['x-federation-timestamp'], 'GET', '/federation/export', digest].join('.')).digest('hex')); + expect(JSON.stringify(axios.get.mock.calls)).not.toContain(secret); + expect(axios.get.mock.calls[1][1].params.cursor).toBe('first'); + expect(axios.get.mock.calls[2][1].params.cursor).toBe('second'); + expect(JSON.stringify(axios.get.mock.calls)).not.toContain(process.env.API_KEY); + }); + test('unconfigured peers and unsafe destinations never receive credentials', async () => { + for (const url of ['http://peer.example', 'https://user@peer.example', 'https://peer.example/?redirect=elsewhere', 'https://peer.example/#fragment']) { + await expect(forEachExportPage({ nodeId: 'peer-1', url }, () => {})).rejects.toThrow(); + } + await expect(forEachExportPage({ nodeId: 'unknown', url: 'https://peer.example' }, () => {})).rejects.toThrow(); + expect(axios.get).not.toHaveBeenCalled(); + }); + test('old unpaged responses and exported account rows fail closed', async () => { + const consume = jest.fn(); + axios.get.mockResolvedValueOnce(signed({ listings: [], transactions: [], users: [] })); + await expect(forEachExportPage(peer, consume)).rejects.toThrow(/pagination/); + axios.get.mockResolvedValueOnce({ ...signed({ listings: [], transactions: [], users: [{ id: 'u1', password: 'secret' }] }), headers: { ...signed({ listings: [], transactions: [], users: [{ id: 'u1', password: 'secret' }] }).headers, 'x-pagination': 'cursor-v1' } }); + await expect(forEachExportPage(peer, consume)).rejects.toThrow(/account/); + expect(consume).not.toHaveBeenCalled(); + }); + test('rejects a tampered signed page before passing it to the consumer', async () => { + const response = page([]); + response.data.listings.push({ id: 'forged' }); + axios.get.mockResolvedValue(response); + const consume = jest.fn(); + await expect(forEachExportPage('https://peer.example', consume, 'peer-1')).rejects.toThrow(/signature/); + expect(consume).not.toHaveBeenCalled(); + }); + test('a failed later page is not treated as a complete export', async () => { + axios.get.mockResolvedValueOnce(page([{ id: 'l1' }], 'next')).mockRejectedValueOnce(new Error('peer unavailable')); + const consume = jest.fn(); + await expect(forEachExportPage(peer, consume)).rejects.toThrow('peer unavailable'); + expect(consume).toHaveBeenCalledTimes(1); + }); + test('repeating cursors and malformed page shapes stop without looping or consuming bad data', async () => { + axios.get.mockResolvedValue(page([], 'same')); + await expect(forEachExportPage(peer, () => {})).rejects.toThrow(/cursor/); + expect(axios.get).toHaveBeenCalledTimes(2); + axios.get.mockResolvedValue({ ...signed({ listings: {}, transactions: [], users: [] }), headers: { ...signed({ listings: {}, transactions: [], users: [] }).headers, 'x-pagination': 'cursor-v1' } }); + await expect(forEachExportPage(peer, () => {})).rejects.toThrow(/page/); + }); +}); diff --git a/backend/sync/__tests__/federationSyncJob.test.js b/backend/sync/__tests__/federationSyncJob.test.js index cf6c43c6..d699f0e5 100644 --- a/backend/sync/__tests__/federationSyncJob.test.js +++ b/backend/sync/__tests__/federationSyncJob.test.js @@ -1,3 +1,4 @@ +const { configurePeers, signedResponse } = require('../../test-support/federation-pages'); const mockScan = jest.fn(); const mockPut = jest.fn(); @@ -18,32 +19,29 @@ const runFederationSync = require('../federationSyncJob'); describe('runFederationSync', () => { beforeEach(() => { jest.clearAllMocks(); + configurePeers(); }); - it('fetches nodes, imports data, and updates lastSyncAt', async () => { - const nodes = [{ url: 'https://node.example' }]; + it('inspects the full export without importing summaries or updating lastSyncAt', async () => { + const nodes = [{ nodeId: 'peer-1', url: 'https://node.example' }]; const exportData = { listings: [], transactions: [], users: [] }; mockScan.mockReturnValue({ promise: jest.fn().mockResolvedValue({ Items: nodes }) }); - axios.get.mockResolvedValue({ data: exportData }); + axios.get.mockResolvedValue(signedResponse({ data: exportData, headers: { 'x-pagination': 'cursor-v1' } })); importFederatedData.mockResolvedValue({ success: true }); mockPut.mockReturnValue({ promise: jest.fn().mockResolvedValue({}) }); await runFederationSync(); - expect(importFederatedData).toHaveBeenCalledWith(exportData); - expect(mockPut).toHaveBeenCalled(); - - const params = mockPut.mock.calls[0][0]; - expect(params.Item.url).toBe(nodes[0].url); - const timestamp = params.Item.lastSyncAt; - expect(new Date(timestamp).toISOString()).toBe(timestamp); + expect(axios.get).toHaveBeenCalledTimes(1); + expect(importFederatedData).not.toHaveBeenCalled(); + expect(mockPut).not.toHaveBeenCalled(); }); it('logs errors from axios without updating', async () => { - const nodes = [{ url: 'https://node.example' }]; + const nodes = [{ nodeId: 'peer-1', url: 'https://node.example' }]; mockScan.mockReturnValue({ promise: jest.fn().mockResolvedValue({ Items: nodes }) }); diff --git a/backend/sync/__tests__/scheduler.test.js b/backend/sync/__tests__/scheduler.test.js index 6b4d6893..dbbd9aab 100644 --- a/backend/sync/__tests__/scheduler.test.js +++ b/backend/sync/__tests__/scheduler.test.js @@ -1,3 +1,4 @@ +const { configurePeers, signedResponse } = require('../../test-support/federation-pages'); jest.mock('axios'); jest.mock('../../lib/dynamodbClient', () => ({ scan: jest.fn(), @@ -26,6 +27,7 @@ describe('scheduler', () => { beforeEach(() => { jest.resetModules(); jest.clearAllMocks(); + configurePeers(); jest.useFakeTimers(); }); @@ -33,21 +35,22 @@ describe('scheduler', () => { jest.useRealTimers(); }); - test('syncFromPeers upserts data and updates lastSyncAt', async () => { + test('syncFromPeers inspects without mutating canonical records or lastSyncAt', async () => { const dynamodbClient = require('../../lib/dynamodbClient'); dynamodbClient.scan.mockReturnValue({ - promise: () => Promise.resolve({ Items: [{ nodeUrl: 'http://node1' }] }), + promise: () => Promise.resolve({ Items: [{ nodeId: 'peer-1', nodeUrl: 'https://node.example' }] }), }); dynamodbClient.update.mockReturnValue({ promise: () => Promise.resolve() }); const axios = require('axios'); - axios.get.mockResolvedValue({ + axios.get.mockResolvedValue(signedResponse({ data: { listings: [{ _id: 'l1', updatedAt: '2024-01-01T00:00:00.000Z' }], transactions: [{ _id: 't1', updatedAt: '2024-01-01T00:00:00.000Z' }], - users: [{ _id: 'u1', updatedAt: '2024-01-01T00:00:00.000Z' }], + users: [], }, - }); + headers: { 'x-pagination': 'cursor-v1' }, + })); const listingMock = require('../../marketplace/models/listings'); const transactionMock = require('../../models/transaction'); @@ -59,17 +62,11 @@ describe('scheduler', () => { const { syncFromPeers } = require('../scheduler'); await syncFromPeers(); - expect(listingMock.create).toHaveBeenCalledTimes(1); - expect(transactionMock.create).toHaveBeenCalledTimes(1); - expect(userMock.create).toHaveBeenCalledTimes(1); - - expect(dynamodbClient.update).toHaveBeenCalledTimes(1); - const params = dynamodbClient.update.mock.calls[0][0]; - const lastSyncAt = params.ExpressionAttributeValues[':lastSyncAt']; - // Check that lastSyncAt is a valid ISO string and close to now - const lastSyncDate = new Date(lastSyncAt); - expect(lastSyncDate.toISOString()).toBe(lastSyncAt); - expect(Math.abs(Date.now() - lastSyncDate.getTime())).toBeLessThan(5000); // within 5 seconds + expect(axios.get).toHaveBeenCalledTimes(1); + expect(listingMock.create).not.toHaveBeenCalled(); + expect(transactionMock.create).not.toHaveBeenCalled(); + expect(userMock.create).not.toHaveBeenCalled(); + expect(dynamodbClient.update).not.toHaveBeenCalled(); }); test('schedules syncFromPeers at interval', () => { diff --git a/backend/sync/exportPages.js b/backend/sync/exportPages.js new file mode 100644 index 00000000..fd2278fe --- /dev/null +++ b/backend/sync/exportPages.js @@ -0,0 +1,58 @@ +const axios = require('axios'); + +const { createPeerHeaders, verifyPeerResponse } = require('../middleware/federationPeerAuth'); + +function peerCredentials(peer, expectedNodeId) { + const base = (typeof peer === 'string' ? peer : peer.url || peer.nodeUrl).replace(/\/+$/, ''); + const nodeId = typeof peer === 'string' ? expectedNodeId : peer.nodeId; + const url = new URL(base); + if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password || url.search || url.hash) throw new Error('Invalid federation peer URL'); + const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname); + if (url.protocol !== 'https:' && !loopback) throw new Error('Federation peer credentials require HTTPS'); + const localNodeId = process.env.FEDERATION_LOCAL_NODE_ID || process.env.NODE_ID; + if (!localNodeId) throw new Error('Federation local node identity is not configured'); + let keys; + try { keys = JSON.parse(process.env.FEDERATION_OUTBOUND_PEERS_JSON || '{}'); } + catch { throw new Error('Invalid FEDERATION_OUTBOUND_PEERS_JSON configuration'); } + if (!keys || typeof keys !== 'object' || Array.isArray(keys)) throw new Error('Invalid FEDERATION_OUTBOUND_PEERS_JSON configuration'); + const secret = typeof nodeId === 'string' && Object.hasOwn(keys, nodeId) && keys[nodeId]; + if (typeof secret !== 'string' || secret.length < 32) throw new Error('Federation peer has no configured HMAC credential'); + return { base, nodeId, localNodeId, secret }; +} + +// Stream pages to the consumer instead of collecting a complete peer export in +// memory. Completion means a final page under the explicit pagination contract. +async function forEachExportPage(peer, consume, expectedNodeId) { + const { base, nodeId, localNodeId, secret } = peerCredentials(peer, expectedNodeId); + const path = new URL(`${base}/federation/export`).pathname; + let cursor; + for (let pages = 0; pages < 10000; pages++) { + const response = await axios.get(`${base}/federation/export`, { + headers: createPeerHeaders({ nodeId: localNodeId, secret, method: 'GET', path }), params: { limit: 50, ...(cursor && { cursor }) }, + timeout: 5000, maxRedirects: 0, maxContentLength: 16 * 1024 * 1024, + }); + if (!verifyPeerResponse({ headers: response.headers || {}, expectedNodeId: nodeId, secret, method: 'GET', path, body: JSON.stringify(response.data) })) throw new Error('Federation peer response signature is invalid'); + const header = name => response.headers?.get?.(name) ?? response.headers?.[name]; + if (header('x-pagination') !== 'cursor-v1') throw new Error('Peer does not declare cursor pagination'); + const { data } = response; + if (!data || !Array.isArray(data.listings) || !Array.isArray(data.transactions) || !Array.isArray(data.users)) throw new Error('Invalid federation export page'); + if (data.users.length) throw new Error('Peer export contains local account records'); + const next = header('x-next-cursor'); + if (next !== undefined && (typeof next !== 'string' || !/^[\w-]+$/.test(next) || next.length > 16384 || next === cursor)) throw new Error('Invalid federation export cursor'); + await consume({ listings: data.listings, transactions: data.transactions, users: [] }); + if (!next) return; + cursor = next; + } + throw new Error('Federation export exceeded page safety limit; sync is incomplete'); +} + +async function inspectExport(peer) { + const counts = { listings: 0, transactions: 0, users: 0 }; + await forEachExportPage(peer, page => { + counts.listings += page.listings.length; + counts.transactions += page.transactions.length; + }); + return counts; +} + +module.exports = { forEachExportPage, inspectExport }; diff --git a/backend/sync/federationSyncJob.js b/backend/sync/federationSyncJob.js index 92b7f8bb..2d3ee798 100644 --- a/backend/sync/federationSyncJob.js +++ b/backend/sync/federationSyncJob.js @@ -1,18 +1,15 @@ -const axios = require("axios"); -const { getAllNodes, saveNode } = require("../models/nodeRegistry"); -const { importFederatedData } = require("./importHelper"); +const { getAllNodes } = require('../models/nodeRegistry'); +const { inspectExport } = require('./exportPages'); +// Legacy entry point: inspection only. Never pass privacy-filtered summaries +// to the unconditional canonical-record importer. async function runFederationSync() { - const nodes = await getAllNodes(); - for (const node of nodes) { + for (const node of await getAllNodes()) { try { - const { data } = await axios.get(`${node.url}/federation/export`); - await importFederatedData(data); - node.lastSyncAt = new Date().toISOString(); - await saveNode(node); - console.log(`✅ Synced with ${node.url}`); - } catch (err) { - console.error(`❌ Failed to sync with ${node.url}: ${err.message}`); + const counts = await inspectExport(node); + console.info('Federation export inspected; automatic import disabled', { counts }); + } catch (error) { + console.error('Federation export inspection failed; no sync recorded', error.message); } } } diff --git a/backend/sync/scheduler.js b/backend/sync/scheduler.js index 689d7eb8..76a2a527 100644 --- a/backend/sync/scheduler.js +++ b/backend/sync/scheduler.js @@ -1,59 +1,7 @@ -const axios = require("axios"); -const dynamodbClient = require("../lib/dynamodbClient"); -const Listing = require("../marketplace/models/listings"); -const Transaction = require("../models/transaction"); -const User = require("../models/user"); +const inspectPeers = require('./federationSyncJob'); -const NODE_REGISTRY_TABLE = "NodeRegistry"; - -const upsertMany = async (Model, items, key = "_id") => { - for (let item of items) { - const existing = await Model.findOne({ [key]: item[key] }); - if (!existing) { - await Model.create(item); - } else if (new Date(item.updatedAt) > new Date(existing.updatedAt)) { - await Model.updateOne({ [key]: item[key] }, item); - } - } -}; - -const syncFromPeers = async () => { - try { - const { Items: nodes = [] } = await dynamodbClient - .scan({ TableName: NODE_REGISTRY_TABLE }) - .promise(); - - for (let node of nodes) { - try { - const res = await axios.get(`${node.nodeUrl}/federation/export`); - const { listings, transactions, users } = res.data; - - if (listings) await upsertMany(Listing, listings); - if (transactions) await upsertMany(Transaction, transactions); - if (users) await upsertMany(User, users); - - await dynamodbClient - .update({ - TableName: NODE_REGISTRY_TABLE, - Key: { nodeUrl: node.nodeUrl }, - UpdateExpression: "set lastSyncAt = :lastSyncAt", - ExpressionAttributeValues: { - ":lastSyncAt": new Date().toISOString(), - }, - }) - .promise(); - - console.log(`✔ Synced data from node: ${node.nodeUrl}`); - } catch (err) { - console.error(`❌ Federation sync error for node ${node.nodeUrl}:`, err.message); - } - } - } catch (err) { - console.error("❌ Federation sync error:", err.message); - } -}; - -// Example scheduled interval (every 30 mins) -setInterval(syncFromPeers, 30 * 60 * 1000); +// Keep the legacy schedule, but never merge privacy-filtered export pages. +const syncFromPeers = () => inspectPeers(); +setInterval(() => syncFromPeers().catch(() => console.error('Federation registry scan failed')), 30 * 60 * 1000); module.exports = { syncFromPeers }; diff --git a/backend/test-support/federation-pages.js b/backend/test-support/federation-pages.js new file mode 100644 index 00000000..f54f8d7d --- /dev/null +++ b/backend/test-support/federation-pages.js @@ -0,0 +1,12 @@ +const { createPeerResponseHeaders } = require('../middleware/federationPeerAuth'); +const secret = '0123456789abcdef0123456789abcdef'; +function configurePeers() { + process.env.FEDERATION_LOCAL_NODE_ID = 'local-node'; + process.env.FEDERATION_OUTBOUND_PEERS_JSON = JSON.stringify({ 'peer-1': secret, 'peer-2': secret }); +} +function signedResponse(response, nodeId = 'peer-1') { + return { ...response, headers: { ...response.headers, ...createPeerResponseHeaders({ + nodeId, secret, method: 'GET', path: '/federation/export', body: JSON.stringify(response.data) + }) } }; +} +module.exports = { configurePeers, signedResponse }; diff --git a/backend/test-support/message-retry-worker.js b/backend/test-support/message-retry-worker.js index 2b31916f..db21d218 100644 --- a/backend/test-support/message-retry-worker.js +++ b/backend/test-support/message-retry-worker.js @@ -14,7 +14,14 @@ setClient({ chat: { completions: { create: async () => { const app = express(); app.use(express.json()); -app.use('/messages', require('../routes/communicationRoutes')); +if (process.env.TEST_RETRY_SERVICE === 'true') { + // API-key mode verifies the shared retry service, not private-chat access. + app.use(require('../middleware/requestRateLimit')(), require('../middleware/authMiddleware')); + app.post('/messages/:conversationId', require('../utils/asyncHandler')( + require('../controllers/communication_controller').sendMessage)); +} else { + app.use('/messages', require('../routes/communicationRoutes')); +} const server = app.listen(0, '127.0.0.1', () => { process.send({ type: 'ready', url: `http://127.0.0.1:${server.address().port}` }); }); diff --git a/frontend/chat-ui/accountClient.test.js b/frontend/chat-ui/accountClient.test.js new file mode 100644 index 00000000..969faaab --- /dev/null +++ b/frontend/chat-ui/accountClient.test.js @@ -0,0 +1,99 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); + +test('account requests carry bearer headers without putting credentials in URLs', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let received; + const client = createAccountClient({ baseUrl: 'https://backend.example', fetchImpl: async (url, options) => { + received = { url, options }; + return new Response('{"id":"owned"}', { status: 201 }); + } }); + client.setToken('account-token'); + assert.deepEqual(await client.request('POST', '/conversations', { title: 'mine' }), { id: 'owned' }); + assert.equal(received.url, 'https://backend.example/conversations'); + assert.equal(received.options.headers.Authorization, 'Bearer account-token'); + assert.equal(received.options.redirect, 'error'); + assert.equal(received.options.body, '{"title":"mine"}'); +}); + +test('clearing or changing accounts discards in-flight account responses', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let finish; + const client = createAccountClient({ fetchImpl: () => new Promise(resolve => { finish = resolve; }) }); + client.setToken('alice-token'); + const pending = client.request('GET', '/messages/private'); + client.setToken('bob-token'); + finish(new Response('[{"content":"alice secret"}]')); + await assert.rejects(pending, /Account changed/); + client.setToken(null); + assert.equal(client.getToken(), null); +}); + +test('attachment downloads use the same account and reject credential destinations outside uploads', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let received; + const client = createAccountClient({ fetchImpl: async (url, options) => { + received = { url, options }; + return new Response('private file'); + } }); + client.setToken('owner-token'); + const blob = await client.attachment('/uploads/file.txt'); + assert.equal(await blob.text(), 'private file'); + assert.equal(received.options.headers.Authorization, 'Bearer owner-token'); + for (const url of ['https://attacker.example/x', '//attacker.example/x', '/uploads/../users', '/uploads/%2e%2e/users']) { + await assert.rejects(client.attachment(url), /Invalid attachment path/); + } + client.setToken(null); + await assert.rejects(client.attachment('/uploads/file.txt'), /Sign in/); +}); + +test('collection requests follow empty pages and retain the account and query filters', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + const calls = []; + const pages = [ + new Response('[]', { headers: { 'X-Next-Cursor': 'opaque-1' } }), + new Response('[{"id":"mine"}]', { headers: { 'X-Next-Cursor': 'opaque-2' } }), + new Response('[{"id":"also-mine"}]'), + ]; + const client = createAccountClient({ fetchImpl: async (url, options) => { + calls.push({ url, options }); + return pages.shift(); + } }); + client.setToken('alice-token'); + assert.deepEqual(await client.request('GET', '/conversations?limit=1'), [{ id: 'mine' }, { id: 'also-mine' }]); + assert.deepEqual(calls.map(call => call.url), ['/conversations?limit=1', '/conversations?limit=1&cursor=opaque-1', '/conversations?limit=1&cursor=opaque-2']); + assert.ok(calls.every(call => call.options.headers.Authorization === 'Bearer alice-token' && call.options.redirect === 'error')); +}); + +test('changing accounts between pages rejects the collection instead of returning a partial old-account result', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let finish; + let calls = 0; + const client = createAccountClient({ fetchImpl: async () => { + if (++calls === 1) return new Response('[{"id":"alice-private"}]', { headers: { 'X-Next-Cursor': 'next' } }); + return new Promise(resolve => { finish = resolve; }); + } }); + client.setToken('alice-token'); + const pending = client.request('GET', '/conversations'); + for (let wait = 0; !finish && wait < 100; wait++) await new Promise(resolve => setImmediate(resolve)); + assert.equal(typeof finish, 'function', 'client must request the continuation page'); + client.setToken('bob-token'); + finish(new Response('[]')); + await assert.rejects(pending, /Account changed/); +}); + +test('collection pagination refuses a repeated cursor and surfaces later HTTP failures', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let calls = 0; + const client = createAccountClient({ fetchImpl: async () => { + calls++; + return new Response('[]', { headers: { 'X-Next-Cursor': 'same' } }); + } }); + await assert.rejects(client.request('GET', '/conversations'), /cursor/); + assert.equal(calls, 2); + let attempt = 0; + const failing = createAccountClient({ fetchImpl: async () => ++attempt === 1 + ? new Response('[{"id":"first"}]', { headers: { 'X-Next-Cursor': 'next' } }) + : new Response('{"error":"page failed"}', { status: 500 }) }); + await assert.rejects(failing.request('GET', '/conversations'), /page failed/); +}); diff --git a/frontend/chat-ui/basic.test.js b/frontend/chat-ui/basic.test.js index 85b3efc2..3784f216 100644 --- a/frontend/chat-ui/basic.test.js +++ b/frontend/chat-ui/basic.test.js @@ -1,6 +1,14 @@ const { test } = require('node:test'); const assert = require('node:assert/strict'); +test('account changes clear private chat state while retaining display preferences', async () => { + const { reducer } = await import('./src/chatReducer.mjs'); + const state = { messages: [{ content: 'private' }], conversations: [{ id: 'old' }], currentConversation: { id: 'old' }, aiThinking: true, theme: 'dark' }; + assert.deepEqual(reducer(state, { type: 'RESET_ACCOUNT' }), { + messages: [], conversations: [], currentConversation: null, aiThinking: false, theme: 'dark', accountVersion: 1, + }); +}); + test('chat streaming appends content and upsert preserves message metadata', async () => { const { reducer } = await import('./src/chatReducer.mjs'); const initial = { messages: [{ id: 'm1', content: 'Hello', from: 'ai' }] }; diff --git a/frontend/chat-ui/src/accountClient.mjs b/frontend/chat-ui/src/accountClient.mjs new file mode 100644 index 00000000..fd30ea63 --- /dev/null +++ b/frontend/chat-ui/src/accountClient.mjs @@ -0,0 +1,71 @@ +export function createAccountClient({ baseUrl = '', apiKey = '', fetchImpl = (...args) => fetch(...args) } = {}) { + let token = null; + let version = 0; + const changes = new EventTarget(); + function setToken(value) { + token = typeof value === 'string' && value ? value : null; + version++; + changes.dispatchEvent(new Event('change')); + } + function url(path) { + if (!path.startsWith('/') || path.startsWith('//')) throw new Error('Invalid API path'); + return `${baseUrl.replace(/\/$/, '')}${path}`; + } + function headers() { + const result = {}; + if (token) result.Authorization = `Bearer ${token}`; + if (apiKey) result['x-api-key'] = apiKey; + return result; + } + async function readPage(method, path, body, binary, started) { + const options = { method, headers: headers(), redirect: 'error' }; + if (body !== undefined) { + options.headers['Content-Type'] = 'application/json'; + options.body = JSON.stringify(body); + } + const response = await fetchImpl(url(path), options); + let data; + if (binary && response.ok) data = await response.blob(); + else { + const text = await response.text(); + try { data = text ? JSON.parse(text) : null; } catch { data = text; } + } + if (version !== started) throw new Error('Account changed during request'); + if (!response.ok) { + const error = new Error(data?.error || data?.message || `Request failed with status ${response.status}`); + error.status = response.status; + error.data = data; + throw error; + } + return { data, cursor: response.headers.get('X-Next-Cursor') }; + } + async function read(method, path, body, binary = false) { + const started = version; + let page = await readPage(method, path, body, binary, started); + if (method !== 'GET' || binary || !Array.isArray(page.data)) return page.data; + const items = [...page.data]; + let previous; + const [pathname, query = ''] = path.split('?'); + for (let pages = 1; page.cursor; pages++) { + if (pages >= 10000 || page.cursor === previous || !/^[\w-]+$/.test(page.cursor) || page.cursor.length > 16384) throw new Error('Invalid collection cursor'); + if (version !== started) throw new Error('Account changed during request'); + previous = page.cursor; + const params = new URLSearchParams(query); + params.set('cursor', page.cursor); + page = await readPage(method, `${pathname}?${params}`, undefined, false, started); + if (!Array.isArray(page.data)) throw new Error('Invalid collection page'); + items.push(...page.data); + } + return items; + } + async function attachment(path) { + if (!token) throw new Error('Sign in to download attachments'); + // Never forward a bearer to a path supplied by another origin or endpoint. + if (!/^\/uploads\/[^/?#\\]+$/.test(path)) throw new Error('Invalid attachment path'); + let name; + try { name = decodeURIComponent(path.slice('/uploads/'.length)); } catch { /* invalid encoding */ } + if (!name || name === '.' || name === '..' || /[/\\\0]/.test(name)) throw new Error('Invalid attachment path'); + return read('GET', path, undefined, true); + } + return { setToken, getToken: () => token, changes, url, headers, request: read, attachment }; +} diff --git a/frontend/chat-ui/src/api.js b/frontend/chat-ui/src/api.js index 76673fd0..b4c0af30 100644 --- a/frontend/chat-ui/src/api.js +++ b/frontend/chat-ui/src/api.js @@ -1,5 +1,9 @@ +import { createAccountClient } from './accountClient.mjs'; + export const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || ''; const API_KEY = import.meta.env.VITE_API_KEY || ''; +export const accountClient = createAccountClient({ baseUrl: API_BASE_URL, apiKey: API_KEY }); +export const setAccountToken = accountClient.setToken; function buildUrl(path) { const clean = path.startsWith('/') ? path : `/${path}`; @@ -7,33 +11,17 @@ function buildUrl(path) { } async function request(method, path, body) { - const url = buildUrl(path); - const headers = {}; - if (API_KEY) headers['x-api-key'] = API_KEY; - const options = { method, headers }; - if (body !== undefined) { - headers['Content-Type'] = 'application/json'; - options.body = JSON.stringify(body); - } + return accountClient.request(method, path.startsWith('/') ? path : `/${path}`, body); +} - const res = await fetch(url, options); - const text = await res.text(); - let data = null; - if (text) { - try { - data = JSON.parse(text); - } catch (err) { - // non-JSON response; preserve text in data - data = text; - } - } - if (!res.ok) { - const error = new Error(data?.message || `Request failed with status ${res.status}`); - error.status = res.status; - error.data = data; - throw error; - } - return data; +export async function downloadAttachment(file) { + const blob = await accountClient.attachment(file.path); + const objectUrl = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = objectUrl; + link.download = file.originalname || 'attachment'; + link.click(); + setTimeout(() => URL.revokeObjectURL(objectUrl), 0); } export const get = (path) => request('GET', path); diff --git a/frontend/chat-ui/src/chatReducer.mjs b/frontend/chat-ui/src/chatReducer.mjs index 163bc39b..4c30cae3 100644 --- a/frontend/chat-ui/src/chatReducer.mjs +++ b/frontend/chat-ui/src/chatReducer.mjs @@ -1,4 +1,5 @@ const handlers = { + RESET_ACCOUNT: (state) => ({ ...state, messages: [], conversations: [], currentConversation: null, aiThinking: false, accountVersion: (state.accountVersion || 0) + 1 }), ADD_MESSAGE: (state, action) => ({ ...state, messages: [...state.messages, action.message] }), UPSERT_MESSAGE(state, action) { const index = state.messages.findIndex((message) => message.id === action.message.id); diff --git a/frontend/chat-ui/src/components/MessageBubble.jsx b/frontend/chat-ui/src/components/MessageBubble.jsx index 5a629303..8d46792f 100644 --- a/frontend/chat-ui/src/components/MessageBubble.jsx +++ b/frontend/chat-ui/src/components/MessageBubble.jsx @@ -1,5 +1,6 @@ -import React from 'react'; +import React, { useState } from 'react'; import MessageActions from './MessageActions.jsx'; +import { downloadAttachment } from '../api'; /** @typedef {import('../types').Message} Message */ @@ -8,6 +9,12 @@ import MessageActions from './MessageActions.jsx'; * @param {{message: Message}} props */ export default function MessageBubble({ message, index = 0 }) { + const [downloadError, setDownloadError] = useState(''); + const download = async () => { + setDownloadError(''); + try { await downloadAttachment(message.file); } + catch (error) { setDownloadError(error.message); } + }; const isUser = message.role === 'user' || message.sender === 'user' || message.from === 'user'; const alignment = isUser ? 'items-end text-right' : 'items-start text-left'; const bubbleClass = `message-bubble ${isUser ? 'user' : 'assistant'}`; @@ -18,9 +25,9 @@ export default function MessageBubble({ message, index = 0 }) { let content; if (message.type === 'file' && message.file) { content = ( - + ); } else { const text = message.content || message.message || ''; @@ -32,6 +39,7 @@ export default function MessageBubble({ message, index = 0 }) { {!isUser && }
{content} + {downloadError &&

{downloadError}

}
); diff --git a/frontend/chat-ui/src/store.jsx b/frontend/chat-ui/src/store.jsx index 66c55e23..22ed9b26 100644 --- a/frontend/chat-ui/src/store.jsx +++ b/frontend/chat-ui/src/store.jsx @@ -1,5 +1,6 @@ import { reducer } from './chatReducer.mjs'; -import React, { createContext, useContext, useReducer } from 'react'; +import React, { createContext, useContext, useEffect, useReducer } from 'react'; +import { accountClient } from './api'; /** * @typedef {import('./types').Message} Message @@ -53,8 +54,13 @@ const StoreContext = createContext(); */ export function StoreProvider({ children }) { const [state, dispatch] = useReducer(reducer, initialState); + useEffect(() => { + const reset = () => dispatch({ type: 'RESET_ACCOUNT' }); + accountClient.changes.addEventListener('change', reset); + return () => accountClient.changes.removeEventListener('change', reset); + }, []); return ( - + {children} );