From eae4e15e2b0fb33ac2865d791f00e121bd605fbf Mon Sep 17 00:00:00 2001 From: Nirmal Khedkar Date: Mon, 26 Oct 2026 14:16:54 -0700 Subject: [PATCH 1/4] Enforce account ownership for private conversations Signed-off-by: Nirmal Khedkar --- CONFORMANCE.md | 12 ++ backend/__tests__/conversation.test.js | 4 +- backend/__tests__/conversationAccess.test.js | 171 ++++++++++++++++++ .../controllers/conversation_controller.js | 15 +- backend/docs/conversation-ownership.md | 90 +++++++++ backend/middleware/attachmentOwner.js | 19 ++ backend/middleware/chatAuth.js | 19 ++ backend/middleware/conversationOwner.js | 8 + backend/models/conversation.js | 56 ++++-- backend/models/message.js | 19 +- backend/routes/communicationRoutes.js | 7 +- backend/routes/conversationRoutes.js | 10 +- .../scripts/check-conversation-ownership.js | 56 ++++++ backend/server.js | 11 +- frontend/chat-ui/accountClient.test.js | 48 +++++ frontend/chat-ui/basic.test.js | 8 + frontend/chat-ui/src/accountClient.mjs | 53 ++++++ frontend/chat-ui/src/api.js | 40 ++-- frontend/chat-ui/src/chatReducer.mjs | 1 + .../chat-ui/src/components/MessageBubble.jsx | 14 +- frontend/chat-ui/src/store.jsx | 10 +- 21 files changed, 611 insertions(+), 60 deletions(-) create mode 100644 backend/__tests__/conversationAccess.test.js create mode 100644 backend/docs/conversation-ownership.md create mode 100644 backend/middleware/attachmentOwner.js create mode 100644 backend/middleware/chatAuth.js create mode 100644 backend/middleware/conversationOwner.js create mode 100644 backend/scripts/check-conversation-ownership.js create mode 100644 frontend/chat-ui/accountClient.test.js create mode 100644 frontend/chat-ui/src/accountClient.mjs diff --git a/CONFORMANCE.md b/CONFORMANCE.md index f3b9f0a1..a14c2a0b 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -43,3 +43,15 @@ Adopts the covenant scale (LBTAS, v2 floor) and carries its own record instance. ## Product-specific notes (last, per the ordering rule) The reference economy. Its operator/federation model is the named reference for the substrate's frontend-as-operator design target. + +Private application chat uses the verified backend account `userId` as an +immutable conversation owner. Conversation/message routes and attachment +downloads enforce that boundary; shared integration keys supply no participant +identity. Legacy ownerless rows remain inaccessible pending explicit operator +assignment. Bindings: `backend/middleware/chatAuth.js`, +`conversationOwner.js`, `attachmentOwner.js`, the conditional conversation +model writes, and `backend/__tests__/conversationAccess.test.js`. This does not +promote private chat to a sealed commons dialog or change the in-review ledger +claims above. SSE requires the companion authenticated-stream change before +rollout. See `backend/docs/conversation-ownership.md` for migration and client +integration limits. diff --git a/backend/__tests__/conversation.test.js b/backend/__tests__/conversation.test.js index da0bcaa7..2e168e0d 100644 --- a/backend/__tests__/conversation.test.js +++ b/backend/__tests__/conversation.test.js @@ -36,10 +36,10 @@ test('removing a conversation deletes its messages', async () => { Conversation.setDocClient(mockClient); Message.setDocClient(mockClient); - await mockClient.put({ TableName: 'Conversations', Item: { id: '1', title: 'seed', pinned: false, createdAt: new Date().toISOString() } }).promise(); + await mockClient.put({ TableName: 'Conversations', Item: { id: '1', ownerId: 'owner', title: 'seed', pinned: false, createdAt: new Date().toISOString() } }).promise(); await mockClient.put({ TableName: 'Messages', Item: { id: 'm1', conversationId: '1', content: 'hi' } }).promise(); - await Conversation.remove('1'); + await Conversation.remove('1', 'owner'); const msgs = await Message.listMessages('1'); assert.equal(msgs.length, 0); }); diff --git a/backend/__tests__/conversationAccess.test.js b/backend/__tests__/conversationAccess.test.js new file mode 100644 index 00000000..c1ba2644 --- /dev/null +++ b/backend/__tests__/conversationAccess.test.js @@ -0,0 +1,171 @@ +const request = require('supertest'); +const jwt = require('jsonwebtoken'); +const fs = require('fs'); +const path = require('path'); + +const mockTables = { Conversations: new Map(), Messages: new Map() }; +let mockMessageWrites = 0; +let mockMessageReads = 0; +function mockConditionalFailure() { + return Object.assign(new Error('condition failed'), { code: 'ConditionalCheckFailedException' }); +} +jest.mock('../lib/dynamodbClient', () => ({ + put: ({ TableName, Item }) => ({ promise: async () => { + if (TableName === 'Messages') mockMessageWrites++; + mockTables[TableName].set(Item.id, { ...Item }); + return {}; + } }), + get: ({ TableName, Key }) => ({ promise: async () => ({ Item: mockTables[TableName].get(Key.id) }) }), + scan: ({ TableName, ExpressionAttributeValues, ExclusiveStartKey }) => ({ promise: async () => { + let Items = [...mockTables[TableName].values()]; + // Deliberately include a continuation page with no matching items first. + if (ExclusiveStartKey) Items = Items.slice(1); + else if (Items.length > 1) Items = Items.slice(0, 1); + const LastEvaluatedKey = !ExclusiveStartKey && mockTables[TableName].size > 1 ? { id: Items[0].id } : undefined; + if (ExpressionAttributeValues?.[':ownerId']) { + Items = Items.filter(item => item.ownerId === ExpressionAttributeValues[':ownerId']); + } + if (ExpressionAttributeValues?.[':filePath']) { + Items = Items.filter(item => item.file?.path === ExpressionAttributeValues[':filePath']); + } + return { Items, LastEvaluatedKey }; + } }), + update: ({ TableName, Key, ExpressionAttributeNames, ExpressionAttributeValues, ConditionExpression }) => ({ promise: async () => { + const item = mockTables[TableName].get(Key.id); + if (!item || (ConditionExpression.includes('ownerId') && item.ownerId !== ExpressionAttributeValues[':ownerId'])) throw mockConditionalFailure(); + item[ExpressionAttributeNames['#field']] = ExpressionAttributeValues[':value']; + return { Attributes: { ...item } }; + } }), + delete: ({ TableName, Key, ConditionExpression, ExpressionAttributeValues }) => ({ promise: async () => { + const item = mockTables[TableName].get(Key.id); + if (ConditionExpression && (!item || item.ownerId !== ExpressionAttributeValues[':ownerId'])) throw mockConditionalFailure(); + mockTables[TableName].delete(Key.id); + return { Attributes: item }; + } }), + query: ({ ExpressionAttributeValues }) => ({ promise: async () => { + mockMessageReads++; + return { Items: [...mockTables.Messages.values()].filter(item => item.conversationId === ExpressionAttributeValues[':cid']) }; + } }), + batchWrite: ({ RequestItems }) => ({ promise: async () => { + for (const { DeleteRequest: { Key } } of RequestItems.Messages || []) mockTables.Messages.delete(Key.id); + return {}; + } }), +})); +jest.mock('../bull/pingJobs', () => ({ addPingJob: jest.fn() })); +jest.mock('../bull/smsQueue', () => ({ enqueueSms: jest.fn() })); +jest.mock('../federation/federationSyncJob', () => jest.fn()); +jest.mock('../services/openAIResponder', () => ({ generateResponse: async () => ({ from: 'ai', content: 'fixture reply' }) })); + +describe('private conversation HTTP boundary', () => { + let app; + const originalEnv = { ...process.env }; + const globals = Object.fromEntries(['broadcast', 'emitToken', 'emitMessage'].map(key => [key, global[key]])); + const filename = 'ownership-fixture.txt'; + const attachment = `/uploads/${filename}`; + const authorization = (userId, extra = {}) => `Bearer ${jwt.sign({ userId, ...extra }, process.env.JWT_SECRET, { expiresIn: '1h' })}`; + + beforeAll(() => { + process.env.MINIMAL_SERVER = 'false'; + process.env.API_KEY = 'conversation-test-key'; + process.env.JWT_SECRET = 'conversation-test-secret'; + ({ app } = require('../server')); + fs.writeFileSync(path.join(__dirname, '../uploads', filename), 'private attachment'); + }); + beforeEach(() => { + mockTables.Conversations.clear(); + mockTables.Messages.clear(); + mockMessageWrites = 0; + mockMessageReads = 0; + mockTables.Conversations.set('foreign', { id: 'foreign', ownerId: 'bob', title: 'Bobs private chat' }); + mockTables.Conversations.set('mine', { id: 'mine', ownerId: 'alice', title: 'My chat', pinned: false }); + mockTables.Conversations.set('legacy', { id: 'legacy', title: 'Unknown owner' }); + mockTables.Messages.set('secret', { id: 'secret', conversationId: 'foreign', content: 'private', file: { path: attachment } }); + }); + afterAll(() => { + fs.unlinkSync(path.join(__dirname, '../uploads', filename)); + process.env = originalEnv; + for (const [key, value] of Object.entries(globals)) { + if (value === undefined) delete global[key]; else global[key] = value; + } + }); + + test('creation derives immutable ownership from the verified account, ignoring supplied owner fields', async () => { + const result = await request(app).post('/conversations').set('Authorization', authorization('alice')) + .send({ title: 'New chat', ownerId: 'bob', userId: 'bob' }); + expect(result.status).toBe(201); + expect(result.body.ownerId).toBe('alice'); + expect(mockTables.Conversations.get(result.body.id).ownerId).toBe('alice'); + }); + test('listing traverses empty filtered pages without exposing foreign or legacy chats', async () => { + const result = await request(app).get('/conversations').set('Authorization', authorization('alice')); + expect(result.status).toBe(200); + expect(result.body.map(item => item.id)).toEqual(['mine']); + }); + test.each(['foreign', 'legacy', 'missing'])('all ID routes conceal %s and have no message side effects', async id => { + for (const [method, url, body] of [ + ['get', `/messages/${id}`], ['get', `/conversations/${id}`], ['put', `/conversations/${id}`, { title: 'stolen' }], + ['post', `/conversations/${id}/pin`, { pinned: true }], ['delete', `/conversations/${id}`], + ['get', `/messages/${id}`], ['post', `/messages/${id}`, { from: 'user', content: 'private AI trigger' }], + ]) { + const result = await request(app)[method](url).set('Authorization', authorization('alice')).send(body); + expect(result.status).toBe(404); + expect(result.body).toEqual({ error: 'Not found' }); + } + expect(mockMessageReads).toBe(0); + expect(mockMessageWrites).toBe(0); + expect(mockTables.Conversations.get('foreign').title).toBe('Bobs private chat'); + expect(mockTables.Messages.get('secret').content).toBe('private'); + }); + test.each([undefined, '', ' ', 7, ['alice']])('rejects a signed token without a valid userId (%p)', async userId => { + const result = await request(app).get('/conversations').set('Authorization', authorization(userId)); + expect(result.status).toBe(401); + }); + test('a shared API key cannot replace or bypass individual bearer authentication', async () => { + for (const bearer of [undefined, 'Bearer invalid']) { + let call = request(app).get('/conversations').set('x-api-key', process.env.API_KEY); + if (bearer) call = call.set('Authorization', bearer); + expect((await call).status).toBe(401); + } + const result = await request(app).get('/conversations').set('x-api-key', process.env.API_KEY) + .set('Authorization', authorization('alice')); + expect(result.status).toBe(200); + expect(result.body.map(item => item.id)).toEqual(['mine']); + }); + test('the rightful owner can read, rename, pin, send, list and delete', async () => { + const auth = authorization('alice'); + expect((await request(app).get('/conversations/mine').set('Authorization', auth)).status).toBe(200); + expect((await request(app).put('/conversations/mine').set('Authorization', auth).send({ title: 'Renamed', ownerId: 'bob' })).body.title).toBe('Renamed'); + expect((await request(app).post('/conversations/mine/pin').set('Authorization', auth).send({ pinned: true })).body.pinned).toBe(true); + const sent = await request(app).post('/messages/mine').set('Authorization', auth).send({ from: 'alice', content: 'hello', type: 'text' }); + expect(sent.status).toBe(201); + const messages = await request(app).get('/messages/mine').set('Authorization', auth); + expect(messages.body.map(item => item.content)).toEqual(['hello']); + expect((await request(app).delete('/conversations/mine').set('Authorization', auth)).status).toBe(204); + expect(mockTables.Conversations.has('mine')).toBe(false); + expect([...mockTables.Messages.values()].some(item => item.conversationId === 'mine')).toBe(false); + expect(mockTables.Conversations.get('foreign').ownerId).toBe('bob'); + }); + test('attachment paths enforce the same owner boundary, including encoded paths', async () => { + for (const url of [attachment, '/uploads/%6fwnership-fixture.txt']) { + const foreign = await request(app).get(url).set('Authorization', authorization('alice')); + expect(foreign.status).toBe(404); + const owner = await request(app).get(url).set('Authorization', authorization('bob')); + expect(owner.status).toBe(200); + expect(owner.body.toString()).toBe('private attachment'); + expect(owner.headers['cache-control']).toContain('no-store'); + expect(owner.headers['content-disposition']).toBe('attachment'); + } + expect((await request(app).get(attachment).set('x-api-key', process.env.API_KEY)).status).toBe(401); + }); + + test('conditional model writes cannot rename, pin or delete another owner or a legacy row', async () => { + const Conversation = require('../models/conversation'); + for (const id of ['foreign', 'legacy', 'missing']) { + expect(await Conversation.rename(id, 'stolen', 'alice')).toBeNull(); + expect(await Conversation.pin(id, true, 'alice')).toBeNull(); + expect(await Conversation.remove(id, 'alice')).toBe(false); + } + expect(mockTables.Messages.get('secret').content).toBe('private'); + expect(mockTables.Conversations.get('foreign').title).toBe('Bobs private chat'); + }); +}); diff --git a/backend/controllers/conversation_controller.js b/backend/controllers/conversation_controller.js index e4b0d94f..373aa894 100644 --- a/backend/controllers/conversation_controller.js +++ b/backend/controllers/conversation_controller.js @@ -2,33 +2,36 @@ const Conversation = require('../models/conversation'); exports.create = async (req, res) => { const { title } = req.body; - const convo = await Conversation.create(title); + const convo = await Conversation.create(title, req.user.userId); res.status(201).json(convo); }; -exports.list = async (_req, res) => { - const convos = await Conversation.list(); +exports.list = async (req, res) => { + const convos = await Conversation.list(req.user.userId); res.json(convos); }; +exports.get = (req, res) => res.json(req.conversation); + exports.rename = async (req, res) => { const { id } = req.params; const { title } = req.body; - const convo = await Conversation.rename(id, title); + const convo = await Conversation.rename(id, title, req.user.userId); if (!convo) return res.status(404).json({ error: 'Not found' }); res.json(convo); }; exports.remove = async (req, res) => { const { id } = req.params; - await Conversation.remove(id); + const removed = await Conversation.remove(id, req.user.userId); + if (!removed) return res.status(404).json({ error: 'Not found' }); res.status(204).end(); }; exports.pin = async (req, res) => { const { id } = req.params; const { pinned } = req.body; - const convo = await Conversation.pin(id, pinned); + const convo = await Conversation.pin(id, pinned, req.user.userId); if (!convo) return res.status(404).json({ error: 'Not found' }); res.json(convo); }; diff --git a/backend/docs/conversation-ownership.md b/backend/docs/conversation-ownership.md new file mode 100644 index 00000000..bdb7cf5b --- /dev/null +++ b/backend/docs/conversation-ownership.md @@ -0,0 +1,90 @@ +# Private conversation ownership + +## Boundary + +Private chat uses the existing backend login JWT's `userId`, verified as an +HS256 account token. A shared `API_KEY`/`WIX_API_KEY` identifies an integration, +not a participant, and grants no conversation, message or attachment access. +Supplying a valid integration key alongside a bearer never bypasses bearer +verification. No Cognito `sub`, caller-supplied owner, message sender label or +administrative role is treated as a substitute account identity. + +New Conversations rows carry immutable `ownerId`. Listing filters on that +owner across DynamoDB scan pages. Detail reads, rename, pin, delete, message +reads and sends check ownership. Rename/pin/delete also carry a conditional +owner check in DynamoDB. Foreign, missing and ownerless IDs return the same +404. A denied send reaches neither file writes, messages, responders nor +broadcasts. Attachments require a message reference belonging to an owned +conversation; responses disable caching and download as inert attachments. + +This is application-local private chat. It does not seal a JFA dialog or +modify Mycelium commitments, exchange records, reputation, escrow or federation. +The separate status of those mechanisms in CONFORMANCE.md remains accurate. + +## Existing data and deployment + +Back up Conversations and Messages before rollout. Existing rows without an +owner remain stored but inaccessible; the application never grants them to +the first caller or guesses an owner from `from`. An operator must establish +the rightful existing account through independent evidence before making an +explicit, conditional administrative backfill. This PR supplies no public +claim/transfer endpoint and performs no automatic data migration or deletion. +If ownership cannot be established, preserve the inaccessible row for the +operator's data-retention process. Ownership transfer is not supported here. + +The existing Messages deletion helper still has its previously documented +large-history batching/retry limitations. This change establishes who may +invoke deletion; it does not claim complete erasure of large histories or +physical attachment bytes. Stale attachment files become inaccessible when +their owned message/conversation reference no longer exists. + +The deployment needs DynamoDB GetItem/Scan/PutItem/UpdateItem/DeleteItem access +to the configured conversation table and the existing message access, now +including Scan for attachment-reference lookup. Scans require no new index +or table. They prioritize correct isolation; an indexed attachment lookup is +a possible measured follow-up, not supplied by this change. + +**Roll out together with the authenticated-stream companion PR (A2).** A1 +closes REST and attachment access; unauthenticated SSE remains a separate +existing path until A2 lands. Other open hardening work, including admin, +user/federation authorization and attachment write containment, remains +separate and must be assessed before exposing public signup. + +## Browser integration + +The chat UI previously sent only a shared build-time API key and has no +working account-login screen. The host application's account flow must obtain +the existing backend `/api/auth/login` response and pass its token to the +exported `setAccountToken(token)` in `frontend/chat-ui/src/api.js`. Call +`setAccountToken(null)` on logout. Tokens stay in memory; the adapter uses +Authorization headers, never query strings or browser storage. Account changes +clear chat state and discard in-flight responses from the old account. + +Do not substitute an Amplify/Cognito token without an approved backend identity +binding. PR #183 separately makes backend login/registration publicly +reachable; until it lands, login still needs the server's existing integration +authentication. This PR does not claim completed browser onboarding. + +Attachment buttons download through the authenticated adapter rather than +opening uncredentialed URLs. Malformed, external and non-upload download paths +are rejected before forwarding credentials. Redirects are rejected. + +## Verification + +`cd backend && npm test` includes production-server HTTP tests for two users, +legacy records, missing IDs, conditional mutation ownership, malformed subjects, +shared-key bypass attempts and protected attachments. AI provider calls and +background queues are isolated; account verification, mounted routes and the +authorization/model logic remain real. `cd frontend/chat-ui && npm test` +checks credential placement, account changes, attachment destinations and the +existing chat reducer. Database-backed verification uses isolated DynamoDB +Local tables, never live records. + +Run the storage check with `DYNAMODB_ENDPOINT=http://127.0.0.1:18001 node +scripts/check-conversation-ownership.js` from `backend`, after starting +DynamoDB Local on that loopback port. It creates uniquely named disposable +tables and removes them afterward. The check refuses non-loopback endpoints. + +Scope follows the [Fruitful communications overview](https://ntari.atlassian.net/wiki/spaces/Fruitful/pages/44761089) +and [Slack's instruction to use Janus and bylaws](https://ntari.slack.com/archives/C0AENHR5KG8/p1791982812690669). +Current repository invariants take precedence over older architecture summaries. diff --git a/backend/middleware/attachmentOwner.js b/backend/middleware/attachmentOwner.js new file mode 100644 index 00000000..bde65905 --- /dev/null +++ b/backend/middleware/attachmentOwner.js @@ -0,0 +1,19 @@ +const Message = require('../models/message'); +const Conversation = require('../models/conversation'); + +module.exports = async (req, res, next) => { + let filename; + try { filename = decodeURIComponent(req.path).slice(1); } catch { /* invalid URL */ } + if (!filename || filename.includes('/') || filename.includes('\\') || filename.includes('\0') || filename === '.' || filename === '..') { + return res.status(404).json({ error: 'Not found' }); + } + const messages = await Message.findByAttachmentPath(`/uploads/${filename}`); + for (const message of messages) { + if (await Conversation.getOwned(message.conversationId, req.user.userId)) { + // Private response must not be cached across accounts after authorization. + res.set('Cache-Control', 'private, no-store'); + return next(); + } + } + return res.status(404).json({ error: 'Not found' }); +}; diff --git a/backend/middleware/chatAuth.js b/backend/middleware/chatAuth.js new file mode 100644 index 00000000..7dbe46c7 --- /dev/null +++ b/backend/middleware/chatAuth.js @@ -0,0 +1,19 @@ +const jwt = require('jsonwebtoken'); + +// A shared integration key authenticates no individual participant. Verify the +// bearer independently even when the general API middleware accepted that key. +module.exports = (req, res, next) => { + const header = req.headers.authorization; + if (!header || !header.startsWith('Bearer ')) { + return res.status(401).json({ error: 'Account bearer token required' }); + } + try { + const payload = jwt.verify(header.slice(7), process.env.JWT_SECRET, { algorithms: ['HS256'] }); + if (typeof payload.userId !== 'string' || !payload.userId.trim()) throw new Error('Missing account'); + req.user = payload; + res.set('Cache-Control', 'private, no-store'); + return next(); + } catch { + return res.status(401).json({ error: 'Unauthorized: Invalid Token' }); + } +}; diff --git a/backend/middleware/conversationOwner.js b/backend/middleware/conversationOwner.js new file mode 100644 index 00000000..5338198b --- /dev/null +++ b/backend/middleware/conversationOwner.js @@ -0,0 +1,8 @@ +const Conversation = require('../models/conversation'); + +module.exports = async (req, res, next) => { + const conversation = await Conversation.getOwned(req.params.conversationId || req.params.id, req.user.userId); + if (!conversation) return res.status(404).json({ error: 'Not found' }); + req.conversation = conversation; + return next(); +}; diff --git a/backend/models/conversation.js b/backend/models/conversation.js index 79d5080c..c0a28ef0 100644 --- a/backend/models/conversation.js +++ b/backend/models/conversation.js @@ -4,9 +4,11 @@ const Message = require('./message'); const CONVERSATION_TABLE_NAME = process.env.CONVERSATION_TABLE_NAME || 'Conversations'; -async function create(title = 'New Chat') { +async function create(title = 'New Chat', ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) throw new Error('Conversation owner required'); const convo = { id: randomUUID(), + ownerId, title, pinned: false, createdAt: new Date().toISOString(), @@ -15,12 +17,31 @@ async function create(title = 'New Chat') { return convo; } -async function list() { - const res = await docClient.scan({ TableName: CONVERSATION_TABLE_NAME }).promise(); - return res.Items || []; +async function list(ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) return []; + const items = []; + let cursor; + do { + const res = await docClient.scan({ + TableName: CONVERSATION_TABLE_NAME, + FilterExpression: 'ownerId = :ownerId', + ExpressionAttributeValues: { ':ownerId': ownerId }, + ...(cursor && { ExclusiveStartKey: cursor }), + }).promise(); + items.push(...(res.Items || [])); + cursor = res.LastEvaluatedKey; + } while (cursor); + return items; } -async function update(id, field, value) { +async function getOwned(id, ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) return null; + const { Item } = await docClient.get({ TableName: CONVERSATION_TABLE_NAME, Key: { id }, ConsistentRead: true }).promise(); + return Item && Item.ownerId === ownerId ? Item : null; +} + +async function update(id, field, value, ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) return null; try { const res = await docClient .update({ @@ -28,9 +49,9 @@ async function update(id, field, value) { Key: { id }, UpdateExpression: 'SET #field = :value', ExpressionAttributeNames: { '#field': field }, - ExpressionAttributeValues: { ':value': value }, + ExpressionAttributeValues: { ':value': value, ':ownerId': ownerId }, ReturnValues: 'ALL_NEW', - ConditionExpression: 'attribute_exists(id)', + ConditionExpression: 'attribute_exists(id) AND ownerId = :ownerId', }) .promise(); return res.Attributes; @@ -40,17 +61,28 @@ async function update(id, field, value) { } } -function rename(id, title) { return update(id, 'title', title); } +function rename(id, title, ownerId) { return update(id, 'title', title, ownerId); } -async function remove(id) { - await docClient.delete({ TableName: CONVERSATION_TABLE_NAME, Key: { id } }).promise(); +async function remove(id, ownerId) { + if (typeof ownerId !== 'string' || !ownerId.trim()) return false; + try { + await docClient.delete({ + TableName: CONVERSATION_TABLE_NAME, Key: { id }, + ConditionExpression: 'attribute_exists(id) AND ownerId = :ownerId', + ExpressionAttributeValues: { ':ownerId': ownerId }, + }).promise(); + } catch (err) { + if (err.code === 'ConditionalCheckFailedException') return false; + throw err; + } await Message.removeByConversation(id); + return true; } -function pin(id, pinned) { return update(id, 'pinned', pinned); } +function pin(id, pinned, ownerId) { return update(id, 'pinned', pinned, ownerId); } function setDocClient(client) { docClient = client; } -module.exports = { create, list, rename, remove, pin, setDocClient }; +module.exports = { create, list, getOwned, rename, remove, pin, setDocClient }; diff --git a/backend/models/message.js b/backend/models/message.js index 3f0f0353..f37aafa5 100644 --- a/backend/models/message.js +++ b/backend/models/message.js @@ -66,4 +66,21 @@ function setDocClient(client) { docClient = client; } -module.exports = { sendMessage, listMessages, removeByConversation, setDocClient }; \ No newline at end of file +async function findByAttachmentPath(filePath) { + const items = []; + let cursor; + do { + const page = await docClient.scan({ + TableName: MESSAGE_TABLE_NAME, + FilterExpression: '#file.#path = :filePath', + ExpressionAttributeNames: { '#file': 'file', '#path': 'path' }, + ExpressionAttributeValues: { ':filePath': filePath }, + ...(cursor && { ExclusiveStartKey: cursor }), + }).promise(); + items.push(...(page.Items || [])); + cursor = page.LastEvaluatedKey; + } while (cursor); + return items; +} + +module.exports = { sendMessage, listMessages, removeByConversation, findByAttachmentPath, setDocClient }; diff --git a/backend/routes/communicationRoutes.js b/backend/routes/communicationRoutes.js index 3a0367c2..675bce23 100644 --- a/backend/routes/communicationRoutes.js +++ b/backend/routes/communicationRoutes.js @@ -1,11 +1,12 @@ const express = require('express'); const router = express.Router(); const commController = require('../controllers/communication_controller'); -const auth = require('../middleware/authMiddleware'); +const auth = require('../middleware/chatAuth'); +const owner = require('../middleware/conversationOwner'); const asyncHandler = require('../utils/asyncHandler'); router.use(auth); -router.post('/:conversationId', asyncHandler(commController.sendMessage)); -router.get('/:conversationId', asyncHandler(commController.listMessages)); +router.post('/:conversationId', asyncHandler(owner), asyncHandler(commController.sendMessage)); +router.get('/:conversationId', asyncHandler(owner), asyncHandler(commController.listMessages)); module.exports = router; diff --git a/backend/routes/conversationRoutes.js b/backend/routes/conversationRoutes.js index 49739001..d2880d73 100644 --- a/backend/routes/conversationRoutes.js +++ b/backend/routes/conversationRoutes.js @@ -1,14 +1,16 @@ const express = require('express'); const router = express.Router(); -const auth = require('../middleware/authMiddleware'); +const auth = require('../middleware/chatAuth'); +const owner = require('../middleware/conversationOwner'); const ctrl = require('../controllers/conversation_controller'); const asyncHandler = require('../utils/asyncHandler'); router.use(auth); router.post('/', asyncHandler(ctrl.create)); router.get('/', asyncHandler(ctrl.list)); -router.put('/:id', asyncHandler(ctrl.rename)); -router.delete('/:id', asyncHandler(ctrl.remove)); -router.post('/:id/pin', asyncHandler(ctrl.pin)); +router.get('/:id', asyncHandler(owner), ctrl.get); +router.put('/:id', asyncHandler(owner), asyncHandler(ctrl.rename)); +router.delete('/:id', asyncHandler(owner), asyncHandler(ctrl.remove)); +router.post('/:id/pin', asyncHandler(owner), asyncHandler(ctrl.pin)); module.exports = router; diff --git a/backend/scripts/check-conversation-ownership.js b/backend/scripts/check-conversation-ownership.js new file mode 100644 index 00000000..61ba60b9 --- /dev/null +++ b/backend/scripts/check-conversation-ownership.js @@ -0,0 +1,56 @@ +// Explicit integration check: creates and removes isolated DynamoDB Local tables. +const assert = require('node:assert/strict'); +const { randomUUID } = require('node:crypto'); +const { DynamoDBClient, CreateTableCommand, DeleteTableCommand } = require('@aws-sdk/client-dynamodb'); + +async function main() { + const endpoint = process.env.DYNAMODB_ENDPOINT; + const parsed = new URL(endpoint); + if (!['127.0.0.1', 'localhost', '[::1]'].includes(parsed.hostname)) throw new Error('DynamoDB Local loopback endpoint required'); + process.env.AWS_ACCESS_KEY_ID = 'localcheck'; + process.env.AWS_SECRET_ACCESS_KEY = 'localcheck'; + process.env.AWS_REGION = 'us-east-1'; + const suffix = randomUUID().replaceAll('-', ''); + process.env.CONVERSATION_TABLE_NAME = `ChatOwnership_${suffix}`; + process.env.MESSAGE_TABLE_NAME = `ChatMessages_${suffix}`; + const admin = new DynamoDBClient({ endpoint, region: 'us-east-1', credentials: { accessKeyId: 'localcheck', secretAccessKey: 'localcheck' } }); + const tables = []; + try { + for (const [TableName, keys] of [ + [process.env.CONVERSATION_TABLE_NAME, [{ AttributeName: 'id', KeyType: 'HASH' }]], + [process.env.MESSAGE_TABLE_NAME, [{ AttributeName: 'conversationId', KeyType: 'HASH' }, { AttributeName: 'id', KeyType: 'RANGE' }]], + ]) { + await admin.send(new CreateTableCommand({ + TableName, KeySchema: keys, AttributeDefinitions: keys.map(({ AttributeName }) => ({ AttributeName, AttributeType: 'S' })), BillingMode: 'PAY_PER_REQUEST', + })); + tables.push(TableName); + } + const Conversation = require('../models/conversation'); + const Message = require('../models/message'); + const db = require('../lib/dynamodbClient'); + const mine = await Conversation.create('private', 'alice'); + await Conversation.create('foreign', 'bob'); + await db.put({ TableName: process.env.CONVERSATION_TABLE_NAME, Item: { id: 'legacy', title: 'unknown owner' } }).promise(); + assert.equal(await Conversation.getOwned(mine.id, 'bob'), null); + assert.equal(await Conversation.getOwned('legacy', 'alice'), null); + assert.equal(await Conversation.rename(mine.id, 'stolen', 'bob'), null); + assert.equal(await Conversation.pin(mine.id, true, 'bob'), null); + assert.equal(await Conversation.remove(mine.id, 'bob'), false); + assert.equal((await Conversation.rename(mine.id, 'renamed', 'alice')).title, 'renamed'); + assert.equal((await Conversation.pin(mine.id, true, 'alice')).pinned, true); + for (let i = 0; i < 6; i++) await Conversation.create('x'.repeat(230000), i % 2 ? 'alice' : 'bob'); + assert.equal((await Conversation.list('alice')).length, 4); + const message = await Message.sendMessage(mine.id, { from: 'alice', content: 'private content' }); + assert.equal((await Message.listMessages(mine.id))[0].id, message.id); + await db.put({ TableName: process.env.MESSAGE_TABLE_NAME, Item: { id: 'attachment', conversationId: mine.id, file: { path: '/uploads/example.txt' } } }).promise(); + assert.equal((await Message.findByAttachmentPath('/uploads/example.txt'))[0].id, 'attachment'); + assert.equal(await Conversation.remove(mine.id, 'alice'), true); + assert.deepEqual(await Message.listMessages(mine.id), []); + assert.equal(await Conversation.getOwned(mine.id, 'alice'), null); + console.log('PASS: real DynamoDB owner conditions, legacy isolation, multi-page owner listing, message lookup and deletion'); + } finally { + for (const TableName of tables) await admin.send(new DeleteTableCommand({ TableName })); + admin.destroy(); + } +} +main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/backend/server.js b/backend/server.js index 6623eac1..f127e30e 100644 --- a/backend/server.js +++ b/backend/server.js @@ -14,6 +14,8 @@ try { } const path = require("path"); const authMiddleware = require("./middleware/authMiddleware"); +const chatAuth = require('./middleware/chatAuth'); +const asyncHandler = require('./utils/asyncHandler'); // Load environment variables dotenv.config(); @@ -130,7 +132,14 @@ global.emitMessage = emitMessage; // Middleware app.use(authMiddleware); -app.use('/uploads', express.static(path.join(__dirname, 'uploads'))); +app.use('/uploads', chatAuth, asyncHandler(require('./middleware/attachmentOwner')), express.static(path.join(__dirname, 'uploads'), { + dotfiles: 'deny', index: false, + setHeaders(res) { + res.setHeader('Content-Disposition', 'attachment'); + res.setHeader('Content-Type', 'application/octet-stream'); + res.setHeader('X-Content-Type-Options', 'nosniff'); + }, +})); // Routes let runFederationSync; diff --git a/frontend/chat-ui/accountClient.test.js b/frontend/chat-ui/accountClient.test.js new file mode 100644 index 00000000..59197fd0 --- /dev/null +++ b/frontend/chat-ui/accountClient.test.js @@ -0,0 +1,48 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); + +test('account requests carry bearer headers without putting credentials in URLs', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let received; + const client = createAccountClient({ baseUrl: 'https://backend.example', fetchImpl: async (url, options) => { + received = { url, options }; + return new Response('{"id":"owned"}', { status: 201 }); + } }); + client.setToken('account-token'); + assert.deepEqual(await client.request('POST', '/conversations', { title: 'mine' }), { id: 'owned' }); + assert.equal(received.url, 'https://backend.example/conversations'); + assert.equal(received.options.headers.Authorization, 'Bearer account-token'); + assert.equal(received.options.redirect, 'error'); + assert.equal(received.options.body, '{"title":"mine"}'); +}); + +test('clearing or changing accounts discards in-flight account responses', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let finish; + const client = createAccountClient({ fetchImpl: () => new Promise(resolve => { finish = resolve; }) }); + client.setToken('alice-token'); + const pending = client.request('GET', '/messages/private'); + client.setToken('bob-token'); + finish(new Response('[{"content":"alice secret"}]')); + await assert.rejects(pending, /Account changed/); + client.setToken(null); + assert.equal(client.getToken(), null); +}); + +test('attachment downloads use the same account and reject credential destinations outside uploads', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let received; + const client = createAccountClient({ fetchImpl: async (url, options) => { + received = { url, options }; + return new Response('private file'); + } }); + client.setToken('owner-token'); + const blob = await client.attachment('/uploads/file.txt'); + assert.equal(await blob.text(), 'private file'); + assert.equal(received.options.headers.Authorization, 'Bearer owner-token'); + for (const url of ['https://attacker.example/x', '//attacker.example/x', '/uploads/../users', '/uploads/%2e%2e/users']) { + await assert.rejects(client.attachment(url), /Invalid attachment path/); + } + client.setToken(null); + await assert.rejects(client.attachment('/uploads/file.txt'), /Sign in/); +}); diff --git a/frontend/chat-ui/basic.test.js b/frontend/chat-ui/basic.test.js index 85b3efc2..3784f216 100644 --- a/frontend/chat-ui/basic.test.js +++ b/frontend/chat-ui/basic.test.js @@ -1,6 +1,14 @@ const { test } = require('node:test'); const assert = require('node:assert/strict'); +test('account changes clear private chat state while retaining display preferences', async () => { + const { reducer } = await import('./src/chatReducer.mjs'); + const state = { messages: [{ content: 'private' }], conversations: [{ id: 'old' }], currentConversation: { id: 'old' }, aiThinking: true, theme: 'dark' }; + assert.deepEqual(reducer(state, { type: 'RESET_ACCOUNT' }), { + messages: [], conversations: [], currentConversation: null, aiThinking: false, theme: 'dark', accountVersion: 1, + }); +}); + test('chat streaming appends content and upsert preserves message metadata', async () => { const { reducer } = await import('./src/chatReducer.mjs'); const initial = { messages: [{ id: 'm1', content: 'Hello', from: 'ai' }] }; diff --git a/frontend/chat-ui/src/accountClient.mjs b/frontend/chat-ui/src/accountClient.mjs new file mode 100644 index 00000000..0c638415 --- /dev/null +++ b/frontend/chat-ui/src/accountClient.mjs @@ -0,0 +1,53 @@ +export function createAccountClient({ baseUrl = '', apiKey = '', fetchImpl = (...args) => fetch(...args) } = {}) { + let token = null; + let version = 0; + const changes = new EventTarget(); + function setToken(value) { + token = typeof value === 'string' && value ? value : null; + version++; + changes.dispatchEvent(new Event('change')); + } + function url(path) { + if (!path.startsWith('/') || path.startsWith('//')) throw new Error('Invalid API path'); + return `${baseUrl.replace(/\/$/, '')}${path}`; + } + function headers() { + const result = {}; + if (token) result.Authorization = `Bearer ${token}`; + if (apiKey) result['x-api-key'] = apiKey; + return result; + } + async function read(method, path, body, binary = false) { + const started = version; + const options = { method, headers: headers(), redirect: 'error' }; + if (body !== undefined) { + options.headers['Content-Type'] = 'application/json'; + options.body = JSON.stringify(body); + } + const response = await fetchImpl(url(path), options); + let data; + if (binary && response.ok) data = await response.blob(); + else { + const text = await response.text(); + try { data = text ? JSON.parse(text) : null; } catch { data = text; } + } + if (version !== started) throw new Error('Account changed during request'); + if (!response.ok) { + const error = new Error(data?.error || data?.message || `Request failed with status ${response.status}`); + error.status = response.status; + error.data = data; + throw error; + } + return data; + } + async function attachment(path) { + if (!token) throw new Error('Sign in to download attachments'); + // Never forward a bearer to a path supplied by another origin or endpoint. + if (!/^\/uploads\/[^/?#\\]+$/.test(path)) throw new Error('Invalid attachment path'); + let name; + try { name = decodeURIComponent(path.slice('/uploads/'.length)); } catch { /* invalid encoding */ } + if (!name || name === '.' || name === '..' || /[/\\\0]/.test(name)) throw new Error('Invalid attachment path'); + return read('GET', path, undefined, true); + } + return { setToken, getToken: () => token, changes, url, headers, request: read, attachment }; +} diff --git a/frontend/chat-ui/src/api.js b/frontend/chat-ui/src/api.js index 76673fd0..b4c0af30 100644 --- a/frontend/chat-ui/src/api.js +++ b/frontend/chat-ui/src/api.js @@ -1,5 +1,9 @@ +import { createAccountClient } from './accountClient.mjs'; + export const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || ''; const API_KEY = import.meta.env.VITE_API_KEY || ''; +export const accountClient = createAccountClient({ baseUrl: API_BASE_URL, apiKey: API_KEY }); +export const setAccountToken = accountClient.setToken; function buildUrl(path) { const clean = path.startsWith('/') ? path : `/${path}`; @@ -7,33 +11,17 @@ function buildUrl(path) { } async function request(method, path, body) { - const url = buildUrl(path); - const headers = {}; - if (API_KEY) headers['x-api-key'] = API_KEY; - const options = { method, headers }; - if (body !== undefined) { - headers['Content-Type'] = 'application/json'; - options.body = JSON.stringify(body); - } + return accountClient.request(method, path.startsWith('/') ? path : `/${path}`, body); +} - const res = await fetch(url, options); - const text = await res.text(); - let data = null; - if (text) { - try { - data = JSON.parse(text); - } catch (err) { - // non-JSON response; preserve text in data - data = text; - } - } - if (!res.ok) { - const error = new Error(data?.message || `Request failed with status ${res.status}`); - error.status = res.status; - error.data = data; - throw error; - } - return data; +export async function downloadAttachment(file) { + const blob = await accountClient.attachment(file.path); + const objectUrl = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = objectUrl; + link.download = file.originalname || 'attachment'; + link.click(); + setTimeout(() => URL.revokeObjectURL(objectUrl), 0); } export const get = (path) => request('GET', path); diff --git a/frontend/chat-ui/src/chatReducer.mjs b/frontend/chat-ui/src/chatReducer.mjs index 163bc39b..4c30cae3 100644 --- a/frontend/chat-ui/src/chatReducer.mjs +++ b/frontend/chat-ui/src/chatReducer.mjs @@ -1,4 +1,5 @@ const handlers = { + RESET_ACCOUNT: (state) => ({ ...state, messages: [], conversations: [], currentConversation: null, aiThinking: false, accountVersion: (state.accountVersion || 0) + 1 }), ADD_MESSAGE: (state, action) => ({ ...state, messages: [...state.messages, action.message] }), UPSERT_MESSAGE(state, action) { const index = state.messages.findIndex((message) => message.id === action.message.id); diff --git a/frontend/chat-ui/src/components/MessageBubble.jsx b/frontend/chat-ui/src/components/MessageBubble.jsx index 5a629303..8d46792f 100644 --- a/frontend/chat-ui/src/components/MessageBubble.jsx +++ b/frontend/chat-ui/src/components/MessageBubble.jsx @@ -1,5 +1,6 @@ -import React from 'react'; +import React, { useState } from 'react'; import MessageActions from './MessageActions.jsx'; +import { downloadAttachment } from '../api'; /** @typedef {import('../types').Message} Message */ @@ -8,6 +9,12 @@ import MessageActions from './MessageActions.jsx'; * @param {{message: Message}} props */ export default function MessageBubble({ message, index = 0 }) { + const [downloadError, setDownloadError] = useState(''); + const download = async () => { + setDownloadError(''); + try { await downloadAttachment(message.file); } + catch (error) { setDownloadError(error.message); } + }; const isUser = message.role === 'user' || message.sender === 'user' || message.from === 'user'; const alignment = isUser ? 'items-end text-right' : 'items-start text-left'; const bubbleClass = `message-bubble ${isUser ? 'user' : 'assistant'}`; @@ -18,9 +25,9 @@ export default function MessageBubble({ message, index = 0 }) { let content; if (message.type === 'file' && message.file) { content = ( - + ); } else { const text = message.content || message.message || ''; @@ -32,6 +39,7 @@ export default function MessageBubble({ message, index = 0 }) { {!isUser && }
{content} + {downloadError &&

{downloadError}

}
); diff --git a/frontend/chat-ui/src/store.jsx b/frontend/chat-ui/src/store.jsx index 66c55e23..22ed9b26 100644 --- a/frontend/chat-ui/src/store.jsx +++ b/frontend/chat-ui/src/store.jsx @@ -1,5 +1,6 @@ import { reducer } from './chatReducer.mjs'; -import React, { createContext, useContext, useReducer } from 'react'; +import React, { createContext, useContext, useEffect, useReducer } from 'react'; +import { accountClient } from './api'; /** * @typedef {import('./types').Message} Message @@ -53,8 +54,13 @@ const StoreContext = createContext(); */ export function StoreProvider({ children }) { const [state, dispatch] = useReducer(reducer, initialState); + useEffect(() => { + const reset = () => dispatch({ type: 'RESET_ACCOUNT' }); + accountClient.changes.addEventListener('change', reset); + return () => accountClient.changes.removeEventListener('change', reset); + }, []); return ( - + {children} ); From 4c58c3240377ac7faaa039733dfdc3a17c0cd901 Mon Sep 17 00:00:00 2001 From: Nirmal Khedkar Date: Tue, 27 Oct 2026 15:57:00 -0700 Subject: [PATCH 2/4] Page collection reads with scoped privacy boundaries Bound conversation, directory, and federation scans with encrypted continuation tokens. Keep owner and sharing filters on every page, and make bundled clients follow continuation explicitly. Use a separate read-only export credential and explicit peer configuration. Exclude local accounts and narrative, and disable automatic merging of these summaries until source-aware reconciliation is defined. Document rollout dependencies and compatibility changes. Signed-off-by: Nirmal Khedkar --- .github/workflows/collection-pagination.yml | 48 +++++ CONFORMANCE.md | 19 ++ .../__tests__/collectionPagination.test.js | 189 ++++++++++++++++++ backend/__tests__/conversationAccess.test.js | 20 +- .../controllers/conversation_controller.js | 9 +- backend/docs/collection-pagination.md | 159 +++++++++++++++ .../__tests__/federationSyncJob.test.js | 83 ++++---- backend/federation/exportAuth.js | 13 ++ backend/federation/exportFields.js | 18 ++ backend/federation/federationRoutes.js | 34 +++- backend/federation/federationSyncJob.js | 60 ++---- backend/federation/models/nodeRegistry.js | 6 +- backend/lib/collectionPage.js | 80 ++++++++ backend/middleware/authMiddleware.js | 2 + backend/models/conversation.js | 12 +- .../routes/__tests__/federationStatus.test.js | 17 +- backend/routes/__tests__/userRoutes.test.js | 12 +- backend/routes/federationRoutes.js | 44 +--- backend/routes/federationStatus.js | 60 +++--- backend/routes/userRoutes.js | 63 +++--- .../scripts/check-collection-pagination.js | 120 +++++++++++ backend/server.js | 3 +- backend/sync/__tests__/exportPages.test.js | 59 ++++++ .../sync/__tests__/federationSyncJob.test.js | 15 +- backend/sync/__tests__/scheduler.test.js | 24 +-- backend/sync/exportPages.js | 52 +++++ backend/sync/federationSyncJob.js | 21 +- backend/sync/scheduler.js | 60 +----- frontend/chat-ui/accountClient.test.js | 51 +++++ frontend/chat-ui/src/accountClient.mjs | 24 ++- 30 files changed, 1064 insertions(+), 313 deletions(-) create mode 100644 .github/workflows/collection-pagination.yml create mode 100644 backend/__tests__/collectionPagination.test.js create mode 100644 backend/docs/collection-pagination.md create mode 100644 backend/federation/exportAuth.js create mode 100644 backend/federation/exportFields.js create mode 100644 backend/lib/collectionPage.js create mode 100644 backend/scripts/check-collection-pagination.js create mode 100644 backend/sync/__tests__/exportPages.test.js create mode 100644 backend/sync/exportPages.js diff --git a/.github/workflows/collection-pagination.yml b/.github/workflows/collection-pagination.yml new file mode 100644 index 00000000..50883197 --- /dev/null +++ b/.github/workflows/collection-pagination.yml @@ -0,0 +1,48 @@ +name: Collection pagination regression + +on: + pull_request: + paths: + - 'backend/**' + - 'frontend/chat-ui/**' + - '.github/workflows/collection-pagination.yml' + +permissions: + contents: read + +jobs: + collection-pagination: + runs-on: ubuntu-latest + services: + dynamodb: + image: amazon/dynamodb-local@sha256:6ada631ec7aa5fc83446855eb17233e3dad184fec47477e8140c6c569cfe0c86 + ports: + - 18001:8000 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: backend/package-lock.json + - run: npm ci + working-directory: backend + - run: npm test + working-directory: backend + - run: npm ci + working-directory: frontend/chat-ui + - run: npm test + working-directory: frontend/chat-ui + - run: npm run build + working-directory: frontend/chat-ui + - name: Wait for DynamoDB Local + run: | + for attempt in $(seq 1 30); do + if curl --silent --output /dev/null http://127.0.0.1:18001; then exit 0; fi + sleep 1 + done + exit 1 + - run: node scripts/check-collection-pagination.js + working-directory: backend + env: + DYNAMODB_ENDPOINT: http://127.0.0.1:18001 diff --git a/CONFORMANCE.md b/CONFORMANCE.md index a14c2a0b..ff53e492 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -42,6 +42,25 @@ Adopts the covenant scale (LBTAS, v2 floor) and carries its own record instance. ## Product-specific notes (last, per the ordering rule) +### Bounded application reads (A16) + +Conversation pages retain the account/owner boundary documented in +`backend/docs/conversation-ownership.md`. The user directory allowlists local +public fields and sharing-opted-in coarse location; it is not an account +export. Cursors encrypt private scan positions and bind the account/query. +`backend/__tests__/collectionPagination.test.js` and the real DynamoDB check +`backend/scripts/check-collection-pagination.js` exercise these boundaries. + +Federation export is an explicitly read-only application view, using a +separate read-only credential. Account rows, local dialog narrative and +scalar ratings are omitted. Scheduled paths inspect pages without importing +summaries, rewriting canonical records or advancing synchronization time. +Source provenance/conflict policy and safe manual import remain open work; +no witnessed-federation or full-stack privacy conformance is asserted. See +`backend/docs/collection-pagination.md` for exact omissions, compatibility +changes, configuration, rollout dependencies and residuals. Existing record, +covenant and economy status claims above remain unchanged. + The reference economy. Its operator/federation model is the named reference for the substrate's frontend-as-operator design target. Private application chat uses the verified backend account `userId` as an diff --git a/backend/__tests__/collectionPagination.test.js b/backend/__tests__/collectionPagination.test.js new file mode 100644 index 00000000..db2d021f --- /dev/null +++ b/backend/__tests__/collectionPagination.test.js @@ -0,0 +1,189 @@ +const request = require('supertest'); +const jwt = require('jsonwebtoken'); + +const mockTables = {}; +let mockScans = []; +jest.mock('../lib/dynamodbClient', () => ({ + scan: params => ({ promise: async () => { + mockScans.push(params); + const rows = mockTables[params.TableName] || []; + const start = params.ExclusiveStartKey ? rows.findIndex(row => row.id === params.ExclusiveStartKey.id) + 1 : 0; + const evaluated = rows.slice(start, start + (params.Limit || rows.length)); + let Items = evaluated; + const values = params.ExpressionAttributeValues || {}; + if (values[':ownerId']) Items = Items.filter(row => row.ownerId === values[':ownerId']); + if (values[':role']) Items = Items.filter(row => row.role === values[':role']); + if (values[':location']) Items = Items.filter(row => row.location === values[':location'] && row.locationPrivacy === true); + // Return complete records deliberately: response DTOs must also protect + // against unexpected fields, independently of the database projection. + return { Items, ...(start + evaluated.length < rows.length && { + LastEvaluatedKey: { id: evaluated.at(-1).id }, + }) }; + } }), +})); +jest.mock('../bull/pingJobs', () => ({ addPingJob: jest.fn() })); +jest.mock('../bull/smsQueue', () => ({ enqueueSms: jest.fn() })); +jest.mock('../federation/federationSyncJob', () => jest.fn()); + +describe('bounded collection HTTP responses', () => { + let app; + const env = { ...process.env }; + const authorization = userId => `Bearer ${jwt.sign({ userId }, process.env.JWT_SECRET, { expiresIn: '2h' })}`; + beforeAll(() => { + process.env.MINIMAL_SERVER = 'false'; + process.env.JWT_SECRET = 'collection-test-secret'; + process.env.API_KEY = 'collection-operator-key'; + process.env.FEDERATION_EXPORT_KEY = 'collection-read-only-export-key'; + ({ app } = require('../server')); + }); + afterAll(() => { process.env = env; }); + beforeEach(() => { + mockScans = []; + mockTables.Conversations = [ + { id: 'bobs-secret-id', ownerId: 'bob', title: 'private' }, + { id: 'mine-1', ownerId: 'alice', title: 'first' }, + { id: 'legacy', title: 'unowned' }, + { id: 'mine-2', ownerId: 'alice', title: 'second' }, + ]; + mockTables.Users = [ + { id: 'u1', username: 'grower', role: 'producer', location: 'Town', locationPrivacy: false, password: 'hash', verificationCode: 'code', email: 'private@example.invalid', phone: 'private', coordinates: { lat: 1, long: 2 }, futureSecret: 'secret' }, + { id: 'u2', username: 'buyer', role: 'consumer', location: 'Town', locationPrivacy: true }, + { id: 'u3', username: 'seller', role: 'producer', location: 'Town', locationPrivacy: true }, + ]; + mockTables.Listings = [{ id: 'l1', userId: 'u1', title: 'Public listing', price: 2, futureSecret: 'not a listing field' }, { id: 'l2', userId: 'u2', title: 'Second' }]; + mockTables.Transactions = [{ id: 't1', consumerId: 'u1', producerId: 'u2', status: 'pending', dialogNotes: 'private narrative', futureSecret: 'secret' }]; + }); + + async function collect(path, header, value) { + const pages = []; + let cursor; + do { + const result = await request(app).get(path + (cursor ? `&cursor=${encodeURIComponent(cursor)}` : '')).set(header, value); + expect(result.status).toBe(200); + expect(result.headers['x-pagination']).toBe('cursor-v1'); + expect(result.headers['cache-control']).toContain('no-store'); + pages.push(result.body); + cursor = result.headers['x-next-cursor']; + expect(pages.length).toBeLessThan(10); + } while (cursor); + return pages; + } + + test('conversation pages include empty filtered pages but never foreign or ownerless rows', async () => { + const pages = await collect('/conversations?limit=1', 'Authorization', authorization('alice')); + expect(pages).toEqual([[], [{ id: 'mine-1', ownerId: 'alice', title: 'first' }], [], [{ id: 'mine-2', ownerId: 'alice', title: 'second' }]]); + expect(mockScans).toHaveLength(4); + expect(mockScans.every(scan => scan.Limit === 1 && scan.ExpressionAttributeValues[':ownerId'] === 'alice')).toBe(true); + }); + + test('user pages allowlist public fields and do not expose private location by filtering', async () => { + const pages = await collect('/users?limit=1', 'Authorization', authorization('alice')); + expect(pages.flat()).toEqual([ + { id: 'u1', username: 'grower', role: 'producer' }, + { id: 'u2', username: 'buyer', role: 'consumer', location: 'Town' }, + { id: 'u3', username: 'seller', role: 'producer', location: 'Town' }, + ]); + mockScans = []; + const filtered = await collect('/users?limit=1&role=producer&location=Town', 'Authorization', authorization('alice')); + expect(filtered.flat().map(row => row.id)).toEqual(['u3']); + expect(filtered[0]).toEqual([]); + }); + + test('federation pages finish collections independently without exporting accounts or narrative', async () => { + const pages = await collect('/federation/export?limit=1', 'x-federation-key', process.env.FEDERATION_EXPORT_KEY); + expect(pages).toEqual([ + { listings: [{ id: 'l1', userId: 'u1', title: 'Public listing', price: 2 }], transactions: [{ id: 't1', consumerId: 'u1', producerId: 'u2', status: 'pending' }], users: [] }, + { listings: [{ id: 'l2', userId: 'u2', title: 'Second' }], transactions: [], users: [] }, + ]); + expect(mockScans.map(scan => scan.TableName)).toEqual(['Listings', 'Transactions', 'Listings']); + }); + + test('ordinary account tokens cannot export federation data and shared keys cannot list private chats', async () => { + expect((await request(app).get('/federation/export').set('Authorization', authorization('alice'))).status).toBe(403); + expect((await request(app).get('/federation/export').set('x-api-key', process.env.API_KEY)).status).toBe(403); + expect((await request(app).get('/conversations').set('x-api-key', process.env.API_KEY)).status).toBe(401); + expect((await request(app).get('/users').set('x-api-key', process.env.API_KEY)).status).toBe(401); + expect(mockScans).toHaveLength(0); + }); + + test('the export key grants only GET export and cannot be the general write-capable key', async () => { + for (const [method, path] of [['post', '/federation/import'], ['post', '/users'], ['get', '/api/admin/users'], ['get', '/users']]) { + const result = await request(app)[method](path).set('x-federation-key', process.env.FEDERATION_EXPORT_KEY).send({}); + expect(result.status).toBe(403); + } + const key = process.env.FEDERATION_EXPORT_KEY; + try { + process.env.FEDERATION_EXPORT_KEY = process.env.API_KEY; + expect((await request(app).get('/federation/export').set('x-federation-key', process.env.API_KEY)).status).toBe(403); + } finally { process.env.FEDERATION_EXPORT_KEY = key; } + expect(mockScans).toHaveLength(0); + }); + + test('approved browser origins can read pagination headers without exposing private keys', async () => { + const result = await request(app).get('/users?limit=1').set('Authorization', authorization('alice')).set('Origin', 'https://www.ntari.org'); + expect(result.status).toBe(200); + expect(result.headers['access-control-expose-headers']).toContain('X-Next-Cursor'); + expect(result.headers['access-control-expose-headers']).toContain('X-Pagination'); + expect(JSON.stringify(result.body)).not.toContain('private@example.invalid'); + }); + + test.each(['0', '-1', '101', '1junk', '1.5', '', '1&limit=2'])('invalid limit %s fails before a scan', async limit => { + const result = await request(app).get(`/conversations?limit=${limit}`).set('Authorization', authorization('alice')); + expect(result.status).toBe(400); + expect(mockScans).toHaveLength(0); + }); + + test('defaults are bounded and malformed cursors fail before scanning', async () => { + const first = await request(app).get('/users').set('Authorization', authorization('alice')); + expect(first.status).toBe(200); + expect(mockScans[0].Limit).toBe(50); + mockScans = []; + for (const cursor of ['invalid', '%%%','']) { + const result = await request(app).get('/users').query({ cursor }).set('Authorization', authorization('alice')); + expect(result.status).toBe(400); + } + expect(mockScans).toHaveLength(0); + }); + + test('encrypted cursors bind account, collection, filters, and limit', async () => { + const token = authorization('alice'); + mockTables.Users[0].id = 'private-account-continuation-key'; + const first = await request(app).get('/users?limit=1&role=producer').set('Authorization', token); + const cursor = first.headers['x-next-cursor']; + expect(typeof cursor).toBe('string'); + expect(Buffer.from(cursor, 'base64url').toString()).not.toContain('private-account-continuation-key'); + mockScans = []; + for (const [path, auth] of [ + ['/users?limit=1&role=consumer', token], + ['/users?limit=2&role=producer', token], + ['/users?limit=1&role=producer', authorization('bob')], + ['/conversations?limit=1&role=producer', token], + ]) { + const result = await request(app).get(path).query({ cursor }).set('Authorization', auth); + expect(result.status).toBe(400); + } + const changed = cursor.slice(0, 12) + (cursor[12] === 'A' ? 'B' : 'A') + cursor.slice(13); + expect((await request(app).get('/users?limit=1&role=producer').query({ cursor: changed }).set('Authorization', token)).status).toBe(400); + expect(mockScans).toHaveLength(0); + }); + + test('cursors expire and database failures never return a successful partial page', async () => { + const token = authorization('alice'); + const first = await request(app).get('/users?limit=1').set('Authorization', token); + const cursor = first.headers['x-next-cursor']; + const clock = jest.spyOn(Date, 'now').mockReturnValue(Date.now() + 3600001); + try { + const expired = await request(app).get('/users?limit=1').query({ cursor }).set('Authorization', token); + expect(expired.status).toBe(400); + } finally { clock.mockRestore(); } + mockTables.Listings = null; + const db = require('../lib/dynamodbClient'); + const originalScan = db.scan; + db.scan = () => ({ promise: async () => { throw new Error('database unavailable'); } }); + try { + const result = await request(app).get('/federation/export').set('x-federation-key', process.env.FEDERATION_EXPORT_KEY); + expect(result.status).toBe(500); + expect(result.headers['x-next-cursor']).toBeUndefined(); + } finally { db.scan = originalScan; } + }); +}); diff --git a/backend/__tests__/conversationAccess.test.js b/backend/__tests__/conversationAccess.test.js index c1ba2644..580d6a5c 100644 --- a/backend/__tests__/conversationAccess.test.js +++ b/backend/__tests__/conversationAccess.test.js @@ -96,10 +96,22 @@ describe('private conversation HTTP boundary', () => { expect(result.body.ownerId).toBe('alice'); expect(mockTables.Conversations.get(result.body.id).ownerId).toBe('alice'); }); + async function allConversations(userId, apiKey) { + const rows = []; + let cursor; + do { + let call = request(app).get('/conversations').set('Authorization', authorization(userId)); + if (apiKey) call = call.set('x-api-key', apiKey); + if (cursor) call = call.query({ cursor }); + const result = await call; + expect(result.status).toBe(200); + rows.push(...result.body); + cursor = result.headers['x-next-cursor']; + } while (cursor); + return rows; + } test('listing traverses empty filtered pages without exposing foreign or legacy chats', async () => { - const result = await request(app).get('/conversations').set('Authorization', authorization('alice')); - expect(result.status).toBe(200); - expect(result.body.map(item => item.id)).toEqual(['mine']); + expect((await allConversations('alice')).map(item => item.id)).toEqual(['mine']); }); test.each(['foreign', 'legacy', 'missing'])('all ID routes conceal %s and have no message side effects', async id => { for (const [method, url, body] of [ @@ -129,7 +141,7 @@ describe('private conversation HTTP boundary', () => { const result = await request(app).get('/conversations').set('x-api-key', process.env.API_KEY) .set('Authorization', authorization('alice')); expect(result.status).toBe(200); - expect(result.body.map(item => item.id)).toEqual(['mine']); + expect((await allConversations('alice', process.env.API_KEY)).map(item => item.id)).toEqual(['mine']); }); test('the rightful owner can read, rename, pin, send, list and delete', async () => { const auth = authorization('alice'); diff --git a/backend/controllers/conversation_controller.js b/backend/controllers/conversation_controller.js index 373aa894..6db6089c 100644 --- a/backend/controllers/conversation_controller.js +++ b/backend/controllers/conversation_controller.js @@ -1,4 +1,5 @@ const Conversation = require('../models/conversation'); +const { collectionPage, pageError } = require('../lib/collectionPage'); exports.create = async (req, res) => { const { title } = req.body; @@ -7,8 +8,12 @@ exports.create = async (req, res) => { }; exports.list = async (req, res) => { - const convos = await Conversation.list(req.user.userId); - res.json(convos); + try { + const page = collectionPage(req.query, ['conversations', req.user.userId]); + const { items, next } = await Conversation.listPage(req.user.userId, page.limit, page.state?.start); + page.finish(res, next && { start: next }); + res.json(items); + } catch (error) { pageError(res, error, 'Failed to fetch conversations'); } }; exports.get = (req, res) => res.json(req.conversation); diff --git a/backend/docs/collection-pagination.md b/backend/docs/collection-pagination.md new file mode 100644 index 00000000..3bf5ada7 --- /dev/null +++ b/backend/docs/collection-pagination.md @@ -0,0 +1,159 @@ +# Collection pages and read-only federation exports + +## Purpose and scope + +`GET /conversations`, `GET /users`, and `GET /federation/export` now return +bounded DynamoDB scan pages with explicit continuation. Previously these +paths either stopped at one DynamoDB page or assembled an unbounded array. +This change requires the conversation ownership work in Agrinet #191. Its +private-chat rollout still requires the authenticated-stream companion #192. + +Pagination must not make more private account data available. The directory +uses an allowlist, account records are omitted from federation exports, and +the export credential is separate from write-capable application credentials. +These are deliberate compatibility changes, not a claim that all existing +account or federation security gaps are closed. + +## HTTP contract + +JSON response shapes remain arrays for conversations/users and +`{listings: [], transactions: [], users: []}` for federation export. +Pagination metadata is carried in response headers: + +- `X-Pagination: cursor-v1` declares the contract. +- `X-Page-Limit` is the maximum number of records **evaluated**, per table, + in that request. `limit` defaults to 50 and accepts decimal integers 1–100. + Federation export can evaluate that many listings and transactions each. +- `X-Next-Cursor` is present only when there is another scan position. + Pass its exact opaque value as `cursor`, with the original filters and + limit, to the **same endpoint**. An empty result can still have a cursor: + DynamoDB applies filters after evaluating records. Stop only when the + continuation header is absent. +- `Cache-Control: private, no-store` prevents shared caching. + +The approved browser origin can read these headers through CORS. Existing +clients outside this repository must be updated to follow the header; +otherwise they see only the first bounded page. The bundled chat account +client follows array pages, including empty pages, preserves filters and +credentials, and rejects account changes or later-page errors instead of +returning a partial old-account list. It still accumulates the full list in +browser memory, as before; this PR bounds individual server responses and +does not introduce a lazy-loading sidebar. + +Cursors encrypt and authenticate DynamoDB continuation keys using AES-GCM +with a key derived from the existing `JWT_SECRET`. They expire after one +hour and bind endpoint, limit, directory filters and account identity where +applicable. Raw scan keys can identify private rows even on empty pages, so +they are not base64-encoded plaintext. Changing `JWT_SECRET` invalidates +cursors. Missing secret configuration returns 503 before scanning; invalid, +expired, tampered or mismatched parameters return 400. A storage failure +returns 500 without a successful partial page or continuation header. + +Scans do not provide a snapshot or stable sort. For unchanged tables, following +all continuations returns the matching records. Concurrent additions, +updates and deletions can change what is observed; the API does not promise +exactly-once snapshot enumeration or claim an ordering DynamoDB does not give. +Restart from the first page if a cursor expires. + +## Private conversation and directory boundaries + +Conversations retain #191's account-only authentication and owner filter on +every scan. Other owners' and legacy ownerless rows remain inaccessible. +The existing all-results model helper remains for programmatic compatibility; +the HTTP route uses its bounded page counterpart. + +`GET /users` now requires a verified account token; a shared integration key +alone is insufficient. Returned fields are `id`, `username`, and `role`, plus +the existing coarse `location` only when `locationPrivacy === true`. That +boolean is the existing sharing opt-in convention, not an inverted default. +Location filtering also requires that opt-in, preventing the filter itself +from revealing a private location. Missing/false privacy never publishes a +location. Coordinates, email, phone, password, verification code, scalar +reputation values, and unknown future fields are excluded. The database +projection and the response allowlist both enforce this boundary. Existing +role filtering remains available; role and location each accept one string. + +This directory policy does not repair other account/admin/login endpoints, +role assignment, or the separate account-overwrite issue. Do not describe it +as full A3/A4/A5 implementation or as completed public signup hardening. + +## Federation export is read-only inspection + +Configure a strong, private `FEDERATION_EXPORT_KEY` on the exporting backend. +It must differ from `API_KEY` and `WIX_API_KEY`; equal or unset keys fail +closed. Supply it only in `x-federation-key` on `GET /federation/export`. +It grants no `/users`, admin, import or other write access. Ordinary account +JWTs and general API keys do not grant export access. Keep export keys on +servers, out of frontend builds, URLs and logs. There is no default key. + +Export pages contain only explicitly selected advertised listing fields and +structural transaction fields. Existing participant/contract/listing IDs and +lifecycle flags are retained; `dialogNotes`, scalar ratings and all Users +records are excluded. `users` is always an empty compatibility array, not a +claim that the operator has zero members. `X-Federation-Omissions` states +`users, dialogNotes, ratings`. Advertised listing descriptions and locations +are still included; this is not an automatic PII scrubber for public content. +The duplicate legacy router delegates to the mounted router's same policy. + +For outbound inspection, set `FEDERATION_PEER_KEYS` to a JSON object mapping +each approved peer's exact base URL (without trailing slash) to **that peer's +read-only export key**. For example, the structure is: + +```json +{"https://peer.example":""} +``` + +Registry enrollment alone never causes a credential to be sent. Only +explicitly configured destinations are contacted; userinfo, query strings +and fragments in peer base URLs are rejected. HTTPS is required except for +loopback development. Redirects are refused, and continuation is a token, +never a destination URL. The local general API key is never forwarded. +Requests have a five-second timeout, a 16 MiB response limit and a 10,000-page +safety ceiling. Exceeding a limit is an explicit incomplete inspection, not +successful truncation. Older peers without `X-Pagination: cursor-v1`, +malformed pages and any peer page containing Users records are rejected. + +The server-mounted worker and both legacy scheduled paths inspect all pages +and report counts. **Automatic import is disabled, and `lastSyncAt` is not +advanced.** The old importers overwrite canonical records without a source +namespace or auditable conflict policy; importing redacted summaries could +erase local narrative or account data. A16 cannot safely invent those rules. +Source-aware reconciliation remains A17 design work. No temporary unscoped +cache or new authority store is added. The legacy status router (not mounted by the default server) totals all pages +and reports `exportMode: read-only` and `usersExported: false`; its `lastSync` +value is historical data, not proof this inspection synchronized anything. + +The original manual `/federation/import` endpoint and helper remain separate +legacy security debt; this PR does not certify them safe. General API/admin +authorization, collision policy, witnessed checkpoints and independent peers +remain open work. Operators must review existing import access before rollout. + +These exports are interim application views, not commons artifacts, portable +reputation, signed exchange receipts, witnessed federation, or a coordination +wire extension. No ledger entry is edited, deleted or promoted to authority. +Escrow, credit configuration and the existing in-review conformance claims +remain outside the change. + +## Rollout and verification + +Review #191 first, then the A16 commit; deploy private chat with #192. No table +schema migration or new index is required. The application retains existing +Scan permissions. Configure distinct export and per-peer read credentials +before enabling inspection; unset configuration deliberately fails closed. +Update external clients to consume pagination headers before rollout, and +communicate that automated merging and account replication are disabled. + +`cd backend && npm test` covers production-mounted HTTP boundaries, empty +filtered pages, owner and query binding, cursor tampering/expiry, export +credential isolation, partial failures and non-mutating scheduled inspection. +`cd frontend/chat-ui && npm test && npm run build` checks the account adapter +and production bundle. + +With DynamoDB Local on a loopback endpoint, run +`DYNAMODB_ENDPOINT=http://127.0.0.1:18001 node scripts/check-collection-pagination.js` +from `backend`. It creates uniquely named disposable tables and removes them +afterward. Records exceed DynamoDB's 1 MiB scan boundary, so continuation, +privacy projections, filtering, the actual account adapter and read-only +export traversal are checked against real storage. The check confirms local +credentials and narrative remain unchanged. CI runs the same check with a +digest-pinned local database image; no live peer, payment or AI call is made. diff --git a/backend/federation/__tests__/federationSyncJob.test.js b/backend/federation/__tests__/federationSyncJob.test.js index 5b821fb1..4689cbc4 100644 --- a/backend/federation/__tests__/federationSyncJob.test.js +++ b/backend/federation/__tests__/federationSyncJob.test.js @@ -1,52 +1,49 @@ jest.useFakeTimers(); jest.mock('axios'); -jest.mock('../../lib/dynamodbClient', () => ({ - scan: jest.fn(), - update: jest.fn() -})); - +jest.mock('../../lib/dynamodbClient', () => ({ scan: jest.fn(), update: jest.fn(), put: jest.fn() })); const axios = require('axios'); -const dynamodbClient = require('../../lib/dynamodbClient'); -const runFederationSync = require('../federationSyncJob'); +const db = require('../../lib/dynamodbClient'); +const run = require('../federationSyncJob'); -describe('runFederationSync', () => { +describe('scheduled federation inspection', () => { + const env = { ...process.env }; + let info; + let errors; beforeEach(() => { - jest.clearAllMocks(); + jest.resetAllMocks(); + process.env.FEDERATION_PEER_KEYS = JSON.stringify({ 'https://peer.example': 'read-key' }); + db.scan.mockReturnValue({ promise: async () => ({ Items: [{ url: 'https://peer.example' }] }) }); + info = jest.spyOn(console, 'info').mockImplementation(() => {}); + errors = jest.spyOn(console, 'error').mockImplementation(() => {}); }); - - it('updates lastSyncAt on successful sync', async () => { - dynamodbClient.scan.mockReturnValue({ - promise: () => Promise.resolve({ Items: [{ url: 'http://node1.com' }] }) - }); - dynamodbClient.update.mockReturnValue({ - promise: () => Promise.resolve() - }); - axios.get.mockResolvedValue({ data: { foo: 'bar' } }); - axios.post.mockResolvedValue({}); - - await runFederationSync(); - - expect(dynamodbClient.update).toHaveBeenCalledTimes(1); - const params = dynamodbClient.update.mock.calls[0][0]; - expect(params).toMatchObject({ - Key: { url: 'http://node1.com' }, - UpdateExpression: 'set lastSyncAt = :time', - ExpressionAttributeValues: { - ':time': expect.any(String) - } - }); - const timestamp = params.ExpressionAttributeValues[':time']; - expect(new Date(timestamp).toISOString()).toBe(timestamp); + afterEach(() => { info.mockRestore(); errors.mockRestore(); }); + afterAll(() => { process.env = env; jest.useRealTimers(); }); + const page = (listings, next) => ({ data: { listings, transactions: [], users: [] }, headers: { 'x-pagination': 'cursor-v1', ...(next && { 'x-next-cursor': next }) } }); + test('reads every page and reports inspection without canonical writes or a sync timestamp', async () => { + axios.get.mockResolvedValueOnce(page([{ id: 'l1' }], 'next')).mockResolvedValueOnce(page([{ id: 'l2' }])); + await run(); + expect(info).toHaveBeenCalledWith('Federation export inspected; automatic import disabled', { peer: 'https://peer.example', counts: { listings: 2, transactions: 0, users: 0 } }); + expect(db.put).not.toHaveBeenCalled(); + expect(db.update).not.toHaveBeenCalled(); + expect(axios.post).not.toHaveBeenCalled(); }); - - it('does not update lastSyncAt on failed sync', async () => { - dynamodbClient.scan.mockReturnValue({ - promise: () => Promise.resolve({ Items: [{ url: 'http://node1.com' }] }) - }); - axios.get.mockRejectedValue(new Error('fail')); - - await runFederationSync(); - - expect(dynamodbClient.update).not.toHaveBeenCalled(); + test('a failed later page never reports a successful inspection or sync', async () => { + axios.get.mockResolvedValueOnce(page([{ id: 'l1' }], 'next')).mockRejectedValueOnce(new Error('later page failed')); + await run(); + expect(info).not.toHaveBeenCalled(); + expect(errors).toHaveBeenCalled(); + expect(db.put).not.toHaveBeenCalled(); + expect(db.update).not.toHaveBeenCalled(); + }); + test('continues through registry pages even when an earlier peer fails', async () => { + process.env.FEDERATION_PEER_KEYS = JSON.stringify({ 'https://peer.example': 'read-key', 'https://other.example': 'other-read-key' }); + db.scan.mockReturnValueOnce({ promise: async () => ({ Items: [{ url: 'https://peer.example' }], LastEvaluatedKey: { url: 'https://peer.example' } }) }) + .mockReturnValueOnce({ promise: async () => ({ Items: [{ url: 'https://other.example' }] }) }); + axios.get.mockRejectedValueOnce(new Error('unavailable')).mockResolvedValueOnce(page([])); + await run(); + expect(axios.get.mock.calls.map(call => call[0])).toEqual(['https://peer.example/federation/export', 'https://other.example/federation/export']); + expect(db.scan.mock.calls[1][0].ExclusiveStartKey).toEqual({ url: 'https://peer.example' }); + expect(info).toHaveBeenCalledTimes(1); + expect(db.update).not.toHaveBeenCalled(); }); }); diff --git a/backend/federation/exportAuth.js b/backend/federation/exportAuth.js new file mode 100644 index 00000000..d21023e7 --- /dev/null +++ b/backend/federation/exportAuth.js @@ -0,0 +1,13 @@ +const { timingSafeEqual } = require('node:crypto'); + +function validExportKey(req) { + const expected = process.env.FEDERATION_EXPORT_KEY; + const supplied = req.headers['x-federation-key']; + // This key must not accidentally become a write-capable general API key. + if (!expected || expected === process.env.API_KEY || expected === process.env.WIX_API_KEY || typeof supplied !== 'string') return false; + const a = Buffer.from(expected); + const b = Buffer.from(supplied); + return a.length === b.length && timingSafeEqual(a, b); +} + +module.exports = { validExportKey }; diff --git a/backend/federation/exportFields.js b/backend/federation/exportFields.js new file mode 100644 index 00000000..21369554 --- /dev/null +++ b/backend/federation/exportFields.js @@ -0,0 +1,18 @@ +// Interim application export, not a commons record or a reputation transfer. +// Only existing structural transaction fields and advertised listing fields +// cross this boundary. Never spread an entire DynamoDB record into a response. +function fields(item, names) { + return Object.fromEntries(names.filter(name => Object.hasOwn(item, name)).map(name => [name, item[name]])); +} + +const listingFields = ['id', 'userId', 'type', 'title', 'category', 'description', 'price', 'location', + 'media', 'logisticsRange', 'processingCategory', 'compostingService', 'availability', 'tags', 'createdAt']; +const transactionFields = ['id', 'transactionId', 'contractId', 'consumerId', 'producerId', 'buyerId', + 'sellerId', 'listingId', 'status', 'createdAt', 'lastPing', 'pingCount', 'buyerRated', 'sellerRated', + 'ratingGiven', 'dialogConfirmed', 'flaggedForReview', 'escrowLocked', 'escrowReleasedAt']; + +module.exports = { + listingFields, transactionFields, + listing: item => fields(item, listingFields), + transaction: item => fields(item, transactionFields), +}; diff --git a/backend/federation/federationRoutes.js b/backend/federation/federationRoutes.js index 48506004..f158af57 100644 --- a/backend/federation/federationRoutes.js +++ b/backend/federation/federationRoutes.js @@ -3,22 +3,36 @@ const router = express.Router(); const docClient = require('../lib/dynamodbClient'); const { LISTING_TABLE_NAME } = require('../models/listing'); const { TRANSACTION_TABLE_NAME } = require('../models/transaction'); -const { USER_TABLE_NAME } = require('../models/user'); const { importFederatedData } = require('../sync/importHelper'); +const { collectionPage, scanPage, pageError } = require('../lib/collectionPage'); +const exportFields = require('./exportFields'); +const { validExportKey } = require('./exportAuth'); // Export route for peer sync router.get('/export', async (req, res) => { + // A member JWT is not operator permission to bulk-export application data. + if (!validExportKey(req)) return res.status(403).json({ error: 'Read-only federation export key required' }); try { - const listingsRes = await docClient.scan({ TableName: LISTING_TABLE_NAME }).promise(); - const transactionsRes = await docClient.scan({ TableName: TRANSACTION_TABLE_NAME }).promise(); - const usersRes = await docClient.scan({ TableName: USER_TABLE_NAME }).promise(); - res.json({ - listings: listingsRes.Items || [], - transactions: transactionsRes.Items || [], - users: usersRes.Items || [] - }); + const page = collectionPage(req.query, ['federation-export']); + const result = { listings: [], transactions: [], users: [] }; + const next = {}; + for (const [name, table, allowed, present] of [ + ['listings', LISTING_TABLE_NAME, exportFields.listingFields, exportFields.listing], + ['transactions', TRANSACTION_TABLE_NAME, exportFields.transactionFields, exportFields.transaction], + ]) { + if (page.state && page.state[name] === null) { next[name] = null; continue; } + const names = Object.fromEntries(allowed.map((field, i) => [`#f${i}`, field])); + const data = await scanPage(docClient, { + TableName: table, ProjectionExpression: Object.keys(names).join(', '), ExpressionAttributeNames: names, + }, page.limit, page.state?.[name]); + result[name] = data.items.map(present); + next[name] = data.next; + } + page.finish(res, Object.values(next).some(Boolean) ? next : null); + res.set('X-Federation-Omissions', 'users, dialogNotes, ratings'); + res.json(result); } catch (err) { - res.status(500).json({ error: 'Export failed' }); + pageError(res, err, 'Export failed'); } }); diff --git a/backend/federation/federationSyncJob.js b/backend/federation/federationSyncJob.js index 1ffca6c5..817a50e7 100644 --- a/backend/federation/federationSyncJob.js +++ b/backend/federation/federationSyncJob.js @@ -1,50 +1,20 @@ -const axios = require("axios"); -const dynamodbClient = require("../lib/dynamodbClient"); -const { NODE_REGISTRY_TABLE_NAME } = require("./models/nodeRegistry"); - -const BACKEND_URL = process.env.BACKEND_URL || "http://localhost:5000"; - -const runFederationSync = async () => { - // Scan all nodes from DynamoDB table - const { Items: nodes } = await dynamodbClient - .scan({ TableName: NODE_REGISTRY_TABLE_NAME }) - .promise(); -/* -const { getAllNodes, saveNode } = require("./models/nodeRegistry"); - -const runFederationSync = async () => { - const nodes = await getAllNodes(); -*/ - for (const node of nodes) { +const dynamodbClient = require('../lib/dynamodbClient'); +const { NODE_REGISTRY_TABLE_NAME } = require('./models/nodeRegistry'); +const { scanItems } = require('../lib/collectionPage'); +const { inspectExport } = require('../sync/exportPages'); + +async function runFederationSync() { + for await (const node of scanItems(dynamodbClient, { TableName: NODE_REGISTRY_TABLE_NAME })) { + const url = node.url || node.nodeUrl; try { - // Fetch federation/export data from the node - const { data } = await axios.get(`${node.url}/federation/export`); - - // Update lastSyncAt in DynamoDB for this node - await axios.post(`${BACKEND_URL}/import`, data); - - await dynamodbClient - .update({ - TableName: NODE_REGISTRY_TABLE_NAME, - Key: { url: node.url }, - UpdateExpression: "set lastSyncAt = :time", - ExpressionAttributeValues: { - ":time": new Date().toISOString(), - }, - }) - .promise(); - - //node.lastSyncAt = new Date().toISOString(); - //await saveNode(node); - - console.log(`✅ Synced with ${node.url}`); - } catch (err) { - console.error(`❌ Failed to sync with ${node.url}`); + const counts = await inspectExport(url); + // Redacted summaries cannot replace canonical records or local narrative. + console.info('Federation export inspected; automatic import disabled', { peer: url, counts }); + } catch (error) { + console.error('Federation export inspection failed; no sync recorded', { peer: url, error: error.message }); } } -}; - -// Run every 6 hours -setInterval(runFederationSync, 6 * 60 * 60 * 1000); // 6 hours +} +setInterval(() => runFederationSync().catch(() => console.error('Federation registry scan failed')), 6 * 60 * 60 * 1000); module.exports = runFederationSync; diff --git a/backend/federation/models/nodeRegistry.js b/backend/federation/models/nodeRegistry.js index 27f2b609..03368bdb 100644 --- a/backend/federation/models/nodeRegistry.js +++ b/backend/federation/models/nodeRegistry.js @@ -1,4 +1,5 @@ const docClient = require("../../lib/dynamodbClient"); +const { scanItems } = require('../../lib/collectionPage'); const { applyNodeTemplate } = require("./nodeTemplate"); const NODE_REGISTRY_TABLE_NAME = "NodeRegistry"; @@ -58,8 +59,9 @@ module.exports = { * @returns {Promise>} */ async function getAllNodes() { - const data = await docClient.scan({ TableName: NODE_REGISTRY_TABLE_NAME }).promise(); - return data.Items || []; + const nodes = []; + for await (const node of scanItems(docClient, { TableName: NODE_REGISTRY_TABLE_NAME })) nodes.push(node); + return nodes; } /** diff --git a/backend/lib/collectionPage.js b/backend/lib/collectionPage.js new file mode 100644 index 00000000..ff0878c5 --- /dev/null +++ b/backend/lib/collectionPage.js @@ -0,0 +1,80 @@ +const { createHash, randomBytes, createCipheriv, createDecipheriv } = require('node:crypto'); + +function badPage() { + return Object.assign(new Error('Invalid or expired pagination parameters'), { status: 400 }); +} + +function encryptionKey() { + if (!process.env.JWT_SECRET) { + throw Object.assign(new Error('Pagination requires JWT_SECRET configuration'), { status: 503 }); + } + return createHash('sha256').update('agrinet-collection-page-v1\0').update(process.env.JWT_SECRET).digest(); +} + +function encode(state, scope) { + const iv = randomBytes(12); + const cipher = createCipheriv('aes-256-gcm', encryptionKey(), iv); + cipher.setAAD(Buffer.from(scope)); + const bytes = Buffer.concat([ + cipher.update(JSON.stringify({ state, expires: Date.now() + 3600000 })), cipher.final(), + ]); + return Buffer.concat([iv, cipher.getAuthTag(), bytes]).toString('base64url'); +} + +function decode(token, scope) { + const key = encryptionKey(); + try { + if (typeof token !== 'string' || token.length > 16384 || !/^[\w-]+$/.test(token)) throw badPage(); + const bytes = Buffer.from(token, 'base64url'); + if (bytes.length < 29 || bytes.toString('base64url') !== token) throw badPage(); + const decipher = createDecipheriv('aes-256-gcm', key, bytes.subarray(0, 12)); + decipher.setAAD(Buffer.from(scope)); + decipher.setAuthTag(bytes.subarray(12, 28)); + const decoded = JSON.parse(Buffer.concat([decipher.update(bytes.subarray(28)), decipher.final()]).toString()); + if (!Number.isFinite(decoded.expires) || decoded.expires <= Date.now() || !decoded.state || typeof decoded.state !== 'object' || Array.isArray(decoded.state)) throw badPage(); + return decoded.state; + } catch { throw badPage(); } +} + +function collectionPage(query, scope) { + const rawLimit = query.limit === undefined ? '50' : query.limit; + if (typeof rawLimit !== 'string' || !/^[1-9]\d*$/.test(rawLimit)) throw badPage(); + const limit = Number(rawLimit); + if (!Number.isSafeInteger(limit) || limit > 100) throw badPage(); + const boundScope = JSON.stringify([scope, limit]); + // Fail configuration before scanning, including first/last pages with no cursor. + encryptionKey(); + return { + limit, + state: query.cursor === undefined ? undefined : decode(query.cursor, boundScope), + finish(res, state) { + const next = state ? encode(state, boundScope) : null; + res.set('Cache-Control', 'private, no-store'); + res.set('X-Pagination', 'cursor-v1'); + res.set('X-Page-Limit', String(limit)); + if (next) res.set('X-Next-Cursor', next); + }, + }; +} + +async function scanPage(client, params, limit, start) { + const result = await client.scan({ ...params, Limit: limit, ...(start && { ExclusiveStartKey: start }) }).promise(); + const next = result.LastEvaluatedKey; + return { items: result.Items || [], next: next && Object.keys(next).length ? next : null }; +} + +function pageError(res, error, fallback) { + const status = error.status === 400 || error.status === 503 ? error.status : 500; + return res.status(status).json({ error: status === 500 ? fallback : error.message }); +} + +async function* scanItems(client, params) { + let start; + do { + const page = await scanPage(client, params, 50, start); + yield* page.items; + start = page.next; + } while (start); +} + +module.exports = { collectionPage, scanPage, scanItems, pageError }; diff --git a/backend/middleware/authMiddleware.js b/backend/middleware/authMiddleware.js index 275287b3..d2eaaf67 100644 --- a/backend/middleware/authMiddleware.js +++ b/backend/middleware/authMiddleware.js @@ -1,6 +1,8 @@ const jwt = require('../utils/jwt'); +const { validExportKey } = require('../federation/exportAuth'); module.exports = (req, res, next) => { + if (req.method === 'GET' && /^\/federation\/export\/?$/.test(req.path) && validExportKey(req)) return next(); const apiKey = req.headers['x-api-key']; const validKey = process.env.API_KEY || process.env.WIX_API_KEY; if (apiKey && apiKey === validKey) { diff --git a/backend/models/conversation.js b/backend/models/conversation.js index c0a28ef0..ffbf965e 100644 --- a/backend/models/conversation.js +++ b/backend/models/conversation.js @@ -1,6 +1,7 @@ const { randomUUID } = require('crypto'); let docClient = require('../lib/dynamodbClient'); const Message = require('./message'); +const { scanPage } = require('../lib/collectionPage'); const CONVERSATION_TABLE_NAME = process.env.CONVERSATION_TABLE_NAME || 'Conversations'; @@ -34,6 +35,15 @@ async function list(ownerId) { return items; } +async function listPage(ownerId, limit, start) { + if (typeof ownerId !== 'string' || !ownerId.trim()) throw new Error('Conversation owner required'); + return scanPage(docClient, { + TableName: CONVERSATION_TABLE_NAME, + FilterExpression: 'ownerId = :ownerId', + ExpressionAttributeValues: { ':ownerId': ownerId }, + }, limit, start); +} + async function getOwned(id, ownerId) { if (typeof ownerId !== 'string' || !ownerId.trim()) return null; const { Item } = await docClient.get({ TableName: CONVERSATION_TABLE_NAME, Key: { id }, ConsistentRead: true }).promise(); @@ -85,4 +95,4 @@ function setDocClient(client) { docClient = client; } -module.exports = { create, list, getOwned, rename, remove, pin, setDocClient }; +module.exports = { create, list, listPage, getOwned, rename, remove, pin, setDocClient }; diff --git a/backend/routes/__tests__/federationStatus.test.js b/backend/routes/__tests__/federationStatus.test.js index ab33ff3a..84656ad0 100644 --- a/backend/routes/__tests__/federationStatus.test.js +++ b/backend/routes/__tests__/federationStatus.test.js @@ -16,14 +16,15 @@ app.use(federationStatusRouter); describe('GET /federation/status', () => { beforeEach(() => { jest.clearAllMocks(); + process.env.FEDERATION_PEER_KEYS = JSON.stringify({ 'https://node1.com': 'read-key', 'https://node2.com': 'other-read-key' }); }); it('returns federation health with last sync times', async () => { dynamodbClient.scan.mockReturnValue({ promise: () => Promise.resolve({ Items: [ - { nodeUrl: 'http://node1.com', lastSyncAt: '2024-01-01T00:00:00Z' }, - { nodeUrl: 'http://node2.com', lastSyncAt: '2024-01-02T00:00:00Z' } + { nodeUrl: 'https://node1.com', lastSyncAt: '2024-01-01T00:00:00Z' }, + { nodeUrl: 'https://node2.com', lastSyncAt: '2024-01-02T00:00:00Z' } ] }) }); @@ -34,8 +35,9 @@ describe('GET /federation/status', () => { data: { listings: [1, 2], transactions: [1], - users: [1] - } + users: [] + }, + headers: { 'x-pagination': 'cursor-v1' } }); } return Promise.reject(new Error('Network Error')); @@ -45,17 +47,18 @@ describe('GET /federation/status', () => { expect(res.status).toBe(200); expect(res.body).toEqual({ + exportMode: 'read-only', usersExported: false, federationHealth: [ { - node: 'http://node1.com', + node: 'https://node1.com', status: 'ONLINE', listings: 2, transactions: 1, - users: 1, + users: 0, lastSync: '2024-01-01T00:00:00Z' }, { - node: 'http://node2.com', + node: 'https://node2.com', status: 'OFFLINE', error: 'Network Error', listings: 0, diff --git a/backend/routes/__tests__/userRoutes.test.js b/backend/routes/__tests__/userRoutes.test.js index 8e0886fc..897c52b1 100644 --- a/backend/routes/__tests__/userRoutes.test.js +++ b/backend/routes/__tests__/userRoutes.test.js @@ -1,6 +1,7 @@ const request = require('supertest'); const express = require('express'); const bcrypt = require('bcryptjs'); +const jwt = require('jsonwebtoken'); jest.mock('../../lib/dynamodbClient', () => ({ scan: jest.fn(), @@ -15,6 +16,7 @@ describe('userRoutes', () => { beforeEach(() => { app = express(); app.use(express.json()); + process.env.JWT_SECRET = 'directory-test-key'; app.use('/users', userRoutes); docClient.scan.mockReset(); docClient.put.mockReset(); @@ -30,10 +32,10 @@ describe('userRoutes', () => { }) }); - const res = await request(app).get('/users'); + const res = await request(app).get('/users').set('Authorization', `Bearer ${jwt.sign({ userId: 'test' }, process.env.JWT_SECRET)}`); expect(res.status).toBe(200); expect(res.body).toEqual([ - { id: '1', username: 'john', role: 'consumer', location: 'NY' } + { id: '1', username: 'john', role: 'consumer' } ]); expect(res.body[0].password).toBeUndefined(); }); @@ -41,12 +43,12 @@ describe('userRoutes', () => { it('applies role and location filters', async () => { docClient.scan.mockReturnValue({ promise: () => Promise.resolve({ Items: [] }) }); - await request(app).get('/users?role=farmer&location=Texas'); + await request(app).get('/users?role=farmer&location=Texas').set('Authorization', `Bearer ${jwt.sign({ userId: 'test' }, process.env.JWT_SECRET)}`); expect(docClient.scan).toHaveBeenCalledWith( expect.objectContaining({ - FilterExpression: 'role = :role AND location = :location', - ExpressionAttributeValues: { ':role': 'farmer', ':location': 'Texas' } + FilterExpression: '#role = :role AND #location = :location AND #privacy = :public', + ExpressionAttributeValues: { ':role': 'farmer', ':location': 'Texas', ':public': true } }) ); }); diff --git a/backend/routes/federationRoutes.js b/backend/routes/federationRoutes.js index f9a57436..60efd66d 100644 --- a/backend/routes/federationRoutes.js +++ b/backend/routes/federationRoutes.js @@ -1,42 +1,2 @@ -const express = require('express'); -const router = express.Router(); -const docClient = require('../lib/dynamodbClient'); -const { LISTING_TABLE_NAME } = require('../models/listing'); -const { TRANSACTION_TABLE_NAME } = require('../models/transaction'); -const { USER_TABLE_NAME } = require('../models/user'); -const { importFederatedData } = require('../sync/importHelper'); - -// Export core objects -router.get('/export', async (req, res) => { - try { - const listingsRes = await docClient.scan({ TableName: LISTING_TABLE_NAME }).promise(); - const transactionsRes = await docClient.scan({ TableName: TRANSACTION_TABLE_NAME }).promise(); - const usersRes = await docClient - .scan({ - TableName: USER_TABLE_NAME, - ProjectionExpression: 'id, email, reputationScore, role' - }) - .promise(); - - res.json({ - listings: listingsRes.Items || [], - transactions: transactionsRes.Items || [], - users: usersRes.Items || [] - }); - } catch (error) { - res.status(500).json({ error: 'Export error' }); - } -}); - -// Import core objects -router.post('/import', async (req, res) => { - try { - await importFederatedData(req.body); - - res.json({ message: 'Data imported successfully' }); - } catch (error) { - res.status(500).json({ error: 'Import error' }); - } -}); - -module.exports = router; +// Keep the legacy import path on the same export policy as the mounted router. +module.exports = require('../federation/federationRoutes'); diff --git a/backend/routes/federationStatus.js b/backend/routes/federationStatus.js index 7eb46b20..706efd24 100644 --- a/backend/routes/federationStatus.js +++ b/backend/routes/federationStatus.js @@ -1,43 +1,35 @@ const express = require("express"); const router = express.Router(); const dynamodbClient = require("../lib/dynamodbClient"); -const axios = require("axios"); +const { inspectExport } = require('../sync/exportPages'); +const { scanItems } = require('../lib/collectionPage'); router.get("/federation/status", async (req, res) => { try { - const { Items = [] } = await dynamodbClient - .scan({ TableName: "NodeRegistry" }) - .promise(); - - const checks = await Promise.all( - Items.map(async (node) => { - const url = node.nodeUrl; - try { - const { data } = await axios.get(`${url}/federation/export`, { - timeout: 5000, - }); - return { - node: url, - status: "ONLINE", - listings: data.listings?.length || 0, - transactions: data.transactions?.length || 0, - users: data.users?.length || 0, - lastSync: node.lastSyncAt, - }; - } catch (err) { - return { - node: url, - status: "OFFLINE", - error: err.message, - listings: 0, - transactions: 0, - users: 0, - lastSync: node.lastSyncAt, - }; - } - }) - ); - res.json({ federationHealth: checks }); + const checks = []; + for await (const node of scanItems(dynamodbClient, { TableName: 'NodeRegistry' })) { + const url = node.nodeUrl || node.url; + try { + const counts = await inspectExport(url); + checks.push({ + node: url, + status: "ONLINE", + ...counts, + lastSync: node.lastSyncAt, + }); + } catch (err) { + checks.push({ + node: url, + status: "OFFLINE", + error: err.message, + listings: 0, + transactions: 0, + users: 0, + lastSync: node.lastSyncAt, + }); + } + } + res.json({ federationHealth: checks, exportMode: 'read-only', usersExported: false }); } catch (err) { res.status(500).json({ error: "Failed to validate federation status" }); } diff --git a/backend/routes/userRoutes.js b/backend/routes/userRoutes.js index 18e12fa7..4074bd88 100644 --- a/backend/routes/userRoutes.js +++ b/backend/routes/userRoutes.js @@ -3,34 +3,51 @@ const bcrypt = require('../utils/bcrypt'); const router = express.Router(); const docClient = require('../lib/dynamodbClient'); const { USER_TABLE_NAME, createUserItem } = require('../models/user'); +const account = require('../middleware/chatAuth'); +const { collectionPage, scanPage, pageError } = require('../lib/collectionPage'); -// GET /users - fetch all users, omit sensitive data -router.get('/', async (req, res) => { - const params = { - TableName: USER_TABLE_NAME - }; +// GET /users - bounded directory, never account or private location records +router.get('/', account, async (req, res) => { + try { + for (const name of ['role', 'location']) { + if (req.query[name] !== undefined && typeof req.query[name] !== 'string') { + return res.status(400).json({ error: 'Filters must be single strings' }); + } + } + const page = collectionPage(req.query, ['users', req.user.userId, req.query.role || '', req.query.location || '']); + const params = { + TableName: USER_TABLE_NAME, + ProjectionExpression: '#id, #username, #role, #location, #privacy', + ExpressionAttributeNames: { '#id': 'id', '#username': 'username', '#role': 'role', '#location': 'location', '#privacy': 'locationPrivacy' }, + }; - const filters = []; - const values = {}; - if (req.query.role) { - filters.push('role = :role'); - values[':role'] = req.query.role; - } - if (req.query.location) { - filters.push('location = :location'); - values[':location'] = req.query.location; - } - if (filters.length) { - params.FilterExpression = filters.join(' AND '); - params.ExpressionAttributeValues = values; - } + const filters = []; + const values = {}; + if (req.query.role) { + filters.push('#role = :role'); + values[':role'] = req.query.role; + } + if (req.query.location) { + filters.push('#location = :location AND #privacy = :public'); + values[':location'] = req.query.location; + values[':public'] = true; + } + if (filters.length) { + params.FilterExpression = filters.join(' AND '); + params.ExpressionAttributeValues = values; + } - try { - const data = await docClient.scan(params).promise(); - const users = (data.Items || []).map(({ password, ...rest }) => rest); + const { items, next } = await scanPage(docClient, params, page.limit, page.state?.start); + const users = items.map(item => ({ + ...(typeof item.id === 'string' && { id: item.id }), + ...(typeof item.username === 'string' && { username: item.username }), + ...(typeof item.role === 'string' && { role: item.role }), + ...(item.locationPrivacy === true && typeof item.location === 'string' && { location: item.location }), + })); + page.finish(res, next && { start: next }); res.json(users); } catch (err) { - res.status(500).json({ error: 'Failed to fetch users' }); + pageError(res, err, 'Failed to fetch users'); } }); diff --git a/backend/scripts/check-collection-pagination.js b/backend/scripts/check-collection-pagination.js new file mode 100644 index 00000000..feb450b3 --- /dev/null +++ b/backend/scripts/check-collection-pagination.js @@ -0,0 +1,120 @@ +// Disposable local-storage check. No production endpoint or data is accepted. +const assert = require('node:assert/strict'); +const { randomUUID } = require('node:crypto'); +const { DynamoDBClient, CreateTableCommand, DeleteTableCommand } = require('@aws-sdk/client-dynamodb'); +const { DynamoDBDocument, PutCommand, GetCommand } = require('@aws-sdk/lib-dynamodb'); +const express = require('express'); +const jwt = require('jsonwebtoken'); + +async function main() { + const endpoint = new URL(process.env.DYNAMODB_ENDPOINT); + if (!['127.0.0.1', 'localhost', '[::1]'].includes(endpoint.hostname)) throw new Error('DynamoDB Local loopback endpoint required'); + process.env.AWS_ACCESS_KEY_ID = 'localcheck'; + process.env.AWS_SECRET_ACCESS_KEY = 'localcheck'; + process.env.AWS_REGION = 'us-east-1'; + process.env.JWT_SECRET = randomUUID(); + process.env.API_KEY = randomUUID(); + process.env.FEDERATION_EXPORT_KEY = randomUUID(); + const admin = new DynamoDBClient({ endpoint: endpoint.href, region: 'us-east-1', credentials: { accessKeyId: 'localcheck', secretAccessKey: 'localcheck' } }); + const store = DynamoDBDocument.from(admin); + const prefix = `CollectionPages_${randomUUID().replaceAll('-', '')}`; + const tables = Object.fromEntries(['Conversations', 'Users', 'Listings', 'Transactions'].map(name => [name, `${prefix}_${name}`])); + process.env.CONVERSATION_TABLE_NAME = tables.Conversations; + const created = []; + let server; + try { + for (const TableName of Object.values(tables)) { + await admin.send(new CreateTableCommand({ TableName, BillingMode: 'PAY_PER_REQUEST', + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }], AttributeDefinitions: [{ AttributeName: 'id', AttributeType: 'S' }], + })); + created.push(TableName); + } + const put = (table, Item) => store.send(new PutCommand({ TableName: tables[table], Item })); + for (let i = 0; i < 8; i++) { + await put('Conversations', { id: `c${i}`, ownerId: i % 2 ? 'alice' : 'bob', title: 'private'.repeat(33000) }); + await put('Users', { id: `u${i}`, username: `member${i}`, role: i % 2 ? 'producer' : 'consumer', location: 'Town', locationPrivacy: i % 2 === 1, + password: 'private hash', verificationCode: 'private code', email: 'private@example.invalid', coordinates: { lat: 1, long: 2 }, privatePadding: 'x'.repeat(230000) }); + await put('Listings', { id: `l${i}`, userId: `u${i}`, title: `Public listing ${i}`, description: 'advertised text'.repeat(16000), price: i + 1, privateExtra: 'not an advertised field' }); + await put('Transactions', { id: `t${i}`, buyerId: 'alice', sellerId: 'bob', status: 'pending', dialogNotes: 'local narrative'.repeat(15000) }); + } + await put('Conversations', { id: 'legacy', title: 'no owner' }); + + const db = require('../lib/dynamodbClient'); + const scan = db.scan; + const scans = []; + // Only substitute isolated table names. The production SDK adapter, scan + // limit, filters, projections and continuation keys all remain real. + db.scan = params => { + scans.push(params); + return scan({ ...params, TableName: tables[params.TableName] || params.TableName }); + }; + const app = express(); + app.use(express.json()); + app.use(require('../middleware/authMiddleware')); + app.use('/conversations', require('../routes/conversationRoutes')); + app.use('/users', require('../routes/userRoutes')); + app.use('/federation', require('../federation/federationRoutes')); + server = await new Promise(resolve => { const listener = app.listen(0, '127.0.0.1', () => resolve(listener)); }); + const base = `http://127.0.0.1:${server.address().port}`; + const token = jwt.sign({ userId: 'alice' }, process.env.JWT_SECRET, { expiresIn: '2h' }); + const headers = { Authorization: `Bearer ${token}` }; + async function collect(path) { + const result = []; + let cursor; + let pages = 0; + do { + const response = await fetch(`${base}${path}${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`, { headers, redirect: 'error' }); + assert.equal(response.status, 200); + assert.equal(response.headers.get('X-Pagination'), 'cursor-v1'); + result.push(...await response.json()); + cursor = response.headers.get('X-Next-Cursor'); + assert.ok(++pages < 30); + } while (cursor); + return { result, pages }; + } + const directory = await collect('/users?limit=100'); + assert.ok(directory.pages > 1, 'the 1 MiB scan boundary must require continuation even with a projection'); + assert.equal(directory.result.length, 8); + for (const user of directory.result) { + assert.deepEqual(Object.keys(user).sort(), ['id', 'username', 'role', ...(Number(user.id.slice(1)) % 2 ? ['location'] : [])].sort()); + } + const filtered = await collect('/users?limit=1&role=producer&location=Town'); + assert.deepEqual(filtered.result.map(user => user.id).sort(), ['u1', 'u3', 'u5', 'u7']); + const { createAccountClient } = await import('../../frontend/chat-ui/src/accountClient.mjs'); + const client = createAccountClient({ baseUrl: base }); + client.setToken(token); + const conversations = await client.request('GET', '/conversations?limit=100'); + assert.deepEqual(conversations.map(item => item.id).sort(), ['c1', 'c3', 'c5', 'c7']); + const first = await fetch(`${base}/conversations?limit=1`, { headers }); + const cursor = first.headers.get('X-Next-Cursor'); + assert.ok(cursor); + const foreign = await fetch(`${base}/conversations?limit=1&cursor=${cursor}`, { headers: { Authorization: `Bearer ${jwt.sign({ userId: 'bob' }, process.env.JWT_SECRET)}` } }); + assert.equal(foreign.status, 400); + + const { forEachExportPage, inspectExport } = require('../sync/exportPages'); + process.env.FEDERATION_PEER_KEYS = JSON.stringify({ [base]: process.env.FEDERATION_EXPORT_KEY }); + const exported = { listings: [], transactions: [], users: [] }; + let exportPages = 0; + await forEachExportPage(base, page => { + exportPages++; + for (const name of Object.keys(exported)) exported[name].push(...page[name]); + }); + assert.ok(exportPages > 1); + assert.deepEqual(exported.listings.map(item => item.id).sort(), Array.from({ length: 8 }, (_, i) => `l${i}`)); + assert.deepEqual(exported.transactions.map(item => item.id).sort(), Array.from({ length: 8 }, (_, i) => `t${i}`)); + assert.deepEqual(exported.users, []); + assert.ok(exported.listings.every(item => !Object.hasOwn(item, 'privateExtra'))); + assert.ok(exported.transactions.every(item => !Object.hasOwn(item, 'dialogNotes'))); + assert.deepEqual(await inspectExport(base), { listings: 8, transactions: 8, users: 0 }); + assert.equal((await store.send(new GetCommand({ TableName: tables.Users, Key: { id: 'u0' } }))).Item.password, 'private hash'); + assert.ok((await store.send(new GetCommand({ TableName: tables.Transactions, Key: { id: 't0' } }))).Item.dialogNotes.length > 100000); + assert.ok(scans.some(params => params.ExclusiveStartKey)); + assert.ok(scans.every(params => params.Limit > 0 && params.Limit <= 100)); + console.log('PASS: real DynamoDB 1 MiB continuation, ownership, private projections, filters, browser adapter, read-only exports and unchanged local records'); + } finally { + if (server) await new Promise(resolve => { server.close(resolve); server.closeAllConnections(); }); + for (const TableName of created) await admin.send(new DeleteTableCommand({ TableName })); + admin.destroy(); + } +} +main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/backend/server.js b/backend/server.js index f127e30e..dfed1770 100644 --- a/backend/server.js +++ b/backend/server.js @@ -36,7 +36,8 @@ app.use(cors({ }, credentials: true, methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], - allowedHeaders: ['Content-Type', 'Authorization'] + allowedHeaders: ['Content-Type', 'Authorization', 'x-federation-key'], + exposedHeaders: ['X-Pagination', 'X-Next-Cursor', 'X-Page-Limit', 'X-Federation-Omissions'] })); // ----------------------------------------- diff --git a/backend/sync/__tests__/exportPages.test.js b/backend/sync/__tests__/exportPages.test.js new file mode 100644 index 00000000..7608b444 --- /dev/null +++ b/backend/sync/__tests__/exportPages.test.js @@ -0,0 +1,59 @@ +jest.mock('axios'); +const axios = require('axios'); +const { forEachExportPage } = require('../exportPages'); + +describe('federation export traversal', () => { + const env = { ...process.env }; + beforeEach(() => { + jest.resetAllMocks(); + process.env.API_KEY = 'never-send-the-local-write-key'; + process.env.FEDERATION_PEER_KEYS = JSON.stringify({ 'https://peer.example': 'peer-read-only-key' }); + }); + afterAll(() => { process.env = env; }); + const page = (listings, cursor) => ({ + data: { listings, transactions: [], users: [] }, + headers: { 'x-pagination': 'cursor-v1', ...(cursor && { 'x-next-cursor': cursor }) }, + }); + test('imports every page including an empty continuation, using only the configured peer read key', async () => { + axios.get.mockResolvedValueOnce(page([{ id: 'l1' }], 'first')) + .mockResolvedValueOnce(page([], 'second')).mockResolvedValueOnce(page([{ id: 'l2' }])); + const seen = []; + await forEachExportPage('https://peer.example/', data => { seen.push(data); }); + expect(seen.flatMap(data => data.listings)).toEqual([{ id: 'l1' }, { id: 'l2' }]); + expect(axios.get).toHaveBeenCalledTimes(3); + expect(axios.get.mock.calls[0]).toEqual(['https://peer.example/federation/export', { + headers: { 'x-federation-key': 'peer-read-only-key' }, params: { limit: 50 }, + timeout: 5000, maxRedirects: 0, maxContentLength: 16 * 1024 * 1024, + }]); + expect(axios.get.mock.calls[1][1].params.cursor).toBe('first'); + expect(axios.get.mock.calls[2][1].params.cursor).toBe('second'); + expect(JSON.stringify(axios.get.mock.calls)).not.toContain(process.env.API_KEY); + }); + test('unknown destinations and userinfo cannot receive a configured credential', async () => { + for (const url of ['https://unconfigured.example', 'https://peer.example.evil', 'https://user@peer.example', 'https://peer.example/?redirect=elsewhere']) { + await expect(forEachExportPage(url, () => {})).rejects.toThrow(); + } + expect(axios.get).not.toHaveBeenCalled(); + }); + test('old unpaged responses and exported account rows fail closed', async () => { + const consume = jest.fn(); + axios.get.mockResolvedValueOnce({ data: { listings: [], transactions: [], users: [] } }); + await expect(forEachExportPage('https://peer.example', consume)).rejects.toThrow(/pagination/); + axios.get.mockResolvedValueOnce({ ...page([]), data: { listings: [], transactions: [], users: [{ id: 'u1', password: 'secret' }] } }); + await expect(forEachExportPage('https://peer.example', consume)).rejects.toThrow(/account/); + expect(consume).not.toHaveBeenCalled(); + }); + test('a failed later page is not treated as a complete export', async () => { + axios.get.mockResolvedValueOnce(page([{ id: 'l1' }], 'next')).mockRejectedValueOnce(new Error('peer unavailable')); + const consume = jest.fn(); + await expect(forEachExportPage('https://peer.example', consume)).rejects.toThrow('peer unavailable'); + expect(consume).toHaveBeenCalledTimes(1); + }); + test('repeating cursors and malformed page shapes stop without looping or consuming bad data', async () => { + axios.get.mockResolvedValue(page([], 'same')); + await expect(forEachExportPage('https://peer.example', () => {})).rejects.toThrow(/cursor/); + expect(axios.get).toHaveBeenCalledTimes(2); + axios.get.mockResolvedValue({ ...page([]), data: { listings: {}, transactions: [], users: [] } }); + await expect(forEachExportPage('https://peer.example', () => {})).rejects.toThrow(/page/); + }); +}); diff --git a/backend/sync/__tests__/federationSyncJob.test.js b/backend/sync/__tests__/federationSyncJob.test.js index cf6c43c6..6187d540 100644 --- a/backend/sync/__tests__/federationSyncJob.test.js +++ b/backend/sync/__tests__/federationSyncJob.test.js @@ -18,28 +18,25 @@ const runFederationSync = require('../federationSyncJob'); describe('runFederationSync', () => { beforeEach(() => { jest.clearAllMocks(); + process.env.FEDERATION_PEER_KEYS = JSON.stringify({ 'https://node.example': 'read-key' }); }); - it('fetches nodes, imports data, and updates lastSyncAt', async () => { + it('inspects the full export without importing summaries or updating lastSyncAt', async () => { const nodes = [{ url: 'https://node.example' }]; const exportData = { listings: [], transactions: [], users: [] }; mockScan.mockReturnValue({ promise: jest.fn().mockResolvedValue({ Items: nodes }) }); - axios.get.mockResolvedValue({ data: exportData }); + axios.get.mockResolvedValue({ data: exportData, headers: { 'x-pagination': 'cursor-v1' } }); importFederatedData.mockResolvedValue({ success: true }); mockPut.mockReturnValue({ promise: jest.fn().mockResolvedValue({}) }); await runFederationSync(); - expect(importFederatedData).toHaveBeenCalledWith(exportData); - expect(mockPut).toHaveBeenCalled(); - - const params = mockPut.mock.calls[0][0]; - expect(params.Item.url).toBe(nodes[0].url); - const timestamp = params.Item.lastSyncAt; - expect(new Date(timestamp).toISOString()).toBe(timestamp); + expect(axios.get).toHaveBeenCalledTimes(1); + expect(importFederatedData).not.toHaveBeenCalled(); + expect(mockPut).not.toHaveBeenCalled(); }); it('logs errors from axios without updating', async () => { diff --git a/backend/sync/__tests__/scheduler.test.js b/backend/sync/__tests__/scheduler.test.js index 6b4d6893..b52baf90 100644 --- a/backend/sync/__tests__/scheduler.test.js +++ b/backend/sync/__tests__/scheduler.test.js @@ -26,6 +26,7 @@ describe('scheduler', () => { beforeEach(() => { jest.resetModules(); jest.clearAllMocks(); + process.env.FEDERATION_PEER_KEYS = JSON.stringify({ 'https://node.example': 'read-key' }); jest.useFakeTimers(); }); @@ -33,10 +34,10 @@ describe('scheduler', () => { jest.useRealTimers(); }); - test('syncFromPeers upserts data and updates lastSyncAt', async () => { + test('syncFromPeers inspects without mutating canonical records or lastSyncAt', async () => { const dynamodbClient = require('../../lib/dynamodbClient'); dynamodbClient.scan.mockReturnValue({ - promise: () => Promise.resolve({ Items: [{ nodeUrl: 'http://node1' }] }), + promise: () => Promise.resolve({ Items: [{ nodeUrl: 'https://node.example' }] }), }); dynamodbClient.update.mockReturnValue({ promise: () => Promise.resolve() }); @@ -45,8 +46,9 @@ describe('scheduler', () => { data: { listings: [{ _id: 'l1', updatedAt: '2024-01-01T00:00:00.000Z' }], transactions: [{ _id: 't1', updatedAt: '2024-01-01T00:00:00.000Z' }], - users: [{ _id: 'u1', updatedAt: '2024-01-01T00:00:00.000Z' }], + users: [], }, + headers: { 'x-pagination': 'cursor-v1' }, }); const listingMock = require('../../marketplace/models/listings'); @@ -59,17 +61,11 @@ describe('scheduler', () => { const { syncFromPeers } = require('../scheduler'); await syncFromPeers(); - expect(listingMock.create).toHaveBeenCalledTimes(1); - expect(transactionMock.create).toHaveBeenCalledTimes(1); - expect(userMock.create).toHaveBeenCalledTimes(1); - - expect(dynamodbClient.update).toHaveBeenCalledTimes(1); - const params = dynamodbClient.update.mock.calls[0][0]; - const lastSyncAt = params.ExpressionAttributeValues[':lastSyncAt']; - // Check that lastSyncAt is a valid ISO string and close to now - const lastSyncDate = new Date(lastSyncAt); - expect(lastSyncDate.toISOString()).toBe(lastSyncAt); - expect(Math.abs(Date.now() - lastSyncDate.getTime())).toBeLessThan(5000); // within 5 seconds + expect(axios.get).toHaveBeenCalledTimes(1); + expect(listingMock.create).not.toHaveBeenCalled(); + expect(transactionMock.create).not.toHaveBeenCalled(); + expect(userMock.create).not.toHaveBeenCalled(); + expect(dynamodbClient.update).not.toHaveBeenCalled(); }); test('schedules syncFromPeers at interval', () => { diff --git a/backend/sync/exportPages.js b/backend/sync/exportPages.js new file mode 100644 index 00000000..37b522ee --- /dev/null +++ b/backend/sync/exportPages.js @@ -0,0 +1,52 @@ +const axios = require('axios'); + +function peerKey(base) { + const url = new URL(base); + if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password || url.search || url.hash) throw new Error('Invalid federation peer URL'); + let keys; + try { keys = JSON.parse(process.env.FEDERATION_PEER_KEYS || '{}'); } + catch { throw new Error('Invalid FEDERATION_PEER_KEYS configuration'); } + if (!keys || typeof keys !== 'object' || Array.isArray(keys)) throw new Error('Invalid FEDERATION_PEER_KEYS configuration'); + const key = Object.hasOwn(keys, base) && keys[base]; + if (typeof key !== 'string' || !key) throw new Error('Federation peer has no configured read-only key'); + if (key === process.env.API_KEY || key === process.env.WIX_API_KEY) throw new Error('Refusing to forward the local write-capable API key'); + const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname); + if (url.protocol !== 'https:' && !loopback) throw new Error('Federation peer credentials require HTTPS'); + return key; +} + +// Stream pages to the consumer instead of collecting a complete peer export in +// memory. Completion means a final page under the explicit pagination contract. +async function forEachExportPage(peer, consume) { + const base = peer.replace(/\/+$/, ''); + const key = peerKey(base); + let cursor; + for (let pages = 0; pages < 10000; pages++) { + const response = await axios.get(`${base}/federation/export`, { + headers: { 'x-federation-key': key }, params: { limit: 50, ...(cursor && { cursor }) }, + timeout: 5000, maxRedirects: 0, maxContentLength: 16 * 1024 * 1024, + }); + const header = name => response.headers?.get?.(name) ?? response.headers?.[name]; + if (header('x-pagination') !== 'cursor-v1') throw new Error('Peer does not declare cursor pagination'); + const { data } = response; + if (!data || !Array.isArray(data.listings) || !Array.isArray(data.transactions) || !Array.isArray(data.users)) throw new Error('Invalid federation export page'); + if (data.users.length) throw new Error('Peer export contains local account records'); + const next = header('x-next-cursor'); + if (next !== undefined && (typeof next !== 'string' || !/^[\w-]+$/.test(next) || next.length > 16384 || next === cursor)) throw new Error('Invalid federation export cursor'); + await consume({ listings: data.listings, transactions: data.transactions, users: [] }); + if (!next) return; + cursor = next; + } + throw new Error('Federation export exceeded page safety limit; sync is incomplete'); +} + +async function inspectExport(peer) { + const counts = { listings: 0, transactions: 0, users: 0 }; + await forEachExportPage(peer, page => { + counts.listings += page.listings.length; + counts.transactions += page.transactions.length; + }); + return counts; +} + +module.exports = { forEachExportPage, inspectExport }; diff --git a/backend/sync/federationSyncJob.js b/backend/sync/federationSyncJob.js index 92b7f8bb..27aef8f0 100644 --- a/backend/sync/federationSyncJob.js +++ b/backend/sync/federationSyncJob.js @@ -1,18 +1,15 @@ -const axios = require("axios"); -const { getAllNodes, saveNode } = require("../models/nodeRegistry"); -const { importFederatedData } = require("./importHelper"); +const { getAllNodes } = require('../models/nodeRegistry'); +const { inspectExport } = require('./exportPages'); +// Legacy entry point: inspection only. Never pass privacy-filtered summaries +// to the unconditional canonical-record importer. async function runFederationSync() { - const nodes = await getAllNodes(); - for (const node of nodes) { + for (const node of await getAllNodes()) { try { - const { data } = await axios.get(`${node.url}/federation/export`); - await importFederatedData(data); - node.lastSyncAt = new Date().toISOString(); - await saveNode(node); - console.log(`✅ Synced with ${node.url}`); - } catch (err) { - console.error(`❌ Failed to sync with ${node.url}: ${err.message}`); + const counts = await inspectExport(node.url || node.nodeUrl); + console.info('Federation export inspected; automatic import disabled', { counts }); + } catch (error) { + console.error('Federation export inspection failed; no sync recorded', error.message); } } } diff --git a/backend/sync/scheduler.js b/backend/sync/scheduler.js index 689d7eb8..76a2a527 100644 --- a/backend/sync/scheduler.js +++ b/backend/sync/scheduler.js @@ -1,59 +1,7 @@ -const axios = require("axios"); -const dynamodbClient = require("../lib/dynamodbClient"); -const Listing = require("../marketplace/models/listings"); -const Transaction = require("../models/transaction"); -const User = require("../models/user"); +const inspectPeers = require('./federationSyncJob'); -const NODE_REGISTRY_TABLE = "NodeRegistry"; - -const upsertMany = async (Model, items, key = "_id") => { - for (let item of items) { - const existing = await Model.findOne({ [key]: item[key] }); - if (!existing) { - await Model.create(item); - } else if (new Date(item.updatedAt) > new Date(existing.updatedAt)) { - await Model.updateOne({ [key]: item[key] }, item); - } - } -}; - -const syncFromPeers = async () => { - try { - const { Items: nodes = [] } = await dynamodbClient - .scan({ TableName: NODE_REGISTRY_TABLE }) - .promise(); - - for (let node of nodes) { - try { - const res = await axios.get(`${node.nodeUrl}/federation/export`); - const { listings, transactions, users } = res.data; - - if (listings) await upsertMany(Listing, listings); - if (transactions) await upsertMany(Transaction, transactions); - if (users) await upsertMany(User, users); - - await dynamodbClient - .update({ - TableName: NODE_REGISTRY_TABLE, - Key: { nodeUrl: node.nodeUrl }, - UpdateExpression: "set lastSyncAt = :lastSyncAt", - ExpressionAttributeValues: { - ":lastSyncAt": new Date().toISOString(), - }, - }) - .promise(); - - console.log(`✔ Synced data from node: ${node.nodeUrl}`); - } catch (err) { - console.error(`❌ Federation sync error for node ${node.nodeUrl}:`, err.message); - } - } - } catch (err) { - console.error("❌ Federation sync error:", err.message); - } -}; - -// Example scheduled interval (every 30 mins) -setInterval(syncFromPeers, 30 * 60 * 1000); +// Keep the legacy schedule, but never merge privacy-filtered export pages. +const syncFromPeers = () => inspectPeers(); +setInterval(() => syncFromPeers().catch(() => console.error('Federation registry scan failed')), 30 * 60 * 1000); module.exports = { syncFromPeers }; diff --git a/frontend/chat-ui/accountClient.test.js b/frontend/chat-ui/accountClient.test.js index 59197fd0..969faaab 100644 --- a/frontend/chat-ui/accountClient.test.js +++ b/frontend/chat-ui/accountClient.test.js @@ -46,3 +46,54 @@ test('attachment downloads use the same account and reject credential destinatio client.setToken(null); await assert.rejects(client.attachment('/uploads/file.txt'), /Sign in/); }); + +test('collection requests follow empty pages and retain the account and query filters', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + const calls = []; + const pages = [ + new Response('[]', { headers: { 'X-Next-Cursor': 'opaque-1' } }), + new Response('[{"id":"mine"}]', { headers: { 'X-Next-Cursor': 'opaque-2' } }), + new Response('[{"id":"also-mine"}]'), + ]; + const client = createAccountClient({ fetchImpl: async (url, options) => { + calls.push({ url, options }); + return pages.shift(); + } }); + client.setToken('alice-token'); + assert.deepEqual(await client.request('GET', '/conversations?limit=1'), [{ id: 'mine' }, { id: 'also-mine' }]); + assert.deepEqual(calls.map(call => call.url), ['/conversations?limit=1', '/conversations?limit=1&cursor=opaque-1', '/conversations?limit=1&cursor=opaque-2']); + assert.ok(calls.every(call => call.options.headers.Authorization === 'Bearer alice-token' && call.options.redirect === 'error')); +}); + +test('changing accounts between pages rejects the collection instead of returning a partial old-account result', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let finish; + let calls = 0; + const client = createAccountClient({ fetchImpl: async () => { + if (++calls === 1) return new Response('[{"id":"alice-private"}]', { headers: { 'X-Next-Cursor': 'next' } }); + return new Promise(resolve => { finish = resolve; }); + } }); + client.setToken('alice-token'); + const pending = client.request('GET', '/conversations'); + for (let wait = 0; !finish && wait < 100; wait++) await new Promise(resolve => setImmediate(resolve)); + assert.equal(typeof finish, 'function', 'client must request the continuation page'); + client.setToken('bob-token'); + finish(new Response('[]')); + await assert.rejects(pending, /Account changed/); +}); + +test('collection pagination refuses a repeated cursor and surfaces later HTTP failures', async () => { + const { createAccountClient } = await import('./src/accountClient.mjs'); + let calls = 0; + const client = createAccountClient({ fetchImpl: async () => { + calls++; + return new Response('[]', { headers: { 'X-Next-Cursor': 'same' } }); + } }); + await assert.rejects(client.request('GET', '/conversations'), /cursor/); + assert.equal(calls, 2); + let attempt = 0; + const failing = createAccountClient({ fetchImpl: async () => ++attempt === 1 + ? new Response('[{"id":"first"}]', { headers: { 'X-Next-Cursor': 'next' } }) + : new Response('{"error":"page failed"}', { status: 500 }) }); + await assert.rejects(failing.request('GET', '/conversations'), /page failed/); +}); diff --git a/frontend/chat-ui/src/accountClient.mjs b/frontend/chat-ui/src/accountClient.mjs index 0c638415..fd30ea63 100644 --- a/frontend/chat-ui/src/accountClient.mjs +++ b/frontend/chat-ui/src/accountClient.mjs @@ -17,8 +17,7 @@ export function createAccountClient({ baseUrl = '', apiKey = '', fetchImpl = (.. if (apiKey) result['x-api-key'] = apiKey; return result; } - async function read(method, path, body, binary = false) { - const started = version; + async function readPage(method, path, body, binary, started) { const options = { method, headers: headers(), redirect: 'error' }; if (body !== undefined) { options.headers['Content-Type'] = 'application/json'; @@ -38,7 +37,26 @@ export function createAccountClient({ baseUrl = '', apiKey = '', fetchImpl = (.. error.data = data; throw error; } - return data; + return { data, cursor: response.headers.get('X-Next-Cursor') }; + } + async function read(method, path, body, binary = false) { + const started = version; + let page = await readPage(method, path, body, binary, started); + if (method !== 'GET' || binary || !Array.isArray(page.data)) return page.data; + const items = [...page.data]; + let previous; + const [pathname, query = ''] = path.split('?'); + for (let pages = 1; page.cursor; pages++) { + if (pages >= 10000 || page.cursor === previous || !/^[\w-]+$/.test(page.cursor) || page.cursor.length > 16384) throw new Error('Invalid collection cursor'); + if (version !== started) throw new Error('Account changed during request'); + previous = page.cursor; + const params = new URLSearchParams(query); + params.set('cursor', page.cursor); + page = await readPage(method, `${pathname}?${params}`, undefined, false, started); + if (!Array.isArray(page.data)) throw new Error('Invalid collection page'); + items.push(...page.data); + } + return items; } async function attachment(path) { if (!token) throw new Error('Sign in to download attachments'); From 0f86fdd873300d267d4f9f67babb215d732ed47f Mon Sep 17 00:00:00 2001 From: Nirmal Khedkar Date: Fri, 30 Oct 2026 23:12:40 -0700 Subject: [PATCH 3/4] refactor(retries): separate validation and replay Move input checks and saved-receipt replay into focused helpers. Extract verification database setup and worker cleanup without changing request results, receipt keys or lifecycle ordering. Add a regression for a missing stored assistant reply: replay must return 410 without repeating delivery or returning a partial success. Signed-off-by: Nirmal Khedkar --- backend/__tests__/message_retries.test.js | 13 ++++++ backend/scripts/check-message-retries.js | 32 +++++++++------ backend/services/retrySafeMessage.js | 48 ++++++++++++++--------- 3 files changed, 62 insertions(+), 31 deletions(-) diff --git a/backend/__tests__/message_retries.test.js b/backend/__tests__/message_retries.test.js index 81cb1073..2fb47fcb 100644 --- a/backend/__tests__/message_retries.test.js +++ b/backend/__tests__/message_retries.test.js @@ -318,6 +318,19 @@ test('removing original messages never lets a retry resurrect them or retained n assert.equal(aiCalls, 1); }); + +test('a missing persisted assistant reply cannot replay a partial success', async () => { + const first = await send(); + assert.equal(first.status, 201); + const [replyKey] = [...db.tables.get('Messages')].find(([, row]) => row.id === first.body.reply.id); + db.tables.get('Messages').delete(replyKey); + const retry = await send(); + assert.equal(retry.status, 410); + assert.equal(retry.body.code, 'message_send_removed'); + assert.equal(messages().length, 1); + assert.equal(aiCalls, 1); +}); + test('a missing receipt table refuses keyed sends before any message or AI work', async () => { const put = docClient.put; docClient.put = params => params.TableName === 'MessageSends' diff --git a/backend/scripts/check-message-retries.js b/backend/scripts/check-message-retries.js index f7eb8e90..06ce6672 100644 --- a/backend/scripts/check-message-retries.js +++ b/backend/scripts/check-message-retries.js @@ -18,6 +18,24 @@ async function checkPrivateAccess(send, target, apiCaller) { assert.equal((await send(target, { user: 'another-grower' })).status, 404); } +async function createTables(client, definitions, created) { + for (const [name, composite] of definitions) { + await client.send(new CreateTableCommand({ TableName: name, BillingMode: 'PAY_PER_REQUEST', + AttributeDefinitions: composite ? [{ AttributeName: 'conversationId', AttributeType: 'S' }, { AttributeName: 'id', AttributeType: 'S' }] + : [{ AttributeName: 'id', AttributeType: 'S' }], + KeySchema: composite ? [{ AttributeName: 'conversationId', KeyType: 'HASH' }, { AttributeName: 'id', KeyType: 'RANGE' }] + : [{ AttributeName: 'id', KeyType: 'HASH' }], + })); + created.push(name); + } +} + +async function stopWorkers(workers, stop) { + for (const target of workers) { + if (target.child.exitCode === null && target.child.signalCode === null) await stop(target); + } +} + async function main() { const endpoint = process.env.DYNAMODB_TEST_ENDPOINT; if (!endpoint || !['localhost', '127.0.0.1', '[::1]'].includes(new URL(endpoint).hostname)) { @@ -85,15 +103,7 @@ async function main() { } try { - for (const [name, composite] of [[messages, true], [receipts, false], [conversations, false]]) { - await client.send(new CreateTableCommand({ TableName: name, BillingMode: 'PAY_PER_REQUEST', - AttributeDefinitions: composite ? [{ AttributeName: 'conversationId', AttributeType: 'S' }, { AttributeName: 'id', AttributeType: 'S' }] - : [{ AttributeName: 'id', AttributeType: 'S' }], - KeySchema: composite ? [{ AttributeName: 'conversationId', KeyType: 'HASH' }, { AttributeName: 'id', KeyType: 'RANGE' }] - : [{ AttributeName: 'id', KeyType: 'HASH' }], - })); - created.push(name); - } + await createTables(client, [[messages, true], [receipts, false], [conversations, false]], created); await seedConversations(client, conversations); const firstWorker = await worker(true); const secondWorker = await worker(false); @@ -129,9 +139,7 @@ async function main() { assert.equal(afterCrash.Count, 3); console.log('PASS: process death leaves a pending receipt and does not duplicate a message or AI invocation.'); } finally { - for (const target of workers) { - if (target.child.exitCode === null && target.child.signalCode === null) await stop(target); - } + await stopWorkers(workers, stop); for (const name of created.reverse()) await client.send(new DeleteTableCommand({ TableName: name })); client.destroy(); } diff --git a/backend/services/retrySafeMessage.js b/backend/services/retrySafeMessage.js index 1657a837..3f742526 100644 --- a/backend/services/retrySafeMessage.js +++ b/backend/services/retrySafeMessage.js @@ -20,16 +20,41 @@ function problem(status, code, error) { return { status, body: { code, error } } const failed = () => problem(500, 'message_send_failed', 'This send failed and will not be repeated automatically. Inspect the conversation before starting a new send.'); -async function send(req, deliver) { +function readInput(req) { const key = req.headers['idempotency-key']; if (typeof key !== 'string' || !/^[\x21-\x7e]{1,128}$/.test(key)) { - return problem(400, 'invalid_idempotency_key', 'Idempotency-Key must be 1–128 printable ASCII characters without spaces.'); + return { error: problem(400, 'invalid_idempotency_key', 'Idempotency-Key must be 1–128 printable ASCII characters without spaces.') }; } // Auth middleware has already verified the JWT or accepted the configured // API key. Never use body.from (a display/role label) as the caller identity. const userId = typeof req.user?.userId === 'string' && req.user.userId ? req.user.userId : null; const apiKey = req.user ? null : req.headers['x-api-key']; - if (!userId && !apiKey) return problem(400, 'idempotency_identity_required', 'Retry-safe sends require a stable authenticated user ID or API key.'); + if (!userId && !apiKey) { + return { error: problem(400, 'idempotency_identity_required', 'Retry-safe sends require a stable authenticated user ID or API key.') }; + } + return { key, userId, apiKey }; +} + +async function replay(receipt, fingerprint, conversationId) { + if (receipt.fingerprint !== fingerprint) { + return problem(409, 'idempotency_key_conflict', 'This key was already used with a different message payload.'); + } + if (receipt.state !== 'complete') { + return problem(409, 'message_send_pending', 'This send is still running or was interrupted. Retry this key later; inspect the conversation before starting a new send.'); + } + if (receipt.outcome.status === 500) return failed(); + const message = await Message.getMessage(conversationId, receipt.outcome.messageId); + const reply = receipt.outcome.replyId ? await Message.getMessage(conversationId, receipt.outcome.replyId) : null; + if (!message || (receipt.outcome.replyId && !reply)) { + return problem(410, 'message_send_removed', 'The original message or reply is no longer available. It will not be recreated.'); + } + return { status: 201, body: reply ? { message, ai: reply, reply } : { message } }; +} + +async function send(req, deliver) { + const input = readInput(req); + if (input.error) return input.error; + const { key, userId, apiKey } = input; const conversationId = req.params.conversationId; const { from, to, content, type = 'text', file } = req.body; @@ -49,22 +74,7 @@ async function send(req, deliver) { } catch { return problem(503, 'message_send_unavailable', 'Retry protection is unavailable. No send was started by this request. Retry with the same key.'); } - if (!claim.claimed) { - const { receipt } = claim; - if (receipt.fingerprint !== fingerprint) { - return problem(409, 'idempotency_key_conflict', 'This key was already used with a different message payload.'); - } - if (receipt.state !== 'complete') { - return problem(409, 'message_send_pending', 'This send is still running or was interrupted. Retry this key later; inspect the conversation before starting a new send.'); - } - if (receipt.outcome.status === 500) return failed(); - const message = await Message.getMessage(conversationId, receipt.outcome.messageId); - const reply = receipt.outcome.replyId ? await Message.getMessage(conversationId, receipt.outcome.replyId) : null; - if (!message || (receipt.outcome.replyId && !reply)) { - return problem(410, 'message_send_removed', 'The original message or reply is no longer available. It will not be recreated.'); - } - return { status: 201, body: reply ? { message, ai: reply, reply } : { message } }; - } + if (!claim.claimed) return replay(claim.receipt, fingerprint, conversationId); let result, outcome; try { From 05b894f082f03a7e9cf2a6ee3012093999415155 Mon Sep 17 00:00:00 2001 From: Nirmal Khedkar Date: Fri, 6 Nov 2026 09:49:07 -0800 Subject: [PATCH 4/4] fix(chat): rate limit message reads and sends Signed-off-by: Nirmal Khedkar --- backend/__tests__/collectionRateLimit.test.js | 27 +++++++++++++++++-- backend/docs/collection-pagination.md | 7 ++++- backend/routes/communicationRoutes.js | 3 ++- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/backend/__tests__/collectionRateLimit.test.js b/backend/__tests__/collectionRateLimit.test.js index 396982ed..43719bb8 100644 --- a/backend/__tests__/collectionRateLimit.test.js +++ b/backend/__tests__/collectionRateLimit.test.js @@ -5,8 +5,15 @@ const jwt = require('jsonwebtoken'); jest.mock('../bull/pingJobs', () => ({ addPingJob: jest.fn() })); jest.mock('../bull/smsQueue', () => ({ enqueueSms: jest.fn() })); jest.mock('../federation/federationSyncJob', () => jest.fn()); +jest.mock('../lib/dynamodbClient', () => ({ + get: ({ Key }) => ({ promise: async () => ({ + Item: Key.id === 'owned' ? { id: 'owned', ownerId: 'alice' } : undefined, + }) }), + query: () => ({ promise: async () => ({ Items: [] }) }), + put: () => ({ promise: async () => ({}) }), +})); -describe('directory and attachment request budgets', () => { +describe('account collection request budgets', () => { let app, server, agent; const send = (method, url) => request(server)[method](url).agent(agent); const env = { ...process.env }; @@ -32,6 +39,7 @@ describe('directory and attachment request budgets', () => { test.each([ ['/users?limit=0', 400], ['/uploads/bad%00name', 404], + ['/messages/missing', 404], ])('%s preserves normal responses and throttles each account independently', async (url, status) => { const token = authorization('alice'); for (let i = 0; i < 1000; i++) { @@ -44,7 +52,7 @@ describe('directory and attachment request budgets', () => { expect((await send('get', url).set('Authorization', authorization('bob'))).status).toBe(status); }, 20000); - test.each(['/users', '/uploads/bad%00name'])('%s limits API-key-only callers without granting account access', async url => { + test.each(['/users', '/uploads/bad%00name', '/messages/missing'])('%s limits API-key-only callers without granting account access', async url => { for (let i = 0; i < 1000; i++) { expect((await send('get', url).set('x-api-key', process.env.API_KEY)).status).toBe(401); } @@ -56,6 +64,7 @@ describe('directory and attachment request budgets', () => { test.each([ ['/users?limit=0', 400], ['/uploads/bad%00name', 404], + ['/messages/missing', 404], ])('%s cannot gain another account budget by adding an API key', async (url, status) => { const token = authorization('credential-owner'); for (let i = 0; i < 1000; i++) { @@ -65,4 +74,18 @@ describe('directory and attachment request budgets', () => { expect((await send('get', url).set('Authorization', token).set('x-api-key', key)).status).toBe(429); } }, 20000); + + test('message reads and sends share an account budget across conversation IDs and encoded paths', async () => { + const token = authorization('alice'); + for (let i = 0; i < 1000; i++) { + expect((await send('get', '/messages/missing').set('Authorization', token)).status).toBe(404); + } + for (const method of ['get', 'post']) { + const blocked = await send(method, '/messages/%6fwned') + .set('Authorization', token).send({ from: 'bob', content: 'hello' }); + expect(blocked.status).toBe(429); + expect(Number(blocked.headers['retry-after'])).toBeGreaterThan(0); + } + expect((await send('get', '/messages/owned').set('Authorization', authorization('bob'))).status).toBe(404); + }, 20000); }); diff --git a/backend/docs/collection-pagination.md b/backend/docs/collection-pagination.md index c0bfd6b3..862851bf 100644 --- a/backend/docs/collection-pagination.md +++ b/backend/docs/collection-pagination.md @@ -22,7 +22,12 @@ account already verified by outer middleware, or an IP fallback. A separate budget after account authentication enforces 1,000 requests per account even when an API key accompanies the bearer token. Excess requests return HTTP 429 with `Retry-After`. Existing ownership, response fields and pagination remain -unchanged below the limit. Conversation and federation routes are unchanged. +unchanged below the limit. Message reads and sends (`GET` and `POST` +`/messages/:conversationId`) use the same two-guard policy with a separate +1,000-request budget shared across methods and conversation IDs for each +account. The guards run before ownership lookups, message storage and AI +responses. Adding an API key or refreshing a token does not reset an account's +budget. Conversation and federation routes are unchanged. Budgets are local to each server process and reset on restart. Multiple instances need a shared limiter store for a deployment-wide budget. Proxy diff --git a/backend/routes/communicationRoutes.js b/backend/routes/communicationRoutes.js index 675bce23..bc52a3c6 100644 --- a/backend/routes/communicationRoutes.js +++ b/backend/routes/communicationRoutes.js @@ -4,8 +4,9 @@ const commController = require('../controllers/communication_controller'); const auth = require('../middleware/chatAuth'); const owner = require('../middleware/conversationOwner'); const asyncHandler = require('../utils/asyncHandler'); +const requestRateLimit = require('../middleware/requestRateLimit'); -router.use(auth); +router.use(requestRateLimit(), auth, requestRateLimit()); router.post('/:conversationId', asyncHandler(owner), asyncHandler(commController.sendMessage)); router.get('/:conversationId', asyncHandler(owner), asyncHandler(commController.listMessages));