Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 20 additions & 11 deletions CONFORMANCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,26 @@ Adopts the covenant scale (LBTAS, v2 floor) and carries its own record instance.

The reference economy. Its operator/federation model is the named reference for the substrate's frontend-as-operator design target.

Private application chat uses the verified backend account `userId` as an
immutable conversation owner. Conversation/message routes and attachment
downloads enforce that boundary; shared integration keys supply no participant
identity. Legacy ownerless rows remain inaccessible pending explicit operator
assignment. Bindings: `backend/middleware/chatAuth.js`,
`conversationOwner.js`, `attachmentOwner.js`, the conditional conversation
model writes, and `backend/__tests__/conversationAccess.test.js`. This does not
promote private chat to a sealed commons dialog or change the in-review ledger
claims above. SSE uses the same verified owner identity before stream headers,
implemented in `backend/lib/conversationStreams.js` and checked by
`backend/__tests__/conversationStreams.test.js`. The unrestricted global feed
is retired. Streams expire with the account token and close on local parent
deletion after retryable message cleanup. Shared request/capacity guards bind
to the verified account; Redis is required across replicas. Bindings include
`backend/middleware/streamConcurrency.js`, `backend/__tests__/streamCapacity.test.js`
and the optional real-Redis `streamSharedCapacity.test.js`. Shared capacity does
not provide cross-replica event delivery or deletion invalidation. See
`backend/docs/conversation-ownership.md` for migration and client
integration limits.

The application message API supports optional retry receipts, scoped to the
authenticated caller and conversation. These are private operational metadata,
not sealed dialog or commons entries; they store hashes and message references,
Expand All @@ -68,14 +88,3 @@ binding changes, and the in-review statuses above remain in review.
dates, adjacent-crop conflicts, resize impact, portable round-tripping, invalid
imports, and storage errors. This is feature verification, not a claim that the
delegated JFA invariants are now enforced by this repository.
Private application chat uses the verified backend account `userId` as an
immutable conversation owner. Conversation/message routes and attachment
downloads enforce that boundary; shared integration keys supply no participant
identity. Legacy ownerless rows remain inaccessible pending explicit operator
assignment. Bindings: `backend/middleware/chatAuth.js`,
`conversationOwner.js`, `attachmentOwner.js`, the conditional conversation
model writes, and `backend/__tests__/conversationAccess.test.js`. This does not
promote private chat to a sealed commons dialog or change the in-review ledger
claims above. Legacy SSE endpoints remain retired with HTTP 410 until the
companion authenticated-stream change lands. See `backend/docs/conversation-ownership.md` for migration and client
integration limits.
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,14 @@ Additional chat interface endpoints (e.g., `/conversations`, `/messages/:id`, `/
structures are documented inline in the frontend components
[`Sidebar.jsx`](frontend/chat-ui/src/components/Sidebar.jsx) and
[`ChatWindow.jsx`](frontend/chat-ui/src/components/ChatWindow.jsx).
Private chat requires the existing backend account JWT, including for SSE and
attachments. Shared API keys do not identify a chat owner. Legacy ownerless
chats remain inaccessible pending explicit operator assignment. The host
application supplies its login token to `setAccountToken`; the chat UI does
not provide a complete onboarding flow. See
[conversation ownership](backend/docs/conversation-ownership.md) and
[authenticated streams](backend/docs/authenticated-streams.md). The global
`/events` feed is retired; use an authorized `/stream/:conversationId` request.
### User Registration
#### POST /userRegistration
```
Expand Down
2 changes: 1 addition & 1 deletion backend/__tests__/communication_controller.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ test('agrinet responder intercepts Agrinet-specific questions', async () => {
assert.strictEqual(agrinetCalled, true, 'Agrinet guard should evaluate');
const messageBroadcasts = broadcastEvents.filter((evt) => evt.payload.type === 'message');
assert.strictEqual(messageBroadcasts.length, 2, 'both messages should be broadcast');
assert.strictEqual(emitted.length, 2, 'emitMessage should fire for both messages');
assert.strictEqual(emitted.length, 0, 'legacy alias must not duplicate canonical broadcasts');
} finally {
agrinetResponder.shouldHandle = originalShouldHandle;
agrinetResponder.generateResponse = originalGenerateResponse;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,8 @@ test('sendMessage streams tokens and persists ai message', async () => {
assert.equal(jsonResponse.ai.id, aiMessage.id);

assert.equal(tokenEvents.length, 3);
assert.deepEqual(directTokens, ['Hello', ' ', 'world!']);
assert.deepEqual(directTokens, [], 'legacy alias must not duplicate canonical tokens');
assert.equal(messageEvents.length, 2, 'one event per persisted message');
assert(tokenEvents.every((event) => event.id === aiMessage.id));

assert.ok(
Expand Down
180 changes: 180 additions & 0 deletions backend/__tests__/conversationStreams.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
const jwt = require('jsonwebtoken');
const mockConversations = new Map();
let mockCleanupFailure = false;
let mockHoldRead;
jest.mock('../lib/dynamodbClient', () => ({
get: ({ Key }) => ({ promise: async () => {
const Item = mockConversations.get(Key.id);
const hold = mockHoldRead;
mockHoldRead = undefined;
if (hold) { hold.started(); await hold.wait; }
return { Item };
} }),
delete: ({ Key }) => ({ promise: async () => { mockConversations.delete(Key.id); return {}; } }),
query: () => ({ promise: async () => {
if (mockCleanupFailure) throw new Error('fixture cleanup failure');
return { Items: [] };
} }),
}));
jest.mock('../bull/pingJobs', () => ({ addPingJob: jest.fn() }));
jest.mock('../bull/smsQueue', () => ({ enqueueSms: jest.fn() }));
jest.mock('../federation/federationSyncJob', () => jest.fn());

describe('production conversation SSE boundary', () => {
const originalEnv = { ...process.env };
const originalGlobals = Object.fromEntries(['broadcast', 'emitToken', 'emitMessage'].map(key => [key, global[key]]));
let server;
let base;
const controllers = [];
const bearer = (userId, seconds = 60) => `Bearer ${jwt.sign({ userId }, process.env.JWT_SECRET, { expiresIn: seconds })}`;
async function open(path, headers = {}) {
const controller = new AbortController();
controllers.push(controller);
return fetch(base + path, { headers, signal: AbortSignal.any([controller.signal, AbortSignal.timeout(4000)]) });
}
beforeAll(async () => {
process.env.MINIMAL_SERVER = 'false';
process.env.JWT_SECRET = 'stream-fixture-secret';
process.env.API_KEY = 'stream-fixture-key';
({ server } = require('../server'));
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
base = `http://127.0.0.1:${server.address().port}`;
});
beforeEach(() => {
mockCleanupFailure = false;
mockHoldRead = undefined;
mockConversations.clear();
mockConversations.set('mine', { id: 'mine', ownerId: 'alice' });
mockConversations.set('foreign', { id: 'foreign', ownerId: 'bob' });
mockConversations.set('legacy', { id: 'legacy' });
});
afterEach(() => { for (const controller of controllers.splice(0)) controller.abort(); });
afterAll(async () => {
server.closeAllConnections();
await new Promise(resolve => server.close(resolve));
process.env = originalEnv;
for (const [key, value] of Object.entries(originalGlobals)) {
if (value === undefined) delete global[key]; else global[key] = value;
}
});

test.each([
['anonymous', {}, ''],
['shared key', { 'x-api-key': 'stream-fixture-key' }, ''],
['invalid bearer and shared key', { Authorization: 'Bearer invalid', 'x-api-key': 'stream-fixture-key' }, ''],
['query key', {}, '?x-api-key=stream-fixture-key'],
])('rejects %s before opening stream headers', async (_name, headers, query) => {
let response;
try { response = await open('/stream/mine' + query, headers); } catch { response = { status: 'open stream', headers: new Headers() }; }
expect(response.status).toBe(401);
expect(response.headers.get('content-type')).not.toContain('text/event-stream');
});
test.each(['foreign', 'legacy', 'missing'])('conceals %s from an authenticated non-owner', async id => {
let response;
try { response = await open(`/stream/${id}`, { Authorization: bearer('alice') }); } catch { response = { status: 'open stream' }; }
expect(response.status).toBe(404);
expect(await response.json()).toEqual({ error: 'Not found' });
});
test('query bearer is not a substitute for an Authorization header', async () => {
const token = bearer('alice').slice(7);
let response;
try { response = await open(`/stream/mine?access_token=${token}`); } catch { response = { status: 'open stream' }; }
expect(response.status).toBe(401);
});
test('retires the unrestricted global feed rather than exposing transaction events to every account', async () => {
const response = await open('/events', { Authorization: bearer('alice') });
expect(response.status).toBe(410);
expect(response.headers.get('content-type')).not.toContain('text/event-stream');
});
test('owner receives only conversation-scoped events and deletion ends the stream', async () => {
const response = await open('/stream/mine', { Authorization: bearer('alice') });
expect(response.status).toBe(200);
expect(response.headers.get('cache-control')).toContain('no-store');
const reader = response.body.getReader();
global.emitMessage('foreign', { content: 'Bobs secret' });
global.emitMessage('mine', { content: 'Alice answer' });
const chunk = new TextDecoder().decode((await reader.read()).value);
expect(chunk).toContain('Alice answer');
expect(chunk).not.toContain('Bobs secret');
const deletion = await fetch(base + '/conversations/mine', { method: 'DELETE', headers: { Authorization: bearer('alice') } });
expect(deletion.status).toBe(204);
expect((await reader.read()).done).toBe(true);
});
test('a connection closes when its account token expires', async () => {
const response = await open('/stream/mine', { Authorization: `Bearer ${jwt.sign({ userId: 'alice', exp: Math.ceil(Date.now() / 1000) + 1 }, process.env.JWT_SECRET)}` });
expect(response.status).toBe(200);
const read = await response.body.getReader().read();
expect(read.done).toBe(true);
expect((await open('/stream/mine', { Authorization: bearer('alice', -1) })).status).toBe(401);
});

test('failed message cleanup preserves ownership for retry and committed deletion closes streams', async () => {
const response = await open('/stream/mine', { Authorization: bearer('alice') });
const reader = response.body.getReader();
mockCleanupFailure = true;
const expectedError = jest.spyOn(console, 'error').mockImplementation(() => {});
const deletion = await fetch(base + '/conversations/mine', { method: 'DELETE', headers: { Authorization: bearer('alice') } });
expectedError.mockRestore();
expect(deletion.status).toBe(500);
expect(mockConversations.has('mine')).toBe(true);
mockCleanupFailure = false;
const retry = await fetch(base + '/conversations/mine', { method: 'DELETE', headers: { Authorization: bearer('alice') } });
expect(retry.status).toBe(204);
expect(mockConversations.has('mine')).toBe(false);
expect((await reader.read()).done).toBe(true);
});

test('deletion revokes a stream whose ownership lookup is still in flight', async () => {
let release;
let started;
const reading = new Promise(resolve => { started = resolve; });
mockHoldRead = { started, wait: new Promise(resolve => { release = resolve; }) };
const pending = open('/stream/mine', { Authorization: bearer('alice') });
await reading;
const deletion = await fetch(base + '/conversations/mine', { method: 'DELETE', headers: { Authorization: bearer('alice') } });
expect(deletion.status).toBe(204);
release();
const response = await pending;
expect(response.status).toBe(404);
expect(response.headers.get('content-type')).not.toContain('text/event-stream');
});
test('retired global feed returns 410 without account credentials', async () => {
const response = await open('/events');
expect(response.status).toBe(410);
});
test('production stream route shares reservations and reuses released capacity', async () => {
const { EventEmitter } = require('node:events');
const { createConcurrentStreamLimit } = require('../middleware/streamConcurrency');
const reserve = createConcurrentStreamLimit({ maximum: 5 });
const userId = 'shared-capacity-owner';
mockConversations.set('capacity-room', { id: 'capacity-room', ownerId: userId });
const held = Array.from({ length: 5 }, () => new EventEmitter());
try {
for (const res of held) {
await reserve({ user: { userId, exp: Math.ceil(Date.now() / 1000) + 60 } }, res, () => {});
}
const blocked = await open('/stream/capacity-room', { Authorization: bearer(userId) });
expect(blocked.status).toBe(429);
held[0].emit('close');
await new Promise(resolve => setImmediate(resolve));
const admitted = await open('/stream/capacity-room', { Authorization: bearer(userId) });
expect(admitted.status).toBe(200);
expect(admitted.headers.get('content-type')).toContain('text/event-stream');
} finally {
for (const res of held) res.emit('close');
}
});
test('stream opening requests have an account budget before ownership reads', async () => {
const auth = { Authorization: bearer('stream-rate-fixture') };
const first = await open('/stream/missing', auth);
expect(first.status).toBe(404);
expect(first.headers.get('ratelimit-limit')).not.toBeNull();
const limit = Number(first.headers.get('ratelimit-limit'));
for (let i = 1; i < limit; i++) expect((await open('/stream/missing', auth)).status).toBe(404);
const blocked = await open('/stream/missing', auth);
expect(blocked.status).toBe(429);
expect(Number(blocked.headers.get('retry-after'))).toBeGreaterThan(0);
expect((await open('/stream/missing', { Authorization: bearer('other-stream-user') })).status).toBe(404);
});

});
61 changes: 61 additions & 0 deletions backend/__tests__/sseDelivery.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
const { test } = require('node:test');
const assert = require('node:assert/strict');
const { EventEmitter } = require('node:events');
const streams = require('../lib/conversationStreams');
const Conversation = require('../models/conversation');
const Message = require('../models/message');
const dynamo = require('../lib/dynamodbClient');
const controller = require('../controllers/communication_controller');
const { setClient, resetClient } = require('../lib/openaiClient');

function streamResponse(frames) {
const res = new EventEmitter();
res.status = () => res;
res.set = () => res;
res.flushHeaders = () => {};
res.write = frame => frames.push(frame);
res.end = () => { res.writableEnded = true; res.emit('close'); };
return res;
}

async function checkDelivery(t, canonical) {
const originalOwner = Conversation.getOwned;
const originalGlobals = Object.fromEntries(['broadcast', 'emitToken', 'emitMessage'].map(key => [key, global[key]]));
const frames = [];
const saved = [];
const id = canonical ? 'canonical-delivery' : 'legacy-delivery';
t.after(() => {
streams.close(id);
Conversation.getOwned = originalOwner;
Message.setDocClient(dynamo);
resetClient();
for (const [key, value] of Object.entries(originalGlobals)) {
if (value === undefined) delete global[key]; else global[key] = value;
}
});
Conversation.getOwned = async () => ({ id, ownerId: 'alice' });
Message.setDocClient({
put: ({ Item }) => ({ promise: async () => { saved.push(Item); return {}; } }),
query: () => ({ promise: async () => ({ Items: saved }) }),
});
setClient({ chat: { completions: { create: async () => (async function* () {
for (const content of ['Hello', ' ', 'world!']) yield { choices: [{ delta: { content } }] };
}()) } } });
global.emitToken = (conversationId, responseId, token) => streams.send(conversationId, 'token', { id: responseId, token });
global.emitMessage = (conversationId, message) => streams.send(conversationId, 'message', { message });
if (canonical) global.broadcast = (event, data, conversationId) => streams.send(conversationId, event, data);
else delete global.broadcast;
await streams.open({ user: { userId: 'alice', exp: Math.ceil(Date.now() / 1000) + 60 }, params: { conversationId: id } }, streamResponse(frames));
const result = { status() { return this; }, json(body) { this.body = body; } };
await controller.sendMessage({ params: { conversationId: id }, body: { from: 'user', content: 'How are you?', type: 'text' } }, result);
const tokens = frames.filter(frame => frame.startsWith('event: token\n')).map(frame => JSON.parse(frame.split('\n')[1].slice(6)));
const messages = frames.filter(frame => frame.startsWith('event: message\n')).map(frame => JSON.parse(frame.split('\n')[1].slice(6)).message.id);
assert.deepEqual(tokens.map(token => token.token), ['Hello', ' ', 'world!'], 'each streamed token must have one SSE frame');
assert(tokens.every(token => token.id === result.body.ai.id));
assert.deepEqual(messages, [result.body.message.id, result.body.ai.id], 'each persisted message must have one SSE frame');
}

test('canonical and legacy aliases produce one frame per token and message', async t => {
await t.test('canonical broadcast present', t => checkDelivery(t, true));
await t.test('legacy aliases only', t => checkDelivery(t, false));
});
Loading