From 03505e7335029a760a407832d97bc4c0c9361b34 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Nov 2026 23:14:30 +0000 Subject: [PATCH] Add Zigbee gateway diagnostics API and tooling Implement POST /api/diag/zbgw matching ZigbeeGateway-ESP32C6 diag.c wire format (X-ZBGW-Diag token auth, command delivery), SQLite storage next to telemetry, admin feed/ack/command endpoints, zigbee-gateway OTA via the existing firmware coordinator, and telemetry pull/analyse/all scripts. Co-authored-by: James Brown --- .env.example | 7 + Caddyfile.example | 16 + README.md | 70 +++++ package.json | 3 + src/__tests__/zigbee-diag-api.test.ts | 381 ++++++++++++++++++++++++ src/api/routes.ts | 260 +++++++++++++++- src/config.ts | 13 +- src/index.ts | 2 + src/service/ZigbeeDiagnosticsService.ts | 367 +++++++++++++++++++++++ tools/telemetry/all.ts | 53 ++++ tools/telemetry/analyse-zigbee.ts | 294 ++++++++++++++++++ tools/telemetry/pull-zigbee.ts | 169 +++++++++++ 12 files changed, 1633 insertions(+), 2 deletions(-) create mode 100644 src/__tests__/zigbee-diag-api.test.ts create mode 100644 src/service/ZigbeeDiagnosticsService.ts create mode 100644 tools/telemetry/all.ts create mode 100644 tools/telemetry/analyse-zigbee.ts create mode 100644 tools/telemetry/pull-zigbee.ts diff --git a/.env.example b/.env.example index c669357..f453c88 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,13 @@ TELEMETRY_FEED_MAX_LIMIT=5000 # When true, ack endpoint always returns wouldDelete instead of deleting TELEMETRY_ACK_ENABLED=false +# Zigbee gateway diagnostics (POST /api/diag/zbgw, header X-ZBGW-Diag) +# Comma-separated product tokens — never commit real tokens +# Generate: openssl rand -base64 32 +ZBGW_DIAG_TOKENS= +# Optional separate SQLite path (default: sibling of TELEMETRY_DB_PATH → zigbee.db) +# ZIGBEE_DB_PATH=./data/zigbee.db + # Cloudflare R2 Configuration (for camera uploads and firmware storage) # R2_ACCOUNT_ID= # R2_ACCESS_KEY_ID= diff --git a/Caddyfile.example b/Caddyfile.example index b50d9a7..b0063c5 100644 --- a/Caddyfile.example +++ b/Caddyfile.example @@ -16,3 +16,19 @@ respond 403 } } + +# Zigbee gateway diagnostics hostname +# Devices POST to https://zigbeediagnostics.percolate.one/api/diag/zbgw +# Routes are Host-independent; reverse_proxy the same upstream as scriptproxy. +# +# zigbeediagnostics.percolate.one { +# handle /api* { +# reverse_proxy localhost:8081 +# } +# handle { +# respond 403 +# } +# } +# +# Or, if scriptproxy already uses a named upstream / site block, point both +# hostnames at that same reverse_proxy target. diff --git a/README.md b/README.md index b1dec47..08862b1 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,10 @@ Environment variables (or `application.properties` file): - `TELEMETRY_FEED_MAX_LIMIT` - Maximum feed limit (default: 5000) - `TELEMETRY_ACK_ENABLED` - Enable actual deletion via ack (default: false; when false, ack is forced dry-run) +- **Zigbee gateway diagnostics:** + - `ZBGW_DIAG_TOKENS` - Comma-separated product tokens for device `X-ZBGW-Diag` header (REQUIRED for `/api/diag/zbgw`) + - `ZIGBEE_DB_PATH` - Optional SQLite path (default: sibling of telemetry DB → `zigbee.db`) + - **Admin Authentication:** - `ADMIN_ADDRESSES` - Comma-separated Ethereum addresses authorized for admin operations - `TELEMETRY_ADMIN_ADDRESSES` - Additional addresses authorized for telemetry ack @@ -75,10 +79,66 @@ Admin-protected operations require a signed challenge. The flow: 1. **GET /api/admin/challenge** - Returns `{nonce, expiresAt}` (nonce valid 60s) 2. Client signs EIP-191 personal_sign message: `::` - For ack: `ack::` + - For Zigbee ack: `zbgw-ack::` + - For Zigbee command queue: `zbgw-cmd:::` (`restart` | `ota_check`) - For firmware upload: `firmware::::` - For firmware rollback: `current:::` 3. Submit signature with request (nonce is single-use and burned after verification) +### Zigbee Gateway Diagnostics + +Host-independent routes used by ESP32-C6 gateways (firmware `diag.c`). Devices may call +`https://zigbeediagnostics.percolate.one/api/diag/zbgw` (Caddy reverse-proxies that hostname +to the same upstream as `scriptproxy.percolate.one`). + +**POST /api/diag/zbgw** (device) + +Auth: header `X-ZBGW-Diag: ` must match one of `ZBGW_DIAG_TOKENS` (constant-time compare). +Payload capped at 64KB. Arbitrary extra fields are stored as JSON (watchdog counters, reset +reason, recovery stage, unknown-device descriptor dumps, etc.). + +Example body (matches firmware wire format): +```json +{ + "device_id": "aabbccddeeff", + "fw": "1.2.3", + "kind": "boot|error|ok|poll|device|device-added", + "ok": true, + "uptime_s": 120, + "reset": "power|software|panic|watchdog|brownout|sleep|other", + "heap": 120000, + "wifi_rssi": -55, + "mqtt_ok": true, + "zigbee_ok": true, + "devices": 2, + "code": "boot", + "message": "Gateway started" +} +``` + +Response (pending admin-queued commands are delivered once, then marked delivered): +```json +{ "commands": [{ "action": "restart" }] } +``` + +**Admin reads** (Bearer `TELEMETRY_API_KEY`, same as `/telemetry/*`): + +| Endpoint | Description | +|----------|-------------| +| `GET /api/diag/zbgw/gateways` | List known gateways (mac, fw, first/last seen, last IP, status) | +| `GET /api/diag/zbgw/events?afterId=&limit=` | Event feed with after-id paging | +| `POST /api/diag/zbgw/ack` | Ack/delete events up to id (`zbgw-ack::`; respects `TELEMETRY_ACK_ENABLED`) | +| `POST /api/diag/zbgw/commands` | Queue `restart` or `ota_check` (`zbgw-cmd:::`) | + +SQLite tables live in `zigbee.db` next to the telemetry DB (`zigbee_gateways`, `zigbee_events`, +`zigbee_commands`) with auto-migrating schema. Columns `device_pubkey` / `auth_approved` are +reserved for a future per-device P-256 signing scheme (admin-approved pubkey); not implemented yet. + +**OTA:** Upload project `zigbee-gateway` via the existing firmware coordinator / +`npm run firmware:upload zigbee-gateway firmware.bin`. Devices (or tooling) can query +`GET /api/firmware/zigbee-gateway/latest` → `{version,url,sha256,size}`. No special-casing beyond +the project name. + ### Telemetry Feed & Acknowledgement **GET /api/telemetry/feed?afterId=&limit=** @@ -348,6 +408,15 @@ Outputs markdown report with: - Failing/slow API methods - Attribution: proxy vs Web3E TcpBridge/UdpBridge vs device firmware +**Zigbee diagnostics pull & analyse:** + +```bash +npm run telemetry:zigbee:pull # → data/zigbee-YYYYMMDD.jsonl (no ack) +npm run telemetry:zigbee:pull -- --ack # Pull + ack (only when --ack passed) +npm run telemetry:zigbee:analyse # Silent gateways, watchdog, Wi-Fi/MQTT flaps, OTA failures, unknown-device dumps +npm run telemetry:all # SmartLayer pull+analyse, then Zigbee pull+analyse +``` + ### Firmware Upload **Upload firmware binary with admin signature:** @@ -355,6 +424,7 @@ Outputs markdown report with: ```bash # Using npm scripts (recommended) npm run firmware:upload GarageDoor 1.2.3 firmware.bin +npm run firmware:upload zigbee-gateway 1.2.3 firmware.bin # Or using ts-node directly ts-node tools/firmware/upload.ts diff --git a/package.json b/package.json index c304dd8..0da8ca0 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,9 @@ "typecheck:tools": "tsc --project tsconfig.tools.json --noEmit", "telemetry:pull": "ts-node tools/telemetry/pull.ts", "telemetry:analyse": "ts-node tools/telemetry/analyse.ts", + "telemetry:zigbee:pull": "ts-node tools/telemetry/pull-zigbee.ts", + "telemetry:zigbee:analyse": "ts-node tools/telemetry/analyse-zigbee.ts", + "telemetry:all": "ts-node tools/telemetry/all.ts", "firmware:upload": "ts-node tools/firmware/upload.ts" }, "engines": { diff --git a/src/__tests__/zigbee-diag-api.test.ts b/src/__tests__/zigbee-diag-api.test.ts new file mode 100644 index 0000000..faff5a3 --- /dev/null +++ b/src/__tests__/zigbee-diag-api.test.ts @@ -0,0 +1,381 @@ +import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, vi } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import { TelemetryService } from '../service/TelemetryService'; +import { FirmwareService } from '../service/FirmwareService'; +import { ZigbeeDiagnosticsService } from '../service/ZigbeeDiagnosticsService'; +import { AdminAuthService } from '../service/AdminAuthService'; +import { createRouter } from '../api/routes'; +import { unlinkSync, existsSync } from 'fs'; +import { Wallet, HDNodeWallet } from 'ethers'; + +const TEST_TELEMETRY_DB = './test-zbgw-telemetry.db'; +const TEST_ZIGBEE_DB = './test-zbgw-zigbee.db'; +const TEST_FIRMWARE_DB = './test-zbgw-firmware.db'; +const TEST_API_KEY = 'test-telemetry-key-zbgw'; +const TEST_DIAG_TOKEN = 'test-zbgw-diag-token'; +const TEST_DIAG_TOKEN_2 = 'second-zbgw-token'; + +function cleanupDb(path: string): void { + if (existsSync(path)) unlinkSync(path); +} + +describe('Zigbee Gateway Diagnostics API', () => { + let app: express.Application; + let wallet: HDNodeWallet; + + beforeAll(() => { + wallet = Wallet.createRandom(); + process.env.TELEMETRY_API_KEY = TEST_API_KEY; + process.env.ADMIN_ADDRESSES = wallet.address; + process.env.TELEMETRY_ACK_ENABLED = 'true'; + process.env.ZBGW_DIAG_TOKENS = `${TEST_DIAG_TOKEN},${TEST_DIAG_TOKEN_2}`; + process.env.TELEMETRY_DB_PATH = TEST_TELEMETRY_DB; + process.env.ZIGBEE_DB_PATH = TEST_ZIGBEE_DB; + }); + + beforeEach(() => { + cleanupDb(TEST_TELEMETRY_DB); + cleanupDb(TEST_ZIGBEE_DB); + cleanupDb(TEST_FIRMWARE_DB); + + TelemetryService.resetInstance(); + ZigbeeDiagnosticsService.resetInstance(); + FirmwareService.resetInstance(); + AdminAuthService.resetInstance(); + + TelemetryService.getInstance(TEST_TELEMETRY_DB, 30); + ZigbeeDiagnosticsService.getInstance(TEST_ZIGBEE_DB, 30); + + app = express(); + app.use(express.json({ limit: '1mb' })); + + const mockService = { + checkServices: vi.fn(), + getResponse: vi.fn(), + getDeviceAddress: vi.fn(), + } as any; + + const mockTcpService = { + getResponse: vi.fn(), + getDeviceAddress: vi.fn(), + handleDeviceConnection: vi.fn(), + getLoginChallenge: vi.fn(), + getConnectedDevices: vi.fn(() => []), + sendApiCallToDevice: vi.fn(), + } as any; + + app.use('/api', createRouter(mockService, mockTcpService)); + }); + + afterEach(() => { + try { TelemetryService.getInstance().close(); } catch { /* ignore */ } + try { ZigbeeDiagnosticsService.getInstance().close(); } catch { /* ignore */ } + try { FirmwareService.getInstance(TEST_FIRMWARE_DB).close(); } catch { /* ignore */ } + TelemetryService.resetInstance(); + ZigbeeDiagnosticsService.resetInstance(); + FirmwareService.resetInstance(); + AdminAuthService.resetInstance(); + cleanupDb(TEST_TELEMETRY_DB); + cleanupDb(TEST_ZIGBEE_DB); + cleanupDb(TEST_FIRMWARE_DB); + }); + + afterAll(() => { + delete process.env.TELEMETRY_API_KEY; + delete process.env.ADMIN_ADDRESSES; + delete process.env.TELEMETRY_ACK_ENABLED; + delete process.env.ZBGW_DIAG_TOKENS; + delete process.env.TELEMETRY_DB_PATH; + delete process.env.ZIGBEE_DB_PATH; + }); + + describe('POST /api/diag/zbgw (device)', () => { + const sampleBody = { + device_id: 'aabbccddeeff', + fw: '1.2.3', + kind: 'boot', + ok: true, + uptime_s: 12, + reset: 'power', + heap: 120000, + wifi_rssi: -55, + mqtt_ok: true, + zigbee_ok: true, + devices: 2, + code: 'boot', + message: 'Gateway started', + }; + + it('rejects missing X-ZBGW-Diag token', async () => { + const res = await request(app) + .post('/api/diag/zbgw') + .send(sampleBody) + .expect(401); + expect(res.body.error).toBe('Unauthorized'); + }); + + it('rejects invalid token', async () => { + await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', 'wrong-token') + .send(sampleBody) + .expect(401); + }); + + it('accepts any configured comma-separated token', async () => { + await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN_2) + .send(sampleBody) + .expect(200); + }); + + it('returns 503 when tokens not configured', async () => { + delete process.env.ZBGW_DIAG_TOKENS; + const res = await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .send(sampleBody) + .expect(503); + expect(res.body.error).toMatch(/disabled/i); + process.env.ZBGW_DIAG_TOKENS = `${TEST_DIAG_TOKEN},${TEST_DIAG_TOKEN_2}`; + }); + + it('stores boot event and returns empty commands', async () => { + const res = await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .send(sampleBody) + .expect(200); + + expect(res.body).toEqual({ commands: [] }); + + const zigbee = ZigbeeDiagnosticsService.getInstance(); + const gateways = zigbee.listGateways(); + expect(gateways).toHaveLength(1); + expect(gateways[0].mac).toBe('aabbccddeeff'); + expect(gateways[0].firmwareVersion).toBe('1.2.3'); + expect(gateways[0].status).toBe('boot'); + }); + + it('works regardless of Host header', async () => { + await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .set('Host', 'zigbeediagnostics.percolate.one') + .send(sampleBody) + .expect(200); + }); + + it('accepts and stores arbitrary extra fields', async () => { + await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .send({ + ...sampleBody, + kind: 'device-added', + watchdog_count: 3, + recovery_stage: 'mqtt_reconnect', + unknown_device: { + manufacturer: 'Acme', + model: 'Plug-X', + power_source: 'mains', + endpoints: [{ id: 1, clusters: [6, 8] }], + }, + }) + .expect(200); + + const feed = ZigbeeDiagnosticsService.getInstance().getEventsFeed(0, 10); + expect(feed.rows).toHaveLength(1); + expect(feed.rows[0].payload.watchdog_count).toBe(3); + expect((feed.rows[0].payload.unknown_device as any).model).toBe('Plug-X'); + }); + + it('rejects missing device_id', async () => { + const { device_id, ...rest } = sampleBody; + await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .send(rest) + .expect(400); + }); + + it('rejects oversized payloads', async () => { + const big = { + ...sampleBody, + blob: 'x'.repeat(70 * 1024), + }; + await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .send(big) + .expect(413); + }); + + it('delivers queued restart command on poll', async () => { + const zigbee = ZigbeeDiagnosticsService.getInstance(); + zigbee.queueCommand('aabbccddeeff', 'restart', wallet.address.toLowerCase()); + + const res = await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .send({ ...sampleBody, kind: 'poll' }) + .expect(200); + + expect(res.body.commands).toEqual([{ action: 'restart' }]); + + const pending = zigbee.getPendingCommands('aabbccddeeff'); + expect(pending).toHaveLength(0); + + // Second poll should not re-deliver + const res2 = await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .send({ ...sampleBody, kind: 'poll' }) + .expect(200); + expect(res2.body.commands).toEqual([]); + }); + }); + + describe('Admin endpoints', () => { + async function postDeviceEvent(kind = 'ok') { + await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .send({ + device_id: '112233445566', + fw: '2.0.0', + kind, + ok: kind !== 'error', + reset: 'software', + }) + .expect(200); + } + + it('lists gateways with TELEMETRY_API_KEY', async () => { + await postDeviceEvent('boot'); + + const res = await request(app) + .get('/api/diag/zbgw/gateways') + .set('Authorization', `Bearer ${TEST_API_KEY}`) + .expect(200); + + expect(res.body.gateways).toHaveLength(1); + expect(res.body.gateways[0].mac).toBe('112233445566'); + }); + + it('rejects gateway list without bearer', async () => { + await request(app).get('/api/diag/zbgw/gateways').expect(401); + }); + + it('pages events with afterId and supports ack', async () => { + await postDeviceEvent('boot'); + await postDeviceEvent('poll'); + await postDeviceEvent('ok'); + + const page1 = await request(app) + .get('/api/diag/zbgw/events?afterId=0&limit=2') + .set('Authorization', `Bearer ${TEST_API_KEY}`) + .expect(200); + + expect(page1.body.rows).toHaveLength(2); + expect(page1.body.hasMore).toBe(true); + + const lastId = page1.body.rows[1].id; + const page2 = await request(app) + .get(`/api/diag/zbgw/events?afterId=${lastId}&limit=10`) + .set('Authorization', `Bearer ${TEST_API_KEY}`) + .expect(200); + + expect(page2.body.rows).toHaveLength(1); + + const challenge = await request(app).get('/api/admin/challenge').expect(200); + const upToId = page1.body.maxId; + const message = `zbgw-ack:${upToId}:${challenge.body.nonce}`; + const signature = await wallet.signMessage(message); + + const ack = await request(app) + .post('/api/diag/zbgw/ack') + .set('Authorization', `Bearer ${TEST_API_KEY}`) + .send({ upToId, signature, message }) + .expect(200); + + expect(ack.body.deleted).toBe(3); + expect(ack.body.remaining).toBe(0); + }); + + it('queues ota_check via admin-signed endpoint and delivers it', async () => { + await postDeviceEvent('boot'); + + const challenge = await request(app).get('/api/admin/challenge').expect(200); + const mac = '112233445566'; + const action = 'ota_check'; + const message = `zbgw-cmd:${mac}:${action}:${challenge.body.nonce}`; + const signature = await wallet.signMessage(message); + + const queued = await request(app) + .post('/api/diag/zbgw/commands') + .set('Authorization', `Bearer ${TEST_API_KEY}`) + .send({ mac, action, signature, message }) + .expect(200); + + expect(queued.body.success).toBe(true); + expect(queued.body.command.action).toBe('ota_check'); + + const deliver = await request(app) + .post('/api/diag/zbgw') + .set('X-ZBGW-Diag', TEST_DIAG_TOKEN) + .send({ + device_id: mac, + fw: '2.0.0', + kind: 'poll', + ok: true, + }) + .expect(200); + + expect(deliver.body.commands).toEqual([{ action: 'ota_check' }]); + }); + + it('rejects command queue with bad signature', async () => { + const challenge = await request(app).get('/api/admin/challenge').expect(200); + const other = Wallet.createRandom(); + const mac = '112233445566'; + const action = 'restart'; + const message = `zbgw-cmd:${mac}:${action}:${challenge.body.nonce}`; + const signature = await other.signMessage(message); + + await request(app) + .post('/api/diag/zbgw/commands') + .set('Authorization', `Bearer ${TEST_API_KEY}`) + .send({ mac, action, signature, message }) + .expect(403); + }); + }); + + describe('Firmware project zigbee-gateway', () => { + it('serves GET /api/firmware/zigbee-gateway/latest', async () => { + const fw = FirmwareService.getInstance(TEST_FIRMWARE_DB); + fw.addFirmware( + 'zigbee-gateway', + '1.0.1', + 'abc123', + 4096, + 'firmware/zigbee-gateway/1.0.1/firmware.bin', + 'https://example.r2.dev/firmware/zigbee-gateway/1.0.1/firmware.bin' + ); + + const res = await request(app) + .get('/api/firmware/zigbee-gateway/latest') + .expect(200); + + expect(res.body).toMatchObject({ + project: 'zigbee-gateway', + version: '1.0.1', + sha256: 'abc123', + size: 4096, + url: 'https://example.r2.dev/firmware/zigbee-gateway/1.0.1/firmware.bin', + }); + }); + }); +}); diff --git a/src/api/routes.ts b/src/api/routes.ts index 62ece60..d2ba6d1 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -7,6 +7,11 @@ import { randomUUID } from 'crypto'; import { TelemetryService } from '../service/TelemetryService'; import { FirmwareService } from '../service/FirmwareService'; import { AdminAuthService } from '../service/AdminAuthService'; +import { + ZigbeeDiagnosticsService, + ZBGW_MAX_PAYLOAD_BYTES, + isAllowedAction, +} from '../service/ZigbeeDiagnosticsService'; import { loadConfig } from '../config'; import { FirmwareVersionParser } from '../service/FirmwareVersionParser'; @@ -74,6 +79,36 @@ function telemetryAuthMiddleware(req: Request, res: Response, next: NextFunction next(); } +function zbgwTokenMatches(provided: string | undefined, tokens: string[]): boolean { + if (!provided || tokens.length === 0) { + return false; + } + let matched = false; + for (const token of tokens) { + if (AdminAuthService.constantTimeCompare(provided, token)) { + matched = true; + } + } + return matched; +} + +function zbgwDiagAuthMiddleware(req: Request, res: Response, next: NextFunction): void { + const config = loadConfig(); + + if (config.zbgwDiagTokens.length === 0) { + res.status(503).json({ error: 'Zigbee diagnostics endpoints are disabled' }); + return; + } + + const token = req.get('X-ZBGW-Diag') || undefined; + if (!zbgwTokenMatches(token, config.zbgwDiagTokens)) { + res.status(401).json({ error: 'Unauthorized' }); + return; + } + + next(); +} + function getAdminAddresses(): string[] { const config = loadConfig(); return [...config.adminAddresses, ...config.telemetryAdminAddresses]; @@ -84,6 +119,14 @@ export function createRouter(service: ASyncService, tcpService: ASyncTCPService) const telemetry = TelemetryService.getInstance(); const adminAuth = AdminAuthService.getInstance(); const config = loadConfig(); + + let zigbee: ZigbeeDiagnosticsService | null = null; + try { + // Initialized in index.ts (or tests); do not create a side-effect DB here. + zigbee = ZigbeeDiagnosticsService.getInstance(); + } catch (err) { + console.warn('[ZigbeeDiag] Service not initialized:', err); + } let firmware: FirmwareService | null = null; try { @@ -100,7 +143,7 @@ export function createRouter(service: ASyncService, tcpService: ASyncTCPService) res.set('Access-Control-Max-Age', '10000'); res.set('Access-Control-Allow-Credentials', 'false'); res.set('Access-Control-Allow-Methods', 'GET, POST, PUT, OPTIONS'); - res.set('Access-Control-Allow-Headers', 'Content-Type, Authorization'); + res.set('Access-Control-Allow-Headers', 'Content-Type, Authorization, X-ZBGW-Diag'); if (req.method === 'OPTIONS') { res.status(204).end(); return; @@ -940,6 +983,221 @@ export function createRouter(service: ASyncService, tcpService: ASyncTCPService) } }); + // ========== Zigbee gateway diagnostics ========== + // Device POST path matches firmware (diag.c): Host header independent. + router.post('/diag/zbgw', zbgwDiagAuthMiddleware, (req, res) => { + try { + if (!zigbee) { + res.status(503).json({ error: 'Zigbee diagnostics service not initialized' }); + return; + } + + const body = req.body; + if (!body || typeof body !== 'object' || Array.isArray(body)) { + res.status(400).json({ error: 'Expected JSON object body' }); + return; + } + + const payloadBytes = Buffer.byteLength(JSON.stringify(body), 'utf8'); + if (payloadBytes > ZBGW_MAX_PAYLOAD_BYTES) { + res.status(413).json({ error: `Payload exceeds ${ZBGW_MAX_PAYLOAD_BYTES} bytes` }); + return; + } + + const deviceId = body.device_id ?? body.mac ?? body.deviceId; + if (typeof deviceId !== 'string' || !deviceId.trim()) { + res.status(400).json({ error: 'Missing device_id' }); + return; + } + + const kind = typeof body.kind === 'string' && body.kind ? body.kind : 'unknown'; + const clientIp = getClientIp(req); + const result = zigbee.recordEvent(deviceId, kind, body as Record, clientIp); + + // Firmware (diag.c) expects { "commands": [ { "action": "restart" }, ... ] } + res.json({ commands: result.commands }); + } catch (err) { + console.error('[ZigbeeDiag] Device POST error:', err); + res.status(500).json({ error: 'Internal error' }); + } + }); + + router.get('/diag/zbgw/gateways', telemetryAuthMiddleware, (_req, res) => { + try { + if (!zigbee) { + res.status(503).json({ error: 'Zigbee diagnostics service not initialized' }); + return; + } + res.json({ + gateways: zigbee.listGateways(), + serverTime: Date.now(), + }); + } catch (err) { + console.error('[ZigbeeDiag] List gateways error:', err); + res.status(500).json({ error: 'Internal error' }); + } + }); + + router.get('/diag/zbgw/events', telemetryAuthMiddleware, (req, res) => { + try { + if (!zigbee) { + res.status(503).json({ error: 'Zigbee diagnostics service not initialized' }); + return; + } + + const afterId = parseInt(req.query.afterId as string) || 0; + const limit = Math.min( + parseInt(req.query.limit as string) || 1000, + config.telemetryFeedMaxLimit + ); + + const feed = zigbee.getEventsFeed(afterId, limit); + res.json({ + rows: feed.rows, + maxId: feed.maxId, + hasMore: feed.hasMore, + serverTime: Date.now(), + }); + } catch (err) { + console.error('[ZigbeeDiag] Events feed error:', err); + res.status(500).json({ error: 'Internal error' }); + } + }); + + router.post('/diag/zbgw/ack', telemetryAuthMiddleware, async (req, res) => { + try { + if (!zigbee) { + res.status(503).json({ error: 'Zigbee diagnostics service not initialized' }); + return; + } + + const { upToId, signature, message, dryRun } = req.body || {}; + + if (!upToId || typeof upToId !== 'number') { + res.status(400).json({ error: 'Missing or invalid upToId' }); + return; + } + + const adminAddresses = getAdminAddresses(); + if (adminAddresses.length === 0) { + res.status(503).json({ error: 'Admin addresses not configured' }); + return; + } + + if (!signature || !message) { + res.status(400).json({ error: 'Missing signature or message' }); + return; + } + + const nonceMatch = message.match(/:([a-f0-9]{64})$/); + if (!nonceMatch) { + res.status(400).json({ error: 'Message does not contain valid nonce' }); + return; + } + const nonce = nonceMatch[1]; + + // Message format: 'zbgw-ack::' + const expectedMessagePrefix = `zbgw-ack:${upToId}:`; + if (!message.startsWith(expectedMessagePrefix)) { + res.status(400).json({ error: 'Message format mismatch' }); + return; + } + + const result = adminAuth.verifyAndBurnNonce(nonce, message, signature, adminAddresses); + if (!result.valid) { + res.status(403).json({ error: result.error || 'Signature verification failed' }); + return; + } + + const forceDryRun = !config.telemetryAckEnabled; + const effectiveDryRun = dryRun || forceDryRun; + + if (effectiveDryRun) { + res.json({ + wouldDelete: zigbee.countEventsUpTo(upToId), + dryRun: true, + reason: forceDryRun ? 'TELEMETRY_ACK_ENABLED=false' : 'dryRun requested', + }); + return; + } + + const { deleted, remaining } = zigbee.deleteEventsUpTo(upToId); + res.json({ + deleted, + remaining, + deletedBy: result.address, + }); + } catch (err) { + console.error('[ZigbeeDiag] Ack error:', err); + res.status(500).json({ error: 'Internal error' }); + } + }); + + router.post('/diag/zbgw/commands', telemetryAuthMiddleware, async (req, res) => { + try { + if (!zigbee) { + res.status(503).json({ error: 'Zigbee diagnostics service not initialized' }); + return; + } + + const { mac, action, signature, message } = req.body || {}; + + if (!mac || typeof mac !== 'string') { + res.status(400).json({ error: 'Missing mac' }); + return; + } + if (!action || typeof action !== 'string') { + res.status(400).json({ error: 'Missing action' }); + return; + } + if (!isAllowedAction(action)) { + res.status(400).json({ error: `Unsupported action (allowed: restart, ota_check)` }); + return; + } + + const adminAddresses = getAdminAddresses(); + if (adminAddresses.length === 0) { + res.status(503).json({ error: 'Admin addresses not configured' }); + return; + } + + if (!signature || !message) { + res.status(400).json({ error: 'Missing signature or message' }); + return; + } + + const nonceMatch = message.match(/:([a-f0-9]{64})$/); + if (!nonceMatch) { + res.status(400).json({ error: 'Message does not contain valid nonce' }); + return; + } + const nonce = nonceMatch[1]; + + // Message format: 'zbgw-cmd:::' + const expectedMessage = `zbgw-cmd:${mac}:${action}:${nonce}`; + if (message !== expectedMessage) { + res.status(400).json({ error: 'Message format mismatch' }); + return; + } + + const authResult = adminAuth.verifyAndBurnNonce(nonce, message, signature, adminAddresses); + if (!authResult.valid) { + res.status(403).json({ error: authResult.error || 'Signature verification failed' }); + return; + } + + const command = zigbee.queueCommand(mac, action, authResult.address); + res.json({ + success: true, + command, + queuedBy: authResult.address, + }); + } catch (err) { + console.error('[ZigbeeDiag] Queue command error:', err); + res.status(500).json({ error: 'Internal error' }); + } + }); + // ========== Camera upload (called by Remote Camera device) ========== router.post('/camera/upload', express.raw({ type: 'image/jpeg', limit: '3mb' }), async (req, res) => { try { diff --git a/src/config.ts b/src/config.ts index 9ea1d3c..d216df7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -14,6 +14,9 @@ export interface Config { telemetryFeedMaxLimit: number; firmwareUploadMaxSizeMb: number; trustProxy: boolean; + /** Comma-separated shared product tokens for Zigbee gateway diagnostics (X-ZBGW-Diag). */ + zbgwDiagTokens: string[]; + zigbeeDbPath: string; } function loadProperties(): Record { @@ -53,11 +56,12 @@ function parseAddressList(value?: string): string[] { export function loadConfig(): Config { const props = loadProperties(); + const telemetryDbPath = process.env.TELEMETRY_DB_PATH || props['telemetry.db.path'] || './data/telemetry.db'; return { serverPort: parseInt(props['server.port'] || '8081', 10), udpPort: parseInt(props['udp.port'] || '8083', 10), tcpPort: parseInt(props['tcp.port'] || '8082', 10), - telemetryDbPath: process.env.TELEMETRY_DB_PATH || props['telemetry.db.path'] || './data/telemetry.db', + telemetryDbPath, telemetryRetentionDays: parseInt(process.env.TELEMETRY_RETENTION_DAYS || props['telemetry.retention.days'] || '30', 10), telemetryApiKey: process.env.TELEMETRY_API_KEY || props['telemetry.api.key'], telemetryAdminAddresses: parseAddressList( @@ -74,5 +78,12 @@ export function loadConfig(): Config { process.env.FIRMWARE_UPLOAD_MAX_SIZE_MB || props['firmware.upload.max.size.mb'] || '10', 10 ), trustProxy: (process.env.TRUST_PROXY || props['trust.proxy'] || 'false').toLowerCase() === 'true', + zbgwDiagTokens: parseAddressList( + process.env.ZBGW_DIAG_TOKENS || props['zbgw.diag.tokens'] + ), + zigbeeDbPath: + process.env.ZIGBEE_DB_PATH || + props['zigbee.db.path'] || + telemetryDbPath.replace(/telemetry\.db$/, 'zigbee.db'), }; } diff --git a/src/index.ts b/src/index.ts index ad158c4..f9b6711 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,10 +5,12 @@ import { ASyncService } from './service/ASyncService'; import { ASyncTCPService } from './service/ASyncTCPService'; import { loadConfig } from './config'; import { TelemetryService } from './service/TelemetryService'; +import { ZigbeeDiagnosticsService } from './service/ZigbeeDiagnosticsService'; const config = loadConfig(); TelemetryService.getInstance(config.telemetryDbPath, config.telemetryRetentionDays); +ZigbeeDiagnosticsService.getInstance(config.zigbeeDbPath, config.telemetryRetentionDays); const app = express(); const service = new ASyncService(); diff --git a/src/service/ZigbeeDiagnosticsService.ts b/src/service/ZigbeeDiagnosticsService.ts new file mode 100644 index 0000000..d265914 --- /dev/null +++ b/src/service/ZigbeeDiagnosticsService.ts @@ -0,0 +1,367 @@ +/** + * ZigbeeDiagnosticsService - Zigbee gateway diagnostics storage (SQLite) + * + * TODO(auth): Add per-device P-256 signing. Register device pubkey on first + * contact (or via admin), require ADMIN_ADDRESSES-signed challenge approval + * before accepting signed posts, then verify signatures instead of (or in + * addition to) the shared X-ZBGW-Diag product token. + */ + +import Database from 'better-sqlite3'; +import { existsSync, mkdirSync } from 'fs'; +import { dirname } from 'path'; + +export const ZBGW_MAX_PAYLOAD_BYTES = 64 * 1024; +export const ZBGW_ALLOWED_ACTIONS = ['restart', 'ota_check'] as const; +export type ZbgwCommandAction = (typeof ZBGW_ALLOWED_ACTIONS)[number]; + +export interface ZigbeeGateway { + mac: string; + firmwareVersion: string | null; + firstSeen: number; + lastSeen: number; + lastIp: string | null; + status: string | null; + /** Reserved for future per-device P-256 auth (not used yet). */ + devicePubkey: string | null; + /** Reserved for future admin approval of device pubkey (not used yet). */ + authApproved: boolean; +} + +export interface ZigbeeEvent { + id: number; + mac: string; + kind: string; + payload: Record; + receivedAt: number; +} + +export interface ZigbeeCommand { + id: number; + mac: string; + action: string; + queuedAt: number; + deliveredAt: number | null; + queuedBy: string | null; +} + +export function normalizeMac(raw: string): string { + return raw.replace(/[^a-fA-F0-9]/g, '').toLowerCase(); +} + +export function isAllowedAction(action: string): action is ZbgwCommandAction { + return (ZBGW_ALLOWED_ACTIONS as readonly string[]).includes(action); +} + +export class ZigbeeDiagnosticsService { + private static instance: ZigbeeDiagnosticsService; + private db: Database.Database; + private retentionDays: number; + + private constructor(dbPath: string, retentionDays: number = 30) { + const dir = dirname(dbPath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + + this.db = new Database(dbPath); + this.retentionDays = retentionDays; + this.initDatabase(); + this.startCleanupSchedule(); + } + + static getInstance(dbPath?: string, retentionDays?: number): ZigbeeDiagnosticsService { + if (!ZigbeeDiagnosticsService.instance) { + if (!dbPath) { + throw new Error('Database path required for first initialization'); + } + ZigbeeDiagnosticsService.instance = new ZigbeeDiagnosticsService(dbPath, retentionDays); + } + return ZigbeeDiagnosticsService.instance; + } + + static resetInstance(): void { + if (ZigbeeDiagnosticsService.instance) { + ZigbeeDiagnosticsService.instance.close(); + ZigbeeDiagnosticsService.instance = null as any; + } + } + + private initDatabase(): void { + this.db.exec(` + CREATE TABLE IF NOT EXISTS zigbee_gateways ( + mac TEXT PRIMARY KEY, + firmware_version TEXT, + first_seen INTEGER NOT NULL, + last_seen INTEGER NOT NULL, + last_ip TEXT, + status TEXT, + device_pubkey TEXT, + auth_approved INTEGER NOT NULL DEFAULT 0 + ); + + CREATE TABLE IF NOT EXISTS zigbee_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + mac TEXT NOT NULL, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + received_at INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_zigbee_events_id ON zigbee_events(id); + CREATE INDEX IF NOT EXISTS idx_zigbee_events_mac ON zigbee_events(mac, received_at); + CREATE INDEX IF NOT EXISTS idx_zigbee_events_received ON zigbee_events(received_at); + + CREATE TABLE IF NOT EXISTS zigbee_commands ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + mac TEXT NOT NULL, + action TEXT NOT NULL, + queued_at INTEGER NOT NULL, + delivered_at INTEGER, + queued_by TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_zigbee_commands_pending + ON zigbee_commands(mac, delivered_at); + `); + + this.migrateSchema(); + } + + private migrateSchema(): void { + const gatewayCols = this.db.prepare('PRAGMA table_info(zigbee_gateways)').all() as Array<{ name: string }>; + const gatewayNames = new Set(gatewayCols.map(c => c.name)); + const gatewayRequired = [ + { name: 'firmware_version', type: 'TEXT' }, + { name: 'first_seen', type: 'INTEGER' }, + { name: 'last_seen', type: 'INTEGER' }, + { name: 'last_ip', type: 'TEXT' }, + { name: 'status', type: 'TEXT' }, + { name: 'device_pubkey', type: 'TEXT' }, + { name: 'auth_approved', type: 'INTEGER NOT NULL DEFAULT 0' }, + ]; + for (const col of gatewayRequired) { + if (!gatewayNames.has(col.name)) { + console.log(`[ZigbeeDiag] Migrating schema: adding zigbee_gateways.${col.name}`); + this.db.exec(`ALTER TABLE zigbee_gateways ADD COLUMN ${col.name} ${col.type}`); + } + } + + const cmdCols = this.db.prepare('PRAGMA table_info(zigbee_commands)').all() as Array<{ name: string }>; + const cmdNames = new Set(cmdCols.map(c => c.name)); + if (!cmdNames.has('queued_by')) { + console.log('[ZigbeeDiag] Migrating schema: adding zigbee_commands.queued_by'); + this.db.exec('ALTER TABLE zigbee_commands ADD COLUMN queued_by TEXT'); + } + } + + private startCleanupSchedule(): void { + setInterval(() => this.cleanupOldEvents(), 24 * 60 * 60 * 1000); + } + + private cleanupOldEvents(): void { + const cutoff = Date.now() - this.retentionDays * 24 * 60 * 60 * 1000; + const result = this.db.prepare('DELETE FROM zigbee_events WHERE received_at < ?').run(cutoff); + if (result.changes > 0) { + console.log(`[ZigbeeDiag] Cleaned up ${result.changes} events older than ${this.retentionDays} days`); + } + } + + recordEvent( + macRaw: string, + kind: string, + payload: Record, + ipAddress?: string + ): { eventId: number; commands: Array<{ action: string }> } { + const mac = normalizeMac(macRaw); + if (!mac) { + throw new Error('Invalid device MAC'); + } + + const now = Date.now(); + const firmwareVersion = + typeof payload.fw === 'string' ? payload.fw : + typeof payload.firmware_version === 'string' ? payload.firmware_version : + null; + const status = kind || 'unknown'; + + const upsert = this.db.transaction(() => { + const existing = this.db.prepare('SELECT mac FROM zigbee_gateways WHERE mac = ?').get(mac) as + | { mac: string } + | undefined; + + if (existing) { + this.db.prepare(` + UPDATE zigbee_gateways + SET firmware_version = COALESCE(?, firmware_version), + last_seen = ?, + last_ip = COALESCE(?, last_ip), + status = ? + WHERE mac = ? + `).run(firmwareVersion, now, ipAddress || null, status, mac); + } else { + this.db.prepare(` + INSERT INTO zigbee_gateways + (mac, firmware_version, first_seen, last_seen, last_ip, status, device_pubkey, auth_approved) + VALUES (?, ?, ?, ?, ?, ?, NULL, 0) + `).run(mac, firmwareVersion, now, now, ipAddress || null, status); + } + + const insert = this.db.prepare(` + INSERT INTO zigbee_events (mac, kind, payload, received_at) + VALUES (?, ?, ?, ?) + `); + const info = insert.run(mac, kind, JSON.stringify(payload), now); + + const pending = this.db.prepare(` + SELECT id, action FROM zigbee_commands + WHERE mac = ? AND delivered_at IS NULL + ORDER BY id ASC + `).all(mac) as Array<{ id: number; action: string }>; + + const mark = this.db.prepare(` + UPDATE zigbee_commands SET delivered_at = ? WHERE id = ? + `); + for (const cmd of pending) { + mark.run(now, cmd.id); + } + + return { + eventId: Number(info.lastInsertRowid), + commands: pending.map(c => ({ action: c.action })), + }; + }); + + return upsert(); + } + + listGateways(): ZigbeeGateway[] { + const rows = this.db.prepare(` + SELECT + mac, + firmware_version as firmwareVersion, + first_seen as firstSeen, + last_seen as lastSeen, + last_ip as lastIp, + status, + device_pubkey as devicePubkey, + auth_approved as authApproved + FROM zigbee_gateways + ORDER BY last_seen DESC + `).all() as any[]; + + return rows.map(r => ({ + ...r, + authApproved: r.authApproved === 1, + })); + } + + getEventsFeed(afterId: number, limit: number): { + rows: ZigbeeEvent[]; + maxId: number; + hasMore: boolean; + } { + const rows = this.db.prepare(` + SELECT id, mac, kind, payload, received_at as receivedAt + FROM zigbee_events + WHERE id > ? + ORDER BY id ASC + LIMIT ? + `).all(afterId, limit) as any[]; + + const { maxId } = this.db.prepare('SELECT MAX(id) as maxId FROM zigbee_events').get() as { + maxId: number | null; + }; + + const parsed: ZigbeeEvent[] = rows.map(r => ({ + id: r.id, + mac: r.mac, + kind: r.kind, + payload: JSON.parse(r.payload), + receivedAt: r.receivedAt, + })); + + const lastId = parsed[parsed.length - 1]?.id || 0; + const hasMore = parsed.length === limit && (maxId || 0) > lastId; + + return { + rows: parsed, + maxId: maxId || 0, + hasMore, + }; + } + + countEventsUpTo(upToId: number): number { + const { count } = this.db.prepare( + 'SELECT COUNT(*) as count FROM zigbee_events WHERE id <= ?' + ).get(upToId) as { count: number }; + return count; + } + + deleteEventsUpTo(upToId: number): { deleted: number; remaining: number } { + const deleteResult = this.db.prepare('DELETE FROM zigbee_events WHERE id <= ?').run(upToId); + const { count: remaining } = this.db.prepare( + 'SELECT COUNT(*) as count FROM zigbee_events' + ).get() as { count: number }; + + if (deleteResult.changes > 1000) { + try { + this.db.pragma('incremental_vacuum'); + } catch (vacErr) { + console.warn('[ZigbeeDiag] Vacuum warning:', vacErr); + } + } + + return { deleted: deleteResult.changes, remaining }; + } + + queueCommand(macRaw: string, action: string, queuedBy?: string): ZigbeeCommand { + const mac = normalizeMac(macRaw); + if (!mac) { + throw new Error('Invalid device MAC'); + } + if (!isAllowedAction(action)) { + throw new Error(`Unsupported action: ${action}`); + } + + const now = Date.now(); + const info = this.db.prepare(` + INSERT INTO zigbee_commands (mac, action, queued_at, delivered_at, queued_by) + VALUES (?, ?, ?, NULL, ?) + `).run(mac, action, now, queuedBy || null); + + return { + id: Number(info.lastInsertRowid), + mac, + action, + queuedAt: now, + deliveredAt: null, + queuedBy: queuedBy || null, + }; + } + + getPendingCommands(macRaw: string): ZigbeeCommand[] { + const mac = normalizeMac(macRaw); + const rows = this.db.prepare(` + SELECT + id, mac, action, + queued_at as queuedAt, + delivered_at as deliveredAt, + queued_by as queuedBy + FROM zigbee_commands + WHERE mac = ? AND delivered_at IS NULL + ORDER BY id ASC + `).all(mac) as ZigbeeCommand[]; + return rows; + } + + close(): void { + this.db.close(); + } + + reset(): void { + this.db.exec('DELETE FROM zigbee_events'); + this.db.exec('DELETE FROM zigbee_commands'); + this.db.exec('DELETE FROM zigbee_gateways'); + } +} diff --git a/tools/telemetry/all.ts b/tools/telemetry/all.ts new file mode 100644 index 0000000..4b92653 --- /dev/null +++ b/tools/telemetry/all.ts @@ -0,0 +1,53 @@ +#!/usr/bin/env ts-node +/** + * Combined telemetry + Zigbee diagnostics pull and analyse. + * + * Runs: + * 1. SmartLayer telemetry pull + * 2. SmartLayer telemetry analyse + * 3. Zigbee diagnostics pull + * 4. Zigbee diagnostics analyse + * + * Pass-through flags (e.g. --ack, --dry-run) are forwarded to both pull scripts. + * Zigbee events are only acked when --ack is present (same as pull-zigbee.ts). + * + * Usage: + * npm run telemetry:all + * npm run telemetry:all -- --ack --dry-run + */ + +import { spawnSync } from 'child_process'; +import { join } from 'path'; + +const root = join(__dirname, '..', '..'); +const args = process.argv.slice(2); + +function run(script: string, extraArgs: string[] = []): void { + const result = spawnSync( + process.execPath, + ['-r', 'ts-node/register', join(__dirname, script), ...extraArgs], + { + cwd: root, + stdio: 'inherit', + env: process.env, + } + ); + + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +console.log('=== SmartLayer telemetry: pull ==='); +run('pull.ts', args); + +console.log('\n=== SmartLayer telemetry: analyse ==='); +run('analyse.ts'); + +console.log('\n=== Zigbee diagnostics: pull ==='); +run('pull-zigbee.ts', args); + +console.log('\n=== Zigbee diagnostics: analyse ==='); +run('analyse-zigbee.ts'); + +console.log('\nDone.'); diff --git a/tools/telemetry/analyse-zigbee.ts b/tools/telemetry/analyse-zigbee.ts new file mode 100644 index 0000000..192ac1b --- /dev/null +++ b/tools/telemetry/analyse-zigbee.ts @@ -0,0 +1,294 @@ +#!/usr/bin/env ts-node +/** + * Zigbee Diagnostics Analysis Tool + * + * Analyzes Zigbee gateway JSONL for: + * - Silent gateways (no recent poll/ok/boot) + * - Watchdog / reset reasons + * - Wi-Fi / MQTT flaps + * - OTA failures + * - Unknown-device descriptor dumps + * + * Usage: + * npm run telemetry:zigbee:analyse [-- data/zigbee-YYYYMMDD.jsonl] + * OR: ts-node tools/telemetry/analyse-zigbee.ts [data/zigbee-YYYYMMDD.jsonl] + */ + +import { config } from 'dotenv'; +import { readFileSync, readdirSync, existsSync } from 'fs'; +import { join } from 'path'; + +const envPaths = [ + join(__dirname, '.env'), + join(__dirname, '..', '..', '.env'), +]; +for (const envPath of envPaths) { + if (existsSync(envPath)) { + config({ path: envPath }); + break; + } +} + +interface ZigbeeEventRow { + id: number; + mac: string; + kind: string; + payload: Record; + receivedAt: number; +} + +interface GatewayAnalysis { + mac: string; + eventCount: number; + lastSeen: number; + lastKind: string; + firmwareVersion?: string; + silent: boolean; + silentHours: number; + resetReasons: Map; + watchdogRestarts: number; + wifiFlaps: number; + mqttFlaps: number; + otaFailures: Array<{ code?: string; message?: string; at: number }>; + unknownDevices: Array<{ at: number; dump: Record }>; + errorCodes: Map; +} + +const SILENT_THRESHOLD_MS = 10 * 60 * 1000; // ~5 poll intervals of 2 min + +function parseJsonlFile(path: string): ZigbeeEventRow[] { + const content = readFileSync(path, 'utf-8'); + return content + .split('\n') + .filter(line => line.trim()) + .map(line => JSON.parse(line)); +} + +function findLatestFile(): string | null { + const dataDir = join(__dirname, '..', '..', 'data'); + try { + const files = readdirSync(dataDir) + .filter(f => f.startsWith('zigbee-') && f.endsWith('.jsonl')) + .sort() + .reverse(); + return files.length > 0 ? join(dataDir, files[0]) : null; + } catch { + return null; + } +} + +function analyzeGateway(mac: string, events: ZigbeeEventRow[], now: number): GatewayAnalysis { + const sorted = [...events].sort((a, b) => a.receivedAt - b.receivedAt); + const last = sorted[sorted.length - 1]; + const resetReasons = new Map(); + const errorCodes = new Map(); + const otaFailures: GatewayAnalysis['otaFailures'] = []; + const unknownDevices: GatewayAnalysis['unknownDevices'] = []; + + let wifiFlaps = 0; + let mqttFlaps = 0; + let prevMqtt: boolean | undefined; + let prevWifiOk: boolean | undefined; + let watchdogRestarts = 0; + let firmwareVersion: string | undefined; + + for (const ev of sorted) { + const p = ev.payload || {}; + if (typeof p.fw === 'string') firmwareVersion = p.fw; + + const reset = typeof p.reset === 'string' ? p.reset : undefined; + if (reset) { + resetReasons.set(reset, (resetReasons.get(reset) || 0) + 1); + if (reset === 'watchdog' || /wdt/i.test(reset)) { + watchdogRestarts++; + } + } + + if (ev.kind === 'boot' && reset === 'watchdog') { + watchdogRestarts++; + } + + if (typeof p.mqtt_ok === 'boolean') { + if (prevMqtt === true && p.mqtt_ok === false) mqttFlaps++; + prevMqtt = p.mqtt_ok; + } + + const wifiDown = typeof p.wifi_rssi === 'number' ? p.wifi_rssi === 0 : undefined; + if (wifiDown !== undefined) { + if (prevWifiOk === true && wifiDown) wifiFlaps++; + prevWifiOk = !wifiDown; + } + + const code = typeof p.code === 'string' ? p.code : undefined; + if (ev.kind === 'error' && code) { + errorCodes.set(code, (errorCodes.get(code) || 0) + 1); + if (/^ota_/i.test(code)) { + otaFailures.push({ + code, + message: typeof p.message === 'string' ? p.message : undefined, + at: ev.receivedAt, + }); + } + } + + const unknown = + p.unknown_device || + p.unknownDevice || + (ev.kind === 'device-added' || ev.kind === 'device' ? p.device || p.descriptor : undefined); + + if (unknown && typeof unknown === 'object') { + unknownDevices.push({ at: ev.receivedAt, dump: unknown }); + } else if ( + (ev.kind === 'device-added' || (ev.kind === 'device' && code === 'device-added')) && + (p.endpoints || p.clusters || p.manufacturer || p.model) + ) { + unknownDevices.push({ + at: ev.receivedAt, + dump: { + endpoints: p.endpoints, + clusters: p.clusters, + manufacturer: p.manufacturer, + model: p.model, + power_source: p.power_source || p.powerSource, + message: p.message, + code: p.code, + }, + }); + } + } + + const silentDuration = now - (last?.receivedAt || 0); + const silent = !last || silentDuration > SILENT_THRESHOLD_MS; + + return { + mac, + eventCount: sorted.length, + lastSeen: last?.receivedAt || 0, + lastKind: last?.kind || 'none', + firmwareVersion, + silent, + silentHours: silentDuration / (60 * 60 * 1000), + resetReasons, + watchdogRestarts, + wifiFlaps, + mqttFlaps, + otaFailures, + unknownDevices, + errorCodes, + }; +} + +function generateReport(analyses: GatewayAnalysis[], now: number): string { + let md = '# Zigbee Gateway Diagnostics Report\n\n'; + md += `Generated: ${new Date(now).toISOString()}\n\n`; + md += `Total gateways: ${analyses.length}\n\n`; + + const silent = analyses.filter(a => a.silent); + const withWatchdog = analyses.filter(a => a.watchdogRestarts > 0); + const withOtaFail = analyses.filter(a => a.otaFailures.length > 0); + const withUnknown = analyses.filter(a => a.unknownDevices.length > 0); + + md += '## Summary\n\n'; + md += `- Silent gateways (>${SILENT_THRESHOLD_MS / 60000} min since last event): ${silent.length}\n`; + md += `- Watchdog restarts observed: ${withWatchdog.length} gateway(s)\n`; + md += `- OTA failures: ${withOtaFail.length} gateway(s)\n`; + md += `- Unknown-device dumps: ${withUnknown.length} gateway(s)\n\n`; + + if (silent.length > 0) { + md += '## Silent Gateways\n\n'; + for (const a of silent.sort((x, y) => y.silentHours - x.silentHours)) { + md += `### \`${a.mac}\`\n\n`; + md += `- Silent for: ${a.silentHours.toFixed(1)} hours\n`; + md += `- Last seen: ${a.lastSeen ? new Date(a.lastSeen).toISOString() : 'never'} (${a.lastKind})\n`; + if (a.firmwareVersion) md += `- Firmware: ${a.firmwareVersion}\n`; + md += '\n'; + } + } + + md += '## Gateway Details\n\n'; + for (const a of analyses.sort((x, y) => y.lastSeen - x.lastSeen)) { + md += `### \`${a.mac}\`\n\n`; + if (a.firmwareVersion) md += `**Firmware:** ${a.firmwareVersion}\n\n`; + md += `**Events:** ${a.eventCount} | **Last:** ${a.lastSeen ? new Date(a.lastSeen).toISOString() : 'n/a'} (${a.lastKind})\n\n`; + + if (a.resetReasons.size > 0) { + md += '**Reset reasons:**\n'; + for (const [reason, count] of Array.from(a.resetReasons.entries()).sort((x, y) => y[1] - x[1])) { + md += `- ${reason}: ${count}\n`; + } + md += '\n'; + } + + if (a.watchdogRestarts > 0) { + md += `**Watchdog restarts:** ${a.watchdogRestarts}\n\n`; + } + + if (a.wifiFlaps > 0 || a.mqttFlaps > 0) { + md += `**Connectivity flaps:** Wi-Fi ${a.wifiFlaps}, MQTT ${a.mqttFlaps}\n\n`; + } + + if (a.errorCodes.size > 0) { + md += '**Error codes:**\n'; + for (const [code, count] of Array.from(a.errorCodes.entries()).sort((x, y) => y[1] - x[1])) { + md += `- \`${code}\`: ${count}\n`; + } + md += '\n'; + } + + if (a.otaFailures.length > 0) { + md += '**OTA failures:**\n'; + for (const f of a.otaFailures.slice(-5)) { + md += `- ${new Date(f.at).toISOString()}: ${f.code || '?'}${f.message ? ` — ${f.message}` : ''}\n`; + } + md += '\n'; + } + + if (a.unknownDevices.length > 0) { + md += '**Unknown-device dumps:**\n'; + for (const u of a.unknownDevices.slice(-5)) { + md += `- ${new Date(u.at).toISOString()}:\n`; + md += '```json\n' + JSON.stringify(u.dump, null, 2) + '\n```\n'; + } + md += '\n'; + } + + md += '---\n\n'; + } + + return md; +} + +async function main() { + const args = process.argv.slice(2).filter(a => !a.startsWith('--')); + let inputFile = args[0]; + + if (!inputFile) { + const latest = findLatestFile(); + if (!latest) { + console.error('Error: No zigbee-*.jsonl file found. Run telemetry:zigbee:pull first.'); + process.exit(1); + } + inputFile = latest; + console.error(`Using latest file: ${inputFile}\n`); + } + + console.error(`Analyzing: ${inputFile}\n`); + const events = parseJsonlFile(inputFile); + console.error(`Loaded ${events.length} events\n`); + + const byMac = new Map(); + for (const ev of events) { + const list = byMac.get(ev.mac) || []; + list.push(ev); + byMac.set(ev.mac, list); + } + + const now = Date.now(); + const analyses = Array.from(byMac.entries()).map(([mac, evs]) => analyzeGateway(mac, evs, now)); + console.log(generateReport(analyses, now)); +} + +main().catch(err => { + console.error('Error:', err); + process.exit(1); +}); diff --git a/tools/telemetry/pull-zigbee.ts b/tools/telemetry/pull-zigbee.ts new file mode 100644 index 0000000..2127cd9 --- /dev/null +++ b/tools/telemetry/pull-zigbee.ts @@ -0,0 +1,169 @@ +#!/usr/bin/env ts-node +/** + * Zigbee Diagnostics Pull Tool + * + * Pulls Zigbee gateway events from /api/diag/zbgw/events into data/zigbee-YYYYMMDD.jsonl + * Optionally acknowledges (deletes) events from the server with --ack (admin signature). + * Does NOT ack unless --ack is passed. + * + * Usage: + * npm run telemetry:zigbee:pull [-- --ack] [-- --dry-run] + * OR: ts-node tools/telemetry/pull-zigbee.ts [--ack] [--dry-run] + * + * Requires .env with: + * PROXY_BASE_URL=https://your-proxy.com + * TELEMETRY_API_KEY=your-api-key + * TELEMETRY_ADMIN_KEY=0x... (private key, only for --ack) + */ + +import { config } from 'dotenv'; +import { existsSync, mkdirSync, appendFileSync } from 'fs'; +import { join } from 'path'; +import { Wallet } from 'ethers'; + +const envPaths = [ + join(__dirname, '.env'), + join(__dirname, '..', '..', '.env'), +]; + +for (const envPath of envPaths) { + if (existsSync(envPath)) { + config({ path: envPath }); + break; + } +} + +const PROXY_BASE_URL = process.env.PROXY_BASE_URL || 'http://localhost:8081'; +const TELEMETRY_API_KEY = process.env.TELEMETRY_API_KEY || ''; +const TELEMETRY_ADMIN_KEY = process.env.TELEMETRY_ADMIN_KEY || ''; + +interface FeedResponse { + rows: any[]; + maxId: number; + hasMore: boolean; + serverTime: number; +} + +async function fetchFeed(afterId: number, limit = 1000): Promise { + const url = `${PROXY_BASE_URL}/api/diag/zbgw/events?afterId=${afterId}&limit=${limit}`; + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${TELEMETRY_API_KEY}`, + }, + }); + + if (!response.ok) { + throw new Error(`Zigbee feed request failed: ${response.status} ${response.statusText}`); + } + + return response.json() as Promise; +} + +async function acknowledgeEvents(upToId: number, dryRun: boolean): Promise { + if (!TELEMETRY_ADMIN_KEY) { + throw new Error('TELEMETRY_ADMIN_KEY not set in .env'); + } + + const challengeResp = await fetch(`${PROXY_BASE_URL}/api/admin/challenge`); + if (!challengeResp.ok) { + throw new Error(`Challenge request failed: ${challengeResp.status}`); + } + const { nonce } = await challengeResp.json() as { nonce: string }; + + const wallet = new Wallet(TELEMETRY_ADMIN_KEY); + const message = `zbgw-ack:${upToId}:${nonce}`; + const signature = await wallet.signMessage(message); + + const ackResp = await fetch(`${PROXY_BASE_URL}/api/diag/zbgw/ack`, { + method: 'POST', + headers: { + Authorization: `Bearer ${TELEMETRY_API_KEY}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + upToId, + signature, + message, + dryRun, + }), + }); + + if (!ackResp.ok) { + const error = await ackResp.text(); + throw new Error(`Zigbee ack request failed: ${ackResp.status} - ${error}`); + } + + return ackResp.json() as Promise; +} + +async function main() { + const args = process.argv.slice(2); + const shouldAck = args.includes('--ack'); + const dryRun = args.includes('--dry-run'); + + if (!TELEMETRY_API_KEY) { + console.error('Error: TELEMETRY_API_KEY not set in .env'); + process.exit(1); + } + + const dataDir = join(__dirname, '..', '..', 'data'); + if (!existsSync(dataDir)) { + mkdirSync(dataDir, { recursive: true }); + } + + const today = new Date().toISOString().split('T')[0].replace(/-/g, ''); + const outputFile = join(dataDir, `zigbee-${today}.jsonl`); + + let afterId = 0; + let totalRows = 0; + let maxIdSeen = 0; + + console.log(`Pulling Zigbee diagnostics from ${PROXY_BASE_URL}`); + console.log(`Writing to: ${outputFile}`); + + while (true) { + console.log(`Fetching events after ID ${afterId}...`); + const feed = await fetchFeed(afterId, 1000); + + if (feed.rows.length === 0) { + console.log('No more events.'); + break; + } + + for (const row of feed.rows) { + appendFileSync(outputFile, JSON.stringify(row) + '\n'); + maxIdSeen = Math.max(maxIdSeen, row.id); + } + + totalRows += feed.rows.length; + console.log(` Retrieved ${feed.rows.length} events (total: ${totalRows})`); + + if (!feed.hasMore) { + console.log('Reached end of feed.'); + break; + } + + afterId = feed.rows[feed.rows.length - 1].id; + } + + console.log(`\nTotal events written: ${totalRows}`); + console.log(`Max ID: ${maxIdSeen}`); + + if (shouldAck && maxIdSeen > 0) { + console.log(`\n${dryRun ? '[DRY RUN]' : ''} Acknowledging Zigbee events up to ID ${maxIdSeen}...`); + try { + const ackResult = await acknowledgeEvents(maxIdSeen, dryRun); + console.log('Ack result:', JSON.stringify(ackResult, null, 2)); + } catch (err) { + console.error('Ack failed:', err); + process.exit(1); + } + } else if (shouldAck) { + console.log('\nNo events to acknowledge.'); + } +} + +main().catch(err => { + console.error('Error:', err); + process.exit(1); +});