diff --git a/ProtoDef b/ProtoDef index 8e07785..349d52e 160000 --- a/ProtoDef +++ b/ProtoDef @@ -1 +1 @@ -Subproject commit 8e07785e94626882fa3333184bb366e3f6625356 +Subproject commit 349d52ee67a48c174968bb73ae59c50eebfada10 diff --git a/doc/compiler.md b/doc/compiler.md index a97a078..a92f6f9 100644 --- a/doc/compiler.md +++ b/doc/compiler.md @@ -228,4 +228,27 @@ compiledProto.setVariable('noArraySizeCheck', true); // Use it as if it were a normal ProtoDef const buffer = compiledProto.createPacketBuffer('mainType', result) const result = compiledProto.parsePacketBuffer('mainType', buffer) -``` \ No newline at end of file +``` +### Sizing inside a writer + +A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code calling the sizer for `type`, which the sizeOf context already holds since it is generated first. It is only available when the types are compiled through `ProtoDefCompiler`, and `type` has to be a named type, since an anonymous one has no function in that context to call. + +A datatype that needs a helper function in its generated code registers it as a context type, which copies the function's source into the compiled output, rather than reaching for something outside it. The `hash` datatype is built on both: + +```javascript +Write: { + _crc32c: ['context', crc32c], + hash: ['parametrizable', (compiler, { alg, type, body }) => { + // Throws unless `type` is u32, i32, lu32 or li32, directly or through an alias + const signed = hashTypeIsSigned(compiler.resolveType(type), type) + let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` + code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` + code += `const hash = ctx._${alg}(bodyBuffer)\n` + // A CRC is unsigned; a signed `type` takes its two's complement + code += 'return ' + compiler.callType(signed ? 'hash | 0' : 'hash', type) + return compiler.wrapCode(code) + }] +} +``` + +The context is shared with the protocol's own type names, and a context entry wins over a type of the same name, so a helper's name is underscored to keep it out of the way. `compiler.resolveType(type)` follows aliases to the type they name, so the check that a hash is written as a 32 bit integer also accepts a name aliasing one, and fails at compile time otherwise. diff --git a/src/compiler.js b/src/compiler.js index 9c8f057..19aa404 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -12,6 +12,7 @@ class ProtoDefCompiler { this.readCompiler = new ReadCompiler() this.writeCompiler = new WriteCompiler() this.sizeOfCompiler = new SizeOfCompiler() + this.writeCompiler.sizeOfCompiler = this.sizeOfCompiler } addTypes (types) { @@ -62,6 +63,8 @@ class CompiledProtodef { this.sizeOfCtx = sizeOfCtx this.writeCtx = writeCtx this.readCtx = readCtx + // Code from callTypeSize runs against the sizeOf context + writeCtx.sizeOfCtx = sizeOfCtx } read (buffer, cursor, type) { @@ -181,6 +184,15 @@ class Compiler { } } + /** + * `type` with aliases followed down to the type they name, which is + * returned as is when it isn't an alias + */ + resolveType (type) { + while (typeof type === 'string' && typeof this.types[type] === 'string' && this.types[type] !== 'native') type = this.types[type] + return type + } + addProtocol (protocolData, path) { const self = this function recursiveAddTypes (protocolData, path) { @@ -361,11 +373,26 @@ class WriteCompiler extends Compiler { if (args.length > 0) return '(' + code + `)(${value}, buffer, ${offsetExpr}, ` + args.map(name => this.getField(name)).join(', ') + ')' return '(' + code + `)(${value}, buffer, ${offsetExpr})` } + + /** + * Code computing the size of `value` as `type`, for writers that need to + * serialize part of a value before they can write it. The sizer is a + * function in the sizeOf context, which is generated first, so `type` has + * to be a named one for there to be a function to call. + */ + callTypeSize (value, type) { + if (!this.sizeOfCompiler) throw new Error('sizeOfCtx is only available when compiling with ProtoDefCompiler') + if (typeof type !== 'string' || !this.sizeOfCompiler.types[type]) { + throw new Error('cannot size ' + JSON.stringify(type) + ' from a writer, it is not a named type') + } + return `ctx.sizeOfCtx.${type}(${value})` + } } class SizeOfCompiler extends Compiler { constructor () { super() + this.constants = {} this.addTypes(conditionalDatatypes.SizeOf) this.addTypes(structuresDatatypes.SizeOf) @@ -390,12 +417,21 @@ class SizeOfCompiler extends Compiler { this.primitiveTypes[type] = `native.${type}` if (!isNaN(fn)) { this.native[type] = (value) => { return fn } + this.constants[type] = fn } else { this.native[type] = fn } this.types[type] = 'native' } + /** + * The size of `type` when it doesn't depend on the value, following + * aliases down to a fixed-size native; undefined otherwise + */ + constantSize (type) { + return this.constants[this.resolveType(type)] + } + compileType (type) { if (type instanceof Array) { if (this.parameterizableTypes[type[0]]) { return this.parameterizableTypes[type[0]](this, type[1]) } diff --git a/src/datatypes/compiler-utils.js b/src/datatypes/compiler-utils.js index c74afec..4fa5dde 100644 --- a/src/datatypes/compiler-utils.js +++ b/src/datatypes/compiler-utils.js @@ -1,3 +1,5 @@ +const { algorithms: hashAlgorithms, hashTypeIsSigned } = require('./hash') + module.exports = { Read: { pstring: ['parametrizable', (compiler, string) => { @@ -83,10 +85,18 @@ return { value, size } let code = 'const { value, size } = ' + compiler.callType(mapper.type) + '\n' code += 'return { value: ' + JSON.stringify(sanitizeMappings(mapper.mappings)) + '[value] || value, size }' return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { type }) => { + hashTypeIsSigned(compiler.resolveType(type), type) + return compiler.wrapCode('return ' + compiler.callType(type)) }] }, Write: { + // A hash digest is taken in generated code, so the digest function is + // copied into the compiled context rather than reached for outside it. + // Underscored: the context is shared with the protocol's own type names. + _crc32c: ['context', hashAlgorithms.crc32c.digest], pstring: ['parametrizable', (compiler, string) => { let code = `const length = Buffer.byteLength(value, "${string.encoding || 'utf8'}")\n` if (string.countType) { @@ -167,6 +177,16 @@ return (ctx.${type})(val, buffer, offset) code += '}\n' code += 'return ' + compiler.callType('mapped', mapper.type) return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { alg, type, body }) => { + if (!hashAlgorithms[alg]) throw new Error('Unknown hash algorithm: ' + alg) + const signed = hashTypeIsSigned(compiler.resolveType(type), type) + let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` + code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` + code += `const hash = ctx._${alg}(bodyBuffer)\n` + // A CRC is unsigned; a signed `type` takes its two's complement + code += 'return ' + compiler.callType(signed ? 'hash | 0' : 'hash', type) + return compiler.wrapCode(code) }] }, @@ -225,6 +245,11 @@ return (ctx.${type})(val) code += '}\n' code += 'return ' + compiler.callType('mapped', mapper.type) return compiler.wrapCode(code) + }], + // The digest has a fixed width, so a hash is sized without hashing + hash: ['parametrizable', (compiler, { alg, type }) => { + hashTypeIsSigned(compiler.resolveType(type), type) + return String(hashAlgorithms[alg].bytes) }] } } diff --git a/src/datatypes/hash.js b/src/datatypes/hash.js new file mode 100644 index 0000000..9590ce2 --- /dev/null +++ b/src/datatypes/hash.js @@ -0,0 +1,77 @@ +// CRC-32C (Castagnoli): reflected table-driven, all-ones init and final xor. +// The compiler copies this function into the code it generates, so it has to +// stand on its own: no imports, no module scope, its table cached on itself. +function crc32c (buffer) { + let table = crc32c.table + if (!table) { + table = crc32c.table = new Int32Array(256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0x82F63B78 ^ (c >>> 1) : c >>> 1 + table[n] = c + } + } + let c = -1 + for (let i = 0; i < buffer.length; i++) c = table[(c ^ buffer[i]) & 0xff] ^ (c >>> 8) + return (c ^ -1) >>> 0 +} + +// `alg` is an explicit list in the spec, so a protocol means the same thing in +// every implementation; adding an algorithm here is a spec change. The width of +// the digest is what lets a hash be sized without hashing. +const algorithms = { + crc32c: { bytes: 4, digest: crc32c } +} + +function digest (alg, buffer) { + const algorithm = algorithms[alg] + if (!algorithm) throw new Error('Unknown hash algorithm: ' + alg) + return algorithm.digest(buffer) +} + +// The spec allows exactly these as `type`, directly or through an alias: the +// digest is an unsigned 32 bit integer, and a signed type holds the same bits +// read as two's complement. +const hashTypes = { u32: false, i32: true, lu32: false, li32: true } + +// Checks the native type an alias resolved to, and returns whether it is signed +function hashTypeIsSigned (native, type) { + if (!Object.prototype.hasOwnProperty.call(hashTypes, native)) { + throw new Error('hash type must be a 32 bit integer (' + Object.keys(hashTypes).join(', ') + '), ' + JSON.stringify(type) + ' is not') + } + return hashTypes[native] +} + +// An alias is added with its native's functions, so it resolves by identity +function resolveHashType (proto, type) { + const functions = typeof type === 'string' ? proto.types[type] : undefined + const native = functions && Object.keys(hashTypes).find(name => proto.types[name] === functions) + return hashTypeIsSigned(native, type) +} + +function readHash (buffer, offset, { type }, rootNode) { + resolveHashType(this, type) + return this.read(buffer, offset, type, rootNode) +} + +function writeHash (value, buffer, offset, { alg, type, body }, rootNode) { + const signed = resolveHashType(this, type) + if (typeof body !== 'string') throw new Error('hash body must be a named type, ' + JSON.stringify(body) + ' is not') + const bodyBuffer = Buffer.alloc(this.sizeOf(value, body, rootNode)) + this.write(value, bodyBuffer, 0, body, rootNode) + const hash = digest(alg, bodyBuffer) + return this.write(signed ? hash | 0 : hash, buffer, offset, type, rootNode) +} + +// The digest has a fixed width, so a hash is sized without hashing +function sizeOfHash (value, { alg, type }) { + resolveHashType(this, type) + return algorithms[alg].bytes +} + +module.exports = { + digest, + algorithms, + hashTypeIsSigned, + hash: [readHash, writeHash, sizeOfHash, require('../../ProtoDef/schemas/utils.json').hash] +} diff --git a/src/datatypes/utils.js b/src/datatypes/utils.js index 40adce8..a44d045 100644 --- a/src/datatypes/utils.js +++ b/src/datatypes/utils.js @@ -9,6 +9,7 @@ module.exports = { bitflags: [readBitflags, writeBitflags, sizeOfBitflags, require('../../ProtoDef/schemas/utils.json').bitflags], cstring: [readCString, writeCString, sizeOfCString, require('../../ProtoDef/schemas/utils.json').cstring], mapper: [readMapper, writeMapper, sizeOfMapper, require('../../ProtoDef/schemas/utils.json').mapper], + hash: require('./hash').hash, ...require('./varint') } diff --git a/test/misc.js b/test/misc.js index 3ae1325..5bdb8ed 100644 --- a/test/misc.js +++ b/test/misc.js @@ -102,3 +102,94 @@ describe('FullPacketParser', () => { }) } }) + +describe('hash', () => { + const { digest } = require('../src/datatypes/hash') + const inlineBody = ['hash', { alg: 'crc32c', type: 'u32', body: ['buffer', { count: 9 }] }] + const types = { + Body: ['buffer', { count: 9 }], + crc32c: ['hash', { alg: 'crc32c', type: 'u32', body: 'Body' }], + signed: ['hash', { alg: 'crc32c', type: 'HashCode', body: 'Body' }], + HashCode: 'i32', + littleSigned: ['hash', { alg: 'crc32c', type: 'LittleHashCode', body: 'Body' }], + LittleHashCode: 'li32', + // A hash over a list of hashes + entry: ['container', [{ name: 'key', type: ['pstring', { countType: 'u8' }] }, { name: 'value', type: 'li32' }]], + list: ['array', { countType: 'u8', type: ['hash', { alg: 'crc32c', type: 'lu32', body: 'entry' }] }], + nested: ['hash', { alg: 'crc32c', type: 'lu32', body: 'list' }] + } + const proto = new ProtoDef() + proto.addTypes(types) + const compiler = new ProtoDefCompiler() + compiler.addTypesToCompile(types) + const compiled = compiler.compileProtoDefSync() + const check = Buffer.from('123456789') + const u32 = n => { const b = Buffer.alloc(4); b.writeUInt32BE(n); return b } + const lu32 = n => { const b = Buffer.alloc(4); b.writeUInt32LE(n); return b } + + it('crc32c matches its check value', () => { + assert.strictEqual(digest('crc32c', check), 0xE3069283) + }) + + it('rejects an algorithm the spec does not define', () => { + assert.throws(() => digest('sha256', check), /Unknown hash algorithm/) + const log = console.log // the validator dumps the type it rejected + console.log = () => {} + try { + assert.throws(() => new ProtoDef().addTypes({ bad: ['hash', { alg: 'sha256', type: 'u32', body: 'u8' }] })) + } finally { + console.log = log + } + }) + + it('rejects a body that is not a named type', () => { + assert.throws(() => proto.write(check, Buffer.alloc(4), 0, inlineBody), /named type/) + const c = new ProtoDefCompiler() + c.addTypesToCompile({ withInlineBody: inlineBody }) + assert.throws(() => c.compileProtoDefSync(), /named type/) + }) + + it('rejects a type that is not a 32 bit integer', () => { + for (const type of ['f32', 'u64', 'i64', 'varint']) { + const message = new RegExp('hash type must be a 32 bit integer \\(u32, i32, lu32, li32\\), "' + type + '" is not') + const bad = ['hash', { alg: 'crc32c', type, body: 'Body' }] + assert.throws(() => proto.sizeOf(check, bad), message) + assert.throws(() => proto.write(check, Buffer.alloc(8), 0, bad), message) + assert.throws(() => proto.read(Buffer.alloc(8), 0, bad), message) + const c = new ProtoDefCompiler() + for (const kind of ['readCompiler', 'writeCompiler', 'sizeOfCompiler']) { + assert.throws(() => c[kind].compileType(bad), message, kind) + } + } + }) + + for (const [label, p] of [['interpreted', proto], ['compiled', compiled]]) { + describe(label, () => { + it('writes the hash of the serialized body', () => { + assert.deepStrictEqual(p.createPacketBuffer('crc32c', check), u32(0xE3069283)) + }) + it('reads the hash, not the value', () => { + assert.strictEqual(p.parsePacketBuffer('crc32c', u32(0xE3069283)).data, 0xE3069283) + }) + it('writes a signed type in two\'s complement', () => { + const buffer = p.createPacketBuffer('signed', check) + assert.deepStrictEqual(buffer, u32(0xE3069283)) + assert.strictEqual(p.parsePacketBuffer('signed', buffer).data, 0xE3069283 | 0) + }) + it('follows an alias to a little-endian signed type', () => { + const buffer = p.createPacketBuffer('littleSigned', check) + assert.deepStrictEqual(buffer, lu32(0xE3069283)) + assert.strictEqual(p.parsePacketBuffer('littleSigned', buffer).data, 0xE3069283 | 0) + }) + it('sizes without hashing', () => { + assert.strictEqual(p.sizeOf(check, 'signed'), 4) + }) + it('nests hashes of hashes', () => { + const value = [{ key: 'a', value: 1 }, { key: 'b', value: 2 }] + const list = Buffer.concat([Buffer.from([2]), ...value.map(entry => lu32(digest('crc32c', p.createPacketBuffer('entry', entry))))]) + assert.deepStrictEqual(p.createPacketBuffer('list', value), list) + assert.deepStrictEqual(p.createPacketBuffer('nested', value), lu32(digest('crc32c', list))) + }) + }) + } +})