Add the hash datatype

doc/compiler.md

@@ -228,4 +228,27 @@ compiledProto.setVariable('noArraySizeCheck', true);
 // Use it as if it were a normal ProtoDef
 const buffer = compiledProto.createPacketBuffer('mainType', result)
 const result = compiledProto.parsePacketBuffer('mainType', buffer)
-```
+```
+### Sizing inside a writer
+
+A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code calling the sizer for `type`, which the sizeOf context already holds since it is generated first. It is only available when the types are compiled through `ProtoDefCompiler`, and `type` has to be a named type, since an anonymous one has no function in that context to call.
+
+A datatype that needs a helper function in its generated code registers it as a context type, which copies the function's source into the compiled output, rather than reaching for something outside it. The `hash` datatype is built on both:
+
+```javascript
+Write: {
+  _crc32c: ['context', crc32c],
+  hash: ['parametrizable', (compiler, { alg, type, body }) => {
+    // Throws unless `type` is u32, i32, lu32 or li32, directly or through an alias
+    const signed = hashTypeIsSigned(compiler.resolveType(type), type)
+    let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n`
+    code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n`
+    code += `const hash = ctx._${alg}(bodyBuffer)\n`
+    // A CRC is unsigned; a signed `type` takes its two's complement
+    code += 'return ' + compiler.callType(signed ? 'hash | 0' : 'hash', type)
+    return compiler.wrapCode(code)
+  }]
+}
+```
+
+The context is shared with the protocol's own type names, and a context entry wins over a type of the same name, so a helper's name is underscored to keep it out of the way. `compiler.resolveType(type)` follows aliases to the type they name, so the check that a hash is written as a 32 bit integer also accepts a name aliasing one, and fails at compile time otherwise.

src/compiler.js

@@ -12,6 +12,7 @@ class ProtoDefCompiler {
     this.readCompiler = new ReadCompiler()
     this.writeCompiler = new WriteCompiler()
     this.sizeOfCompiler = new SizeOfCompiler()
+    this.writeCompiler.sizeOfCompiler = this.sizeOfCompiler
   }
 
   addTypes (types) {
@@ -62,6 +63,8 @@ class CompiledProtodef {
     this.sizeOfCtx = sizeOfCtx
     this.writeCtx = writeCtx
     this.readCtx = readCtx
+    // Code from callTypeSize runs against the sizeOf context
+    writeCtx.sizeOfCtx = sizeOfCtx
   }
 
   read (buffer, cursor, type) {
@@ -181,6 +184,15 @@ class Compiler {
     }
   }
 
+  /**
+   * `type` with aliases followed down to the type they name, which is
+   * returned as is when it isn't an alias
+   */
+  resolveType (type) {
+    while (typeof type === 'string' && typeof this.types[type] === 'string' && this.types[type] !== 'native') type = this.types[type]
+    return type
+  }
+
   addProtocol (protocolData, path) {
     const self = this
     function recursiveAddTypes (protocolData, path) {
@@ -361,11 +373,26 @@ class WriteCompiler extends Compiler {
     if (args.length > 0) return '(' + code + `)(${value}, buffer, ${offsetExpr}, ` + args.map(name => this.getField(name)).join(', ') + ')'
     return '(' + code + `)(${value}, buffer, ${offsetExpr})`
   }
+
+  /**
+   * Code computing the size of `value` as `type`, for writers that need to
+   * serialize part of a value before they can write it. The sizer is a
+   * function in the sizeOf context, which is generated first, so `type` has
+   * to be a named one for there to be a function to call.
+   */
+  callTypeSize (value, type) {
+    if (!this.sizeOfCompiler) throw new Error('sizeOfCtx is only available when compiling with ProtoDefCompiler')
+    if (typeof type !== 'string' || !this.sizeOfCompiler.types[type]) {
+      throw new Error('cannot size ' + JSON.stringify(type) + ' from a writer, it is not a named type')
+    }
+    return `ctx.sizeOfCtx.${type}(${value})`
+  }
 }
 
 class SizeOfCompiler extends Compiler {
   constructor () {
     super()
+    this.constants = {}
 
     this.addTypes(conditionalDatatypes.SizeOf)
     this.addTypes(structuresDatatypes.SizeOf)
@@ -390,12 +417,21 @@ class SizeOfCompiler extends Compiler {
     this.primitiveTypes[type] = `native.${type}`
     if (!isNaN(fn)) {
       this.native[type] = (value) => { return fn }
+      this.constants[type] = fn
     } else {
       this.native[type] = fn
     }
     this.types[type] = 'native'
   }
 
+  /**
+   * The size of `type` when it doesn't depend on the value, following
+   * aliases down to a fixed-size native; undefined otherwise
+   */
+  constantSize (type) {
+    return this.constants[this.resolveType(type)]
+  }
+
   compileType (type) {
     if (type instanceof Array) {
       if (this.parameterizableTypes[type[0]]) { return this.parameterizableTypes[type[0]](this, type[1]) }

src/datatypes/compiler-utils.js

@@ -1,3 +1,5 @@
+const { algorithms: hashAlgorithms, hashTypeIsSigned } = require('./hash')
+
 module.exports = {
   Read: {
     pstring: ['parametrizable', (compiler, string) => {
@@ -83,10 +85,18 @@ return { value, size }
       let code = 'const { value, size } = ' + compiler.callType(mapper.type) + '\n'
       code += 'return { value: ' + JSON.stringify(sanitizeMappings(mapper.mappings)) + '[value] || value, size }'
       return compiler.wrapCode(code)
+    }],
+    hash: ['parametrizable', (compiler, { type }) => {
+      hashTypeIsSigned(compiler.resolveType(type), type)
+      return compiler.wrapCode('return ' + compiler.callType(type))
     }]
   },
 
   Write: {
+    // A hash digest is taken in generated code, so the digest function is
+    // copied into the compiled context rather than reached for outside it.
+    // Underscored: the context is shared with the protocol's own type names.
+    _crc32c: ['context', hashAlgorithms.crc32c.digest],
     pstring: ['parametrizable', (compiler, string) => {
       let code = `const length = Buffer.byteLength(value, "${string.encoding || 'utf8'}")\n`
       if (string.countType) {
@@ -167,6 +177,16 @@ return (ctx.${type})(val, buffer, offset)
       code += '}\n'
       code += 'return ' + compiler.callType('mapped', mapper.type)
       return compiler.wrapCode(code)
+    }],
+    hash: ['parametrizable', (compiler, { alg, type, body }) => {
+      if (!hashAlgorithms[alg]) throw new Error('Unknown hash algorithm: ' + alg)
+      const signed = hashTypeIsSigned(compiler.resolveType(type), type)
+      let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n`
+      code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n`
+      code += `const hash = ctx._${alg}(bodyBuffer)\n`
+      // A CRC is unsigned; a signed `type` takes its two's complement
+      code += 'return ' + compiler.callType(signed ? 'hash | 0' : 'hash', type)
+      return compiler.wrapCode(code)
     }]
   },
 
@@ -225,6 +245,11 @@ return (ctx.${type})(val)
       code += '}\n'
       code += 'return ' + compiler.callType('mapped', mapper.type)
       return compiler.wrapCode(code)
+    }],
+    // The digest has a fixed width, so a hash is sized without hashing
+    hash: ['parametrizable', (compiler, { alg, type }) => {
+      hashTypeIsSigned(compiler.resolveType(type), type)
+      return String(hashAlgorithms[alg].bytes)
     }]
   }
 }

src/datatypes/hash.js

@@ -0,0 +1,77 @@
+// CRC-32C (Castagnoli): reflected table-driven, all-ones init and final xor.
+// The compiler copies this function into the code it generates, so it has to
+// stand on its own: no imports, no module scope, its table cached on itself.
+function crc32c (buffer) {
+  let table = crc32c.table
+  if (!table) {
+    table = crc32c.table = new Int32Array(256)
+    for (let n = 0; n < 256; n++) {
+      let c = n
+      for (let k = 0; k < 8; k++) c = c & 1 ? 0x82F63B78 ^ (c >>> 1) : c >>> 1
+      table[n] = c
+    }
+  }
+  let c = -1
+  for (let i = 0; i < buffer.length; i++) c = table[(c ^ buffer[i]) & 0xff] ^ (c >>> 8)
+  return (c ^ -1) >>> 0
+}
+
+// `alg` is an explicit list in the spec, so a protocol means the same thing in
+// every implementation; adding an algorithm here is a spec change. The width of
+// the digest is what lets a hash be sized without hashing.
+const algorithms = {
+  crc32c: { bytes: 4, digest: crc32c }
+}
+
+function digest (alg, buffer) {
+  const algorithm = algorithms[alg]
+  if (!algorithm) throw new Error('Unknown hash algorithm: ' + alg)
+  return algorithm.digest(buffer)
+}
+
+// The spec allows exactly these as `type`, directly or through an alias: the
+// digest is an unsigned 32 bit integer, and a signed type holds the same bits
+// read as two's complement.
+const hashTypes = { u32: false, i32: true, lu32: false, li32: true }
+
+// Checks the native type an alias resolved to, and returns whether it is signed
+function hashTypeIsSigned (native, type) {
+  if (!Object.prototype.hasOwnProperty.call(hashTypes, native)) {
+    throw new Error('hash type must be a 32 bit integer (' + Object.keys(hashTypes).join(', ') + '), ' + JSON.stringify(type) + ' is not')
+  }
+  return hashTypes[native]
+}
+
+// An alias is added with its native's functions, so it resolves by identity
+function resolveHashType (proto, type) {
+  const functions = typeof type === 'string' ? proto.types[type] : undefined
+  const native = functions && Object.keys(hashTypes).find(name => proto.types[name] === functions)
+  return hashTypeIsSigned(native, type)
+}
+
+function readHash (buffer, offset, { type }, rootNode) {
+  resolveHashType(this, type)
+  return this.read(buffer, offset, type, rootNode)
+}
+
+function writeHash (value, buffer, offset, { alg, type, body }, rootNode) {
+  const signed = resolveHashType(this, type)
+  if (typeof body !== 'string') throw new Error('hash body must be a named type, ' + JSON.stringify(body) + ' is not')
+  const bodyBuffer = Buffer.alloc(this.sizeOf(value, body, rootNode))
+  this.write(value, bodyBuffer, 0, body, rootNode)
+  const hash = digest(alg, bodyBuffer)
+  return this.write(signed ? hash | 0 : hash, buffer, offset, type, rootNode)
+}
+
+// The digest has a fixed width, so a hash is sized without hashing
+function sizeOfHash (value, { alg, type }) {
+  resolveHashType(this, type)
+  return algorithms[alg].bytes
+}
+
+module.exports = {
+  digest,
+  algorithms,
+  hashTypeIsSigned,
+  hash: [readHash, writeHash, sizeOfHash, require('../../ProtoDef/schemas/utils.json').hash]
+}

src/datatypes/utils.js

@@ -9,6 +9,7 @@ module.exports = {
   bitflags: [readBitflags, writeBitflags, sizeOfBitflags, require('../../ProtoDef/schemas/utils.json').bitflags],
   cstring: [readCString, writeCString, sizeOfCString, require('../../ProtoDef/schemas/utils.json').cstring],
   mapper: [readMapper, writeMapper, sizeOfMapper, require('../../ProtoDef/schemas/utils.json').mapper],
+  hash: require('./hash').hash,
   ...require('./varint')
 }
 

test/misc.js

@@ -102,3 +102,94 @@ describe('FullPacketParser', () => {
     })
   }
 })
+
+describe('hash', () => {
+  const { digest } = require('../src/datatypes/hash')
+  const inlineBody = ['hash', { alg: 'crc32c', type: 'u32', body: ['buffer', { count: 9 }] }]
+  const types = {
+    Body: ['buffer', { count: 9 }],
+    crc32c: ['hash', { alg: 'crc32c', type: 'u32', body: 'Body' }],
+    signed: ['hash', { alg: 'crc32c', type: 'HashCode', body: 'Body' }],
+    HashCode: 'i32',
+    littleSigned: ['hash', { alg: 'crc32c', type: 'LittleHashCode', body: 'Body' }],
+    LittleHashCode: 'li32',
+    // A hash over a list of hashes
+    entry: ['container', [{ name: 'key', type: ['pstring', { countType: 'u8' }] }, { name: 'value', type: 'li32' }]],
+    list: ['array', { countType: 'u8', type: ['hash', { alg: 'crc32c', type: 'lu32', body: 'entry' }] }],
+    nested: ['hash', { alg: 'crc32c', type: 'lu32', body: 'list' }]
+  }
+  const proto = new ProtoDef()
+  proto.addTypes(types)
+  const compiler = new ProtoDefCompiler()
+  compiler.addTypesToCompile(types)
+  const compiled = compiler.compileProtoDefSync()
+  const check = Buffer.from('123456789')
+  const u32 = n => { const b = Buffer.alloc(4); b.writeUInt32BE(n); return b }
+  const lu32 = n => { const b = Buffer.alloc(4); b.writeUInt32LE(n); return b }
+
+  it('crc32c matches its check value', () => {
+    assert.strictEqual(digest('crc32c', check), 0xE3069283)
+  })
+
+  it('rejects an algorithm the spec does not define', () => {
+    assert.throws(() => digest('sha256', check), /Unknown hash algorithm/)
+    const log = console.log // the validator dumps the type it rejected
+    console.log = () => {}
+    try {
+      assert.throws(() => new ProtoDef().addTypes({ bad: ['hash', { alg: 'sha256', type: 'u32', body: 'u8' }] }))
+    } finally {
+      console.log = log
+    }
+  })
+
+  it('rejects a body that is not a named type', () => {
+    assert.throws(() => proto.write(check, Buffer.alloc(4), 0, inlineBody), /named type/)
+    const c = new ProtoDefCompiler()
+    c.addTypesToCompile({ withInlineBody: inlineBody })
+    assert.throws(() => c.compileProtoDefSync(), /named type/)
+  })
+
+  it('rejects a type that is not a 32 bit integer', () => {
+    for (const type of ['f32', 'u64', 'i64', 'varint']) {
+      const message = new RegExp('hash type must be a 32 bit integer \\(u32, i32, lu32, li32\\), "' + type + '" is not')
+      const bad = ['hash', { alg: 'crc32c', type, body: 'Body' }]
+      assert.throws(() => proto.sizeOf(check, bad), message)
+      assert.throws(() => proto.write(check, Buffer.alloc(8), 0, bad), message)
+      assert.throws(() => proto.read(Buffer.alloc(8), 0, bad), message)
+      const c = new ProtoDefCompiler()
+      for (const kind of ['readCompiler', 'writeCompiler', 'sizeOfCompiler']) {
+        assert.throws(() => c[kind].compileType(bad), message, kind)
+      }
+    }
+  })
+
+  for (const [label, p] of [['interpreted', proto], ['compiled', compiled]]) {
+    describe(label, () => {
+      it('writes the hash of the serialized body', () => {
+        assert.deepStrictEqual(p.createPacketBuffer('crc32c', check), u32(0xE3069283))
+      })
+      it('reads the hash, not the value', () => {
+        assert.strictEqual(p.parsePacketBuffer('crc32c', u32(0xE3069283)).data, 0xE3069283)
+      })
+      it('writes a signed type in two\'s complement', () => {
+        const buffer = p.createPacketBuffer('signed', check)
+        assert.deepStrictEqual(buffer, u32(0xE3069283))
+        assert.strictEqual(p.parsePacketBuffer('signed', buffer).data, 0xE3069283 | 0)
+      })
+      it('follows an alias to a little-endian signed type', () => {
+        const buffer = p.createPacketBuffer('littleSigned', check)
+        assert.deepStrictEqual(buffer, lu32(0xE3069283))
+        assert.strictEqual(p.parsePacketBuffer('littleSigned', buffer).data, 0xE3069283 | 0)
+      })
+      it('sizes without hashing', () => {
+        assert.strictEqual(p.sizeOf(check, 'signed'), 4)
+      })
+      it('nests hashes of hashes', () => {
+        const value = [{ key: 'a', value: 1 }, { key: 'b', value: 2 }]
+        const list = Buffer.concat([Buffer.from([2]), ...value.map(entry => lu32(digest('crc32c', p.createPacketBuffer('entry', entry))))])
+        assert.deepStrictEqual(p.createPacketBuffer('list', value), list)
+        assert.deepStrictEqual(p.createPacketBuffer('nested', value), lu32(digest('crc32c', list)))
+      })
+    })
+  }
+})