format: add virtual regions for computed values

CHANGELOG.md

@@ -53,6 +53,21 @@ The sections do not signal obligations; the prefixes do.
 
 ## Unreleased
 
+### Added
+
+- A region with `"location": "virtual"` holds the bytes of a computed value,
+  e.g., a variable the optimizer removed. It uses the new `fragment`
+  addressing scheme: a width-bearing `expression` supplies the bytes, and
+  optional `offset` and `length` select a range of them ([#411]).
+  - Schemas: **ethdebug/format/pointer/region/virtual**,
+    **ethdebug/format/pointer/scheme/fragment**,
+    **ethdebug/format/pointer/region**
+  - Producers: optional: a pointer may describe a value that no data location
+    stores as a virtual region.
+  - Consumers: required: a region may have `"location": "virtual"`. Its bytes
+    are the fragment of its `expression`'s value, which a consumer must
+    evaluate. A fragment outside the value's bytes is invalid.
+
 ### Changed
 
 - The members of a `define` no longer depend on their order, since JSON
@@ -763,3 +778,4 @@ First published version of the specification.
 [#402]: https://github.com/ethdebug/format/pull/402
 [#404]: https://github.com/ethdebug/format/pull/404
 [#408]: https://github.com/ethdebug/format/pull/408
+[#411]: https://github.com/ethdebug/format/pull/411

packages/format/CHANGELOG.md

@@ -6,6 +6,13 @@ tracked in the root [`CHANGELOG.md`](../../CHANGELOG.md).
 
 ## Unreleased
 
+### Added
+
+- The **ethdebug/format/pointer/region/virtual** and
+  **ethdebug/format/pointer/scheme/fragment** schemas, with the types
+  `Pointer.Region.Virtual` and `Pointer.Scheme.Fragment` and their guards
+  `Pointer.Region.isVirtual` and `Pointer.Scheme.isFragment` ([#411]).
+
 ### Changed
 
 - `schemas` parses each schema from YAML on first access instead of at
@@ -156,6 +163,7 @@ First publication.
 [#323]: https://github.com/ethdebug/format/pull/323
 [#325]: https://github.com/ethdebug/format/pull/325
 [#373]: https://github.com/ethdebug/format/pull/373
+[#411]: https://github.com/ethdebug/format/pull/411
 [`0ef2f37`]: https://github.com/ethdebug/format/commit/0ef2f37
 [`10ab103`]: https://github.com/ethdebug/format/commit/10ab103
 [`21e532e`]: https://github.com/ethdebug/format/commit/21e532e

packages/format/src/types/pointer/pointer.test.ts

@@ -160,6 +160,10 @@ testSchemaGuards("ethdebug/format/pointer", [
     schema: "schema:ethdebug/format/pointer/region/code",
     guard: Pointer.Region.isCode,
   },
+  {
+    schema: "schema:ethdebug/format/pointer/region/virtual",
+    guard: Pointer.Region.isVirtual,
+  },
   {
     schema: "schema:ethdebug/format/pointer/collection/group",
     guard: Pointer.Collection.isGroup,

packages/format/src/types/pointer/pointer.ts

@@ -15,7 +15,8 @@ export namespace Pointer {
     | Region.Calldata
     | Region.Returndata
     | Region.Transient
-    | Region.Code;
+    | Region.Code
+    | Region.Virtual;
 
   export const isRegion = (value: unknown): value is Region =>
     [
@@ -26,6 +27,7 @@ export namespace Pointer {
       Region.isReturndata,
       Region.isTransient,
       Region.isCode,
+      Region.isVirtual,
     ].some((guard) => guard(value));
 
   export namespace Region {
@@ -71,6 +73,10 @@ export namespace Pointer {
     export type Code = Base & Scheme.Slice & { location: "code" };
     export const isCode = (value: unknown): value is Code =>
       isBase(value) && Scheme.isSlice(value) && value.location === "code";
+
+    export type Virtual = Base & Scheme.Fragment & { location: "virtual" };
+    export const isVirtual = (value: unknown): value is Virtual =>
+      isBase(value) && Scheme.isFragment(value) && value.location === "virtual";
   }
 
   export namespace Scheme {
@@ -99,6 +105,20 @@ export namespace Pointer {
       isExpression(value.offset) &&
       "length" in value &&
       isExpression(value.length);
+
+    export interface Fragment {
+      expression: Expression;
+      offset?: Expression;
+      length?: Expression;
+    }
+
+    export const isFragment = (value: unknown): value is Fragment =>
+      !!value &&
+      typeof value === "object" &&
+      "expression" in value &&
+      isExpression(value.expression) &&
+      (!("offset" in value) || isExpression(value.offset)) &&
+      (!("length" in value) || isExpression(value.length));
   }
 
   export type Collection =

packages/pointers/CHANGELOG.md

@@ -6,6 +6,13 @@ itself are tracked in the root [`CHANGELOG.md`](../../CHANGELOG.md).
 
 ## Unreleased
 
+### Added
+
+- `dereference` and `read` support virtual regions. A concrete virtual
+  region keeps the bytes of its `expression` and has `offset` and `length`
+  filled in with their defaults. An expression with no width, or a fragment
+  outside the value, throws ([#411]).
+
 ### Changed
 
 - `dereference` evaluates the members of a scope's `define` in dependency
@@ -155,3 +162,4 @@ First publication.
 [#323]: https://github.com/ethdebug/format/pull/323
 [#325]: https://github.com/ethdebug/format/pull/325
 [#408]: https://github.com/ethdebug/format/pull/408
+[#411]: https://github.com/ethdebug/format/pull/411

packages/pointers/src/cursor.ts

@@ -30,10 +30,11 @@ export namespace Cursor {
 
   /**
    * A Pointer region where all dynamic expressions have been replaced with
-   * concrete bytes values.
+   * concrete bytes values. A virtual region's `expression` becomes the
+   * bytes of its value.
    */
   export type Region<R extends Pointer.Region = Pointer.Region> = {
-    [K in keyof R]: K extends "slot" | "offset" | "length"
+    [K in keyof R]: K extends "slot" | "offset" | "length" | "expression"
       ? R[K] extends Pointer.Expression
         ? Data
         : R[K] extends Pointer.Expression | undefined

packages/pointers/src/dereference/index.test.ts

@@ -893,6 +893,110 @@ describe("dereference", () => {
     expect(regions.lookup["my-slot-value"]).toBeDefined();
   });
 
+  describe("virtual regions", () => {
+    // `a` is on the stack, which reads as 0x11223344
+    const a: Pointer.Region = { name: "a", location: "stack", slot: 2 };
+
+    const view = async (...group: Pointer[]) =>
+      await (await dereference({ group: [a, ...group] })).view(state);
+
+    const x: Pointer.Region = {
+      name: "x",
+      location: "virtual",
+      expression: { "~wordsized": { "~sum": [{ "~read": "a" }, 1] } },
+    };
+
+    it("defaults to the whole value of the expression", async () => {
+      const { regions, read } = await view(x);
+      const value = Data.fromNumber(0x11223345).resizeTo(32);
+
+      expect(regions.lookup["x"]).toEqual({
+        name: "x",
+        location: "virtual",
+        expression: value,
+        offset: Data.fromUint(0n),
+        length: Data.fromNumber(32),
+      });
+      expect(await read(regions.lookup["x"])).toEqual(value);
+    });
+
+    it("reads the fragment at an explicit offset and length", async () => {
+      const { regions, read } = await view(
+        { ...x, offset: 29, length: 2 },
+        {
+          name: "rest",
+          location: "virtual",
+          expression: "0xdeadbeef",
+          offset: 1,
+        },
+      );
+
+      expect(await read(regions.lookup["x"])).toEqual(Data.fromHex("0x2233"));
+      expect(await read(regions.lookup["rest"])).toEqual(
+        Data.fromHex("0xadbeef"),
+      );
+      expect(regions.lookup["rest"].length).toEqual(Data.fromNumber(3));
+    });
+
+    it("allows lookups of offset and length from a later region", async () => {
+      const { regions } = await view(
+        {
+          name: "tag",
+          location: "virtual",
+          expression: "0xdeadbeef",
+          offset: 1,
+        },
+        {
+          location: "memory",
+          offset: { ".offset": "tag" },
+          length: { ".length": "tag" },
+        },
+      );
+
+      expect(regions[2]).toEqual({
+        location: "memory",
+        offset: Data.fromNumber(1),
+        length: Data.fromNumber(3),
+      });
+    });
+
+    it("allows reading a virtual region in another expression", async () => {
+      const { regions, read } = await view(x, {
+        name: "y",
+        location: "virtual",
+        expression: { "~concat": ["0x00", { "~read": "x" }] },
+        length: 2,
+      });
+
+      expect(await read(regions.lookup["y"])).toEqual(Data.fromHex("0x0000"));
+      expect(regions.lookup["y"]).toHaveProperty("expression.length", 33);
+    });
+
+    it("rejects a lookup of slot", async () => {
+      await expect(
+        view(x, { location: "storage", slot: { ".slot": "x" } }),
+      ).rejects.toThrow("Region named x does not have slot needed by lookup");
+    });
+
+    it("rejects an expression with no width", async () => {
+      await expect(
+        view({ location: "virtual", expression: { "~sum": [1, 2] } }),
+      ).rejects.toThrow("evaluates to the integer 3, which has no byte width");
+    });
+
+    it("rejects a fragment outside the value", async () => {
+      for (const fragment of [
+        { offset: 5 },
+        { offset: 2, length: 3 },
+        { length: 5 },
+      ]) {
+        await expect(
+          view({ location: "virtual", expression: "0xdeadbeef", ...fragment }),
+        ).rejects.toThrow("extends outside the 4 bytes");
+      }
+    });
+  });
+
   describe("names that start with $", () => {
     it("resolves a region named $this, not the current region", async () => {
       const pointer: Pointer = {

packages/pointers/src/dereference/region.ts

@@ -1,6 +1,6 @@
 import { Pointer } from "@ethdebug/format";
 import type { Cursor } from "#cursor";
-import type { Data } from "#data";
+import { Data } from "#data";
 import { evaluate, Value, type EvaluateOptions } from "#evaluate";
 
 /**
@@ -86,12 +86,61 @@ export async function evaluateRegion<R extends Pointer.Region>(
     }
   }
 
+  if (Pointer.Region.isVirtual(region)) {
+    const fragment = await evaluateFragment(
+      region,
+      evaluatedProperties,
+      options,
+    );
+    return fragment as Cursor.Region<R>;
+  }
+
   return {
     ...region,
     ...evaluatedProperties,
   } as Cursor.Region<R>;
 }
 
+/**
+ * Evaluate a virtual region's `expression` to the bytes it addresses into,
+ * and fill in the default `offset` and `length` so that later regions may
+ * look them up
+ */
+async function evaluateFragment(
+  region: Pointer.Region.Virtual,
+  { offset, length }: { offset?: Data; length?: Data },
+  options: EvaluateOptions,
+): Promise<Cursor.Region<Pointer.Region.Virtual>> {
+  const value = await evaluate(region.expression, options);
+  if (Value.isInteger(value)) {
+    throw new Error(
+      [
+        `Expression of virtual region (${JSON.stringify(region.expression)}) `,
+        `evaluates to the integer ${value.value}, which has no byte `,
+        `width; give it a width with ~wordsized or ~sizedN`,
+      ].join(""),
+    );
+  }
+
+  const width = BigInt(value.data.length);
+  const start = offset?.asUint() ?? 0n;
+  const end = length ? start + length.asUint() : width;
+  if (start > width || end > width) {
+    const fields = length ? `${start}, length ${length.asUint()}` : start;
+    throw new Error(
+      `Fragment of virtual region (offset ${fields}) extends outside ` +
+        `the ${width} bytes of its expression's value`,
+    );
+  }
+
+  return {
+    ...region,
+    expression: value.data,
+    offset: Data.fromUint(start),
+    length: Data.fromUint(end - start),
+  };
+}
+
 /**
  * Detect a stack region and modify its `slot` expression to include the
  * appropriate sum or difference based on the machine stack length change

packages/pointers/src/read.test.ts

@@ -455,4 +455,23 @@ describe("read (segment carry)", () => {
     });
     expect(result).toEqual(bytes([0xa2, 2], [0xa3, 2]));
   });
+
+  it("reads the fragment of a virtual region", async () => {
+    const expression = Data.fromHex("0xdeadbeef");
+
+    expect(
+      await read(
+        {
+          location: "virtual",
+          expression,
+          offset: Data.fromNumber(1),
+          length: Data.fromNumber(2),
+        },
+        options,
+      ),
+    ).toEqual(Data.fromHex("0xadbe"));
+    expect(await read({ location: "virtual", expression }, options)).toEqual(
+      expression,
+    );
+  });
 });

packages/pointers/src/read.ts

@@ -84,6 +84,15 @@ export async function read(
 
       return await state.code.read({ slice: { offset, length } });
     }
+    case "virtual": {
+      const { offset = 0n, length } = withPropertiesAsUints(
+        ["offset", "length"],
+        region,
+      );
+      const end = length === undefined ? undefined : Number(offset + length);
+
+      return Data.fromBytes(region.expression.subarray(Number(offset), end));
+    }
   }
 }