diff --git a/packages/conformance/README.md b/packages/conformance/README.md index 8ebb94b997..aa0d652eb1 100644 --- a/packages/conformance/README.md +++ b/packages/conformance/README.md @@ -49,3 +49,21 @@ To run the full Solidity -> SolDB resources path, set `ETHDEBUG_CONFORMANCE_SOLC and `ETHDEBUG_CONFORMANCE_SOLDB`. If `anvil` and `cast` are on `PATH`, the SolDB breakpoint test runs as well; the executable paths can be overridden with `ETHDEBUG_CONFORMANCE_ANVIL` and `ETHDEBUG_CONFORMANCE_CAST`. + +## Expected failures + +When the format and a tool disagree, the tests do not wait for the tool. This +applies whichever side moved: a format change that a tool does not follow yet, +or a tool that does not follow the format. Each such divergence, per tool, goes +through these steps: + +1. Open one tracking issue on ethdebug/format. It says what the tool does and + where in the tool's code. +2. Add a shim and a tripwire to the test. The shim makes the test pass now. The + tripwire fails once the tool conforms, and its message names the shim and + the issue. When the format change causes the divergence, the shim lands in + the same PR as the format change, so main stays green. +3. When the tripwire fires, one PR removes the shim and the tripwire and closes + the issue. + +Prefer a shim in the test to a pinned tool version. diff --git a/packages/conformance/test/conformance.test.ts b/packages/conformance/test/conformance.test.ts index 07484902c6..b4805902c8 100644 --- a/packages/conformance/test/conformance.test.ts +++ b/packages/conformance/test/conformance.test.ts @@ -198,6 +198,24 @@ function withSolcTildeVocabulary(artifact: EthdebugArtifact): EthdebugArtifact { }; } +// Expected failure (#406): SolDB drops the `ethdebug` stamp from the +// resources object it reads. Until SolDB keeps it, the SolDB tests compare +// its resources with a copy of solc's resources without the stamp, so the +// rest of resources is still checked. The check below fails once SolDB +// keeps the stamp; then remove this function and compare the resources as +// they are. +function withoutResourcesStamp( + resources: EthdebugArtifact["resources"], + soldbResources: { ethdebug?: unknown }, +) { + expect( + soldbResources.ethdebug, + "SolDB now keeps the resources stamp; remove withoutResourcesStamp (#406)", + ).toBeUndefined(); + const { ethdebug: _, ...rest } = resources ?? {}; + return rest; +} + function validArtifact( overrides: Partial = {}, ): EthdebugArtifact { @@ -276,7 +294,9 @@ describe("@ethdebug/conformance", () => { expect(result.exitCode).toBe(0); const json = result.json as any; expect(json.contracts[0].name).toBe("Counter"); - expect(json.contracts[0].resources).toEqual(artifact.resources); + expect(json.contracts[0].resources).toEqual( + withoutResourcesStamp(artifact.resources, json.contracts[0].resources), + ); expect( json.contracts[0].resources.compilation.sources[0].contents, ).toContain("contract Counter");