diff --git a/packages/bugc/src/irgen/debug/pointers.test.ts b/packages/bugc/src/irgen/debug/pointers.test.ts new file mode 100644 index 0000000000..e600e00fe9 --- /dev/null +++ b/packages/bugc/src/irgen/debug/pointers.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect } from "vitest"; + +import { mappingAccess, arrayElementAccess } from "./pointers.js"; + +/** + * `$keccak256` operands must be width-bearing bytes: the EVM hashes + * key‖slot as two 32-byte words, so a bare integer slot operand would + * hash the wrong number of bytes. + */ +describe("mappingAccess", () => { + it("wordsizes a literal slot operand", () => { + expect(mappingAccess(0, 0x1234)).toEqual({ + $keccak256: [{ $wordsized: 0x1234 }, { $wordsized: 0 }], + }); + }); + + it("wordsizes an arithmetic slot operand", () => { + expect(mappingAccess({ $sum: [3, 1] }, 0x1234)).toEqual({ + $keccak256: [{ $wordsized: 0x1234 }, { $wordsized: { $sum: [3, 1] } }], + }); + }); + + it("does not re-wrap a nested keccak256 slot operand", () => { + const inner = mappingAccess(1, 0xaaaa); + expect(mappingAccess(inner, 0xbbbb)).toEqual({ + $keccak256: [{ $wordsized: 0xbbbb }, inner], + }); + }); +}); + +describe("arrayElementAccess", () => { + it("wordsizes the base slot of a dynamic array", () => { + expect(arrayElementAccess(2, "i", true)).toEqual({ + $sum: [{ $keccak256: [{ $wordsized: 2 }] }, "i"], + }); + }); + + it("does not hash the base slot of a fixed array", () => { + expect(arrayElementAccess(2, "i", false)).toEqual({ + $sum: [2, "i"], + }); + }); +}); diff --git a/packages/bugc/src/irgen/debug/pointers.ts b/packages/bugc/src/irgen/debug/pointers.ts index 3b015d1aaa..b6644f7387 100644 --- a/packages/bugc/src/irgen/debug/pointers.ts +++ b/packages/bugc/src/irgen/debug/pointers.ts @@ -121,21 +121,19 @@ export function translateComputeSlotChain( const inst = step.instruction; if (inst.slotKind === "mapping") { - // Mapping access: keccak256(wordsized(key), slot) + // Mapping access: keccak256(wordsized(key), wordsized(slot)) // Try to convert key to expression const keyExpr = valueToExpression(step.key); if (keyExpr !== null) { - expr = { - $keccak256: [{ $wordsized: keyExpr }, expr], - }; + expr = mappingAccess(expr, keyExpr); } // If we can't convert the key, skip this step (use current expr) } else if (inst.slotKind === "array") { - // Array base: keccak256(slot) + // Array base: keccak256(wordsized(slot)) // Note: actual element access is done with binary.add afterward // which we don't see in the compute_slot chain expr = { - $keccak256: [expr], + $keccak256: [wordsized(expr)], }; } else if (inst.slotKind === "field") { // Struct field: slot + fieldSlotOffset @@ -182,17 +180,34 @@ function valueToExpression( return null; } +/** + * Give an expression a 32-byte width for use as a `$keccak256` operand. + * + * `$keccak256` operands must be width-bearing bytes; a bare integer + * (literal, `$sum`, ...) is invalid there. A `$keccak256` result is + * already 32 bytes wide, so it is passed through unwrapped. + */ +function wordsized( + expression: Format.Pointer.Expression, +): Format.Pointer.Expression { + if (typeof expression === "object" && "$keccak256" in expression) { + return expression; + } + return { $wordsized: expression }; +} + /** * Helper to create pointer expression for mapping access * - * Generates: keccak256(concat(key, slot)) + * Generates: keccak256(wordsized(key) ++ wordsized(slot)), matching the + * EVM's hash over key‖slot as two 32-byte words */ export function mappingAccess( slot: number | Format.Pointer.Expression, key: Format.Pointer.Expression, ): Format.Pointer.Expression { return { - $keccak256: [{ $wordsized: key }, slot], + $keccak256: [{ $wordsized: key }, wordsized(slot)], }; } @@ -208,9 +223,9 @@ export function arrayElementAccess( isDynamic: boolean, ): Format.Pointer.Expression { if (isDynamic) { - // Dynamic array: keccak256(slot) + index + // Dynamic array: keccak256(wordsized(slot)) + index return { - $sum: [{ $keccak256: [baseSlot] }, index], + $sum: [{ $keccak256: [wordsized(baseSlot)] }, index], }; } else { // Fixed array: slot + index diff --git a/packages/bugc/src/irgen/debug/storage-analysis.test.ts b/packages/bugc/src/irgen/debug/storage-analysis.test.ts index 37c86f8ab6..15de5ee705 100644 --- a/packages/bugc/src/irgen/debug/storage-analysis.test.ts +++ b/packages/bugc/src/irgen/debug/storage-analysis.test.ts @@ -426,9 +426,9 @@ describe("storage-analysis", () => { const pointer = translateComputeSlotChain(chain!); - // Should be: keccak256(wordsized(0x1234), 0) + // Should be: keccak256(wordsized(0x1234), wordsized(0)) expect(pointer).toEqual({ - $keccak256: [{ $wordsized: 0x1234 }, 0], + $keccak256: [{ $wordsized: 0x1234 }, { $wordsized: 0 }], }); }); @@ -446,12 +446,17 @@ describe("storage-analysis", () => { const pointer = translateComputeSlotChain(chain!); - // Should be: keccak256(wordsized(0xbbbb), keccak256(wordsized(0xaaaa), 1)) + // Should be: + // keccak256( + // wordsized(0xbbbb), + // keccak256(wordsized(0xaaaa), wordsized(1)), + // ) + // The inner keccak256 is already 32-byte bytes, so it is not wrapped. expect(pointer).toEqual({ $keccak256: [ { $wordsized: 0xbbbb }, { - $keccak256: [{ $wordsized: 0xaaaa }, 1], + $keccak256: [{ $wordsized: 0xaaaa }, { $wordsized: 1 }], }, ], }); @@ -470,9 +475,9 @@ describe("storage-analysis", () => { const pointer = translateComputeSlotChain(chain!); - // Should be: keccak256(2) + // Should be: keccak256(wordsized(2)) expect(pointer).toEqual({ - $keccak256: [2], + $keccak256: [{ $wordsized: 2 }], }); }); @@ -509,11 +514,11 @@ describe("storage-analysis", () => { const pointer = translateComputeSlotChain(chain!); - // Should be: sum(keccak256(wordsized(0xaaaa), 4), 2) + // Should be: sum(keccak256(wordsized(0xaaaa), wordsized(4)), 2) expect(pointer).toEqual({ $sum: [ { - $keccak256: [{ $wordsized: 0xaaaa }, 4], + $keccak256: [{ $wordsized: 0xaaaa }, { $wordsized: 4 }], }, 2, ], diff --git a/packages/pointers/src/dereference/generate.ts b/packages/pointers/src/dereference/generate.ts index f6e1b522c9..cbd45c6349 100644 --- a/packages/pointers/src/dereference/generate.ts +++ b/packages/pointers/src/dereference/generate.ts @@ -1,7 +1,7 @@ import type { Pointer } from "@ethdebug/format"; import type { Machine } from "#machine"; import type { Cursor } from "#cursor"; -import type { Data } from "#data"; +import type { Value } from "#evaluate"; import { Memo } from "./memo.js"; import { processPointer, type ProcessOptions } from "./process.js"; @@ -129,7 +129,7 @@ async function initializeProcessOptions({ const stackLengthChange = currentStackLength - initialStackLength; const regions: Record = {}; - const variables: Record = {}; + const variables: Record = {}; return { templates, diff --git a/packages/pointers/src/dereference/index.test.ts b/packages/pointers/src/dereference/index.test.ts index b151462844..72f96dbf21 100644 --- a/packages/pointers/src/dereference/index.test.ts +++ b/packages/pointers/src/dereference/index.test.ts @@ -135,9 +135,7 @@ describe("dereference", () => { expect(region).toEqual({ name: "item", location: "memory", - offset: Data.fromUint( - Data.fromNumber(index).asUint() * 32n, - ).padUntilAtLeast(1), + offset: Data.fromUint(BigInt(index) * 32n), length: Data.fromNumber(32), }); } diff --git a/packages/pointers/src/dereference/memo.ts b/packages/pointers/src/dereference/memo.ts index 09ad3631e3..55671172ff 100644 --- a/packages/pointers/src/dereference/memo.ts +++ b/packages/pointers/src/dereference/memo.ts @@ -1,6 +1,6 @@ import type { Pointer } from "@ethdebug/format"; import type { Cursor } from "#cursor"; -import type { Data } from "#data"; +import type { Value } from "#evaluate"; /** * A single state transition for processing on a stack @@ -62,14 +62,14 @@ export namespace Memo { */ export interface SaveVariables { kind: "save-variables"; - variables: Record; + variables: Record; } /** * Initialize a SaveVariables memo */ export const saveVariables = ( - variables: Record, + variables: Record, ): SaveVariables => ({ kind: "save-variables", variables, diff --git a/packages/pointers/src/dereference/process.ts b/packages/pointers/src/dereference/process.ts index 2a729c962f..cd1ebc33c1 100644 --- a/packages/pointers/src/dereference/process.ts +++ b/packages/pointers/src/dereference/process.ts @@ -1,8 +1,7 @@ import { Pointer } from "@ethdebug/format"; import type { Machine } from "#machine"; import type { Cursor } from "#cursor"; -import { Data } from "#data"; -import { evaluate } from "#evaluate"; +import { evaluate, Value } from "#evaluate"; import { Memo } from "./memo.js"; import { adjustStackLength, evaluateRegion } from "./region.js"; @@ -15,7 +14,7 @@ export interface ProcessOptions { state: Machine.State; stackLengthChange: bigint; regions: Record; - variables: Record; + variables: Record; } /** @@ -101,13 +100,13 @@ async function* processList( const { list } = collection; const { count: countExpression, each, is } = list; - const count = (await evaluate(countExpression, options)).asUint(); + const count = Value.toInteger(await evaluate(countExpression, options)); const memos: Memo[] = []; for (let index = 0n; index < count; index++) { memos.push( Memo.saveVariables({ - [each]: Data.fromUint(index), + [each]: Value.integer(index), }), ); @@ -123,7 +122,7 @@ async function* processConditional( ): Process { const { if: ifExpression, then: then_, else: else_ } = collection; - const if_ = (await evaluate(ifExpression, options)).asUint(); + const if_ = Value.toInteger(await evaluate(ifExpression, options)); if (if_) { return [Memo.dereferencePointer(then_)]; @@ -142,15 +141,15 @@ async function* processScope( const allVariables = { ...options.variables, }; - const newVariables: { [identifier: string]: Data } = {}; + const newVariables: { [identifier: string]: Value } = {}; for (const [identifier, expression] of Object.entries(variableExpressions)) { - const data = await evaluate(expression, { + const value = await evaluate(expression, { ...options, variables: allVariables, }); - allVariables[identifier] = data; - newVariables[identifier] = data; + allVariables[identifier] = value; + newVariables[identifier] = value; } return [Memo.saveVariables(newVariables), Memo.dereferencePointer(in_)]; diff --git a/packages/pointers/src/dereference/region.ts b/packages/pointers/src/dereference/region.ts index 2a623c9917..a492c9d479 100644 --- a/packages/pointers/src/dereference/region.ts +++ b/packages/pointers/src/dereference/region.ts @@ -1,7 +1,7 @@ import { Pointer } from "@ethdebug/format"; import type { Cursor } from "#cursor"; import type { Data } from "#data"; -import { evaluate, type EvaluateOptions } from "#evaluate"; +import { evaluate, Value, type EvaluateOptions } from "#evaluate"; /** * Evaluate all Pointer.Expression-value properties on a given region @@ -55,7 +55,7 @@ export async function evaluateRegion( const [property, expression] = expressionQueue.shift()!; try { - const data = await evaluate(expression, { + const value = await evaluate(expression, { ...options, regions: { ...options.regions, @@ -63,7 +63,7 @@ export async function evaluateRegion( }, }); - evaluatedProperties[property as keyof R] = data; + evaluatedProperties[property as keyof R] = Value.toData(value); } catch (error) { if ( error instanceof Error && diff --git a/packages/pointers/src/evaluate.examples.test.ts b/packages/pointers/src/evaluate.examples.test.ts new file mode 100644 index 0000000000..98e3486532 --- /dev/null +++ b/packages/pointers/src/evaluate.examples.test.ts @@ -0,0 +1,128 @@ +import { expect, describe, it } from "vitest"; + +import { Pointer, schemaIds, schemas } from "@ethdebug/format"; + +import type { Machine } from "#machine"; +import { Data } from "#data"; +import type { Cursor } from "#cursor"; +import { evaluate, Value, type EvaluateOptions } from "./evaluate.js"; + +/** + * Every `$keccak256` / `$concat` expression appearing in the examples of + * the pointer schemas, along with the `define`d variable names seen along + * the way (so that a variable can be stubbed with the sort of its defining + * expression). + * + * Only the operand sorts are of interest here, so regions and variables + * are stubbed rather than dereferenced. Undefined variables (e.g. those a + * template `expect`s) are stubbed as integers, the stricter sort. + */ +interface Occurrence { + schemaId: string; + expression: Pointer.Expression.Keccak256 | Pointer.Expression.Concat; +} + +const isWidthSensitive = (value: unknown): value is Occurrence["expression"] => + Pointer.Expression.isKeccak256(value) || Pointer.Expression.isConcat(value); + +const occurrences: Occurrence[] = []; +const definedVariables: { [identifier: string]: Value } = {}; + +// stub value for a variable, by the syntactic sort of its defining expression +function stubValue(expression: Pointer.Expression): Value { + if ( + Pointer.Expression.isResize(expression) || + Pointer.Expression.isRead(expression) || + isWidthSensitive(expression) + ) { + return Value.bytes(Data.fromNumber(0).resizeTo(32)); + } + + if (typeof expression === "string" && expression.startsWith("0x")) { + return (expression.length - 2) % 2 === 0 + ? Value.bytes(Data.fromHex(expression)) + : Value.integer(BigInt(expression)); + } + + return Value.integer(1n); +} + +function collect(schemaId: string, node: unknown, withinExamples: boolean) { + if (Array.isArray(node)) { + for (const item of node) { + collect(schemaId, item, withinExamples); + } + return; + } + + if (typeof node !== "object" || node === null) { + return; + } + + const record = node as { [key: string]: unknown }; + + if (withinExamples) { + if (isWidthSensitive(record)) { + occurrences.push({ schemaId, expression: record }); + } + + if (Pointer.Collection.isScope(record)) { + for (const [identifier, expression] of Object.entries(record.define)) { + definedVariables[identifier] = stubValue(expression); + } + } + } + + for (const [key, value] of Object.entries(record)) { + collect(schemaId, value, withinExamples || key === "examples"); + } +} + +for (const schemaId of schemaIds) { + if (schemaId.startsWith("schema:ethdebug/format/pointer")) { + collect(schemaId, schemas[schemaId], false); + } +} + +const zeros = async ({ + slice: { length }, +}: { + slice: Machine.State.Slice; +}): Promise => Data.fromBytes(new Uint8Array(Number(length))); + +const state = { + memory: { read: zeros }, +} as unknown as Machine.State; + +// any region name resolves to a stub memory region +const regions = new Proxy({} as { [identifier: string]: Cursor.Region }, { + get: (_target, name): Cursor.Region => ({ + name: String(name), + location: "memory", + offset: Data.fromNumber(0), + length: Data.fromNumber(32), + }), +}); + +// any variable name resolves to its defined stub, or else an integer +const variables = new Proxy(definedVariables, { + get: (target, name): Value => target[String(name)] ?? Value.integer(1n), +}); + +const options: EvaluateOptions = { state, regions, variables }; + +describe("pointer schema examples", () => { + it("include width-sensitive expressions", () => { + expect(occurrences.length).toBeGreaterThan(0); + }); + + for (const { schemaId, expression } of occurrences) { + const title = `${schemaId}: ${JSON.stringify(expression)}`; + + it(`give every operand a width in ${title}`, async () => { + const result = await evaluate(expression, options); + + expect(Value.isBytes(result)).toBe(true); + }); + } +}); diff --git a/packages/pointers/src/evaluate.test.ts b/packages/pointers/src/evaluate.test.ts index 5a25394c62..9216dae711 100644 --- a/packages/pointers/src/evaluate.test.ts +++ b/packages/pointers/src/evaluate.test.ts @@ -1,14 +1,13 @@ -import { expect, describe, it, beforeEach } from "vitest"; +import { vitest, expect, describe, it, beforeEach } from "vitest"; import { keccak256 } from "ethereum-cryptography/keccak"; -import { toHex } from "ethereum-cryptography/utils"; import { Pointer } from "@ethdebug/format"; import { Machine } from "#machine"; import { Data } from "#data"; import { Cursor } from "#cursor"; -import { evaluate, type EvaluateOptions } from "./evaluate.js"; +import { evaluate, Value, type EvaluateOptions } from "./evaluate.js"; // Create a stub for the Machine.State interface const state: Machine.State = { @@ -18,7 +17,11 @@ const state: Machine.State = { stack: { length: 50n, } as any, - memory: {} as any, + memory: { + read: vitest.fn(async ({ slice: { length } }) => + Data.fromBytes(new Uint8Array(Number(length)).fill(0xee)), + ), + } as any, storage: {} as any, calldata: {} as any, returndata: {} as any, @@ -26,16 +29,18 @@ const state: Machine.State = { code: {} as any, }; +const word = (byte: number): Data => + Data.fromBytes(new Uint8Array(32).fill(byte)); + describe("evaluate", () => { let regions: { [identifier: string]: Cursor.Region }; - let variables: { [identifier: string]: Data }; - let _cursor: Cursor; + let variables: { [identifier: string]: Value }; let options: EvaluateOptions; beforeEach(() => { variables = { - foo: Data.fromNumber(42), - bar: Data.fromHex("0x1f"), + foo: Value.integer(42n), + bar: Value.bytes(Data.fromHex("0x1f")), }; regions = { @@ -61,249 +66,355 @@ describe("evaluate", () => { }; }); - it("evaluates literal expressions", async () => { - expect(await evaluate(42, options)).toEqual(Data.fromNumber(42)); + describe("literals", () => { + it("evaluates a JSON number to an integer", async () => { + expect(await evaluate(42, options)).toEqual(Value.integer(42n)); + expect(await evaluate(0, options)).toEqual(Value.integer(0n)); + }); + + it("evaluates even-digit hex to bytes of that width", async () => { + expect(await evaluate("0x1f", options)).toEqual( + Value.bytes(Data.fromHex("0x1f")), + ); + + const zeros = await evaluate("0x0000", options); + expect(zeros).toEqual(Value.bytes(Data.fromHex("0x0000"))); + expect(Value.isBytes(zeros) && zeros.data.length).toBe(2); + }); - expect(await evaluate("0x1f", options)).toEqual(Data.fromHex("0x1f")); + it("evaluates an odd-digit hex string to an integer", async () => { + expect(await evaluate("0x1", options)).toEqual(Value.integer(1n)); + expect(await evaluate("0xabc", options)).toEqual(Value.integer(0xabcn)); + }); }); - it("evaluates constant expressions", async () => { - expect(await evaluate("$wordsize", options)).toEqual(Data.fromHex("0x20")); + it("evaluates $wordsize to the integer 32", async () => { + expect(await evaluate("$wordsize", options)).toEqual(Value.integer(32n)); }); - it("evaluates variable expressions", async () => { - expect(await evaluate("foo", options)).toEqual(Data.fromNumber(42)); + it("evaluates variables to their values, preserving sort", async () => { + expect(await evaluate("foo", options)).toEqual(Value.integer(42n)); - expect(await evaluate("bar", options)).toEqual(Data.fromHex("0x1f")); + expect(await evaluate("bar", options)).toEqual( + Value.bytes(Data.fromHex("0x1f")), + ); }); - it("evaluates sum expressions", async () => { - const expression: Pointer.Expression = { - $sum: [42, "0x1f", "foo", "bar"], - }; - - expect(await evaluate(expression, options)).toEqual( - Data.fromUint(42n + 0x1fn + 42n + 0x1fn), + it("throws for unknown variables", async () => { + await expect(evaluate("baz", options)).rejects.toThrow( + "Unknown variable with identifier baz", ); }); - it("evaluates difference expressions", async () => { - const expression: Pointer.Expression = { - $difference: ["foo", "bar"], - }; + describe("arithmetic", () => { + it("evaluates sums to an integer, coercing bytes operands", async () => { + const expression: Pointer.Expression = { + $sum: [42, "0x1f", "foo", "bar"], + }; - expect(await evaluate(expression, options)).toEqual( - Data.fromUint(42n - 0x1fn), - ); + expect(await evaluate(expression, options)).toEqual( + Value.integer(42n + 0x1fn + 42n + 0x1fn), + ); + }); + + it("evaluates differences", async () => { + expect(await evaluate({ $difference: ["foo", "bar"] }, options)).toEqual( + Value.integer(42n - 0x1fn), + ); + }); + + it("clamps differences at zero", async () => { + expect(await evaluate({ $difference: ["bar", "foo"] }, options)).toEqual( + Value.integer(0n), + ); + }); + + it("evaluates products", async () => { + const expression: Pointer.Expression = { + $product: [42, "0x1f", "foo", "bar"], + }; + + expect(await evaluate(expression, options)).toEqual( + Value.integer(42n * 0x1fn * 42n * 0x1fn), + ); + }); + + it("evaluates quotients", async () => { + expect(await evaluate({ $quotient: ["foo", "bar"] }, options)).toEqual( + Value.integer(42n / 0x1fn), + ); + }); + + it("evaluates remainders", async () => { + expect(await evaluate({ $remainder: ["foo", "bar"] }, options)).toEqual( + Value.integer(42n % 0x1fn), + ); + }); + + it("reads bytes operands as big-endian integers", async () => { + expect(await evaluate({ $sum: ["0x0100", "0x00"] }, options)).toEqual( + Value.integer(256n), + ); + }); + + it("produces integers with no width, even from wide operands", async () => { + expect( + await evaluate({ $difference: ["0x0000", "0x0000"] }, options), + ).toEqual(Value.integer(0n)); + }); }); - it("evaluates product expressions", async () => { - const expression: Pointer.Expression = { - $product: [42, "0x1f", "foo", "bar"], - }; + describe("lookups", () => { + it("evaluates offset lookups to an integer", async () => { + expect(await evaluate({ ".offset": "stack" }, options)).toEqual( + Value.integer(0x60n), + ); + }); - expect(await evaluate(expression, options)).toEqual( - Data.fromUint(42n * 0x1fn * 42n * 0x1fn), - ); + it("evaluates offset lookups with $this", async () => { + const $this = { + name: "$this", + location: "memory", + offset: Data.fromNumber(0x120), + length: Data.fromNumber(0x40), + } as const; + + expect( + await evaluate( + { ".offset": "$this" }, + { + ...options, + regions: { + ...regions, + $this, + }, + }, + ), + ).toEqual(Value.integer(0x120n)); + }); + + it("evaluates length lookups", async () => { + expect(await evaluate({ ".length": "memory" }, options)).toEqual( + Value.integer(11n), + ); + }); + + it("evaluates slot lookups", async () => { + expect(await evaluate({ ".slot": "stack" }, options)).toEqual( + Value.integer(42n), + ); + }); + + it("throws for lookups of unknown regions", async () => { + await expect(evaluate({ ".slot": "nope" }, options)).rejects.toThrow( + "Region not found: nope", + ); + }); }); - it("evaluates quotient expressions", async () => { - const expression: Pointer.Expression = { - $quotient: ["foo", "bar"], - }; + it("evaluates $read to bytes of the region's length", async () => { + const result = await evaluate({ $read: "memory" }, options); - expect(await evaluate(expression, options)).toEqual( - Data.fromUint(42n / 0x1fn), + expect(result).toEqual( + Value.bytes(Data.fromBytes(new Uint8Array(11).fill(0xee))), ); }); - it("evaluates remainder expressions", async () => { - const expression: Pointer.Expression = { - $remainder: ["foo", "bar"], - }; + describe("resize", () => { + it("gives an integer a width", async () => { + expect(await evaluate({ $sized1: 0 }, options)).toEqual( + Value.bytes(Data.fromHex("0x00")), + ); - expect(await evaluate(expression, options)).toEqual( - Data.fromUint(42n % 0x1fn), - ); + expect(await evaluate({ $sized2: 42 }, options)).toEqual( + Value.bytes(Data.fromHex("0x002a")), + ); + + expect(await evaluate({ $wordsized: 0xabcd }, options)).toEqual( + Value.bytes(Data.fromNumber(0xabcd).resizeTo(32)), + ); + }); + + it("resizes bytes, padding or truncating on the left", async () => { + expect(await evaluate({ $sized1: "0xabcd" }, options)).toEqual( + Value.bytes(Data.fromHex("0xcd")), + ); + + expect(await evaluate({ $sized4: "0xabcd" }, options)).toEqual( + Value.bytes(Data.fromHex("0x0000abcd")), + ); + + expect(await evaluate({ $wordsized: "0xabcd" }, options)).toEqual( + Value.bytes(Data.fromHex("0xabcd").resizeTo(32)), + ); + }); + + it("truncates an integer too large for the requested width", async () => { + expect(await evaluate({ $sized1: 0x1234 }, options)).toEqual( + Value.bytes(Data.fromHex("0x34")), + ); + }); + + it("gives arithmetic results a width", async () => { + expect(await evaluate({ $sized2: { $sum: [1, 2] } }, options)).toEqual( + Value.bytes(Data.fromHex("0x0003")), + ); + }); }); - describe("evaluates concat expressions", () => { + describe("$concat", () => { it("concatenates hex literals", async () => { - const expression: Pointer.Expression = { - $concat: ["0x00", "0x00"], - }; - expect(await evaluate(expression, options)).toEqual( - Data.fromHex("0x0000"), + expect(await evaluate({ $concat: ["0x00", "0x00"] }, options)).toEqual( + Value.bytes(Data.fromHex("0x0000")), ); }); it("concatenates multiple values preserving byte widths", async () => { - const expression: Pointer.Expression = { - $concat: ["0xdead", "0xbeef"], - }; - expect(await evaluate(expression, options)).toEqual( - Data.fromHex("0xdeadbeef"), + expect( + await evaluate({ $concat: ["0xdead", "0xbeef"] }, options), + ).toEqual(Value.bytes(Data.fromHex("0xdeadbeef"))); + }); + + it("returns empty bytes for an empty operand list", async () => { + expect(await evaluate({ $concat: [] }, options)).toEqual( + Value.bytes(Data.zero()), ); }); - it("returns empty data for empty operand list", async () => { - const expression: Pointer.Expression = { - $concat: [], - }; - expect(await evaluate(expression, options)).toEqual(Data.zero()); + it("preserves a single operand unchanged", async () => { + expect(await evaluate({ $concat: ["0xabcdef"] }, options)).toEqual( + Value.bytes(Data.fromHex("0xabcdef")), + ); + }); + + it("preserves leading zeros in hex literals", async () => { + const result = await evaluate({ $concat: ["0x0001", "0x0002"] }, options); + + expect(result).toEqual(Value.bytes(Data.fromHex("0x00010002"))); }); - it("preserves single operand unchanged", async () => { + it("concatenates bytes-valued variables and resized integers", async () => { const expression: Pointer.Expression = { - $concat: ["0xabcdef"], + $concat: [{ $sized2: "foo" }, "bar", { $sized1: { $sum: [1, 2] } }], }; + expect(await evaluate(expression, options)).toEqual( - Data.fromHex("0xabcdef"), + Value.bytes(Data.fromHex("0x002a1f03")), ); }); - it("concatenates variables", async () => { - const expression: Pointer.Expression = { - $concat: ["foo", "bar"], - }; - // foo = 0x2a (42), bar = 0x1f - expect(await evaluate(expression, options)).toEqual( - Data.fromHex("0x2a1f"), + it("rejects a JSON number operand", async () => { + await expect( + evaluate({ $concat: ["0xdead", 0] }, options), + ).rejects.toThrow( + "Operand 1 of $concat (0) evaluates to the integer 0, which has no " + + "byte width; give it a width with $wordsized or $sizedN", ); }); - it("concatenates nested expressions", async () => { - const expression: Pointer.Expression = { - $concat: [ - { $sum: [1, 2] }, // 3 = 0x03 - "0xff", - ], - }; - expect(await evaluate(expression, options)).toEqual( - Data.fromHex("0x03ff"), + it("rejects an integer-valued variable operand", async () => { + await expect( + evaluate({ $concat: ["foo", "bar"] }, options), + ).rejects.toThrow( + 'Operand 0 of $concat ("foo") evaluates to the integer 42', ); }); - it("preserves leading zeros in hex literals", async () => { - const expression: Pointer.Expression = { - $concat: ["0x0001", "0x0002"], - }; - const result = await evaluate(expression, options); - expect(result).toEqual(Data.fromHex("0x00010002")); - expect(result.length).toBe(4); + it("rejects an arithmetic result operand", async () => { + await expect( + evaluate({ $concat: [{ $sum: [1, 2] }, "0xff"] }, options), + ).rejects.toThrow("evaluates to the integer 3"); }); - }); - // skipped because test does not perform proper padding - it.skip("evaluates keccak256 expressions", async () => { - const expression: Pointer.Expression = { - $keccak256: ["foo", "bar", 42, "0x1f"], - }; + it("rejects an odd-digit hex literal operand", async () => { + await expect(evaluate({ $concat: ["0x1"] }, options)).rejects.toThrow( + "evaluates to the integer 1", + ); + }); - const expectedHash = keccak256( - new Uint8Array( - Buffer.from( - toHex(Data.fromNumber(42)).slice(2) + - toHex(Data.fromHex("0x1f")).slice(2) + - toHex(variables.foo).slice(2) + - toHex(variables.bar).slice(2), - "hex", - ), - ), - ); + it("rejects $wordsize and lookups as operands", async () => { + await expect( + evaluate({ $concat: ["$wordsize"] }, options), + ).rejects.toThrow("evaluates to the integer 32"); - expect(await evaluate(expression, options)).toEqual( - Data.fromBytes(expectedHash), - ); + await expect( + evaluate({ $concat: [{ ".slot": "stack" }] }, options), + ).rejects.toThrow("evaluates to the integer 42"); + }); }); - it("evaluates offset lookup expressions", async () => { - const expression: Pointer.Expression = { - ".offset": "stack", - }; - - expect(await evaluate(expression, options)).toEqual(Data.fromUint(0x60n)); - }); + describe("$keccak256", () => { + it("hashes the concatenation of bytes operands", async () => { + const expression: Pointer.Expression = { + $keccak256: [{ $wordsized: "foo" }, "bar", { $sized1: 42 }, "0x1f"], + }; - it("evaluates offset lookup expressions with $this", async () => { - const expression: Pointer.Expression = { - ".offset": "$this", - }; + const preimage = Data.fromNumber(42) + .resizeTo(32) + .concat( + Data.fromHex("0x1f"), + Data.fromHex("0x2a"), + Data.fromHex("0x1f"), + ); - const $this = { - name: "$this", - location: "memory", - offset: Data.fromNumber(0x120), - length: Data.fromNumber(0x40), - } as const; - - expect( - await evaluate(expression, { - ...options, - regions: { - ...regions, - $this, - }, - }), - ).toEqual(Data.fromUint(0x120n)); - }); + expect(await evaluate(expression, options)).toEqual( + Value.bytes(Data.fromBytes(keccak256(preimage))), + ); + }); - it("evaluates length lookup expressions", async () => { - const expression: Pointer.Expression = { - ".length": "memory", - }; + it("produces 32 bytes", async () => { + const result = await evaluate({ $keccak256: [] }, options); - expect(await evaluate(expression, options)).toEqual(Data.fromUint(11n)); - }); + expect(Value.isBytes(result) && result.data.length).toBe(32); + expect(result).toEqual( + Value.bytes(Data.fromBytes(keccak256(new Uint8Array(0)))), + ); + }); - it("evaluates slot lookup expressions", async () => { - const expression: Pointer.Expression = { - ".slot": "stack", - }; + it("hashes a word-sized key and slot over 64 bytes", async () => { + const expression: Pointer.Expression = { + $keccak256: [{ $wordsized: "0x1234" }, { $wordsized: 0 }], + }; - expect(await evaluate(expression, options)).toEqual(Data.fromNumber(42)); - }); + const preimage = Data.fromHex("0x1234").resizeTo(32).concat(word(0)); + expect(preimage).toHaveLength(64); - describe("resulting bytes widths", () => { - it("uses the fewest bytes necessary for a literal", async () => { - expect(await evaluate(0, options)).toHaveLength(0); - expect(await evaluate("0x00", options)).toHaveLength(1); - expect(await evaluate("0x0000", options)).toHaveLength(2); - expect(await evaluate(0xffff, options)).toHaveLength(2); + expect(await evaluate(expression, options)).toEqual( + Value.bytes(Data.fromBytes(keccak256(preimage))), + ); }); - it("uses at least the largest bytes width amongst arithmetic operands", async () => { - expect(await evaluate({ $sum: [0, 0] }, options)).toHaveLength(0); - - expect( - await evaluate({ $difference: ["0x00", "0x00"] }, options), - ).toHaveLength(1); - - expect( - await evaluate({ $remainder: ["0x0001", "0x01"] }, options), - ).toHaveLength(2); + it("rejects a bare integer slot operand", async () => { + // the shape `{ $keccak256: [{ $wordsized: key }, slot] }` with a bare + // integer slot would hash 32 bytes instead of 64 + await expect( + evaluate({ $keccak256: [{ $wordsized: "0x1234" }, 0] }, options), + ).rejects.toThrow( + "Operand 1 of $keccak256 (0) evaluates to the integer 0, which has " + + "no byte width; give it a width with $wordsized or $sizedN", + ); }); - it("uses exactly as many bytes necessary to avoid arithmetic overflow", async () => { - expect( - await evaluate({ $product: ["0xffff", "0xff"] }, options), - ).toHaveLength(3); + it("rejects an integer-valued variable operand", async () => { + await expect(evaluate({ $keccak256: ["foo"] }, options)).rejects.toThrow( + 'Operand 0 of $keccak256 ("foo")', + ); }); }); +}); - it("evaluates resize expressions", async () => { - expect(await evaluate({ $sized1: 0 }, options)).toHaveLength(1); - - { - const data = await evaluate({ $sized1: "0xabcd" }, options); - expect(data).toHaveLength(1); - expect(data).toEqual(Data.fromNumber(0xcd)); - } +describe("Value", () => { + it("coerces bytes to a big-endian integer", () => { + expect(Value.toInteger(Value.bytes(Data.fromHex("0x0100")))).toBe(256n); + expect(Value.toInteger(Value.bytes(Data.zero()))).toBe(0n); + expect(Value.toInteger(Value.integer(7n))).toBe(7n); + }); - { - const data = await evaluate({ $wordsized: "0xabcd" }, options); - expect(data).toHaveLength(32); - expect(data).toEqual(Data.fromNumber(0xabcd).resizeTo(32)); - } + it("encodes an integer as minimal big-endian bytes for region data", () => { + expect(Value.toData(Value.integer(0n))).toEqual(Data.zero()); + expect(Value.toData(Value.integer(256n))).toEqual(Data.fromHex("0x0100")); + expect(Value.toData(Value.bytes(Data.fromHex("0x0000")))).toEqual( + Data.fromHex("0x0000"), + ); }); }); diff --git a/packages/pointers/src/evaluate.ts b/packages/pointers/src/evaluate.ts index 2dc3400f0f..310e249f58 100644 --- a/packages/pointers/src/evaluate.ts +++ b/packages/pointers/src/evaluate.ts @@ -5,20 +5,76 @@ import type { Cursor } from "#cursor"; import { read } from "#read"; import { keccak256 } from "ethereum-cryptography/keccak"; +/** + * The result of evaluating an expression: one of two sorts of value. + * + * An **integer** is an unbounded non-negative integer with no width; it + * is produced by JSON-number literals, `$wordsize`, lookups, arithmetic, + * and odd-digit hex literals. + * + * **Bytes** are a byte sequence with a definite width; they are produced + * by even-digit hex literals, `$read`, and the resize forms. + * + * Variables carry the sort of the expression that defined them. + */ +export type Value = Value.Integer | Value.Bytes; + +export namespace Value { + export interface Integer { + sort: "integer"; + value: bigint; + } + + export interface Bytes { + sort: "bytes"; + data: Data; + } + + export const integer = (value: bigint): Integer => ({ + sort: "integer", + value, + }); + + export const bytes = (data: Data): Bytes => ({ sort: "bytes", data }); + + export const isInteger = (value: Value): value is Integer => + value.sort === "integer"; + + export const isBytes = (value: Value): value is Bytes => + value.sort === "bytes"; + + /** + * Coerce to an integer, for positions where an integer is expected + * (arithmetic operands, list counts, segment slot/offset/length). Bytes + * are read as the non-negative integer they encode big-endian. + */ + export const toInteger = (value: Value): bigint => + isInteger(value) ? value.value : value.data.asUint(); + + /** + * Represent as `Data` for storage on a concrete `Cursor.Region`. Bytes + * keep their width; an integer is encoded as its minimal big-endian + * bytes (a region's slot/offset/length are integers, so this width is + * not significant). + */ + export const toData = (value: Value): Data => + isBytes(value) ? value.data : Data.fromUint(value.value); +} + export interface EvaluateOptions { state: Machine.State; regions: { [identifier: string]: Cursor.Region; }; variables: { - [identifier: string]: Data; + [identifier: string]: Value; }; } export async function evaluate( expression: Pointer.Expression, options: EvaluateOptions, -): Promise { +): Promise { if (Pointer.Expression.isLiteral(expression)) { return evaluateLiteral(expression); } @@ -32,25 +88,7 @@ export async function evaluate( } if (Pointer.Expression.isArithmetic(expression)) { - if (Pointer.Expression.Arithmetic.isSum(expression)) { - return evaluateArithmeticSum(expression, options); - } - - if (Pointer.Expression.Arithmetic.isDifference(expression)) { - return evaluateArithmeticDifference(expression, options); - } - - if (Pointer.Expression.Arithmetic.isProduct(expression)) { - return evaluateArithmeticProduct(expression, options); - } - - if (Pointer.Expression.Arithmetic.isQuotient(expression)) { - return evaluateArithmeticQuotient(expression, options); - } - - if (Pointer.Expression.Arithmetic.isRemainder(expression)) { - return evaluateArithmeticRemainder(expression, options); - } + return evaluateArithmetic(expression, options); } if (Pointer.Expression.isKeccak256(expression)) { @@ -90,186 +128,169 @@ export async function evaluate( ); } +/** + * Evaluate an expression where an integer is expected, coercing bytes + */ +async function evaluateInteger( + expression: Pointer.Expression, + options: EvaluateOptions, +): Promise { + return Value.toInteger(await evaluate(expression, options)); +} + +/** + * Evaluate the operands of a width-sensitive operation (`$concat`, + * `$keccak256`), each of which must evaluate to bytes + */ +async function evaluateBytesOperands( + operation: "$concat" | "$keccak256", + operands: Pointer.Expression[], + options: EvaluateOptions, +): Promise { + return await Promise.all( + operands.map(async (operand, index) => { + const value = await evaluate(operand, options); + + if (Value.isInteger(value)) { + throw new Error( + [ + `Operand ${index} of ${operation} (${JSON.stringify(operand)}) `, + `evaluates to the integer ${value.value}, which has no byte `, + `width; give it a width with $wordsized or $sizedN`, + ].join(""), + ); + } + + return value.data; + }), + ); +} + async function evaluateLiteral( literal: Pointer.Expression.Literal, -): Promise { +): Promise { switch (typeof literal) { - case "string": - return Data.fromHex(literal); + case "string": { + const digits = literal.slice(2); + + // an odd number of digits has no whole-byte width + if (digits.length % 2 === 1) { + return Value.integer(BigInt(literal)); + } + + return Value.bytes(Data.fromHex(literal)); + } case "number": - return Data.fromNumber(literal); + return Value.integer(BigInt(literal)); } } async function evaluateConstant( constant: Pointer.Expression.Constant, -): Promise { +): Promise { switch (constant) { case "$wordsize": - return Data.fromHex("0x20"); + return Value.integer(32n); } } async function evaluateVariable( identifier: Pointer.Expression.Variable, { variables }: EvaluateOptions, -): Promise { - const data = variables[identifier]; - if (typeof data === "undefined") { +): Promise { + const value = variables[identifier]; + if (typeof value === "undefined") { throw new Error(`Unknown variable with identifier ${identifier}`); } - return data; -} - -async function evaluateArithmeticSum( - expression: Pointer.Expression.Arithmetic.Sum, - options: EvaluateOptions, -): Promise { - const operands = await Promise.all( - expression.$sum.map( - async (expression) => await evaluate(expression, options), - ), - ); - - const maxLength = operands.reduce( - (max, { length }) => (length > max ? length : max), - 0, - ); - - const data = Data.fromUint( - operands.reduce((sum, data) => sum + data.asUint(), 0n), - ).padUntilAtLeast(maxLength); - - return data; + return value; } -async function evaluateArithmeticDifference( - expression: Pointer.Expression.Arithmetic.Difference, +async function evaluateArithmetic( + expression: Pointer.Expression.Arithmetic, options: EvaluateOptions, -): Promise { - const [a, b] = await Promise.all( - expression.$difference.map( - async (expression) => await evaluate(expression, options), - ), - ); - - const maxLength = a.length > b.length ? a.length : b.length; +): Promise { + const [[operation, operandExpressions]] = Object.entries(expression) as [ + string, + Pointer.Expression[], + ][]; - const unpadded = - a.asUint() > b.asUint() - ? Data.fromUint(a.asUint() - b.asUint()) - : Data.fromNumber(0); - - const data = unpadded.padUntilAtLeast(maxLength); - return data; -} - -async function evaluateArithmeticProduct( - expression: Pointer.Expression.Arithmetic.Product, - options: EvaluateOptions, -): Promise { const operands = await Promise.all( - expression.$product.map( - async (expression) => await evaluate(expression, options), - ), + operandExpressions.map((operand) => evaluateInteger(operand, options)), ); - const maxLength = operands.reduce( - (max, { length }) => (length > max ? length : max), - 0, - ); - - return Data.fromUint( - operands.reduce((product, data) => product * data.asUint(), 1n), - ).padUntilAtLeast(maxLength); -} - -async function evaluateArithmeticQuotient( - expression: Pointer.Expression.Arithmetic.Quotient, - options: EvaluateOptions, -): Promise { - const [a, b] = await Promise.all( - expression.$quotient.map( - async (expression) => await evaluate(expression, options), - ), - ); - - const maxLength = a.length > b.length ? a.length : b.length; - - const data = Data.fromUint(a.asUint() / b.asUint()).padUntilAtLeast( - maxLength, - ); - - return data; -} - -async function evaluateArithmeticRemainder( - expression: Pointer.Expression.Arithmetic.Remainder, - options: EvaluateOptions, -): Promise { - const [a, b] = await Promise.all( - expression.$remainder.map( - async (expression) => await evaluate(expression, options), - ), - ); - - const maxLength = a.length > b.length ? a.length : b.length; - - const data = Data.fromUint(a.asUint() % b.asUint()).padUntilAtLeast( - maxLength, - ); + switch (operation) { + case "$sum": + return Value.integer(operands.reduce((sum, value) => sum + value, 0n)); + case "$difference": { + const [a, b] = operands; + return Value.integer(a > b ? a - b : 0n); + } + case "$product": + return Value.integer( + operands.reduce((product, value) => product * value, 1n), + ); + case "$quotient": { + const [a, b] = operands; + return Value.integer(a / b); + } + case "$remainder": { + const [a, b] = operands; + return Value.integer(a % b); + } + } - return data; + throw new Error(`Unknown arithmetic operation ${operation}`); } async function evaluateKeccak256( expression: Pointer.Expression.Keccak256, options: EvaluateOptions, -): Promise { - const operands = await Promise.all( - expression.$keccak256.map( - async (expression) => await evaluate(expression, options), - ), +): Promise { + const operands = await evaluateBytesOperands( + "$keccak256", + expression.$keccak256, + options, ); const preimage = Data.zero().concat(...operands); - const hash = Data.fromBytes(keccak256(preimage)); - return hash; + return Value.bytes(Data.fromBytes(keccak256(preimage))); } async function evaluateConcat( expression: Pointer.Expression.Concat, options: EvaluateOptions, -): Promise { - const operands = await Promise.all( - expression.$concat.map( - async (expression) => await evaluate(expression, options), - ), +): Promise { + const operands = await evaluateBytesOperands( + "$concat", + expression.$concat, + options, ); - return Data.zero().concat(...operands); + return Value.bytes(Data.zero().concat(...operands)); } async function evaluateResize( expression: Pointer.Expression.Resize, options: EvaluateOptions, -): Promise { +): Promise { const [[operation, subexpression]] = Object.entries(expression); const newLength = Pointer.Expression.Resize.isToNumber(expression) ? Number(operation.match(/^\$sized([1-9]+[0-9]*)$/)![1]) : 32; - return (await evaluate(subexpression, options)).resizeTo(newLength); + const value = await evaluate(subexpression, options); + + return Value.bytes(Value.toData(value).resizeTo(newLength)); } async function evaluateLookup( operation: O, lookup: Pointer.Expression.Lookup.ForOperation, options: EvaluateOptions, -): Promise { +): Promise { const { regions } = options; const identifier = lookup[operation]; @@ -288,13 +309,13 @@ async function evaluateLookup( ); } - return data; + return Value.integer(data.asUint()); } async function evaluateRead( expression: Pointer.Expression.Read, options: EvaluateOptions, -): Promise { +): Promise { const { state: _state, regions } = options; const identifier = expression.$read; @@ -303,5 +324,5 @@ async function evaluateRead( throw new Error(`Region not found: ${identifier}`); } - return await read(region, options); + return Value.bytes(await read(region, options)); } diff --git a/packages/web/docs/core-schemas/pointers/expressions.mdx b/packages/web/docs/core-schemas/pointers/expressions.mdx index ed60fb7870..8ce19fb53b 100644 --- a/packages/web/docs/core-schemas/pointers/expressions.mdx +++ b/packages/web/docs/core-schemas/pointers/expressions.mdx @@ -32,6 +32,29 @@ A static pointer can't capture this, but an expression can — for example, `array-start + index × 32` computes the element's offset from the array's base and the index. +## Integers and bytes + +Expressions produce two kinds of value. Arithmetic (`$sum`, `$product`, and +so on) is unbounded integer math: the result is a number with no byte width. +`$read`, hexadecimal literals with an even number of digits, and the resize +operations produce **bytes** of a definite width. + +The distinction matters for hashing and concatenation. `$keccak256` and +`$concat` work on bytes, and their results depend on how wide each operand +is, so every operand must already be bytes. A bare number like `5` or an +arithmetic result is not; wrap it in `$wordsized` (or `$sized`) first. +That applies to mapping keys _and_ to slot numbers before hashing: + + + {`{ "$keccak256": [{ "$wordsized": "key" }, { "$wordsized": "slot" }] }`} + + +Resizes are the only way to turn an integer into bytes; nothing pads +implicitly. Going the other way is automatic: where a number is expected (an +arithmetic operand, a `slot`, `offset`, `length`, or list count), bytes are +read as the big-endian integer they encode, so a hash can be used directly as +a slot or added to. + ## Arithmetic expressions Basic math operations for computing addresses: @@ -60,9 +83,10 @@ Remainder after division. ### `$read` — Read from a named region -Reads the bytes from a previously defined region. For example, a group can -name an `array-length-slot` region and then use `{ $read: "array-length-slot" }` -to retrieve the array's length at runtime and use it in a later computation. +Reads the bytes from a previously defined region; the result is bytes as +wide as the region. For example, a group can name an `array-length-slot` +region and then use `{ $read: "array-length-slot" }` to retrieve the array's +length at runtime and use it in a later computation. ## Region property lookups @@ -105,23 +129,48 @@ Solidity uses keccak256 hashing to compute storage locations for dynamic data. ### Array element slots For a dynamic array at slot `n`, elements start at `keccak256(n)`, so element -`i` lives at `keccak256(n) + i`. +`i` lives at `keccak256(n) + i`. The slot is word-sized before hashing, and +the 32-byte hash is then read as an integer by `$sum`: + + + {`{ + "$sum": [ + { "$keccak256": [{ "$wordsized": 5 }] }, + "element-index" + ] +}`} + ### Mapping value slots -For a mapping at slot `n`, the value for key `k` is at `keccak256(k, n)`. +For a mapping at slot `n`, the value for key `k` is at `keccak256(k, n)`, with +both the key and the slot word-sized: + + + {`{ "$keccak256": [{ "$wordsized": "key" }, { "$wordsized": 3 }] }`} + ### Nested mappings For `mapping(address => mapping(uint => uint))` at slot 2, the value is at -`keccak256(inner_key, keccak256(outer_key, 2))`. +`keccak256(inner_key, keccak256(outer_key, 2))`. The inner hash is already 32 +bytes and needs no resize; the keys and the literal slot do: + + + {`{ + "$keccak256": [ + { "$wordsized": "inner-key" }, + { "$keccak256": [{ "$wordsized": "outer-key" }, { "$wordsized": 2 }] } + ] +}`} + ## Data manipulation ### `$concat` — Concatenate bytes -Joins byte sequences without padding. Useful for building hash inputs from -multiple values. +Joins byte sequences without padding. Every operand must be bytes. Useful +for building hash inputs from multiple values. ### `$sized` — resize to N bytes @@ -143,7 +192,8 @@ count-1, computing each element's slot. To read element `i` from `uint256[] storage arr` at slot 5, a pointer: 1. Defines the array's base slot -2. Computes the element's slot: `keccak256(5) + element_index` +2. Computes the element's slot: `keccak256(5) + element_index`, word-sizing + `5` before hashing 3. Returns that storage location ## Learn more diff --git a/packages/web/docs/implementation-guides/pointers/evaluating-expressions.mdx b/packages/web/docs/implementation-guides/pointers/evaluating-expressions.mdx index 6a6f7ccb26..acbe87d586 100644 --- a/packages/web/docs/implementation-guides/pointers/evaluating-expressions.mdx +++ b/packages/web/docs/implementation-guides/pointers/evaluating-expressions.mdx @@ -9,7 +9,44 @@ import CodeListing from "@site/src/components/CodeListing"; Expression evaluation is a bit more interesting than reading raw region data, but, still, performing this evaluation becomes relatively straightforward -if variable and region references are pre-evaluated: +if variable and region references are pre-evaluated. + +## Two sorts of value + +The schema defines every expression to evaluate to one of two sorts of +value: an **integer**, which has a numeric value but no width, or **bytes**, +which have a definite width. The distinction matters because `$keccak256` +and `$concat` produce results that depend on the widths of their operands, +so the schema requires those operands to be bytes. A bare integer (a JSON +number, `$wordsize`, an arithmetic result, or a lookup) must first be given +a width with `$sized` or `$wordsized`; the resize forms are the only +bridge from an integer to bytes. + +This reference implementation represents the two sorts as a tagged union: + + sourceFile.getExportedDeclarations().get("Value")[0]} +/> + +The accompanying `Value` namespace provides constructors, type guards, and +the two conversions the rest of the implementation needs: `toInteger()`, +for positions where the schema expects an integer (bytes are read as the +non-negative integer they encode big-endian), and `toData()`, for storing +a value as a concrete region's `slot`, `offset`, or `length`: + + sourceFile.getExportedDeclarations().get("Value")[1]} +/> + +## Evaluation options + +Variables carry the sort of the expression that defined them, so the +`variables` map holds `Value`s, while the `regions` map holds pre-evaluated +concrete regions: sourceFile.getFunction("evaluateConstant")} /> -Evaluating literals involves detecting hex string vs. number and converting -appropriate to bytes: +Literals follow the schema's sorting rule: a JSON number is an integer, a +hex string with an even number of digits is bytes of that width, and a hex +string with an odd number of digits (which has no whole-byte width) is an +integer: Variable lookups, of course, require consulting the `variables` map passed -in `EvaluateOptions`: +in `EvaluateOptions`, yielding whichever sort of value the variable was +defined with: sourceFile.getFunction("evaluateArithmeticSum")} + extract={(sourceFile) => sourceFile.getFunction("evaluateInteger")} /> -Evaluating products: +With operands as integers, the five operations differ only in how they +combine them. Note that sums and products accept any number of operands, +while differences, quotients, and remainders take exactly two: sourceFile.getFunction("evaluateArithmeticProduct")} + extract={(sourceFile) => sourceFile.getFunction("evaluateArithmetic")} /> -Evaluating differences: - - - sourceFile.getFunction("evaluateArithmeticDifference") - } -/> - -**Note** how this function operates on unsigned values only by bounding the -result below at 0. - -Evaluating quotients: - - sourceFile.getFunction("evaluateArithmeticQuotient")} -/> - -(Quotients of course use integer division only.) - -Evaluating remainders: - - - sourceFile.getFunction("evaluateArithmeticRemainder") - } -/> +**Note** how `$difference` operates on unsigned values only by bounding the +result below at 0, and how `$quotient` uses integer division only. ## Evaluating resize expressions -This schema provides the `{ "$sized": }` construct to allow -explicitly resizing a subexpression. This implementation uses the +This schema provides the `{ "$sized": }` and +`{ "$wordsized": }` constructs to allow explicitly resizing a +subexpression. A resize always produces bytes of the requested width, and +so is the way to give an integer a width; this implementation encodes an +integer as its minimal big-endian bytes and then uses the [`Data.prototype.resizeTo()`](/docs/implementation-guides/pointers/types/data-and-machines) -method to perform this operation. +method for both sorts. sourceFile.getFunction("evaluateBytesOperands")} +/> + +With every operand's width guaranteed, hashing is a matter of concatenating +and applying the hash function: + ` function: -## Literal values +## Integers and bytes + +Every expression evaluates to a value of one of two sorts. An **integer** is +an unbounded, non-negative number with no width: JSON numbers, `$wordsize`, +variables, lookups, arithmetic results, and odd-digit hexadecimal literals +are integers. **Bytes** are a finite sequence with a definite width: +even-digit hexadecimal literals, `$read`, `$keccak256`, `$concat`, and the +resize forms produce bytes. -An expression can be a literal value. +Where an integer is expected (arithmetic operands, a list `count`, a +segment's `slot`, `offset`, or `length`), a bytes value is read as the +non-negative integer its bytes encode, big-endian. Where bytes are expected +(the operands of `$concat` and `$keccak256`, whose results depend on operand +widths), the operand **must** be width-bearing; a bare integer there is +invalid and **must** first be given a width with `$sized` or +`$wordsized`. There is no implicit widening: the resize forms are the only +bridge from an integer to bytes. + +## Literal values -Literal values **must** be represented either as JSON numbers or as -`0x`-prefixed hexadecimal strings. Hexadecimal strings always represent a -literal string of bytes. +An expression can be a literal value, written either as a JSON number or as +a `0x`-prefixed hexadecimal string. -For convenience, this schema does not restrict hexadecimal string -representations to those that specify an even-number of digits (i.e., those -that specify complete byte pairs); odd numbers of hexadecimal digits are fine. +A JSON number is an **integer**. A hexadecimal string with an **even** number +of digits is **bytes**, whose width is the number of bytes written: `"0x00"` +is one zero byte and `"0xdead"` is two bytes. A hexadecimal string with an +**odd** number of digits has no whole-byte width and is therefore an +**integer** equal to the value its digits denote: `"0x1"` is the integer `1`, +not bytes. -Hexadecimal string representations **may** omit leading zeroes; values are -assumed to be left-padded to the bytes width appropriate for the context. +Widths are never inferred from context, so a literal intended as bytes of a +particular width **must** be written with that many digits or wrapped in a +resize form. : [...] }`, where `` -denotes an arithmetic operation. +denotes an arithmetic operation. Operands are taken as integers, and the +result is an **integer** with no width. " }`, where The value of such an expression is the concatenation of bytes present in the running machine state that correspond to the bytes addressed by the referenced -region. +region: **bytes** whose width is the length of that region. ": "" }`, to denote that this expression is equivalent to the defined value for the property named `` inside the region referenced as - ``. + ``. The value is an **integer** (a region's `.offset`, + `.length`, or `.slot`). `` **must** be a valid and present property on the corresponding region, or it **must** correspond to an optional property @@ -136,7 +191,8 @@ $defs: description: | An object of the form `{ "$read": "" }`. The value of this expression equals the raw bytes present in the running machine state - in the referenced region. + in the referenced region. The result is **bytes** whose width is the + length of the region read. type: object properties: $read: @@ -169,9 +225,16 @@ $defs: Keccak256: title: Keccak256 hash description: | - An object of the form `{ "$keccak256": [...values] }`, indicating that this - expression evaluates to the Solidity-style keccak256 hash of the - tightly-packed bytes encoded by `values`. + An object of the form `{ "$keccak256": [...values] }`, indicating + that this expression evaluates to the Solidity-style keccak256 hash + of the tightly-packed bytes encoded by `values`. The result is + **bytes** of width 32. + + Because the hash is taken over the concatenation of the operands' + bytes, each operand **must** be width-bearing (bytes): a bare integer + is not valid here and must be given a width first with `$sizedN` or + `$wordsized`. This is why a mapping-slot computation word-sizes its key + and slot before hashing. type: object properties: $keccak256: @@ -184,7 +247,7 @@ $defs: - $keccak256 examples: - $keccak256: - - 0 + - $wordsized: 0 - "0x00" Concat: @@ -193,7 +256,12 @@ $defs: An object of the form `{ "$concat": [...values] }`, indicating that this expression evaluates to the concatenation of bytes from each value. The byte width of each operand is preserved; no padding is added or - removed between operands. + removed between operands. The result is **bytes** whose width is the + sum of the operand widths. + + Each operand **must** be width-bearing (bytes): a bare integer is not + valid here and must be given a width first with `$sizedN` or + `$wordsized`. type: object properties: $concat: @@ -216,6 +284,10 @@ $defs: Resize: title: Resize data description: | + A resize operation produces **bytes** of a definite width, and is the + bridge from an integer to bytes: give it an integer (or bytes) and it + yields bytes of the requested width. + A resize operation expression is either an object of the form `{ "$sized": }` or an object of the form `{ "$wordsized": }`, where `` is an expression @@ -265,5 +337,6 @@ examples: - .length: "array-start" - 1 - $keccak256: - - 5 - - .offset: "array-start" + - $wordsized: 5 + - $wordsized: + .offset: "array-start"