From e6f1976e3a747d74f6639c37e1ccabe4e21bb6a5 Mon Sep 17 00:00:00 2001 From: Jacob Dent Date: Mon, 28 Sep 2026 18:37:06 -0500 Subject: [PATCH 1/2] Unbiased ints: reject the 2^32 mod max biased values (Lemire); cap max at 2^32 --- CHANGELOG.md | 17 +++++++++++++++++ README.md | 17 +++++++++++------ provable.js | 1 + test.js | 19 +++++++++++++++++++ utils.js | 20 +++++++++++++++++--- 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cc336d..abe91df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,23 @@ requires a major version**, however small it looks. ## [Unreleased] +### Breaking + +- `ints()` is unbiased. Each draw reads 4 bytes as a big-endian u32 and returns + `min + floor(u * max / 2^32)` (Lemire), identical to the old float mapping + for accepted values, but rejects the `2^32 mod max` values that made some + results one draw more likely and reads the next 4 bytes instead. Ranges that + divide 2^32 (and in practice nearly all small ranges) give the same numbers + as 2.x; a draw only changes when it hits a rejected value (for example 24 of + 60,000 five-int draws at `max = 1e6`, about 10% at `max = 1e9`). +- `max` above 2^32 now throws. It used to return only multiples of + `2^(bits above 32)`. + +### Unchanged + +- `ByteGenerator`, `floats()`, `bytesToFloat` and `floatToInt` produce the + same values as 2.x. + ## [2.0.0] - 2026-09-17 ### Breaking diff --git a/README.md b/README.md index 1308a9b..9b359f9 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,11 @@ For a draw with nonce `N`: 2. Bytes are consumed four at a time. Each group becomes a float: `b0/256 + b1/256^2 + b2/256^3 + b3/256^4`, a value in `[0, 1)` with 32 bits of precision. -3. `ints(count, max, min)` maps each float to `floor(min + float * max)`, an - integer in `[min, min + max - 1]`. **`max` is the size of the range, not an - upper bound.** +3. `ints(count, max, min)` reads each 4 bytes as a big-endian u32 `u` and + returns `min + floor(u * max / 2^32)`, an integer in `[min, min + max - 1]`. + If `(u * max) mod 2^32 < 2^32 mod max` the value is rejected and the next 4 + bytes are read instead, so every integer is exactly equally likely. + **`max` is the size of the range, not an upper bound**, and at most 2^32. 4. After the draw the nonce becomes `N + 1` and the new state is emitted. Every draw uses the nonce as it stands and then advances it, so the outcome @@ -194,9 +196,12 @@ const { utils } = require("@provableio/provable-core"); yielding the HMAC byte stream for a fixed nonce. Call it, do not `new` it. - `FloatGenerator(rng, count)`: generator function turning four bytes at a time from `rng` into floats. -- `floats(rng, count)`, `ints(rng, count, max, min = 0)`: array forms of the above. -- `bytesToFloat([b0, b1, b2, b3])`, `floatToInt(float, max, min = 0)`: the two - conversions described in "How outcomes are generated". +- `floats(rng, count)`: array form of the above. +- `ints(rng, count, max, min = 0)`: unbiased integers as described in "How + outcomes are generated" (may read more than `4 * count` bytes). +- `bytesToFloat([b0, b1, b2, b3])`: four bytes to a float in `[0, 1)`. +- `floatToInt(float, max, min = 0)`: the plain float-to-int mapping, kept for + 2.x replays; slightly biased when `max` does not divide 2^32. - `toInteger(value, name)`, `assertSeed(value, name)`: the validators used throughout. diff --git a/provable.js b/provable.js index d46d03a..e47b433 100644 --- a/provable.js +++ b/provable.js @@ -61,6 +61,7 @@ module.exports = ints(count, max, min = 0) { count = positiveInteger(count, "count"); max = positiveInteger(max, "max"); + assert(max <= 2 ** 32, "max must be 2^32 or less"); min = toInteger(min, "min"); return draw((rng) => ints(rng, count, max, min)); }, diff --git a/test.js b/test.js index 1919f69..6bec0d1 100644 --- a/test.js +++ b/test.js @@ -84,6 +84,25 @@ test("provable", (t) => { t.end(); }); + t.test("ints reject biased u32 values instead of skewing the result", (t) => { + const bytes = (list) => list[Symbol.iterator](); + // max 3: 2^32 % 3 = 1, so only u = 0 (high product bits 0, low bits 0) is rejected. + t.deepEqual(utils.ints(bytes([0, 0, 0, 0, 255, 255, 255, 255]), 1, 3, 0), [2], "u=0 redrawn"); + t.deepEqual(utils.ints(bytes([0, 0, 0, 1]), 1, 3, 0), [0], "u=1 accepted"); + // Accepted draws match the float mapping exactly. + // max 1000: u = 2^31 lands on a biased slot (low bits 0 < 2^32 % 1000) and is redrawn. + t.deepEqual(utils.ints(bytes([128, 0, 0, 0, 0, 0, 0, 1]), 1, 1000, 5), [5], "u=2^31 redrawn"); + for (const u of [1, 2 ** 31 + 1, 2 ** 32 - 1, 123456789]) { + const b = [u >>> 24, (u >>> 16) & 255, (u >>> 8) & 255, u & 255]; + t.equal(utils.ints(bytes(b), 1, 1000, 5)[0], utils.floatToInt(utils.bytesToFloat(b), 1000, 5)); + } + // A power-of-two range never rejects and reaches every value. + t.deepEqual(utils.ints(bytes([255, 255, 255, 255]), 1, 2 ** 32, 0), [2 ** 32 - 1]); + t.throws(() => Provable()(base).ints(1, 2 ** 32 + 1), /max/); + t.throws(() => utils.ints(bytes([0, 0, 0, 0]), 1, 2 ** 40, 0), /max/); + t.end(); + }); + t.test("cursor offsets the byte stream", (t) => { t.deepEqual(Provable()({ ...base, cursor: 4 }).ints(1, 10001, 0), [632]); t.deepEqual(Provable()({ ...base, cursor: 32 }).ints(1, 10001, 0), Provable()(base).ints(9, 10001, 0).slice(8)); diff --git a/utils.js b/utils.js index e3921ff..c3366d8 100644 --- a/utils.js +++ b/utils.js @@ -88,6 +88,7 @@ function bytesToFloat(bytes) { // Maps a float in [0, 1) onto the `max` integers starting at `min`, i.e. the // range [min, min + max - 1]. `max` is a range size, not an upper bound. +// Slightly biased when `max` does not divide 2^32; `ints` is the unbiased form. function floatToInt(val, max, min = 0) { return Math.floor(min + val * max); } @@ -106,11 +107,24 @@ function floats(rng, count) { return [...FloatGenerator(rng, count)]; } +const TWO_32 = 2 ** 32; + +// Unbiased integers in [min, min + max - 1] (Lemire's multiply-shift with +// rejection). Each draw reads 4 bytes as a big-endian u32 and takes the high +// 32 bits of u * max, which equals floatToInt(bytesToFloat(bytes), max, min). +// The few u32 values that would over-represent some results are rejected and +// the next 4 bytes are drawn instead, so every result is exactly equally likely. function ints(rng, count, max, min = 0) { + assert(Number.isSafeInteger(max) && max >= 1 && max <= TWO_32, "max must be 1 to 2^32"); + const range = BigInt(max); + const threshold = BigInt(TWO_32 % max); const result = []; - const gen = FloatGenerator(rng, count); - for (let i = 0; i < count; i++) { - result.push(floatToInt(gen.next().value, max, min)); + while (result.length < count) { + let u = 0n; + for (let j = 0; j < BYTES_PER_FLOAT; j++) u = (u << 8n) | BigInt(rng.next().value); + const m = u * range; + if ((m & 0xffffffffn) < threshold) continue; // biased slot: draw again + result.push(min + Number(m >> 32n)); } return result; } From 6bba7e638effdf53a782ea9a3624c156aa9e9d0b Mon Sep 17 00:00:00 2001 From: Jacob Dent Date: Mon, 28 Sep 2026 18:47:07 -0500 Subject: [PATCH 2/2] Changelog: exact per-integer redraw rates --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abe91df..fe97d76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,10 @@ requires a major version**, however small it looks. `min + floor(u * max / 2^32)` (Lemire), identical to the old float mapping for accepted values, but rejects the `2^32 mod max` values that made some results one draw more likely and reads the next 4 bytes instead. Ranges that - divide 2^32 (and in practice nearly all small ranges) give the same numbers - as 2.x; a draw only changes when it hits a rejected value (for example 24 of - 60,000 five-int draws at `max = 1e6`, about 10% at `max = 1e9`). + divide 2^32 give the same numbers as 2.x, and so does every other draw + that isn't rejected. The rejected share per integer is `(2^32 mod max) / 2^32`: + about 1 in a billion for `max = 6`, 0.0225% for `max = 1e6`, 6.9% for + `max = 1e9`, 30% for `max = 3e9`. - `max` above 2^32 now throws. It used to return only multiples of `2^(bits above 32)`.