fix(serializer): reject unsigned 64-bit overflow in BigVLQ encode and decode

.changeset/tidy-bigvlq-ulongs.md

@@ -0,0 +1,5 @@
+---
+"@fleet-sdk/serializer": patch
+---
+
+fix(serializer): reject unsigned 64-bit overflow in BigVLQ encode and decode

packages/serializer/src/coders/numRanges.ts

@@ -1,6 +1,8 @@
 export const MIN_U8 = 0x00;
 export const MAX_U8 = 0xff;
 
+export const MAX_U64 = BigInt("0xffffffffffffffff");
+
 export const MIN_I8 = -0x80;
 export const MAX_I8 = 0x7f;
 

packages/serializer/src/coders/vlq.spec.ts

@@ -90,10 +90,54 @@ describe("Big VLQ encoding/decoding", () => {
     }).toThrow();
   });
 
+  it("Should encode the maximum unsigned 64-bit value", () => {
+    expect(toBigVLQBytes(18446744073709551615n)).toEqual(
+      Uint8Array.from([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01])
+    );
+  });
+
+  it("Should fail trying to encode values above the unsigned 64-bit range", () => {
+    expect(() => {
+      toBigVLQBytes(18446744073709551616n);
+    }).toThrow(RangeError);
+  });
+
   test.each(testVectors)("Should decode", (tv) => {
     expect(readBigVLQ(new SigmaByteReader(tv.bytes))).toEqual(tv.uint);
   });
 
+  it("Should decode the maximum unsigned 64-bit value", () => {
+    expect(
+      readBigVLQ(
+        new SigmaByteReader(
+          Uint8Array.from([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01])
+        )
+      )
+    ).toEqual(18446744073709551615n);
+  });
+
+  it("Should fail trying to decode values above the unsigned 64-bit range", () => {
+    // 2^64, encoded as VLQ
+    expect(() => {
+      readBigVLQ(
+        new SigmaByteReader(
+          Uint8Array.from([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02])
+        )
+      );
+    }).toThrow(RangeError);
+  });
+
+  it("Should fail trying to decode encodings longer than 10 bytes", () => {
+    // 1, encoded in 11 bytes: a 64-bit VLQ never exceeds 10 bytes
+    expect(() => {
+      readBigVLQ(
+        new SigmaByteReader(
+          Uint8Array.from([0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00])
+        )
+      );
+    }).toThrow(RangeError);
+  });
+
   it("Should decode empty Buffer to 0", () => {
     expect(readBigVLQ(new SigmaByteReader(Uint8Array.from([])))).toEqual(0n);
   });
@@ -104,7 +148,7 @@ describe("Big VLQ encoding/decoding", () => {
 
   it("Should encode/decode radom numbers", () => {
     fc.assert(
-      fc.property(fc.bigInt({ min: 0n, max: BigInt(Number.MAX_SAFE_INTEGER) }), (n) => {
+      fc.property(fc.bigInt({ min: 0n, max: 18446744073709551615n }), (n) => {
         expect(readBigVLQ(new SigmaByteReader(toBigVLQBytes(n)))).to.be.equal(n);
       })
     );

packages/serializer/src/coders/vlq.ts

@@ -1,4 +1,5 @@
 import { _0n, _7n, _127n, _128n, ensureBigInt } from "@fleet-sdk/common";
+import { MAX_U64 } from "./numRanges";
 import type { SigmaByteReader } from "./sigmaByteReader";
 import type { SigmaByteWriter } from "./sigmaByteWriter";
 
@@ -70,6 +71,9 @@ export function writeBigVLQ(writer: SigmaByteWriter, value: bigint): SigmaByteWr
   if (value < _0n) {
     throw new RangeError("Variable Length Quantity not supported for negative numbers");
   }
+  if (value > MAX_U64) {
+    throw new RangeError(`Value ${value} is out of range for an unsigned 64-bit integer`);
+  }
 
   let val = value;
   do {
@@ -92,12 +96,21 @@ export function readBigVLQ(reader: SigmaByteReader): bigint {
 
   let value = _0n;
   let shift = _0n;
+  let byteCount = 0;
   let lower7bits: bigint;
 
   do {
     lower7bits = BigInt(reader.readByte());
+    byteCount++;
     value |= (lower7bits & _127n) << shift;
     shift += _7n;
+
+    // The reference `getULong` reads at most 10 bytes (shifts 0..63) and
+    // returns a 64-bit value: longer encodings are an error there, and a
+    // 10-byte encoding whose value exceeds 2^64 - 1 does not round-trip.
+    if (byteCount > 10 || value > MAX_U64) {
+      throw new RangeError(`Value ${value} is out of range for an unsigned 64-bit integer`);
+    }
   } while ((lower7bits & _128n) !== _0n);
 
   return value;