test: derive slow tests' timeouts from the work they do

scripts/json-schema.fixtures.ts

@@ -5,6 +5,7 @@ import { expect } from "vitest";
 
 import { and, or } from "trilean";
 
+import { budgetFor } from "../test/budget";
 import type { Comparable, Condition } from "../src/condition";
 import { allOf, anyOf, defineModel, is, isSet, isUnset, ModelError, not, oneOf } from "../src/index";
 import type { Rule } from "../src/rule";
@@ -20,6 +21,12 @@ import { buildJsonSchema, PACKAGE_VERSION } from "./json-schema";
 export const SEED = 20_261_029;
 /** Runs per property: enough for every mutation to meet every node kind of a condition and every form of a rule many times over. */
 export const RUNS = 600;
+/** The time one run of a property may take, in milliseconds: at least two hundred times what the heaviest of them takes on an idle machine, so a machine under heavy load still passes and only a property that stops terminating fails. */
+const RUN_BUDGET_MS = 150;
+/** The options of a test that checks one property over `RUNS` runs. */
+export const ONE_PROPERTY = budgetFor(RUNS, RUN_BUDGET_MS);
+/** The options of a test that checks two properties over `RUNS` runs each, as `expectAgreement` does. */
+export const TWO_PROPERTIES = budgetFor(2 * RUNS, RUN_BUDGET_MS);
 
 export const schema = buildJsonSchema(PACKAGE_VERSION);
 const ajv = new Ajv2020({ strict: true, allowUnionTypes: true });

scripts/json-schema.unit.test.ts

@@ -9,7 +9,7 @@ import type { Rule } from "../src/rule";
 import type { Scalar } from "../src/value";
 import { valueKey } from "../src/value";
 import type { Mentionable, Mutation } from "./json-schema.fixtures";
-import { accepts, BEYOND_A_SCHEMA, condition, expectAgreement, MAX_OPERANDS, model, MUTATIONS, nearMiss, RUNS, refusalsOf, rule, ruleOver, rules, schema, SEED } from "./json-schema.fixtures";
+import { accepts, BEYOND_A_SCHEMA, condition, expectAgreement, MAX_OPERANDS, model, MUTATIONS, nearMiss, ONE_PROPERTY, refusalsOf, rule, ruleOver, rules, RUNS, schema, SEED, TWO_PROPERTIES } from "./json-schema.fixtures";
 import { PACKAGE_VERSION } from "./json-schema";
 
 /**
@@ -71,19 +71,19 @@ function ruleFitsDomains(candidate: Rule): boolean {
 }
 
 describe("the JSON Schema and the runtime guards", () => {
-  it("agree on every condition and near miss", () => {
+  it("agree on every condition and near miss", TWO_PROPERTIES, () => {
     expectAgreement(condition, isCondition, accepts("PredicateNode"));
   });
 
-  it("agree on every rule and near miss", () => {
+  it("agree on every rule and near miss", TWO_PROPERTIES, () => {
     expectAgreement(rule, isRule, accepts("Rule"));
   });
 
-  it("agree on a list of rules: the guard accepts an array of rules exactly when the schema does", () => {
+  it("agree on a list of rules: the guard accepts an array of rules exactly when the schema does", TWO_PROPERTIES, () => {
     expectAgreement(rules, (value) => Array.isArray(value) && value.every(isRule), accepts("Rules"));
   });
 
-  it("agree on a list of rules with `parseRules`, which refuses what the schema refuses and, of what the schema accepts, exactly what names a dimension or value the model lacks", () => {
+  it("agree on a list of rules with `parseRules`, which refuses what the schema refuses and, of what the schema accepts, exactly what names a dimension or value the model lacks", TWO_PROPERTIES, () => {
     const dimensions = [choice("a", ["x", "y"]), choice("b", [true, null]), firstValid("c", ["y"], { activeWhen: isSet("a") })] as const;
     const known: Mentionable[] = [
       { name: "a", domain: ["x", "y"], unsettable: false },
@@ -103,7 +103,7 @@ describe("the JSON Schema and the runtime guards", () => {
     expectAgreement(fc.array(ruleOver(known), { maxLength: MAX_OPERANDS }), parses, accepts("Rules"), fits);
   });
 
-  it("accept every model `defineModel` accepts and refuse only models it refuses, and `defineModel` refuses what the schema accepts only for what a schema cannot state", () => {
+  it("accept every model `defineModel` accepts and refuse only models it refuses, and `defineModel` refuses what the schema accepts only for what a schema cannot state", TWO_PROPERTIES, () => {
     const valid = accepts("Model");
     fc.assert(
       fc.property(model, (data) => {
@@ -134,7 +134,7 @@ describe("the JSON Schema and the runtime guards", () => {
     expect([...refused].sort()).toEqual([...MUTATIONS].sort());
   });
 
-  it("accept at the root exactly what one of the four definitions accepts", () => {
+  it("accept at the root exactly what one of the four definitions accepts", ONE_PROPERTY, () => {
     const root = accepts(undefined);
     const definitions = [accepts("Model"), accepts("Rules"), accepts("Rule"), accepts("PredicateNode")];
     const anything = fc.oneof(condition, rule, rules, model, nearMiss(fc.oneof(condition, rule, rules, model)).map(({ value }) => value), fc.jsonValue());

scripts/parse-model.unit.test.ts

@@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest";
 import type { Dimension, ModelData } from "../src/index";
 import { modelShapeIssues } from "../src/shape";
 import { choice, collect, count, defineModel, firstValid, isModelData, isNonEmpty, ModelError, parseModel } from "../src/index";
-import { accepts, definesModel, expectAgreement, model, nearMiss, RUNS, SEED } from "./json-schema.fixtures";
+import { accepts, definesModel, expectAgreement, model, nearMiss, ONE_PROPERTY, RUNS, SEED, TWO_PROPERTIES } from "./json-schema.fixtures";
 
 /**
  * `isModelData` and `parseModel` against the published JSON Schema and against `defineModel`, over the seeded models and near misses the schema suite draws. The guard must agree with the schema exactly, and the parser must accept exactly what the schema accepts and `defineModel` accepts, building the model `defineModel` builds from the same definition written in code.
@@ -33,11 +33,11 @@ function inCode(data: ModelData["dimensions"][number]): Dimension {
 }
 
 describe("isModelData", () => {
-  it("agrees with the JSON Schema on every model and near miss", () => {
+  it("agrees with the JSON Schema on every model and near miss", TWO_PROPERTIES, () => {
     expectAgreement(model, isModelData, accepts("Model"));
   });
 
-  it("accepts every value the schema's Model accepts, and every value it accepts the schema's root accepts", () => {
+  it("accepts every value the schema's Model accepts, and every value it accepts the schema's root accepts", ONE_PROPERTY, () => {
     const validModel = accepts("Model");
     const validRoot = accepts(undefined);
     const anything = fc.oneof(model, nearMiss(model).map(({ value }) => value), fc.jsonValue());
@@ -61,7 +61,7 @@ describe("isModelData", () => {
     expect(refused.map((value) => [isModelData(value), validModel(value)])).toEqual(refused.map(() => [false, false]));
   });
 
-  it("fails the agreement for a guard with a rule weakened, so that agreement is not by construction", () => {
+  it("fails the agreement for a guard with a rule weakened, so that agreement is not by construction", TWO_PROPERTIES, () => {
     // `modelShapeIssues` is the shape check `defineModel` runs first: the same guard without the empty-domain, duplicate-value, unset-value, computed and constraints rules.
     const weakened = (value: unknown): boolean => modelShapeIssues(value).length === 0;
     expect(() => {
@@ -71,11 +71,11 @@ describe("isModelData", () => {
 });
 
 describe("parseModel", () => {
-  it("accepts exactly what the schema accepts and `defineModel` accepts, and refuses the rest with a `ModelError`", () => {
+  it("accepts exactly what the schema accepts and `defineModel` accepts, and refuses the rest with a `ModelError`", TWO_PROPERTIES, () => {
     expectAgreement(model, parses, accepts("Model"), definesModel);
   });
 
-  it("enumerates the same combinations, in the same order, as the model written in code", () => {
+  it("enumerates the same combinations, in the same order, as the model written in code", ONE_PROPERTY, () => {
     fc.assert(
       fc.property(model, (data) => {
         if (!isModelData(data)) throw new Error("a generated model is model data");
@@ -91,7 +91,7 @@ describe("parseModel", () => {
     );
   });
 
-  it("is refused with every issue for data of the wrong shape, as the guard refuses it", () => {
+  it("is refused with every issue for data of the wrong shape, as the guard refuses it", ONE_PROPERTY, () => {
     fc.assert(
       fc.property(nearMiss(model), ({ value }) => {
         if (isModelData(value)) return;

src/cancellation.unit.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from "vitest";
 
+import { budgetFor } from "../test/budget";
 import { STEPS_PER_ABORT_CHECK } from "./cancellation";
 import type { AbortSignalLike, Strategy } from "./index";
 import { choice, collect, combinations, count, defineModel, diagnose, full, isNonEmpty, predicate } from "./index";
@@ -8,6 +9,10 @@ const REASON = new Error("stopped by the caller");
 const SIDE = 10;
 const DIMENSION_COUNT = 4;
 const SPACE = SIDE ** DIMENSION_COUNT;
+/** The searches of the whole space that the test of a signal that never aborts makes: a count, then a collect and a diagnose, each of the last two once more without a signal. */
+const WHOLE_SPACE_SEARCHES = 5;
+/** The time a search may spend per combination of the space, in milliseconds, whatever the number of searches that visit it: at least two hundred times what it takes on an idle machine, so a machine under heavy load still passes and only a search that stops terminating fails. */
+const COMBINATION_BUDGET_MS = 2;
 /** Predicate evaluations before the signal aborts: well inside the first leaf of the search, so the rest of the space is still to do. */
 const ABORT_AT = 5;
 
@@ -139,7 +144,7 @@ describe("a signal that aborts mid-search", () => {
     expect(evaluations.total - before).toBeLessThan(STEPS_PER_ABORT_CHECK);
   });
 
-  it("completes with the same result as an unsignalled run when the signal never aborts", () => {
+  it("completes with the same result as an unsignalled run when the signal never aborts", budgetFor(SPACE * WHOLE_SPACE_SEARCHES, COMBINATION_BUDGET_MS), () => {
     const { model } = abortingModel(SPACE + 1);
     const signal = new AbortController().signal;
     expect(count(model, { signal })).toBe(SPACE);

src/condition.unit.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from "vitest";
 
+import { budgetFor } from "../test/budget";
 import type { Condition, Constraint, Rule, Value, Verdict } from "./index";
 import { allOf, anyOf, choice, defineModel, is, isSet, isUnset, not, oneOf, ruleTree } from "./index";
 
@@ -102,10 +103,16 @@ describe("a condition evaluated by the engine", () => {
     expect(verdict(anyOf(), new Map())).toBe("invalid");
   });
 
-  it("is monotone: a definite answer on a partial assignment is the answer on every completion", () => {
+  const everyCondition = conditions(2);
+  const cases = partialAssignments().map((assignment) => ({ assignment, completions: completions(assignment) }));
+  // Each condition is evaluated on every partial assignment and on each of its completions.
+  const evaluations = everyCondition.length * cases.reduce((total, { completions: everyCompletion }) => total + 1 + everyCompletion.length, 0);
+  /** The time one evaluation may take, in milliseconds: at least two hundred times what it takes on an idle machine, so a machine under heavy load still passes and only an evaluation that stops terminating fails. */
+  const EVALUATION_BUDGET_MS = 1;
+
+  it("is monotone: a definite answer on a partial assignment is the answer on every completion", budgetFor(evaluations, EVALUATION_BUDGET_MS), () => {
     const violations: string[] = [];
-    const cases = partialAssignments().map((assignment) => ({ assignment, completions: completions(assignment) }));
-    for (const condition of conditions(2)) {
+    for (const condition of everyCondition) {
       const constraint = constraintOf(condition);
       for (const { assignment, completions: everyCompletion } of cases) {
         const answer = constraint.evaluate(assignment);

src/counting.unit.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from "vitest";
 
+import { budgetFor, candidateCount } from "../test/budget";
 import { hashTest, Random, randomModel, seeds, VALUE_POOL } from "../test/random-model";
 import type { Constraint, Diagnosis, Dimension, Model, ModelParts, Value } from "./index";
 import { at, choice, computed, count, defineModel, diagnose, firstValid, indexOf, isNonEmpty, isSet, LimitExceededError, predicate } from "./index";
@@ -59,13 +60,34 @@ function lateModel(seed: number): AnyModel {
   return defineModel({ dimensions, constraints });
 }
 
+interface Labelled {
+  readonly label: string;
+  readonly model: AnyModel;
+}
+
+const randomModels: readonly Labelled[] = seeds.map((seed) => ({ label: `random ${String(seed)}`, model: randomModel(seed) }));
+const lateModels: readonly Labelled[] = seeds.map((seed) => ({ label: `late ${String(seed)}`, model: lateModel(seed) }));
 /** Every model the equivalence properties are checked against: the generated models of every feature, and the late-constrained ones. */
-const models: readonly { readonly label: string; readonly model: AnyModel }[] = [...seeds.map((seed) => ({ label: `random ${String(seed)}`, model: randomModel(seed) })), ...seeds.map((seed) => ({ label: `late ${String(seed)}`, model: lateModel(seed) }))];
+const models: readonly Labelled[] = [...randomModels, ...lateModels];
+
+/** The combinations the dimensions of `labelled` models could take with the constraints ignored, summed: an upper bound on the leaves a search of each model reaches, and so the work of a test that searches every one of them. */
+function candidatesOf(labelled: readonly Labelled[]): number {
+  return labelled.reduce((total, { model }) => total + candidateCount(model), 0);
+}
+
+/** The time one candidate may take a test, in milliseconds, whatever the number of searches the test makes through it: at least two hundred times what the heaviest of the tests takes on an idle machine, so a machine under heavy load still passes and only a search that stops terminating fails. */
+const CANDIDATE_BUDGET_MS = 4;
+/** The options of a test that searches every model. */
+const EVERY_MODEL = budgetFor(candidatesOf(models), CANDIDATE_BUDGET_MS);
+/** The options of a test that searches the late-constrained models alone. */
+const LATE_MODELS = budgetFor(candidatesOf(lateModels), CANDIDATE_BUDGET_MS);
 
-/** The time one rank or unrank may take, in milliseconds: about forty times what it takes unloaded, so a machine under heavy load still passes and only a search that stops terminating fails. */
-const RANK_OPERATION_BUDGET_MS = 10;
+/** The time one rank or unrank may take, in milliseconds: at least two hundred times what it takes on an idle machine, so a machine under heavy load still passes and only a search that stops terminating fails. */
+const RANK_OPERATION_BUDGET_MS = 125;
 /** The ranks and unranks the exhaustive agreement test makes: one per index of every model, and one more past the last. */
 const RANK_OPERATIONS = models.reduce((total, { model }) => total + count(model) + 1, 0);
+/** The options of the exhaustive agreement test. */
+const EVERY_INDEX = budgetFor(RANK_OPERATIONS, RANK_OPERATION_BUDGET_MS);
 
 /** The diagnosis by the plain search: every combination produced and its values collected, as `diagnose` worked before it counted through the memo. `"limit"` when the solution set has more than `limit` combinations. */
 function referenceDiagnosis(model: ModelParts, limit?: number): Diagnosis | "limit" {
@@ -105,13 +127,13 @@ function countDisagreements(searchOf: (model: ModelParts) => CountingSearch): st
 }
 
 describe("the memoised count", () => {
-  it("generates late-constrained models whose solution sets are neither all empty nor all full", () => {
+  it("generates late-constrained models whose solution sets are neither all empty nor all full", LATE_MODELS, () => {
     const sizes = seeds.map((seed) => count(lateModel(seed)));
     expect(sizes.some((size) => size === 0)).toBe(true);
     expect(sizes.some((size) => size > 1)).toBe(true);
   });
 
-  it("counts what the naive search counts, and refuses past a limit at the same boundary, on every model", () => {
+  it("counts what the naive search counts, and refuses past a limit at the same boundary, on every model", EVERY_MODEL, () => {
     const disagreements = models.filter(({ model }) => {
       const size = countSearch(model, undefined, { completions: naiveCompletions });
 
@@ -120,7 +142,7 @@ describe("the memoised count", () => {
     expect(disagreements).toEqual([]);
   });
 
-  it("diagnoses what the naive search diagnoses, the status, count and dead values alike, and refuses past a limit where it does", () => {
+  it("diagnoses what the naive search diagnoses, the status, count and dead values alike, and refuses past a limit where it does", EVERY_MODEL, () => {
     const disagreements = models.filter(({ model }) => {
       const expected = referenceDiagnosis(model);
       const size = expected === "limit" || expected.status === "empty" ? 0 : expected.count;
@@ -130,7 +152,7 @@ describe("the memoised count", () => {
     expect(disagreements).toEqual([]);
   });
 
-  it("ranks and unranks as the naive count does, at every index and for every combination", { timeout: RANK_OPERATIONS * RANK_OPERATION_BUDGET_MS }, () => {
+  it("ranks and unranks as the naive count does, at every index and for every combination", EVERY_INDEX, () => {
     const disagreements = models.filter(({ model }) => {
       const size = count(model);
       for (let index = 0; index <= size; index += 1) {
@@ -144,7 +166,7 @@ describe("the memoised count", () => {
     expect(disagreements).toEqual([]);
   });
 
-  it("is caught by the equivalence property when the frontier leaves out a dimension it reads", () => {
+  it("is caught by the equivalence property when the frontier leaves out a dimension it reads", EVERY_MODEL, () => {
     const dropFirst = (model: ModelParts): CountingSearch => {
       const search = countingSearch(model, undefined);