diff --git a/scripts/json-schema.fixtures.ts b/scripts/json-schema.fixtures.ts index f55eef3..43b6f14 100644 --- a/scripts/json-schema.fixtures.ts +++ b/scripts/json-schema.fixtures.ts @@ -5,6 +5,7 @@ import { expect } from "vitest"; import { and, or } from "trilean"; +import { budgetFor } from "../test/budget"; import type { Comparable, Condition } from "../src/condition"; import { allOf, anyOf, defineModel, is, isSet, isUnset, ModelError, not, oneOf } from "../src/index"; import type { Rule } from "../src/rule"; @@ -20,6 +21,12 @@ import { buildJsonSchema, PACKAGE_VERSION } from "./json-schema"; export const SEED = 20_261_029; /** Runs per property: enough for every mutation to meet every node kind of a condition and every form of a rule many times over. */ export const RUNS = 600; +/** The time one run of a property may take, in milliseconds: at least two hundred times what the heaviest of them takes on an idle machine, so a machine under heavy load still passes and only a property that stops terminating fails. */ +const RUN_BUDGET_MS = 150; +/** The options of a test that checks one property over `RUNS` runs. */ +export const ONE_PROPERTY = budgetFor(RUNS, RUN_BUDGET_MS); +/** The options of a test that checks two properties over `RUNS` runs each, as `expectAgreement` does. */ +export const TWO_PROPERTIES = budgetFor(2 * RUNS, RUN_BUDGET_MS); export const schema = buildJsonSchema(PACKAGE_VERSION); const ajv = new Ajv2020({ strict: true, allowUnionTypes: true }); diff --git a/scripts/json-schema.unit.test.ts b/scripts/json-schema.unit.test.ts index 050f9f1..0f8c702 100644 --- a/scripts/json-schema.unit.test.ts +++ b/scripts/json-schema.unit.test.ts @@ -9,7 +9,7 @@ import type { Rule } from "../src/rule"; import type { Scalar } from "../src/value"; import { valueKey } from "../src/value"; import type { Mentionable, Mutation } from "./json-schema.fixtures"; -import { accepts, BEYOND_A_SCHEMA, condition, expectAgreement, MAX_OPERANDS, model, MUTATIONS, nearMiss, RUNS, refusalsOf, rule, ruleOver, rules, schema, SEED } from "./json-schema.fixtures"; +import { accepts, BEYOND_A_SCHEMA, condition, expectAgreement, MAX_OPERANDS, model, MUTATIONS, nearMiss, ONE_PROPERTY, refusalsOf, rule, ruleOver, rules, RUNS, schema, SEED, TWO_PROPERTIES } from "./json-schema.fixtures"; import { PACKAGE_VERSION } from "./json-schema"; /** @@ -71,19 +71,19 @@ function ruleFitsDomains(candidate: Rule): boolean { } describe("the JSON Schema and the runtime guards", () => { - it("agree on every condition and near miss", () => { + it("agree on every condition and near miss", TWO_PROPERTIES, () => { expectAgreement(condition, isCondition, accepts("PredicateNode")); }); - it("agree on every rule and near miss", () => { + it("agree on every rule and near miss", TWO_PROPERTIES, () => { expectAgreement(rule, isRule, accepts("Rule")); }); - it("agree on a list of rules: the guard accepts an array of rules exactly when the schema does", () => { + it("agree on a list of rules: the guard accepts an array of rules exactly when the schema does", TWO_PROPERTIES, () => { expectAgreement(rules, (value) => Array.isArray(value) && value.every(isRule), accepts("Rules")); }); - it("agree on a list of rules with `parseRules`, which refuses what the schema refuses and, of what the schema accepts, exactly what names a dimension or value the model lacks", () => { + it("agree on a list of rules with `parseRules`, which refuses what the schema refuses and, of what the schema accepts, exactly what names a dimension or value the model lacks", TWO_PROPERTIES, () => { const dimensions = [choice("a", ["x", "y"]), choice("b", [true, null]), firstValid("c", ["y"], { activeWhen: isSet("a") })] as const; const known: Mentionable[] = [ { name: "a", domain: ["x", "y"], unsettable: false }, @@ -103,7 +103,7 @@ describe("the JSON Schema and the runtime guards", () => { expectAgreement(fc.array(ruleOver(known), { maxLength: MAX_OPERANDS }), parses, accepts("Rules"), fits); }); - it("accept every model `defineModel` accepts and refuse only models it refuses, and `defineModel` refuses what the schema accepts only for what a schema cannot state", () => { + it("accept every model `defineModel` accepts and refuse only models it refuses, and `defineModel` refuses what the schema accepts only for what a schema cannot state", TWO_PROPERTIES, () => { const valid = accepts("Model"); fc.assert( fc.property(model, (data) => { @@ -134,7 +134,7 @@ describe("the JSON Schema and the runtime guards", () => { expect([...refused].sort()).toEqual([...MUTATIONS].sort()); }); - it("accept at the root exactly what one of the four definitions accepts", () => { + it("accept at the root exactly what one of the four definitions accepts", ONE_PROPERTY, () => { const root = accepts(undefined); const definitions = [accepts("Model"), accepts("Rules"), accepts("Rule"), accepts("PredicateNode")]; const anything = fc.oneof(condition, rule, rules, model, nearMiss(fc.oneof(condition, rule, rules, model)).map(({ value }) => value), fc.jsonValue()); diff --git a/scripts/parse-model.unit.test.ts b/scripts/parse-model.unit.test.ts index cbf5966..a9b6a4c 100644 --- a/scripts/parse-model.unit.test.ts +++ b/scripts/parse-model.unit.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest"; import type { Dimension, ModelData } from "../src/index"; import { modelShapeIssues } from "../src/shape"; import { choice, collect, count, defineModel, firstValid, isModelData, isNonEmpty, ModelError, parseModel } from "../src/index"; -import { accepts, definesModel, expectAgreement, model, nearMiss, RUNS, SEED } from "./json-schema.fixtures"; +import { accepts, definesModel, expectAgreement, model, nearMiss, ONE_PROPERTY, RUNS, SEED, TWO_PROPERTIES } from "./json-schema.fixtures"; /** * `isModelData` and `parseModel` against the published JSON Schema and against `defineModel`, over the seeded models and near misses the schema suite draws. The guard must agree with the schema exactly, and the parser must accept exactly what the schema accepts and `defineModel` accepts, building the model `defineModel` builds from the same definition written in code. @@ -33,11 +33,11 @@ function inCode(data: ModelData["dimensions"][number]): Dimension { } describe("isModelData", () => { - it("agrees with the JSON Schema on every model and near miss", () => { + it("agrees with the JSON Schema on every model and near miss", TWO_PROPERTIES, () => { expectAgreement(model, isModelData, accepts("Model")); }); - it("accepts every value the schema's Model accepts, and every value it accepts the schema's root accepts", () => { + it("accepts every value the schema's Model accepts, and every value it accepts the schema's root accepts", ONE_PROPERTY, () => { const validModel = accepts("Model"); const validRoot = accepts(undefined); const anything = fc.oneof(model, nearMiss(model).map(({ value }) => value), fc.jsonValue()); @@ -61,7 +61,7 @@ describe("isModelData", () => { expect(refused.map((value) => [isModelData(value), validModel(value)])).toEqual(refused.map(() => [false, false])); }); - it("fails the agreement for a guard with a rule weakened, so that agreement is not by construction", () => { + it("fails the agreement for a guard with a rule weakened, so that agreement is not by construction", TWO_PROPERTIES, () => { // `modelShapeIssues` is the shape check `defineModel` runs first: the same guard without the empty-domain, duplicate-value, unset-value, computed and constraints rules. const weakened = (value: unknown): boolean => modelShapeIssues(value).length === 0; expect(() => { @@ -71,11 +71,11 @@ describe("isModelData", () => { }); describe("parseModel", () => { - it("accepts exactly what the schema accepts and `defineModel` accepts, and refuses the rest with a `ModelError`", () => { + it("accepts exactly what the schema accepts and `defineModel` accepts, and refuses the rest with a `ModelError`", TWO_PROPERTIES, () => { expectAgreement(model, parses, accepts("Model"), definesModel); }); - it("enumerates the same combinations, in the same order, as the model written in code", () => { + it("enumerates the same combinations, in the same order, as the model written in code", ONE_PROPERTY, () => { fc.assert( fc.property(model, (data) => { if (!isModelData(data)) throw new Error("a generated model is model data"); @@ -91,7 +91,7 @@ describe("parseModel", () => { ); }); - it("is refused with every issue for data of the wrong shape, as the guard refuses it", () => { + it("is refused with every issue for data of the wrong shape, as the guard refuses it", ONE_PROPERTY, () => { fc.assert( fc.property(nearMiss(model), ({ value }) => { if (isModelData(value)) return; diff --git a/src/cancellation.unit.test.ts b/src/cancellation.unit.test.ts index 797b219..0b04c70 100644 --- a/src/cancellation.unit.test.ts +++ b/src/cancellation.unit.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { budgetFor } from "../test/budget"; import { STEPS_PER_ABORT_CHECK } from "./cancellation"; import type { AbortSignalLike, Strategy } from "./index"; import { choice, collect, combinations, count, defineModel, diagnose, full, isNonEmpty, predicate } from "./index"; @@ -8,6 +9,10 @@ const REASON = new Error("stopped by the caller"); const SIDE = 10; const DIMENSION_COUNT = 4; const SPACE = SIDE ** DIMENSION_COUNT; +/** The searches of the whole space that the test of a signal that never aborts makes: a count, then a collect and a diagnose, each of the last two once more without a signal. */ +const WHOLE_SPACE_SEARCHES = 5; +/** The time a search may spend per combination of the space, in milliseconds, whatever the number of searches that visit it: at least two hundred times what it takes on an idle machine, so a machine under heavy load still passes and only a search that stops terminating fails. */ +const COMBINATION_BUDGET_MS = 2; /** Predicate evaluations before the signal aborts: well inside the first leaf of the search, so the rest of the space is still to do. */ const ABORT_AT = 5; @@ -139,7 +144,7 @@ describe("a signal that aborts mid-search", () => { expect(evaluations.total - before).toBeLessThan(STEPS_PER_ABORT_CHECK); }); - it("completes with the same result as an unsignalled run when the signal never aborts", () => { + it("completes with the same result as an unsignalled run when the signal never aborts", budgetFor(SPACE * WHOLE_SPACE_SEARCHES, COMBINATION_BUDGET_MS), () => { const { model } = abortingModel(SPACE + 1); const signal = new AbortController().signal; expect(count(model, { signal })).toBe(SPACE); diff --git a/src/condition.unit.test.ts b/src/condition.unit.test.ts index 9563978..b2cd553 100644 --- a/src/condition.unit.test.ts +++ b/src/condition.unit.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { budgetFor } from "../test/budget"; import type { Condition, Constraint, Rule, Value, Verdict } from "./index"; import { allOf, anyOf, choice, defineModel, is, isSet, isUnset, not, oneOf, ruleTree } from "./index"; @@ -102,10 +103,16 @@ describe("a condition evaluated by the engine", () => { expect(verdict(anyOf(), new Map())).toBe("invalid"); }); - it("is monotone: a definite answer on a partial assignment is the answer on every completion", () => { + const everyCondition = conditions(2); + const cases = partialAssignments().map((assignment) => ({ assignment, completions: completions(assignment) })); + // Each condition is evaluated on every partial assignment and on each of its completions. + const evaluations = everyCondition.length * cases.reduce((total, { completions: everyCompletion }) => total + 1 + everyCompletion.length, 0); + /** The time one evaluation may take, in milliseconds: at least two hundred times what it takes on an idle machine, so a machine under heavy load still passes and only an evaluation that stops terminating fails. */ + const EVALUATION_BUDGET_MS = 1; + + it("is monotone: a definite answer on a partial assignment is the answer on every completion", budgetFor(evaluations, EVALUATION_BUDGET_MS), () => { const violations: string[] = []; - const cases = partialAssignments().map((assignment) => ({ assignment, completions: completions(assignment) })); - for (const condition of conditions(2)) { + for (const condition of everyCondition) { const constraint = constraintOf(condition); for (const { assignment, completions: everyCompletion } of cases) { const answer = constraint.evaluate(assignment); diff --git a/src/counting.unit.test.ts b/src/counting.unit.test.ts index f8a7c53..5d90eb0 100644 --- a/src/counting.unit.test.ts +++ b/src/counting.unit.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { budgetFor, candidateCount } from "../test/budget"; import { hashTest, Random, randomModel, seeds, VALUE_POOL } from "../test/random-model"; import type { Constraint, Diagnosis, Dimension, Model, ModelParts, Value } from "./index"; import { at, choice, computed, count, defineModel, diagnose, firstValid, indexOf, isNonEmpty, isSet, LimitExceededError, predicate } from "./index"; @@ -59,13 +60,34 @@ function lateModel(seed: number): AnyModel { return defineModel({ dimensions, constraints }); } +interface Labelled { + readonly label: string; + readonly model: AnyModel; +} + +const randomModels: readonly Labelled[] = seeds.map((seed) => ({ label: `random ${String(seed)}`, model: randomModel(seed) })); +const lateModels: readonly Labelled[] = seeds.map((seed) => ({ label: `late ${String(seed)}`, model: lateModel(seed) })); /** Every model the equivalence properties are checked against: the generated models of every feature, and the late-constrained ones. */ -const models: readonly { readonly label: string; readonly model: AnyModel }[] = [...seeds.map((seed) => ({ label: `random ${String(seed)}`, model: randomModel(seed) })), ...seeds.map((seed) => ({ label: `late ${String(seed)}`, model: lateModel(seed) }))]; +const models: readonly Labelled[] = [...randomModels, ...lateModels]; + +/** The combinations the dimensions of `labelled` models could take with the constraints ignored, summed: an upper bound on the leaves a search of each model reaches, and so the work of a test that searches every one of them. */ +function candidatesOf(labelled: readonly Labelled[]): number { + return labelled.reduce((total, { model }) => total + candidateCount(model), 0); +} + +/** The time one candidate may take a test, in milliseconds, whatever the number of searches the test makes through it: at least two hundred times what the heaviest of the tests takes on an idle machine, so a machine under heavy load still passes and only a search that stops terminating fails. */ +const CANDIDATE_BUDGET_MS = 4; +/** The options of a test that searches every model. */ +const EVERY_MODEL = budgetFor(candidatesOf(models), CANDIDATE_BUDGET_MS); +/** The options of a test that searches the late-constrained models alone. */ +const LATE_MODELS = budgetFor(candidatesOf(lateModels), CANDIDATE_BUDGET_MS); -/** The time one rank or unrank may take, in milliseconds: about forty times what it takes unloaded, so a machine under heavy load still passes and only a search that stops terminating fails. */ -const RANK_OPERATION_BUDGET_MS = 10; +/** The time one rank or unrank may take, in milliseconds: at least two hundred times what it takes on an idle machine, so a machine under heavy load still passes and only a search that stops terminating fails. */ +const RANK_OPERATION_BUDGET_MS = 125; /** The ranks and unranks the exhaustive agreement test makes: one per index of every model, and one more past the last. */ const RANK_OPERATIONS = models.reduce((total, { model }) => total + count(model) + 1, 0); +/** The options of the exhaustive agreement test. */ +const EVERY_INDEX = budgetFor(RANK_OPERATIONS, RANK_OPERATION_BUDGET_MS); /** The diagnosis by the plain search: every combination produced and its values collected, as `diagnose` worked before it counted through the memo. `"limit"` when the solution set has more than `limit` combinations. */ function referenceDiagnosis(model: ModelParts, limit?: number): Diagnosis | "limit" { @@ -105,13 +127,13 @@ function countDisagreements(searchOf: (model: ModelParts) => CountingSearch): st } describe("the memoised count", () => { - it("generates late-constrained models whose solution sets are neither all empty nor all full", () => { + it("generates late-constrained models whose solution sets are neither all empty nor all full", LATE_MODELS, () => { const sizes = seeds.map((seed) => count(lateModel(seed))); expect(sizes.some((size) => size === 0)).toBe(true); expect(sizes.some((size) => size > 1)).toBe(true); }); - it("counts what the naive search counts, and refuses past a limit at the same boundary, on every model", () => { + it("counts what the naive search counts, and refuses past a limit at the same boundary, on every model", EVERY_MODEL, () => { const disagreements = models.filter(({ model }) => { const size = countSearch(model, undefined, { completions: naiveCompletions }); @@ -120,7 +142,7 @@ describe("the memoised count", () => { expect(disagreements).toEqual([]); }); - it("diagnoses what the naive search diagnoses, the status, count and dead values alike, and refuses past a limit where it does", () => { + it("diagnoses what the naive search diagnoses, the status, count and dead values alike, and refuses past a limit where it does", EVERY_MODEL, () => { const disagreements = models.filter(({ model }) => { const expected = referenceDiagnosis(model); const size = expected === "limit" || expected.status === "empty" ? 0 : expected.count; @@ -130,7 +152,7 @@ describe("the memoised count", () => { expect(disagreements).toEqual([]); }); - it("ranks and unranks as the naive count does, at every index and for every combination", { timeout: RANK_OPERATIONS * RANK_OPERATION_BUDGET_MS }, () => { + it("ranks and unranks as the naive count does, at every index and for every combination", EVERY_INDEX, () => { const disagreements = models.filter(({ model }) => { const size = count(model); for (let index = 0; index <= size; index += 1) { @@ -144,7 +166,7 @@ describe("the memoised count", () => { expect(disagreements).toEqual([]); }); - it("is caught by the equivalence property when the frontier leaves out a dimension it reads", () => { + it("is caught by the equivalence property when the frontier leaves out a dimension it reads", EVERY_MODEL, () => { const dropFirst = (model: ModelParts): CountingSearch => { const search = countingSearch(model, undefined); diff --git a/src/covering.unit.test.ts b/src/covering.unit.test.ts index 1f781b6..03f7fb6 100644 --- a/src/covering.unit.test.ts +++ b/src/covering.unit.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { budgetFor, candidateCount, subsetCount } from "../test/budget"; import { greedyCoveringArray } from "../test/greedy-reference"; import { randomModel, seeds } from "../test/random-model"; import type { Model, Strategy, Value } from "./index"; @@ -16,8 +17,20 @@ const MAX_STRENGTH = 3; const STRENGTHS = Array.from({ length: MAX_STRENGTH }, (_, index) => index + 1); /** The share of the generated models the slower whole-model checks run on. */ const SAMPLE_DIVISOR = 10; +/** How many generated models the slower whole-model checks run on. */ +const SAMPLE = MODELS / SAMPLE_DIVISOR; +/** The strength of the pairwise checks. */ +const PAIRWISE = 2; const SIZES = ["small", "medium", "large"] as const; +/** The tuples a test of the given strength observes in the first `models` generated models at most: each combination any of them could take, once for every set of that many dimensions it holds a tuple of. */ +function tupleObservations(strength: number, models = MODELS): number { + return generated().slice(0, models).reduce((total, { model }) => total + candidateCount(model) * subsetCount(model.dimensions.length, Math.min(strength, model.dimensions.length)), 0); +} + +/** The time a test may spend per tuple observed, in milliseconds: at least two hundred times what the heaviest of them takes on an idle machine, so a machine under heavy load still passes and only a construction that stops terminating fails. */ +const OBSERVATION_BUDGET_MS = 4; + type Row = Readonly>; function rowsOf(model: Model, strategy: Strategy): Row[] { @@ -145,25 +158,23 @@ describe("coveringArray", () => { ); }); - it("never reads the solution set: the combinations it asks the engine for are bounded by the t-tuple count", () => { - // Unconstrained, so the solution set is the full product of the domains while the pairs grow only with the square of the dimension count. - const dimensionCount = 14; - const domain = [0, 1, 2] as const; - const strength = 2; + // Unconstrained, so the solution set is the full product of the domains while the pairs grow only with the square of the dimension count. + const dimensionCount = 14; + const domain = [0, 1, 2] as const; + const strength = 2; + const tupleCount = subsetCount(dimensionCount, strength) * domain.length ** strength; + // The construction's engine queries, from its structure: one per candidate tuple when the tuples are listed (the first t dimensions' included), and per later dimension at most one per row and value plus one fallback in horizontal growth and one per row and uncovered tuple in vertical growth, plus one completion per row. Every row holds a tuple no earlier row did, so there are at most tupleCount rows. + const queryBound = tupleCount + dimensionCount * tupleCount * (domain.length + 1) + tupleCount * tupleCount + tupleCount; + /** The time one engine query may take, in milliseconds: at least two hundred times what the construction spends per query of the bound on an idle machine, so a machine under heavy load still passes and only a construction that stops terminating fails. */ + const QUERY_BUDGET_MS = 0.1; + + it("never reads the solution set: the combinations it asks the engine for are bounded by the t-tuple count", budgetFor(queryBound, QUERY_BUDGET_MS), () => { const model = defineModel({ dimensions: Array.from({ length: dimensionCount }, (_, index) => choice(`d${String(index)}`, domain), ), }); const solutions = count(model, { limit: Number.MAX_SAFE_INTEGER }); - const tupleCount = - subsetCount(dimensionCount, strength) * domain.length ** strength; - // The construction's engine queries, from its structure: one per candidate tuple when the tuples are listed (the first t dimensions' included), and per later dimension at most one per row and value plus one fallback in horizontal growth and one per row and uncovered tuple in vertical growth, plus one completion per row. Every row holds a tuple no earlier row did, so there are at most tupleCount rows. - const queryBound = - tupleCount + - dimensionCount * tupleCount * (domain.length + 1) + - tupleCount * tupleCount + - tupleCount; expect(queryBound).toBeLessThan(solutions); const work = { queries: 0, combinations: 0 }; @@ -193,37 +204,30 @@ describe("coveringArray", () => { }); }); -/** The number of ways to choose `size` of `total` items. */ -function subsetCount(total: number, size: number): number { - let result = 1; - for (let chosen = 0; chosen < size; chosen += 1) { - result = (result * (total - chosen)) / (chosen + 1); - } - - return result; -} - describe("coveringArray against the greedy reference", () => { - it.each(STRENGTHS)( - "is no larger in total than the greedy reference over the generated models at strength %i, and never more rows than coverable tuples", - (strength) => { - let rowsTotal = 0; - let referenceTotal = 0; - for (const { model, seed } of generated()) { - const rows = rowsOf(model, coveringArray({ strength })); - const reference = rowsOf( - model, - greedyCoveringArray({ strength, limit: LIMIT }), - ); - rowsTotal += rows.length; - referenceTotal += reference.length; - expect(rows.length, `seed ${String(seed)}`).toBeLessThanOrEqual( - coverableTupleCount(model, strength), - ); - } - expect(rowsTotal).toBeLessThanOrEqual(referenceTotal); - }, - ); + for (const strength of STRENGTHS) { + it( + `is no larger in total than the greedy reference over the generated models at strength ${String(strength)}, and never more rows than coverable tuples`, + budgetFor(tupleObservations(strength), OBSERVATION_BUDGET_MS), + () => { + let rowsTotal = 0; + let referenceTotal = 0; + for (const { model, seed } of generated()) { + const rows = rowsOf(model, coveringArray({ strength })); + const reference = rowsOf( + model, + greedyCoveringArray({ strength, limit: LIMIT }), + ); + rowsTotal += rows.length; + referenceTotal += reference.length; + expect(rows.length, `seed ${String(seed)}`).toBeLessThanOrEqual( + coverableTupleCount(model, strength), + ); + } + expect(rowsTotal).toBeLessThanOrEqual(referenceTotal); + }, + ); + } }); /** The number of distinct t-tuples the model's solution set contains, read from the whole solution set. */ @@ -241,34 +245,38 @@ function coverableTupleCount(model: Model, strength: number): number { } describe("coveringArray against the isCoveringArray oracle", () => { - it.each(STRENGTHS)( - "is a covering array of strength %i on every generated model, of valid, distinct rows", - (strength) => { - for (const { model, seed } of generated()) { - const rows = rowsOf(model, coveringArray({ strength, limit: LIMIT })); - expect( - isCoveringArray(rows, model, { strength, limit: LIMIT }), - `seed ${String(seed)}`, - ).toBe(true); - // `tupleKey` keeps unset and `null` apart, which `JSON.stringify` of an `undefined` value would not. - expect( - new Set(rows.map((row) => tupleKey(Object.entries(row)))).size, - `seed ${String(seed)}`, - ).toBe(rows.length); - expect(rows.length).toBeLessThanOrEqual(count(model, { limit: LIMIT })); - } - }, - ); + for (const strength of STRENGTHS) { + it( + `is a covering array of strength ${String(strength)} on every generated model, of valid, distinct rows`, + budgetFor(tupleObservations(strength), OBSERVATION_BUDGET_MS), + () => { + for (const { model, seed } of generated()) { + const rows = rowsOf(model, coveringArray({ strength, limit: LIMIT })); + expect( + isCoveringArray(rows, model, { strength, limit: LIMIT }), + `seed ${String(seed)}`, + ).toBe(true); + // `tupleKey` keeps unset and `null` apart, which `JSON.stringify` of an `undefined` value would not. + expect( + new Set(rows.map((row) => tupleKey(Object.entries(row)))).size, + `seed ${String(seed)}`, + ).toBe(rows.length); + expect(rows.length).toBeLessThanOrEqual(count(model, { limit: LIMIT })); + } + }, + ); + } - it("yields the same rows in the same order every time", () => { - for (const { model } of generated().slice(0, MODELS / SAMPLE_DIVISOR)) { + it("yields the same rows in the same order every time", budgetFor(tupleObservations(PAIRWISE, SAMPLE), OBSERVATION_BUDGET_MS), () => { + for (const { model } of generated().slice(0, SAMPLE)) { const strategy = coveringArray({ strength: 2, limit: LIMIT }); expect(rowsOf(model, strategy)).toEqual(rowsOf(model, strategy)); } }); - it("yields the whole solution set at a strength of at least the dimension count", () => { - for (const { model } of generated().slice(0, MODELS / SAMPLE_DIVISOR)) { + // A strength equal to the dimension count observes each combination once, at most what the strongest strength checked does. + it("yields the whole solution set at a strength of at least the dimension count", budgetFor(tupleObservations(MAX_STRENGTH, SAMPLE), OBSERVATION_BUDGET_MS), () => { + for (const { model } of generated().slice(0, SAMPLE)) { const strength = model.dimensions.length; expect( rowsOf(model, coveringArray({ strength, limit: LIMIT })), @@ -276,12 +284,9 @@ describe("coveringArray against the isCoveringArray oracle", () => { } }); - it("the oracle refuses the strategy's output with any one row dropped", () => { + it("the oracle refuses the strategy's output with any one row dropped", budgetFor(tupleObservations(PAIRWISE, SAMPLE), OBSERVATION_BUDGET_MS), () => { let checked = 0; - for (const { model, seed } of generated().slice( - 0, - MODELS / SAMPLE_DIVISOR, - )) { + for (const { model, seed } of generated().slice(0, SAMPLE)) { const rows = rowsOf(model, coveringArray({ strength: 2, limit: LIMIT })); // Redundant rows are removed, so every row holds a tuple no other row does, and without it that tuple is uncovered. for (const [index] of rows.entries()) { diff --git a/src/rank.unit.test.ts b/src/rank.unit.test.ts index 942857e..1b92cb2 100644 --- a/src/rank.unit.test.ts +++ b/src/rank.unit.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { budgetFor, GENERATED_CANDIDATES } from "../test/budget"; import { MAX_FREE_DIMENSIONS, Random, randomModel, seeds, VALUE_POOL } from "../test/random-model"; import type { Combination, Model, Value } from "./index"; import { at, choice, combinations, computed, count, defineModel, explainData, firstValid, indexOf, indexOfData, is, isNonEmpty, isSet, predicate, samples } from "./index"; @@ -9,6 +10,11 @@ import { valueKey } from "./value"; type AnyModel = Model>>; +/** The time a test may spend per candidate combination of the generated models, in milliseconds, whatever the number of times the test ranks, unranks or scans it: at least two hundred times what the heaviest of them takes on an idle machine, so a machine under heavy load still passes and only a search that stops terminating fails. */ +const CANDIDATE_BUDGET_MS = 25; +/** The options of a test that visits every generated model. */ +const GENERATED_MODELS = budgetFor(GENERATED_CANDIDATES, CANDIDATE_BUDGET_MS); + /** How many draws each sampling property takes from a generated model. */ const DRAWS = 8; /** The values a generated computed dimension's function, which counts the set free dimensions it reads, can return. */ @@ -80,7 +86,7 @@ const offByOne: CompletionCount = (search, depth, assignment) => { }; describe("at", () => { - it("is the combination combinations produces at each index, with its id, for every index of every generated model", () => { + it("is the combination combinations produces at each index, with its id, for every index of every generated model", GENERATED_MODELS, () => { const disagreements = seeds.filter((seed) => { const model = randomModel(seed); const expected = expectedOf(model); @@ -90,7 +96,7 @@ describe("at", () => { expect(disagreements).toEqual([]); }); - it("reports an index at or past the end as undefined, the end being count(model)", () => { + it("reports an index at or past the end as undefined, the end being count(model)", GENERATED_MODELS, () => { for (const seed of seeds) { const model = randomModel(seed); const size = count(model); @@ -105,7 +111,7 @@ describe("at", () => { for (const index of [-1, FRACTION, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1]) expect(() => at(model, index)).toThrow(RangeError); }); - it("is caught by the property when a subtree count is off by one", () => { + it("is caught by the property when a subtree count is off by one", GENERATED_MODELS, () => { const caught = seeds.filter((seed) => { const model = randomModel(seed); const expected = expectedOf(model); @@ -127,7 +133,7 @@ describe("at", () => { }); describe("indexOf and indexOfData", () => { - it("invert at: the index of the combination at each index is that index, for every generated model", () => { + it("invert at: the index of the combination at each index is that index, for every generated model", GENERATED_MODELS, () => { const disagreements = seeds.filter((seed) => { const model = randomModel(seed); @@ -136,7 +142,7 @@ describe("indexOf and indexOfData", () => { expect(disagreements).toEqual([]); }); - it("are the linear-scan index of every near miss of every combination, -1 exactly when explainData says it is not valid", () => { + it("are the linear-scan index of every near miss of every combination, -1 exactly when explainData says it is not valid", GENERATED_MODELS, () => { const disagreements = seeds.filter((seed) => { const model = randomModel(seed); const expected = expectedOf(model); @@ -153,7 +159,7 @@ describe("indexOf and indexOfData", () => { expect(disagreements).toEqual([]); }); - it("accept data that leaves out the derived dimensions, as explainData does", () => { + it("accept data that leaves out the derived dimensions, as explainData does", GENERATED_MODELS, () => { for (const seed of seeds) { const model = randomModel(seed); const derived = model.dimensions.filter((dimension) => dimension.kind !== "choice").map((dimension) => dimension.name); @@ -174,7 +180,7 @@ describe("indexOf and indexOfData", () => { } }); - it("are caught by the property when a subtree count is off by one", () => { + it("are caught by the property when a subtree count is off by one", GENERATED_MODELS, () => { const caught = seeds.filter((seed) => { const model = randomModel(seed); @@ -185,7 +191,7 @@ describe("indexOf and indexOfData", () => { }); describe("samples", () => { - it("draws the combinations at the indices a seeded source returns, each bounded by count(model)", () => { + it("draws the combinations at the indices a seeded source returns, each bounded by count(model)", GENERATED_MODELS, () => { for (const seed of seeds) { const model = randomModel(seed); const expected = expectedOf(model); diff --git a/src/search.unit.test.ts b/src/search.unit.test.ts index c631459..d35171f 100644 --- a/src/search.unit.test.ts +++ b/src/search.unit.test.ts @@ -1,11 +1,17 @@ import { describe, expect, it } from "vitest"; +import { budgetFor, GENERATED_CANDIDATES } from "../test/budget"; import { randomModel, seeds } from "../test/random-model"; import type { ModelParts, Value } from "./index"; import { choice, defineModel, isNonEmpty, predicate } from "./index"; import { referenceCombinations } from "./reference"; import { countSearch, searchCombinations } from "./search"; +/** The time a test may spend per candidate combination of the generated models, in milliseconds, whatever the number of times the test searches it or enumerates it by the reference: at least two hundred times what the heaviest of them takes on an idle machine, so a machine under heavy load still passes and only a search that stops terminating fails. */ +const CANDIDATE_BUDGET_MS = 10; +/** The options of a test that visits every generated model. */ +const GENERATED_MODELS = budgetFor(GENERATED_CANDIDATES, CANDIDATE_BUDGET_MS); + /** Whether a search agrees with the reference on a model: the same records in the same order. */ function agrees(search: (model: ModelParts) => Iterable>, model: ModelParts, reference: ModelParts = model): boolean { const expected = JSON.stringify([...referenceCombinations(reference)].map((record) => Object.entries(record))); @@ -23,18 +29,18 @@ function prunesOnUnknown(model: ModelParts): ModelParts { describe("searchCombinations", () => { - it("yields exactly the reference's combinations, in the reference's order, for every generated model", () => { + it("yields exactly the reference's combinations, in the reference's order, for every generated model", GENERATED_MODELS, () => { const disagreements = seeds.filter((seed) => !agrees(searchCombinations, randomModel(seed))); expect(disagreements).toEqual([]); }); - it("generates models whose solution sets are neither all empty nor all full", () => { + it("generates models whose solution sets are neither all empty nor all full", GENERATED_MODELS, () => { const sizes = seeds.map((seed) => [...referenceCombinations(randomModel(seed))].length); expect(sizes.some((size) => size === 0)).toBe(true); expect(sizes.some((size) => size > 1)).toBe(true); }); - it("is caught by the property when its pruning is broken", () => { + it("is caught by the property when its pruning is broken", GENERATED_MODELS, () => { const caught = seeds.filter((seed) => { const model = randomModel(seed); @@ -43,7 +49,7 @@ describe("searchCombinations", () => { expect(caught.length).toBeGreaterThan(0); }); - it("restricts each dimension a fixed assignment names to that value", () => { + it("restricts each dimension a fixed assignment names to that value", GENERATED_MODELS, () => { for (const seed of seeds) { const model = randomModel(seed); const [first] = model.dimensions; @@ -59,7 +65,7 @@ describe("searchCombinations", () => { }); describe("countSearch", () => { - it("counts exactly the reference's combinations for every generated model", () => { + it("counts exactly the reference's combinations for every generated model", GENERATED_MODELS, () => { const disagreements = seeds.filter((seed) => { const model = randomModel(seed); @@ -68,7 +74,7 @@ describe("countSearch", () => { expect(disagreements).toEqual([]); }); - it("reports going past the limit, and not reaching it", () => { + it("reports going past the limit, and not reaching it", GENERATED_MODELS, () => { for (const seed of seeds) { const model = randomModel(seed); const size = [...referenceCombinations(model)].length; diff --git a/src/trilean.unit.test.ts b/src/trilean.unit.test.ts index c098611..033ebab 100644 --- a/src/trilean.unit.test.ts +++ b/src/trilean.unit.test.ts @@ -2,6 +2,7 @@ import type { ExpressionNode, PredicateNode } from "trilean"; import { DEFAULT_MAX_EVALUATION_NODES, DEFAULT_MAX_NESTING_DEPTH } from "trilean/evaluator-budget"; import { describe, expect, it } from "vitest"; +import { budgetFor } from "../test/budget"; import type { Condition, Dimension, Value } from "./index"; import { choice, collect, computed, ConstraintContractError, defineModel, firstValid, is, isCondition, isSet, ModelError, UndecidedConstraintError, UnmappableValueError } from "./index"; @@ -106,7 +107,12 @@ describe("defineModel's refusal of a tree no assignment of dimensions can decide expect(issuesOf(isSet("m"), [choice<"m", "a" | 1>("m", ["a", 1])])).toEqual([]); }); - it("refuses a tree past trilean's evaluation budget, by node count and by depth", () => { + // The nodes the two trees below hold: the wide one is a conjunction of `exists` over a reference, and the deep one the same under negations. + const treeNodes = DEFAULT_MAX_EVALUATION_NODES * 2 + 1 + DEFAULT_MAX_NESTING_DEPTH + 1; + /** The time one node of a tree may take to check, in milliseconds: at least two hundred times what it takes on an idle machine, so a machine under heavy load still passes and only a check that stops terminating fails. */ + const NODE_BUDGET_MS = 2; + + it("refuses a tree past trilean's evaluation budget, by node count and by depth", budgetFor(treeNodes, NODE_BUDGET_MS), () => { const wide: Condition = { kind: "allOf", operands: Array.from({ length: DEFAULT_MAX_EVALUATION_NODES }, () => exists("mode")) }; expect(issuesOf(wide)).toEqual([`rule 0 has ${String(DEFAULT_MAX_EVALUATION_NODES * 2 + 1)} nodes, more than the ${String(DEFAULT_MAX_EVALUATION_NODES)} one trilean evaluation may visit`]); let deep: Condition = exists("mode"); diff --git a/test/budget.ts b/test/budget.ts new file mode 100644 index 0000000..6ac2b18 --- /dev/null +++ b/test/budget.ts @@ -0,0 +1,35 @@ +import { domainOf } from "../src/dimension"; +import type { ModelParts } from "../src/index"; +import { randomModel, seeds } from "./random-model"; + +/** + * The options argument of `it` for a test that does `operations` units of work, each allowed `perOperationMs`: a timeout derived from the work, so that a loaded machine passes and only a test that does far more work per unit than its budget names, such as a search that stops terminating, fails. + * + * A timeout of zero disables vitest's timeout, so an operation count or budget that is not positive is refused rather than turned into a test that never times out. + */ +export function budgetFor(operations: number, perOperationMs: number): { readonly timeout: number } { + const timeout = Math.ceil(operations * perOperationMs); + if (!(timeout > 0) || !Number.isFinite(timeout)) throw new RangeError(`a test budget needs a positive, finite number of operations and milliseconds each, not ${String(operations)} and ${String(perOperationMs)}`); + + return { timeout }; +} + +/** The number of ways to choose `size` of `total` items. */ +export function subsetCount(total: number, size: number): number { + let result = 1; + for (let chosen = 0; chosen < size; chosen += 1) { + result = (result * (total - chosen)) / (chosen + 1); + } + + return result; +} + +/** + * The combinations the model's dimensions could take with every constraint ignored: the product of the options each dimension offers, unset counted for one that is conditionally active and a derived-by-function dimension counted once. An upper bound on the leaves any search of the model reaches, and so on the combinations it yields. + */ +export function candidateCount(model: ModelParts): number { + return model.dimensions.reduce((total, dimension) => total * Math.max(1, domainOf(dimension).length + (dimension.activeWhen === undefined ? 0 : 1)), 1); +} + +/** The candidates of every model `randomModel` generates, summed: the work of a test that visits each generated model once, for a test that visits it more than once to scale by its own passes. */ +export const GENERATED_CANDIDATES = seeds.reduce((total, seed) => total + candidateCount(randomModel(seed)), 0);