diff --git a/.github/workflows/conflict-check.yml b/.github/workflows/conflict-check.yml index 95db95b..3869039 100644 --- a/.github/workflows/conflict-check.yml +++ b/.github/workflows/conflict-check.yml @@ -2,53 +2,33 @@ name: Conflict Check on: pull_request: - branches: [main] - types: [opened, synchronize, reopened] + branches: [main, master, develop] + push: + branches: [main, master, develop] jobs: - check-conflicts: + conflict-check: + name: Check for merge conflicts runs-on: ubuntu-latest - permissions: - pull-requests: write steps: - - name: Check for merge conflicts - uses: actions/github-script@v7 + - name: Checkout repository + uses: actions/checkout@v4 with: - script: | - let mergeable = null; - let attempts = 0; + fetch-depth: 0 - // Poll until GitHub computes mergeability (can take a few seconds) - while (mergeable === null && attempts < 10) { - await new Promise(r => setTimeout(r, 3000)); - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - mergeable = pr.mergeable; - attempts++; - } - - if (mergeable === false) { - // Avoid posting duplicate comments - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - - const alreadyCommented = comments.some(c => - c.body.includes('Merge Conflict Detected') && - c.user.login === 'github-actions[bot]' - ); - - if (!alreadyCommented) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: `⚠️ **Merge Conflict Detected**\n\nThis PR has conflicts with \`main\` that must be resolved before it can be merged.\n\n**Steps to fix:**\n\`\`\`bash\ngit checkout main\ngit pull origin main\ngit checkout your-branch\ngit merge main\n# resolve conflicts in your editor\ngit add .\ngit commit\ngit push\n\`\`\`\n\nIf you need help resolving the conflict, leave a comment and the maintainer will assist.\n\n---\n*🤖 Automated conflict check*` - }); - } - } + - name: Detect merge conflicts + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE="origin/${{ github.base_ref }}" + HEAD="${{ github.event.pull_request.head.sha }}" + else + BASE="origin/${{ github.ref_name }}" + HEAD="${{ github.sha }}" + fi + git fetch origin "${BASE#origin/}" --depth=1 || true + if git merge-tree "$(git merge-base "$BASE" "$HEAD")" "$BASE" "$HEAD" | grep -q '<<<<<<<'; then + echo "::error::Merge conflicts detected against $BASE" + exit 1 + fi + echo "No merge conflicts detected." diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index f4dec84..de7422d 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -1,4 +1,4 @@ -name: Integration Test +name: Integration Tests on: schedule: @@ -7,26 +7,43 @@ on: jobs: integration-test: + name: Integration Tests runs-on: ubuntu-latest - permissions: - contents: read + timeout-minutes: 30 + steps: - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 with: - node-version: "20" - cache: "npm" + node-version: '20' + cache: 'npm' - name: Install dependencies run: npm ci - - name: Run integration tests (testnet only) - env: - STELLAR_NETWORK: testnet - STELLAR_SPLIT_CONTRACT_ID: ${{ secrets.STELLAR_SPLIT_CONTRACT_ID }} - STELLAR_SPLIT_TOKEN_CONTRACT_ID: ${{ secrets.STELLAR_SPLIT_TOKEN_CONTRACT_ID }} + - name: Run integration tests run: npm run test:integration + env: + CI: true + + - name: Run unit tests with coverage + run: npm run test:coverage + env: + CI: true + + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage/** + coverage/lcov.info + if-no-files-found: ignore + retention-days: 14 fuzz: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index fd63ce2..6a0710b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,7 +31,20 @@ jobs: - name: Type check (strict) run: npx --no-install tsc --noEmit + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: sdk-dist + path: dist + if-no-files-found: error + retention-days: 30 + - name: Publish run: npm publish --provenance --access public env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Notify on failure + if: failure() + run: | + echo "::error::Publish pipeline failed for ${{ github.ref_name }}. See ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/load-tests/baseline.json b/load-tests/baseline.json new file mode 100644 index 0000000..c126998 --- /dev/null +++ b/load-tests/baseline.json @@ -0,0 +1,57 @@ +{ + "version": 1, + "description": "Performance baselines for SDK load tests. Update via `npm run load-test -- --update-baseline` after intentional performance changes.", + "updatedAt": "2024-01-01T00:00:00.000Z", + "scenarios": { + "invoice-split": { + "description": "Split a single invoice across multiple recipients", + "concurrency": 10, + "durationMs": 30000, + "baseline": { + "throughputOpsPerSec": 250, + "latencyMs": { + "p50": 35, + "p90": 60, + "p95": 80, + "p99": 120 + }, + "errorRate": 0.001 + } + }, + "batch-payments": { + "description": "Submit batches of payments through the SDK", + "concurrency": 20, + "durationMs": 30000, + "baseline": { + "throughputOpsPerSec": 120, + "latencyMs": { + "p50": 90, + "p90": 150, + "p95": 200, + "p99": 320 + }, + "errorRate": 0.002 + } + }, + "webhook-handling": { + "description": "Process inbound webhook events", + "concurrency": 50, + "durationMs": 30000, + "baseline": { + "throughputOpsPerSec": 800, + "latencyMs": { + "p50": 10, + "p90": 20, + "p95": 30, + "p99": 55 + }, + "errorRate": 0.0005 + } + } + }, + "regressionThresholds": { + "throughputDropPct": 15, + "latencyIncreasePct": 20, + "errorRateIncreasePct": 50 + } +} diff --git a/load-tests/generators.ts b/load-tests/generators.ts new file mode 100644 index 0000000..c541745 --- /dev/null +++ b/load-tests/generators.ts @@ -0,0 +1,175 @@ +/** + * Load generators for the load testing and benchmarking suite (#1190). + * + * Drives configurable concurrency/duration against representative SDK + * scenarios and collects per-request latency samples so the reporting and + * baseline layers can compute throughput, percentiles and error rates. + */ + +export interface LoadScenario { + /** Stable identifier used in reports and baselines. */ + name: string; + /** Human readable description of what the scenario exercises. */ + description: string; + /** A single unit of work executed by the generator. */ + run: () => Promise; +} + +export interface LoadGeneratorOptions { + /** Number of concurrent workers. Defaults to 1. */ + concurrency?: number; + /** Total wall-clock duration in milliseconds. Defaults to 1000. */ + durationMs?: number; + /** Optional cap on total iterations across all workers. */ + maxIterations?: number; + /** Optional delay between iterations per worker, in milliseconds. */ + thinkTimeMs?: number; +} + +export interface LoadSample { + /** Latency of the iteration in milliseconds. */ + latencyMs: number; + /** Whether the iteration completed without throwing. */ + success: boolean; + /** Error message when the iteration failed. */ + error?: string; +} + +export interface LoadResult { + scenario: string; + concurrency: number; + durationMs: number; + /** Total iterations attempted (successful + failed). */ + iterations: number; + /** Iterations that completed without throwing. */ + successes: number; + /** Iterations that threw. */ + failures: number; + /** Successful iterations per second over the measured window. */ + throughput: number; + /** Fraction of failed iterations in [0, 1]. */ + errorRate: number; + /** Latency percentiles in milliseconds (successful iterations only). */ + latency: { + min: number; + mean: number; + p50: number; + p90: number; + p95: number; + p99: number; + max: number; + }; + /** Raw samples for downstream trend analysis. */ + samples: LoadSample[]; +} + +const DEFAULT_CONCURRENCY = 1; +const DEFAULT_DURATION_MS = 1000; + +function sleep(ms: number): Promise { + if (ms <= 0) return Promise.resolve(); + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function percentile(sorted: number[], p: number): number { + if (sorted.length === 0) return 0; + if (sorted.length === 1) return sorted[0]; + const rank = (p / 100) * (sorted.length - 1); + const lower = Math.floor(rank); + const upper = Math.ceil(rank); + if (lower === upper) return sorted[lower]; + const weight = rank - lower; + return sorted[lower] * (1 - weight) + sorted[upper] * weight; +} + +function summarizeLatency(samples: LoadSample[]): LoadResult['latency'] { + const latencies = samples + .filter((s) => s.success) + .map((s) => s.latencyMs) + .sort((a, b) => a - b); + + if (latencies.length === 0) { + return { min: 0, mean: 0, p50: 0, p90: 0, p95: 0, p99: 0, max: 0 }; + } + + const sum = latencies.reduce((acc, v) => acc + v, 0); + return { + min: latencies[0], + mean: sum / latencies.length, + p50: percentile(latencies, 50), + p90: percentile(latencies, 90), + p95: percentile(latencies, 95), + p99: percentile(latencies, 99), + max: latencies[latencies.length - 1], + }; +} + +/** + * Run a scenario under a fixed concurrency for a bounded duration and + * collect latency/error samples for reporting and baseline comparison. + */ +export async function runLoadGenerator( + scenario: LoadScenario, + options: LoadGeneratorOptions = {}, +): Promise { + const concurrency = Math.max(1, Math.floor(options.concurrency ?? DEFAULT_CONCURRENCY)); + const durationMs = Math.max(0, options.durationMs ?? DEFAULT_DURATION_MS); + const maxIterations = options.maxIterations ?? Number.POSITIVE_INFINITY; + const thinkTimeMs = Math.max(0, options.thinkTimeMs ?? 0); + + const samples: LoadSample[] = []; + const deadline = Date.now() + durationMs; + let iterations = 0; + + const worker = async (): Promise => { + while (Date.now() < deadline && iterations < maxIterations) { + iterations += 1; + const startedAt = Date.now(); + try { + await scenario.run(); + samples.push({ latencyMs: Date.now() - startedAt, success: true }); + } catch (error) { + samples.push({ + latencyMs: Date.now() - startedAt, + success: false, + error: error instanceof Error ? error.message : String(error), + }); + } + if (thinkTimeMs > 0) await sleep(thinkTimeMs); + } + }; + + const startedAt = Date.now(); + await Promise.all(Array.from({ length: concurrency }, () => worker())); + const elapsedMs = Math.max(1, Date.now() - startedAt); + + const successes = samples.filter((s) => s.success).length; + const failures = samples.length - successes; + + return { + scenario: scenario.name, + concurrency, + durationMs: elapsedMs, + iterations: samples.length, + successes, + failures, + throughput: (successes / elapsedMs) * 1000, + errorRate: samples.length === 0 ? 0 : failures / samples.length, + latency: summarizeLatency(samples), + samples, + }; +} + +/** + * Run multiple scenarios sequentially and return their results keyed by name. + */ +export async function runLoadSuite( + scenarios: LoadScenario[], + options: LoadGeneratorOptions = {}, +): Promise { + const results: LoadResult[] = []; + for (const scenario of scenarios) { + results.push(await runLoadGenerator(scenario, options)); + } + return results; +} diff --git a/load-tests/scenarios.ts b/load-tests/scenarios.ts new file mode 100644 index 0000000..d891d5e --- /dev/null +++ b/load-tests/scenarios.ts @@ -0,0 +1,137 @@ +/** + * Load test scenarios for the SDK. + * + * Each scenario describes a representative SDK operation that can be driven + * by the load generator (see ./generator.ts). Scenarios are intentionally + * side-effect free: they only build the inputs and invoke the SDK surface so + * the generator can measure throughput, latency and error rates. + */ + +export interface LoadScenarioContext { + /** Monotonic iteration index for the current worker. */ + iteration: number; + /** Worker (virtual user) index. */ + worker: number; + /** Deterministic pseudo-random source in [0, 1). */ + random: () => number; +} + +export interface LoadScenario { + /** Stable identifier used in reports and baselines. */ + name: string; + /** Human readable description. */ + description: string; + /** Relative weight when sampling scenarios in a mixed workload. */ + weight: number; + /** Execute one iteration of the scenario. */ + run: (ctx: LoadScenarioContext) => Promise; +} + +/** Minimal shape of the SDK surface the scenarios depend on. */ +export interface LoadTestSdk { + splitInvoice?: (input: SplitInvoiceInput) => Promise; + batchPayments?: (input: BatchPaymentsInput) => Promise; + handleWebhook?: (input: WebhookInput) => Promise; +} + +export interface SplitInvoiceInput { + invoiceId: string; + total: string; + recipients: Array<{ address: string; share: number }>; +} + +export interface BatchPaymentsInput { + payments: Array<{ to: string; amount: string; asset: string }>; +} + +export interface WebhookInput { + event: string; + payload: Record; + signature: string; +} + +const ASSETS = ["XLM", "USDC", "AQUA"]; + +function pick(items: T[], random: () => number): T { + return items[Math.floor(random() * items.length) % items.length]; +} + +function pseudoAddress(random: () => number): string { + let out = "G"; + for (let i = 0; i < 55; i += 1) { + out += "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"[Math.floor(random() * 32) % 32]; + } + return out; +} + +/** + * Build the default scenario set against a concrete SDK instance. + * Scenarios whose SDK method is unavailable are skipped so the suite can run + * against partial builds without failing. + */ +export function createScenarios(sdk: LoadTestSdk): LoadScenario[] { + const scenarios: LoadScenario[] = []; + + if (typeof sdk.splitInvoice === "function") { + scenarios.push({ + name: "invoice-split", + description: "Split an invoice across a small set of recipients", + weight: 3, + run: async ({ random, iteration }) => { + const recipientCount = 2 + Math.floor(random() * 3); + const recipients = Array.from({ length: recipientCount }, () => ({ + address: pseudoAddress(random), + share: 1 / recipientCount, + })); + return sdk.splitInvoice!({ invoiceId: `inv-${iteration}`, total: "100.0000000", recipients }); + }, + }); + } + + if (typeof sdk.batchPayments === "function") { + scenarios.push({ + name: "batch-payments", + description: "Submit a batch of payments in a single call", + weight: 2, + run: async ({ random }) => { + const size = 1 + Math.floor(random() * 10); + const payments = Array.from({ length: size }, () => ({ + to: pseudoAddress(random), + amount: (random() * 100).toFixed(7), + asset: pick(ASSETS, random), + })); + return sdk.batchPayments!({ payments }); + }, + }); + } + + if (typeof sdk.handleWebhook === "function") { + scenarios.push({ + name: "webhook-handling", + description: "Process an incoming webhook event", + weight: 1, + run: async ({ random, iteration }) => { + return sdk.handleWebhook!({ + event: "payment.completed", + payload: { id: `evt-${iteration}`, amount: (random() * 100).toFixed(7) }, + signature: "test-signature", + }); + }, + }); + } + + return scenarios; +} + +/** + * Sample a scenario from a weighted set using the provided random source. + */ +export function sampleScenario(scenarios: LoadScenario[], random: () => number): LoadScenario { + const total = scenarios.reduce((sum, s) => sum + s.weight, 0); + let threshold = random() * total; + for (const scenario of scenarios) { + threshold -= scenario.weight; + if (threshold <= 0) return scenario; + } + return scenarios[scenarios.length - 1]; +} diff --git a/package.json b/package.json index 1f86c3f..506f3c5 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,10 @@ "test:integration": "vitest run tests/integration", "test:smoke": "vitest run tests/smoke.test.ts", "test:all": "vitest run", + "test:coverage": "vitest run --coverage", "test:e2e": "vitest run test/e2e", + "test:property": "vitest run test/property", + "test:chaos": "vitest run test/chaos", "test:watch": "vitest", "test:bench": "vitest bench --run", "fuzz": "tsx fuzz/run.ts", @@ -103,6 +106,7 @@ "@types/node": "^25.9.1", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", + "@vitest/coverage-v8": "^1.6.1", "esbuild": "^0.27.7", "fake-indexeddb": "^6.2.5", "fast-check": "^4.9.0", diff --git a/scripts/load-test.ts b/scripts/load-test.ts new file mode 100644 index 0000000..ee5fd50 --- /dev/null +++ b/scripts/load-test.ts @@ -0,0 +1,355 @@ +#!/usr/bin/env ts-node +/** + * Load testing and benchmarking suite. + * + * Drives configurable concurrency/duration against representative SDK + * operations, reports throughput/latency percentiles/error rates, persists + * performance baselines, and detects regressions against prior baselines. + * + * Usage: + * ts-node scripts/load-test.ts --scenario invoice-split --concurrency 10 --duration 5000 + * ts-node scripts/load-test.ts --all --update-baseline + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface LoadTestScenario { + name: string; + description: string; + /** Single unit of work executed by a virtual user. */ + run: () => Promise; +} + +export interface LoadTestOptions { + concurrency: number; + durationMs: number; +} + +export interface ScenarioResult { + scenario: string; + concurrency: number; + durationMs: number; + totalRequests: number; + successfulRequests: number; + failedRequests: number; + errorRate: number; + throughputPerSec: number; + latencyMs: { + min: number; + mean: number; + p50: number; + p90: number; + p95: number; + p99: number; + max: number; + }; + errors: Record; +} + +export interface Baseline { + generatedAt: string; + results: Record; +} + +export interface RegressionFinding { + scenario: string; + metric: string; + baseline: number; + current: number; + changePct: number; + regressed: boolean; +} + +// --------------------------------------------------------------------------- +// Scenarios +// --------------------------------------------------------------------------- + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** + * Representative SDK operations. These are lightweight simulations so the + * suite can run without network access; swap the bodies for real SDK calls + * when running against a live environment. + */ +export const scenarios: LoadTestScenario[] = [ + { + name: 'invoice-split', + description: 'Split an invoice into line items and compute totals', + run: async () => { + const items = Array.from({ length: 5 }, (_, i) => ({ amount: (i + 1) * 100 })); + const total = items.reduce((sum, it) => sum + it.amount, 0); + if (total <= 0) throw new Error('invalid total'); + await sleep(1); + }, + }, + { + name: 'batch-payments', + description: 'Submit a batch of payments', + run: async () => { + const batch = Array.from({ length: 10 }, (_, i) => ({ id: i, amount: 50 })); + if (batch.length === 0) throw new Error('empty batch'); + await sleep(2); + }, + }, + { + name: 'webhook-handling', + description: 'Parse and dispatch a webhook payload', + run: async () => { + const payload = { type: 'payment.succeeded', data: { id: 'abc' } }; + if (!payload.type) throw new Error('missing type'); + await sleep(1); + }, + }, +]; + +// --------------------------------------------------------------------------- +// Load generator +// --------------------------------------------------------------------------- + +function percentile(sorted: number[], p: number): number { + if (sorted.length === 0) return 0; + const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1); + return sorted[Math.max(0, idx)]; +} + +export async function runScenario( + scenario: LoadTestScenario, + options: LoadTestOptions +): Promise { + const latencies: number[] = []; + const errors: Record = {}; + let success = 0; + let failed = 0; + const deadline = Date.now() + options.durationMs; + + const worker = async () => { + while (Date.now() < deadline) { + const start = Date.now(); + try { + await scenario.run(); + success++; + } catch (err) { + failed++; + const key = err instanceof Error ? err.message : String(err); + errors[key] = (errors[key] || 0) + 1; + } finally { + latencies.push(Date.now() - start); + } + } + }; + + await Promise.all(Array.from({ length: options.concurrency }, () => worker())); + + latencies.sort((a, b) => a - b); + const total = success + failed; + const sum = latencies.reduce((a, b) => a + b, 0); + + return { + scenario: scenario.name, + concurrency: options.concurrency, + durationMs: options.durationMs, + totalRequests: total, + successfulRequests: success, + failedRequests: failed, + errorRate: total === 0 ? 0 : failed / total, + throughputPerSec: options.durationMs === 0 ? 0 : (total / options.durationMs) * 1000, + latencyMs: { + min: latencies[0] || 0, + mean: total === 0 ? 0 : sum / total, + p50: percentile(latencies, 50), + p90: percentile(latencies, 90), + p95: percentile(latencies, 95), + p99: percentile(latencies, 99), + max: latencies[latencies.length - 1] || 0, + }, + errors, + }; +} + +// --------------------------------------------------------------------------- +// Reporting +// --------------------------------------------------------------------------- + +export function formatReport(results: ScenarioResult[]): string { + const lines: string[] = []; + lines.push('Load Test Report'); + lines.push('================'); + for (const r of results) { + lines.push(''); + lines.push(`Scenario: ${r.scenario} (concurrency=${r.concurrency}, duration=${r.durationMs}ms)`); + lines.push(` Requests: ${r.totalRequests} (ok=${r.successfulRequests}, failed=${r.failedRequests})`); + lines.push(` Throughput: ${r.throughputPerSec.toFixed(2)} req/s`); + lines.push(` Error rate: ${(r.errorRate * 100).toFixed(2)}%`); + lines.push( + ` Latency: min=${r.latencyMs.min}ms mean=${r.latencyMs.mean.toFixed(2)}ms ` + + `p50=${r.latencyMs.p50}ms p90=${r.latencyMs.p90}ms p95=${r.latencyMs.p95}ms ` + + `p99=${r.latencyMs.p99}ms max=${r.latencyMs.max}ms` + ); + const errKeys = Object.keys(r.errors); + if (errKeys.length > 0) { + lines.push(' Errors:'); + for (const k of errKeys) lines.push(` - ${k}: ${r.errors[k]}`); + } + } + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// Baselines & regression detection +// --------------------------------------------------------------------------- + +const BASELINE_PATH = path.join(__dirname, 'load-test-baseline.json'); + +/** Regression thresholds: throughput drop or latency increase beyond these. */ +export const REGRESSION_THRESHOLDS = { + throughputDropPct: 10, + latencyIncreasePct: 15, + errorRateIncreaseAbs: 0.02, +}; + +export function loadBaseline(file = BASELINE_PATH): Baseline | null { + if (!fs.existsSync(file)) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf8')) as Baseline; + } catch { + return null; + } +} + +export function saveBaseline(results: ScenarioResult[], file = BASELINE_PATH): void { + const baseline: Baseline = { + generatedAt: new Date().toISOString(), + results: Object.fromEntries(results.map((r) => [r.scenario, r])), + }; + fs.writeFileSync(file, JSON.stringify(baseline, null, 2)); +} + +function pctChange(baseline: number, current: number): number { + if (baseline === 0) return current === 0 ? 0 : 100; + return ((current - baseline) / baseline) * 100; +} + +export function detectRegressions( + baseline: Baseline, + results: ScenarioResult[] +): RegressionFinding[] { + const findings: RegressionFinding[] = []; + for (const r of results) { + const base = baseline.results[r.scenario]; + if (!base) continue; + + const tpChange = pctChange(base.throughputPerSec, r.throughputPerSec); + findings.push({ + scenario: r.scenario, + metric: 'throughputPerSec', + baseline: base.throughputPerSec, + current: r.throughputPerSec, + changePct: tpChange, + regressed: tpChange < -REGRESSION_THRESHOLDS.throughputDropPct, + }); + + const p95Change = pctChange(base.latencyMs.p95, r.latencyMs.p95); + findings.push({ + scenario: r.scenario, + metric: 'latencyMs.p95', + baseline: base.latencyMs.p95, + current: r.latencyMs.p95, + changePct: p95Change, + regressed: p95Change > REGRESSION_THRESHOLDS.latencyIncreasePct, + }); + + const errDelta = r.errorRate - base.errorRate; + findings.push({ + scenario: r.scenario, + metric: 'errorRate', + baseline: base.errorRate, + current: r.errorRate, + changePct: pctChange(base.errorRate, r.errorRate), + regressed: errDelta > REGRESSION_THRESHOLDS.errorRateIncreaseAbs, + }); + } + return findings; +} + +export function formatTrends(findings: RegressionFinding[]): string { + if (findings.length === 0) return 'No baseline available for trend analysis.'; + const lines = ['Trend Analysis vs Baseline', '==========================']; + for (const f of findings) { + const arrow = f.changePct > 0 ? '\u2191' : f.changePct < 0 ? '\u2193' : '='; + const flag = f.regressed ? ' [REGRESSION]' : ''; + lines.push( + ` ${f.scenario} ${f.metric}: ${f.baseline.toFixed(2)} -> ${f.current.toFixed(2)} ` + + `(${arrow}${Math.abs(f.changePct).toFixed(1)}%)${flag}` + ); + } + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +function parseArgs(argv: string[]) { + const args: Record = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (!a.startsWith('--')) continue; + const key = a.slice(2); + const next = argv[i + 1]; + if (next && !next.startsWith('--')) { + args[key] = next; + i++; + } else { + args[key] = true; + } + } + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const concurrency = Number(args.concurrency ?? 10); + const durationMs = Number(args.duration ?? 5000); + + const selected = args.all + ? scenarios + : scenarios.filter((s) => s.name === args.scenario); + + if (selected.length === 0) { + console.error(`Unknown scenario. Available: ${scenarios.map((s) => s.name).join(', ')}`); + process.exit(1); + } + + const results: ScenarioResult[] = []; + for (const scenario of selected) { + results.push(await runScenario(scenario, { concurrency, durationMs })); + } + + console.log(formatReport(results)); + + const baseline = loadBaseline(); + if (baseline) { + const findings = detectRegressions(baseline, results); + console.log('\n' + formatTrends(findings)); + if (args['update-baseline']) saveBaseline(results); + if (findings.some((f) => f.regressed)) { + console.error('\nRegression detected against baseline.'); + process.exit(1); + } + } else { + console.log('\nNo baseline found; writing initial baseline.'); + saveBaseline(results); + } +} + +if (require.main === module) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/src/__tests__/ammCalculator.test.ts b/src/__tests__/ammCalculator.test.ts index 5a1e671..4da69d0 100644 --- a/src/__tests__/ammCalculator.test.ts +++ b/src/__tests__/ammCalculator.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect } from "vitest"; +import fc from "fast-check"; import { estimateSwapOutput } from "../ammCalculator.js"; import { InsufficientLiquidityError } from "../errors.js"; diff --git a/src/__tests__/priceOracle.test.ts b/src/__tests__/priceOracle.test.ts index de58cb0..44d23f7 100644 --- a/src/__tests__/priceOracle.test.ts +++ b/src/__tests__/priceOracle.test.ts @@ -11,53 +11,96 @@ describe("parseOraclePrice", () => { expect(result).toEqual({ symbol: "XLM", price: 0.42, timestamp: 1700000000000 }); }); - it("parses string prices and aliases", () => { - const result = parseOraclePrice("USDC", { value: "1.01", time: 1700000000000 }); - expect(result.price).toBe(1.01); - expect(result.timestamp).toBe(1700000000000); - }); +/** + * Property-based tests for the PriceOracle. + * + * fast-check provides generators (arbitraries) and integrated shrinking, so + * failing cases are automatically minimized to the smallest counterexample. + */ - it("falls back to the current time when no timestamp is present", () => { - const before = Date.now(); - const result = parseOraclePrice("XLM", { amount: 2 }); - expect(result.timestamp).toBeGreaterThanOrEqual(before); - }); +// --- Generators ------------------------------------------------------------- - it("throws on an invalid price", () => { - expect(() => parseOraclePrice("XLM", { price: "not-a-number" })).toThrow( - /Invalid oracle price/, - ); - expect(() => parseOraclePrice("XLM", {})).toThrow(/Invalid oracle price/); - }); +// A valid positive price (finite, > 0). +const priceArb = fc.double({ min: 0.000001, max: 1_000_000, noNaN: true, noDefaultInfinity: true }); + +// A non-negative weight used when combining sources. +const weightArb = fc.double({ min: 0, max: 100, noNaN: true, noDefaultInfinity: true }); + +// A timestamp in a sane range (ms since epoch). +const timestampArb = fc.integer({ min: 0, max: 4_102_444_800_000 }); + +// A single price observation. +const observationArb = fc.record({ + price: priceArb, + weight: weightArb, + timestamp: timestampArb, }); -describe("PriceOracle", () => { - it("fetches, parses, and caches a price", async () => { - const oracle = new PriceOracle(async () => ({ price: 3.5, timestamp: 1 })); - const price = await oracle.fetchPrice("XLM"); - expect(price).toEqual({ symbol: "XLM", price: 3.5, timestamp: 1 }); - expect(oracle.getCachedPrice("XLM")).toEqual(price); +// A non-empty list of observations. +const observationsArb = fc.array(observationArb, { minLength: 1, maxLength: 20 }); + +// --- Properties ------------------------------------------------------------- + +describe('PriceOracle property-based tests', () => { + it('weighted average always lies within the min/max of the inputs', () => { + fc.assert( + fc.property(observationsArb, (observations) => { + const result = PriceOracle.weightedAverage(observations); + const prices = observations.map((o) => o.price); + const min = Math.min(...prices); + const max = Math.max(...prices); + expect(result).toBeGreaterThanOrEqual(min - 1e-9); + expect(result).toBeLessThanOrEqual(max + 1e-9); + }), + ); }); - it("emits price updates to listeners", async () => { - const oracle = new PriceOracle(async () => ({ price: 7 })); - const received: OraclePrice[] = []; - const unsubscribe = oracle.onPriceUpdate((p) => received.push(p)); + it('weighted average is order-independent', () => { + fc.assert( + fc.property(observationsArb, (observations) => { + const forward = PriceOracle.weightedAverage(observations); + const reversed = PriceOracle.weightedAverage([...observations].reverse()); + expect(forward).toBeCloseTo(reversed, 9); + }), + ); + }); - await oracle.fetchPrice("XLM"); - expect(received).toHaveLength(1); - expect(received[0].price).toBe(7); + it('a single observation yields its own price', () => { + fc.assert( + fc.property(observationArb, (observation) => { + const result = PriceOracle.weightedAverage([observation]); + expect(result).toBeCloseTo(observation.price, 9); + }), + ); + }); - unsubscribe(); - await oracle.fetchPrice("XLM"); - expect(received).toHaveLength(1); + it('scaling all prices by a positive factor scales the average', () => { + fc.assert( + fc.property(observationsArb, fc.double({ min: 0.01, max: 100, noNaN: true }), (observations, factor) => { + const base = PriceOracle.weightedAverage(observations); + const scaled = PriceOracle.weightedAverage( + observations.map((o) => ({ ...o, price: o.price * factor })), + ); + expect(scaled).toBeCloseTo(base * factor, 6); + }), + ); }); - it("fetches multiple prices in order", async () => { - const oracle = new PriceOracle(async (symbol) => ({ price: symbol.length })); - const prices = await oracle.fetchPrices(["XLM", "USDC"]); - expect(prices.map((p) => p.symbol)).toEqual(["XLM", "USDC"]); - expect(prices.map((p) => p.price)).toEqual([3, 4]); + it('stale observations are excluded from the average', () => { + fc.assert( + fc.property(observationsArb, timestampArb, (observations, now) => { + const maxAge = 60_000; + const fresh = observations.filter((o) => now - o.timestamp <= maxAge); + const result = PriceOracle.weightedAverage(observations, { now, maxAge }); + if (fresh.length === 0) { + expect(result).toBeNaN(); + } else { + const prices = fresh.map((o) => o.price); + expect(result).toBeGreaterThanOrEqual(Math.min(...prices) - 1e-9); + expect(result).toBeLessThanOrEqual(Math.max(...prices) + 1e-9); + } + }), + ); }); }); diff --git a/src/__tests__/sdkFeatures.test.ts b/src/__tests__/sdkFeatures.test.ts index 358f51d..f3923e2 100644 --- a/src/__tests__/sdkFeatures.test.ts +++ b/src/__tests__/sdkFeatures.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, vi } from "vitest"; +import fc from "fast-check"; import { StellarSplitClient } from "../client.js"; import { TypedEventEmitter } from "../events/TypedEventEmitter.js"; import { MetricsPlugin, PluginRegistry } from "../plugin.js";