diff --git a/.github/workflows/build-status.yml b/.github/workflows/build-status.yml index de2773373..24aef1071 100644 --- a/.github/workflows/build-status.yml +++ b/.github/workflows/build-status.yml @@ -152,17 +152,6 @@ jobs: path: packages/**/package.tgz if-no-files-found: error retention-days: 1 - - name: Prune old main bundle of fast-check caches - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - env: - GH_TOKEN: ${{ github.token }} - run: gh cache list --key bundle-main- --json key -q '.[].key' | xargs -rn1 gh cache delete || true - - name: Cache bundle of fast-check for main - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: packages/fast-check/package.tgz - key: bundle-main-${{ github.sha }} # Jobs requesting warmup_pnpm_cache AND production_packages typecheck: @@ -508,7 +497,6 @@ jobs: # Optional jobs bench: name: 'Benchmark' - if: github.event_name == 'pull_request' && github.base_ref == 'main' needs: - warmup_pnpm_cache - production_packages @@ -526,15 +514,13 @@ jobs: cache: 'pnpm' - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts - - name: Restore main bundle of fast-check from cache + - name: Restore results from last benchmark execution on main uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - path: packages/fast-check/package.tgz + path: benchmark.json # Non-matching key so restore-keys picks the most recent main cache - key: bundle-main-will-not-match - restore-keys: bundle-main- - - name: Install version from
- run: pnpm --filter fast-check add fast-check-main@file:./package.tgz --ignore-scripts + key: bench-main-will-not-match + restore-keys: bench-main- - name: Download production packages uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -558,6 +544,17 @@ jobs: path: bench-output.txt if-no-files-found: error retention-days: 1 + - name: Prune old benchmark execution from caches + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + GH_TOKEN: ${{ github.token }} + run: gh cache list --key bench-main- --json key -q '.[].key' | xargs -rn1 gh cache delete || true + - name: Cache last benchmark execution for main + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: benchmark.json + key: bench-main-${{ github.sha }} comment_bench: name: 'Comment bench results' needs: bench diff --git a/.gitignore b/.gitignore index c7d320eb3..16e46d57f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ website/docs/api/ deopt.out v8.log v8.out +benchmark.json # IDE related files .idea diff --git a/package.json b/package.json index 4fe49dd78..ee98ca4c4 100644 --- a/package.json +++ b/package.json @@ -26,8 +26,7 @@ "build-ci:all": "pnpm --filter {./packages/**} run build-ci", "test": "vitest", "test:coverage": "vitest --coverage", - "bench": "vitest bench", - "bench:setup": "pnpm --filter fast-check add fast-check-main@https://pkg.pr.new/dubzzz/fast-check@main --ignore-scripts", + "bench": "vitest bench --outputJson benchmark.json --compare benchmark.json", "typecheck:all": "pnpm --parallel run typecheck", "format": "prettier --experimental-cli --cache --write .", "format:file": "prettier --experimental-cli --cache --write", diff --git a/packages/fast-check/test/bench/__test-helpers__/Imports.ts b/packages/fast-check/test/bench/__test-helpers__/Imports.ts index 24b1ac728..748224a5b 100644 --- a/packages/fast-check/test/bench/__test-helpers__/Imports.ts +++ b/packages/fast-check/test/bench/__test-helpers__/Imports.ts @@ -1,16 +1,9 @@ import { xorshift128plus } from 'pure-rand/generator/xorshift128plus'; -import type * as fcCurrentFromSrc from '../../../src/fast-check.js'; +import type * as fcFromSrc from '../../../src/fast-check.js'; // oxlint-disable-next-line typescript/ban-ts-comment // @ts-ignore - May not be available except if built -import * as fcCurrentImport from '../../../lib/fast-check.js'; -// oxlint-disable-next-line typescript/ban-ts-comment -// @ts-ignore - May not be available except if explicitely installed -import * as fcMainImport from 'fast-check-main'; - -export const fcCurrent: typeof fcCurrentFromSrc = fcCurrentImport as unknown as typeof fcCurrentFromSrc; -export const fcMain: typeof fcCurrentFromSrc = fcMainImport as unknown as typeof fcCurrentFromSrc; +import * as fcImport from '../../../lib/fast-check.js'; -const mrng = xorshift128plus(0); -export const mrngCurrent: fcCurrentFromSrc.Random = new fcCurrent.Random(mrng); -export const mrngMain: fcCurrentFromSrc.Random = new fcMain.Random(mrng); +export const fc: typeof fcFromSrc = fcImport as unknown as typeof fcFromSrc; +export const mrng: fcFromSrc.Random = new fc.Random(xorshift128plus(0)); diff --git a/packages/fast-check/test/bench/arbitraries.bench.ts b/packages/fast-check/test/bench/arbitraries.bench.ts index a7b73ff01..ad2d8990f 100644 --- a/packages/fast-check/test/bench/arbitraries.bench.ts +++ b/packages/fast-check/test/bench/arbitraries.bench.ts @@ -1,150 +1,108 @@ import { describe, bench } from 'vitest'; -import type { Arbitrary, Memo, Value } from '../../src/fast-check.js'; -import { fcCurrent, fcMain, mrngCurrent, mrngMain } from './__test-helpers__/Imports.js'; - -type Fc = typeof fcCurrent; +import type { Arbitrary, Memo } from '../../src/fast-check.js'; +import { fc, mrng } from './__test-helpers__/Imports.js'; type BenchCase = { - /** Arbitrary expression, used as the benchmark group label */ name: string; - /** Builds the arbitrary from the provided fast-check module (either `current` or `main`) */ - build: (fc: Fc) => Arbitrary; + arbitrary: Arbitrary; }; const largeLength = { maxLength: 100, size: 'max' as const }; +const letrec = () => { + const { tree } = fc.letrec((tie) => ({ + tree: fc.oneof(tie('leaf'), tie('node')), + node: fc.record({ left: tie('tree'), right: tie('tree') }), + leaf: fc.nat(), + })); + return tree; +}; +const memo = () => { + const leaf = fc.nat; + // oxlint-disable-next-line no-use-before-define -- `tree` and `node` reference each other + const tree: Memo = fc.memo((n) => fc.oneof(node(n), leaf())); + const node: Memo = fc.memo((n) => { + if (n <= 1) return fc.record({ left: leaf(), right: leaf() }); + return fc.record({ left: tree(), right: tree() }); + }); + return tree(3); +}; + const benchCases: BenchCase[] = [ // Numeric and primitive values - { name: 'boolean()', build: (fc) => fc.boolean() }, - { name: 'integer()', build: (fc) => fc.integer() }, - { name: 'maxSafeInteger()', build: (fc) => fc.maxSafeInteger() }, - { name: 'bigInt()', build: (fc) => fc.bigInt() }, - { name: 'float()', build: (fc) => fc.float() }, - { name: 'double()', build: (fc) => fc.double() }, - { name: 'date()', build: (fc) => fc.date() }, + { name: 'boolean()', arbitrary: fc.boolean() }, + { name: 'integer()', arbitrary: fc.integer() }, + { name: 'maxSafeInteger()', arbitrary: fc.maxSafeInteger() }, + { name: 'bigInt()', arbitrary: fc.bigInt() }, + { name: 'float()', arbitrary: fc.float() }, + { name: 'double()', arbitrary: fc.double() }, + { name: 'date()', arbitrary: fc.date() }, // Strings (default small size, an explicit larger size and a non-ASCII unit) - { name: 'string()', build: (fc) => fc.string() }, - { name: "string({ maxLength: 100, size: 'max' })", build: (fc) => fc.string(largeLength) }, - { name: "string({ unit: 'grapheme' })", build: (fc) => fc.string({ unit: 'grapheme' }) }, - { name: 'base64String()', build: (fc) => fc.base64String() }, + { name: 'string()', arbitrary: fc.string() }, + { name: "string({ maxLength: 100, size: 'max' })", arbitrary: fc.string(largeLength) }, + { name: "string({ unit: 'grapheme' })", arbitrary: fc.string({ unit: 'grapheme' }) }, + { name: 'base64String()', arbitrary: fc.base64String() }, // Collections (default small size and an explicit larger size for the bulk path) - { name: 'array(integer())', build: (fc) => fc.array(fc.integer()) }, - { name: "array(integer(), { maxLength: 100, size: 'max' })", build: (fc) => fc.array(fc.integer(), largeLength) }, - { name: 'uniqueArray(integer())', build: (fc) => fc.uniqueArray(fc.integer()) }, - { name: 'set(integer())', build: (fc) => fc.set(fc.integer()) }, - { name: 'tuple(integer())', build: (fc) => fc.tuple(fc.integer()) }, - { name: 'record({ a: integer() })', build: (fc) => fc.record({ a: fc.integer() }) }, - { name: 'dictionary(string(), integer())', build: (fc) => fc.dictionary(fc.string(), fc.integer()) }, - { name: 'map(string(), integer())', build: (fc) => fc.map(fc.string(), fc.integer()) }, + { name: 'array(integer())', arbitrary: fc.array(fc.integer()) }, + { name: "array(integer(), { maxLength: 100, size: 'max' })", arbitrary: fc.array(fc.integer(), largeLength) }, + { name: 'uniqueArray(integer())', arbitrary: fc.uniqueArray(fc.integer()) }, + { name: 'set(integer())', arbitrary: fc.set(fc.integer()) }, + { name: 'tuple(integer())', arbitrary: fc.tuple(fc.integer()) }, + { name: 'record({ a: integer() })', arbitrary: fc.record({ a: fc.integer() }) }, + { name: 'dictionary(string(), integer())', arbitrary: fc.dictionary(fc.string(), fc.integer()) }, + { name: 'map(string(), integer())', arbitrary: fc.map(fc.string(), fc.integer()) }, // Choice and combinators - { name: 'constantFrom(...)', build: (fc) => fc.constantFrom('a', 'b', 'c', 'd', 'e') }, - { name: 'oneof(integer(), integer())', build: (fc) => fc.oneof(fc.integer(), fc.integer()) }, + { name: 'constantFrom(...)', arbitrary: fc.constantFrom('a', 'b', 'c', 'd', 'e') }, + { name: 'oneof(integer(), integer())', arbitrary: fc.oneof(fc.integer(), fc.integer()) }, { name: 'oneof({ weight, arbitrary }, ...)', - build: (fc) => fc.oneof({ arbitrary: fc.integer(), weight: 1 }, { arbitrary: fc.integer(), weight: 2 }), + arbitrary: fc.oneof({ arbitrary: fc.integer(), weight: 1 }, { arbitrary: fc.integer(), weight: 2 }), }, - { name: 'option(integer())', build: (fc) => fc.option(fc.integer()) }, - { name: 'subarray([1, 2, 3, 4, 5])', build: (fc) => fc.subarray([1, 2, 3, 4, 5]) }, + { name: 'option(integer())', arbitrary: fc.option(fc.integer()) }, + { name: 'subarray([1, 2, 3, 4, 5])', arbitrary: fc.subarray([1, 2, 3, 4, 5]) }, // Recursive structures - { - name: 'letrec(tree)', - build: (fc) => { - const { tree } = fc.letrec((tie) => ({ - tree: fc.oneof(tie('leaf'), tie('node')), - node: fc.record({ left: tie('tree'), right: tie('tree') }), - leaf: fc.nat(), - })); - return tree; - }, - }, - { - name: 'memo(tree)', - build: (fc) => { - const leaf = fc.nat; - // oxlint-disable-next-line no-use-before-define -- `tree` and `node` reference each other - const tree: Memo = fc.memo((n) => fc.oneof(node(n), leaf())); - const node: Memo = fc.memo((n) => { - if (n <= 1) return fc.record({ left: leaf(), right: leaf() }); - return fc.record({ left: tree(), right: tree() }); - }); - return tree(3); - }, - }, + { name: 'letrec(tree)', arbitrary: letrec() }, + { name: 'memo(tree)', arbitrary: memo() }, // Structured data - { name: 'anything()', build: (fc) => fc.anything() }, - { name: 'json()', build: (fc) => fc.json() }, + { name: 'anything()', arbitrary: fc.anything() }, + { name: 'json()', arbitrary: fc.json() }, { name: 'entityGraph(node -> node)', - build: (fc) => - fc.entityGraph({ node: { id: fc.integer() } }, { node: { linkTo: { arity: 'many', type: 'node' } } }), + arbitrary: fc.entityGraph({ node: { id: fc.integer() } }, { node: { linkTo: { arity: 'many', type: 'node' } } }), }, // Formatted strings (web and identifiers) - { name: 'emailAddress()', build: (fc) => fc.emailAddress() }, - { name: 'webUrl()', build: (fc) => fc.webUrl() }, - { name: 'ipV4()', build: (fc) => fc.ipV4() }, - { name: 'ipV6()', build: (fc) => fc.ipV6() }, - { name: 'uuid()', build: (fc) => fc.uuid() }, - { name: 'stringMatching(/^[a-zA-Z0-9]+$/)', build: (fc) => fc.stringMatching(/^[a-zA-Z0-9]+$/) }, - { name: 'mixedCase(string())', build: (fc) => fc.mixedCase(fc.string()) }, + { name: 'emailAddress()', arbitrary: fc.emailAddress() }, + { name: 'webUrl()', arbitrary: fc.webUrl() }, + { name: 'ipV4()', arbitrary: fc.ipV4() }, + { name: 'ipV6()', arbitrary: fc.ipV6() }, + { name: 'uuid()', arbitrary: fc.uuid() }, + { name: 'stringMatching(/^[a-zA-Z0-9]+$/)', arbitrary: fc.stringMatching(/^[a-zA-Z0-9]+$/) }, + { name: 'mixedCase(string())', arbitrary: fc.mixedCase(fc.string()) }, // Operators chained on top of another arbitrary - { name: 'integer().map(.)', build: (fc) => fc.integer().map((n) => n + 1) }, - { name: 'integer().chain(.)', build: (fc) => fc.integer().chain(() => fc.integer()) }, - { name: 'integer().filter(.)', build: (fc) => fc.integer().filter((n) => n % 2 === 0) }, + { name: 'integer().map(.)', arbitrary: fc.integer().map((n) => n + 1) }, + { name: 'integer().chain(.)', arbitrary: fc.integer().chain(() => fc.integer()) }, + { name: 'integer().filter(.)', arbitrary: fc.integer().filter((n) => n % 2 === 0) }, ]; const biasFactor = 3; -const numReplicas = 3; +// Pre warm-up for (const benchCase of benchCases) { - describe(benchCase.name, () => { - const current = benchCase.build(fcCurrent); - const main = benchCase.build(fcMain); - - describe('generate', () => { - for (let i = 0; i !== numReplicas; ++i) { - bench(`current-${i}`, () => { - current.generate(mrngCurrent, biasFactor); - }); - bench(`main-${i}`, () => { - main.generate(mrngMain, biasFactor); - }); - } - }); + benchCase.arbitrary.generate(mrng, biasFactor); +} - describe('shrink', () => { - for (let i = 0; i !== numReplicas; ++i) { - let seedCurrent: Value; - let seedMain: Value; - bench( - `current-${i}`, - () => { - current.shrink(seedCurrent.value, seedCurrent.context).next(); - }, - { - setup() { - seedCurrent = current.generate(mrngCurrent, biasFactor); - }, - }, - ); - bench( - `main-${i}`, - () => { - main.shrink(seedMain.value, seedMain.context).next(); - }, - { - setup() { - seedMain = main.generate(mrngMain, biasFactor); - }, - }, - ); - } +// Benchmark +describe('generate', () => { + for (const benchCase of benchCases) { + bench(benchCase.name, () => { + benchCase.arbitrary.generate(mrng, biasFactor); }); - }); -} + } +});