benchmark-gate.js

// The regression rule behind `benchmark-check.js`, kept free of file and process
// access so it can be tested.
//
// Only the Devup UI rows of benchmark-manifest.json (`gate: true`) can fail;
// every other library's row is informational.
//
// Build time is compared as a ratio to a control row measured in the same run
// (Tailwind on the same bundler), because the shared CI runner drifts by about
// 25% between runs while the ratio drifts by a few percent. A row fails when
//   ratio > baseline ratio * (1 + timeTolerance)
// where the tolerance is at least 10% and at least twice the spread the ratio
// showed over the calibration runs stored with the baseline, so ordinary noise
// stays well inside it and a real slowdown does not.
//
// Output sizes are deterministic to about 0.05% between runs. A row fails when
// its bytes grow beyond baseline * 1.01, or its CSS bytes beyond the baseline
// plus the larger of 5% and 32 bytes. A change that grows a size on purpose
// updates benchmark-baseline.json in the same pull request.

export const MIN_TIME_TOLERANCE = 0.1
export const SPREAD_FACTOR = 2
export const BYTES_TOLERANCE = 0.01
export const CSS_TOLERANCE = 0.05
export const CSS_MIN_SLACK = 32
export const KEPT_CALIBRATION_RUNS = 10

export function median(values) {
  const sorted = values.toSorted((a, b) => a - b)
  const middle = Math.floor(sorted.length / 2)
  return sorted.length % 2
    ? sorted[middle]
    : (sorted[middle - 1] + sorted[middle]) / 2
}

/** The targets in the order of one round: the list, started
ound steps of its length / rounds further on */
export function roundOrder(names, round, rounds) {
  const offset = Math.floor((names.length * round) / rounds)
  return [...names.slice(offset), ...names.slice(0, offset)]
}

/** (max - min) / median */
export function relativeSpread(values) {
  return (Math.max(...values) - Math.min(...values)) / median(values)
}

/** The time of a row as a ratio to the control of its group in the same run */
export function timeRatio(run, target, control) {
  return run.rows[target].median / run.rows[control].median
}

export function timeTolerance(calibration) {
  return Math.max(
    MIN_TIME_TOLERANCE,
    SPREAD_FACTOR * relativeSpread(calibration),
  )
}

/** Checks the run against the baseline: one entry per gated row, and whether any failed */
export function evaluate(run, baseline) {
  const results = []
  if (run.samples !== baseline.samples)
    results.push({
      target: '(policy)',
      failures: [
        `the run took ${run.samples} samples per row, the baseline was made with ${baseline.samples}`,
      ],
    })
  for (const [target, expected] of Object.entries(baseline.rows)) {
    const row = run.rows[target]
    const failures = []
    if (!row || !run.rows[expected.control]) {
      failures.push('missing from the run')
    } else {
      const ratio = timeRatio(run, target, expected.control)
      const limit = expected.ratio * (1 + expected.timeTolerance)
      if (ratio > limit)
        failures.push(
          `build time is ${ratio.toFixed(3)}x ${expected.control}, over the limit ${limit.toFixed(3)}x (baseline ${expected.ratio.toFixed(3)}x + ${(expected.timeTolerance * 100).toFixed(0)}%)`,
        )
      const bytesLimit = Math.floor(expected.bytes * (1 + BYTES_TOLERANCE))
      if (row.bytes > bytesLimit)
        failures.push(
          `output is ${row.bytes.toLocaleString()} bytes, over the limit ${bytesLimit.toLocaleString()} (baseline ${expected.bytes.toLocaleString()} + 1%)`,
        )
      const cssLimit =
        expected.cssBytes +
        Math.max(Math.floor(expected.cssBytes * CSS_TOLERANCE), CSS_MIN_SLACK)
      if (row.cssBytes > cssLimit)
        failures.push(
          `CSS is ${row.cssBytes.toLocaleString()} bytes, over the limit ${cssLimit.toLocaleString()} (baseline ${expected.cssBytes.toLocaleString()})`,
        )
    }
    results.push({ target, failures })
  }
  return { results, failed: results.some((entry) => entry.failures.length > 0) }
}

/** The baseline for this run: the stored calibration runs plus this one, and what they say */
export function nextBaseline(run, manifest, previous) {
  const gated = manifest.targets.filter((target) => target.gate)
  const rows = {}
  for (const { name, group } of gated) {
    const control = manifest.controls[group]
    const calibration = [
      ...(previous?.rows[name]?.calibration ?? []),
      timeRatio(run, name, control),
    ].slice(-KEPT_CALIBRATION_RUNS)
    rows[name] = {
      control,
      ratio: median(calibration),
      calibration,
      timeTolerance: timeTolerance(calibration),
      bytes: run.rows[name].bytes,
      cssBytes: run.rows[name].cssBytes,
    }
  }
  return { samples: run.samples, rows }
}

/** Replace only named, approved CSS baselines; default to per-file, never recalibrate. */
export function nextCssBaseline(run, previous, targets = ['devup-ui']) {
  if (
    targets.length === 0 ||
    targets.some(
      (target) =>
        !['devup-ui', 'devup-ui-single', 'vinext-devup-ui'].includes(target),
    )
  )
    throw new TypeError('CSS-only baseline requires approved named targets')
  const rows = { ...previous.rows }
  for (const target of targets) {
    const expected = previous.rows[target]
    if (!expected)
      throw new TypeError(
        `CSS-only baseline requires an existing ${target} row`,
      )
    rows[target] = { ...expected, cssBytes: run.rows[target].cssBytes }
  }
  return { ...previous, rows }
}

/** One line per gated row for the job log */
export function report(run, baseline, evaluation) {
  return evaluation.results
    .map(({ target, failures }) => {
      const expected = baseline.rows[target]
      const row = run.rows[target]
      const status = failures.length ? 'FAIL' : 'ok  '
      if (!expected || !row)
        return `${status} ${target}: ${failures.join('; ')}`
      const ratio = timeRatio(run, target, expected.control)
      return `${status} ${target}: ${row.median.toFixed(2)}s (${ratio.toFixed(3)}x ${expected.control}, baseline ${expected.ratio.toFixed(3)}x ±${(expected.timeTolerance * 100).toFixed(0)}%), ${row.bytes.toLocaleString()} bytes, css ${row.cssBytes.toLocaleString()} bytes${failures.length ? '\n       ' + failures.join('\n       ') : ''}`
    })
    .join('\n')
}