#!/usr/bin/env node
/**
* Flight Protocol Benchmark Report Generator
*
* Reads vitest bench JSON output (--outputJson), normalizes it, optionally
* compares against a baseline, and produces a markdown report suitable for
* GitHub PR comments.
*
* Usage:
* node __bench__/report.mjs --current bench-raw.json [options]
*
* Options:
* --current <file> Vitest bench JSON output (required)
* --baseline <file> Previous results JSON for comparison
* --output <file> Write normalized JSON results (default: bench-results.json)
* --markdown <file> Write markdown report (default: comment.md)
* --commit <sha> Git commit SHA (auto-detected if omitted)
*/
import { readFileSync, writeFileSync } from "node:fs";
import { execSync } from "node:child_process";
// ── CLI args ────────────────────────────────────────────────────
const args = process.argv.slice(2);
function getArg(name, defaultValue = null) {
const idx = args.indexOf(`--${name}`);
return idx !== -1 && args[idx + 1] ? args[idx + 1] : defaultValue;
}
const currentFile = getArg("current");
const baselineFile = getArg("baseline");
const outputFile = getArg("output", "bench-results.json");
const markdownFile = getArg("markdown", "comment.md");
let commitSha = getArg("commit");
if (!currentFile) {
console.error("Usage: node report.mjs --current <vitest-bench-json>");
process.exit(1);
}
if (!commitSha) {
try {
commitSha = execSync("git rev-parse HEAD", { encoding: "utf8" }).trim();
} catch {
commitSha = "unknown";
}
}
const shortSha = commitSha.slice(0, 7);
// ── Parse vitest bench output ───────────────────────────────────
const raw = JSON.parse(readFileSync(currentFile, "utf8"));
/**
* Vitest bench --outputJson format (v4.x):
* {
* files: [
* {
* filepath: "...",
* groups: [
* {
* fullName: "file > group name",
* benchmarks: [
* {
* name: "bench name",
* hz: number,
* mean: number,
* p75: number,
* p99: number,
* min: number,
* max: number,
* rme: number,
* sampleCount: number,
* ...
* }
* ]
* }
* ]
* }
* ]
* }
*/
function parseVitestBench(data) {
const results = {};
const files = data.files || [];
for (const file of files) {
for (const group of file.groups || []) {
// fullName is "file > group name" — extract just the group name
const fullName = group.fullName || "";
const groupName = fullName.includes(" > ")
? fullName.split(" > ").slice(1).join(" > ")
: fullName;
if (!results[groupName]) results[groupName] = {};
for (const bench of group.benchmarks || []) {
results[groupName][bench.name] = {
hz: bench.hz,
mean: bench.mean,
p75: bench.p75,
p99: bench.p99,
min: bench.min,
max: bench.max,
sampleCount: bench.sampleCount,
rme: bench.rme,
};
}
}
}
return results;
}
const results = parseVitestBench(raw);
// ── Write normalized JSON ───────────────────────────────────────
const normalized = {
commit: commitSha,
shortCommit: shortSha,
date: new Date().toISOString(),
results,
};
writeFileSync(outputFile, JSON.stringify(normalized, null, 2) + "\n");
console.log(`Normalized results written to ${outputFile}`);
// ── Load baseline ───────────────────────────────────────────────
let baseline = null;
if (baselineFile) {
try {
baseline = JSON.parse(readFileSync(baselineFile, "utf8"));
console.log(
`Loaded baseline from ${baselineFile} (${baseline.shortCommit || "unknown"})`
);
} catch (e) {
console.warn(`Warning: could not load baseline: ${e.message}`);
}
}
// ── Group mapping ───────────────────────────────────────────────
/**
* Maps bench group names to library identifiers and operation types.
* Group names come from the describe() blocks in bench files.
*/
const GROUP_MAP = {
"@lazarv/rsc serialize": { lib: "lazarv", op: "serialize" },
"@lazarv/rsc prerender": { lib: "lazarv", op: "prerender" },
"@lazarv/rsc deserialize": { lib: "lazarv", op: "deserialize" },
"@lazarv/rsc roundtrip": { lib: "lazarv", op: "roundtrip" },
"webpack serialize": { lib: "webpack", op: "serialize" },
"webpack deserialize": { lib: "webpack", op: "deserialize" },
"webpack roundtrip": { lib: "webpack", op: "roundtrip" },
};
/**
* Build a lookup: { [op]: { [scenarioName]: { lazarv: benchData, webpack: benchData } } }
*/
function buildComparison(data) {
const comparison = {};
for (const [group, benches] of Object.entries(data)) {
const mapped = GROUP_MAP[group];
if (!mapped) continue;
const { lib, op } = mapped;
if (!comparison[op]) comparison[op] = {};
for (const [name, bench] of Object.entries(benches)) {
if (!comparison[op][name]) comparison[op][name] = {};
comparison[op][name][lib] = bench;
}
}
return comparison;
}
const current = buildComparison(results);
const base = baseline ? buildComparison(baseline.results) : null;
// ── Markdown generation ─────────────────────────────────────────
function fmtHz(hz) {
if (hz >= 1_000_000) return `${(hz / 1_000_000).toFixed(1)}M`;
if (hz >= 1_000) return `${(hz / 1_000).toFixed(1)}K`;
return hz.toFixed(0);
}
function fmtTime(ms) {
if (ms < 0.001) return `${(ms * 1_000_000).toFixed(0)} ns`;
if (ms < 1) return `${(ms * 1_000).toFixed(1)} \u00b5s`;
return `${ms.toFixed(2)} ms`;
}
function fmtDelta(current, base, lowerIsBetter = false) {
if (base == null || base === 0) return "";