601 lines
24 KiB
JavaScript
601 lines
24 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Deterministic QA for the detection overlay's bounded two-lane label pipeline.
|
|
*
|
|
* Run:
|
|
* node scripts/qa-labels.mjs --url http://localhost:4173
|
|
*
|
|
* The harness replaces only the three detection read APIs in the loaded page.
|
|
* It does not enable their network pollers or mutate repository data. The field
|
|
* contains 12,000 stable observations and is sampled while the global camera
|
|
* makes a small orbit.
|
|
*/
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import puppeteer from 'puppeteer';
|
|
|
|
const argv = process.argv.slice(2);
|
|
const getFlag = (name) => argv.includes(name);
|
|
const getOpt = (name, fallback) => {
|
|
const index = argv.indexOf(name);
|
|
return index >= 0 && argv[index + 1] ? argv[index + 1] : fallback;
|
|
};
|
|
|
|
const APP_URL = getOpt('--url', 'http://localhost:4173');
|
|
const APP_ORIGIN = new URL(APP_URL).origin;
|
|
const HEADFUL = getFlag('--headful');
|
|
const SHOT_DIR = path.resolve('qa-shots/labels');
|
|
const STORAGE_KEY = 'gev:detection-allocation:v1';
|
|
const OBSERVATION_COUNT = 12000;
|
|
const SYNTHETIC_SAMPLE_MS = 10000;
|
|
const NORMAL_SAMPLE_MS = 6000;
|
|
const SAMPLE_TIMEOUT_MS = 120000;
|
|
const MIN_SYNTHETIC_SOLVES = 20;
|
|
const MIN_SYNTHETIC_SPAN_MS = SYNTHETIC_SAMPLE_MS;
|
|
const MIN_NORMAL_FRAMES = 20;
|
|
const SETTLED_SOLVE_COUNT = 3;
|
|
const GLOBAL_CAMERA_MIN_HEIGHT_M = 20_000_000;
|
|
const GLOBAL_CAMERA_MAX_HEIGHT_M = 30_000_000;
|
|
const FIELD_COUNTS = Object.freeze({
|
|
flights: 7200,
|
|
military: 1000,
|
|
satellites: 3800,
|
|
});
|
|
const NORMAL_COUNTS = Object.freeze({
|
|
flights: 4300,
|
|
military: 70,
|
|
satellites: 830,
|
|
});
|
|
|
|
const CHROME_EXECUTABLE_CANDIDATES = [
|
|
process.env.PUPPETEER_EXECUTABLE_PATH,
|
|
// Prefer puppeteer's version-pinned Chrome-for-Testing over the system
|
|
// Chrome: /Applications auto-updates underneath the harnesses, and its
|
|
// software-GL behavior shifts across majors (system Chrome 150 blew the
|
|
// tile-gated drain budget under SwiftShader on 2026-07-30 — six
|
|
// false-negative qa-cctv-v2 runs against a healthy build). A deterministic
|
|
// pinned browser beats the newest one for regression harnesses.
|
|
(() => { try { return puppeteer.executablePath(); } catch { return null; } })(),
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
|
|
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
|
].filter(Boolean);
|
|
|
|
function findChromeExecutable() {
|
|
for (const candidate of CHROME_EXECUTABLE_CANDIDATES) {
|
|
try {
|
|
if (fs.existsSync(candidate)) return candidate;
|
|
} catch {
|
|
// Let Puppeteer use its bundled browser.
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
const results = [];
|
|
function record(name, ok, detail) {
|
|
results.push({ name, ok, detail });
|
|
const tag = ok ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';
|
|
console.log(` [${tag}] ${name}${detail ? ` — ${detail}` : ''}`);
|
|
}
|
|
|
|
function percentile(values, pct) {
|
|
const sorted = values.filter(Number.isFinite).sort((a, b) => a - b);
|
|
if (sorted.length === 0) return Number.NaN;
|
|
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * pct) - 1)];
|
|
}
|
|
|
|
function parseJson(value, fallback) {
|
|
try {
|
|
return JSON.parse(value);
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
function analyzeMembership(samples) {
|
|
let replacements = 0;
|
|
let exposure = 0;
|
|
const transitions = new Map();
|
|
|
|
for (let index = 0; index < samples.length; index++) {
|
|
const sample = samples[index];
|
|
const current = new Set(sample.labeledKeys);
|
|
if (index > 0) {
|
|
const previous = new Set(samples[index - 1].labeledKeys);
|
|
const removed = [...previous].filter((key) => !current.has(key)).length;
|
|
const added = [...current].filter((key) => !previous.has(key)).length;
|
|
replacements += Math.max(removed, added);
|
|
const elapsedSec = Math.max(0.001, (sample.t - samples[index - 1].t) / 1000);
|
|
exposure += Math.max(1, (previous.size + current.size) / 2) * elapsedSec;
|
|
}
|
|
for (const key of current) {
|
|
if (!transitions.has(key)) transitions.set(key, []);
|
|
transitions.get(key).push(sample.t);
|
|
}
|
|
}
|
|
|
|
let showHideShow = 0;
|
|
for (const times of transitions.values()) {
|
|
for (let index = 1; index < times.length; index++) {
|
|
const previousIndex = samples.findIndex((sample) => sample.t === times[index - 1]);
|
|
const nextIndex = samples.findIndex((sample) => sample.t === times[index]);
|
|
if (nextIndex - previousIndex > 1 && times[index] - times[index - 1] <= 2000) {
|
|
showHideShow++;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
const churnPctPerSec = exposure > 0
|
|
? replacements / exposure * 100
|
|
: 0;
|
|
return {
|
|
churnPctPerSec,
|
|
showHideShow,
|
|
replacements,
|
|
exposure,
|
|
transitions: Math.max(0, samples.length - 1),
|
|
minimumPopulation: samples.length
|
|
? Math.min(...samples.map((sample) => sample.labeledKeys.length))
|
|
: 0,
|
|
maximumPopulation: samples.length
|
|
? Math.max(...samples.map((sample) => sample.labeledKeys.length))
|
|
: 0,
|
|
};
|
|
}
|
|
|
|
async function waitForConclusiveSamples(page, {
|
|
minimumElapsedMs,
|
|
minimumSolves = 0,
|
|
minimumSolveSpanMs = 0,
|
|
minimumNormalFrames = 0,
|
|
}) {
|
|
const deadline = Date.now() + SAMPLE_TIMEOUT_MS;
|
|
while (Date.now() < deadline) {
|
|
const state = await page.evaluate(() => {
|
|
const qa = window.__LABEL_QA;
|
|
const solves = qa?.solve || [];
|
|
const matchingFrames = (qa?.frames || []).filter(
|
|
(sample) => sample.observationCount === 5200,
|
|
);
|
|
return {
|
|
elapsedMs: performance.now() - (qa?.startedAt || performance.now()),
|
|
solveCount: solves.length,
|
|
solveSpanMs: solves.length > 1 ? solves.at(-1).t - solves[0].t : 0,
|
|
normalFrameCount: matchingFrames.length,
|
|
};
|
|
});
|
|
if (state.elapsedMs >= minimumElapsedMs
|
|
&& state.solveCount >= minimumSolves
|
|
&& state.solveSpanMs >= minimumSolveSpanMs
|
|
&& state.normalFrameCount >= minimumNormalFrames) return state;
|
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function waitForSettledSyntheticField(page) {
|
|
await page.waitForFunction(
|
|
({ observationCount, settledSolveCount, minHeight, maxHeight }) => {
|
|
const solves = window.__LABEL_QA?.solve || [];
|
|
if (solves.length < settledSolveCount) return false;
|
|
return solves.slice(-settledSolveCount).every((sample) => (
|
|
sample.observationCount === observationCount
|
|
&& sample.labeledKeys.length > 0
|
|
&& sample.selectedCount === sample.labeledKeys.length
|
|
&& sample.cameraHeight >= minHeight
|
|
&& sample.cameraHeight <= maxHeight
|
|
));
|
|
},
|
|
{ timeout: SAMPLE_TIMEOUT_MS, polling: 100 },
|
|
{
|
|
observationCount: OBSERVATION_COUNT,
|
|
settledSolveCount: SETTLED_SOLVE_COUNT,
|
|
minHeight: GLOBAL_CAMERA_MIN_HEIGHT_M,
|
|
maxHeight: GLOBAL_CAMERA_MAX_HEIGHT_M,
|
|
},
|
|
);
|
|
}
|
|
|
|
async function main() {
|
|
console.log('\nDetection Label QA');
|
|
console.log(` App URL : ${APP_URL}`);
|
|
console.log(` Observations : ${OBSERVATION_COUNT}`);
|
|
console.log(` Sample window: >=${SYNTHETIC_SAMPLE_MS / 1000}s synthetic + >=${NORMAL_SAMPLE_MS / 1000}s normal\n`);
|
|
|
|
try {
|
|
const response = await fetch(APP_URL);
|
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
|
} catch (error) {
|
|
console.error(`Dev server not reachable at ${APP_URL}: ${error.message}`);
|
|
process.exit(2);
|
|
}
|
|
|
|
fs.mkdirSync(SHOT_DIR, { recursive: true });
|
|
const executablePath = findChromeExecutable();
|
|
const browser = await puppeteer.launch({
|
|
headless: HEADFUL ? false : 'new',
|
|
...(executablePath ? { executablePath } : {}),
|
|
args: [
|
|
'--no-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
'--use-gl=angle',
|
|
'--use-angle=swiftshader',
|
|
'--disable-dev-shm-usage',
|
|
'--disable-web-security',
|
|
'--disable-background-timer-throttling',
|
|
'--disable-renderer-backgrounding',
|
|
'--window-size=1280,800',
|
|
],
|
|
});
|
|
|
|
const consoleErrors = [];
|
|
const failedResponses = [];
|
|
try {
|
|
const page = await browser.newPage();
|
|
await page.setViewport({ width: 1280, height: 800, deviceScaleFactor: 1 });
|
|
page.on('console', (message) => {
|
|
if (message.type() !== 'error') return;
|
|
const text = message.text();
|
|
const sourceUrl = message.location()?.url || '';
|
|
if (!/Failed to load resource.*404/i.test(text)) {
|
|
consoleErrors.push(sourceUrl ? `${text} [${sourceUrl}]` : text);
|
|
}
|
|
});
|
|
page.on('pageerror', (error) => consoleErrors.push(`pageerror: ${error.message}`));
|
|
page.on('response', (response) => {
|
|
if (response.status() >= 500) {
|
|
failedResponses.push(`HTTP ${response.status()} ${response.url()}`);
|
|
}
|
|
});
|
|
|
|
await page.evaluateOnNewDocument((storageKey, appOrigin) => {
|
|
localStorage.setItem(storageKey, 'WEIGHTED');
|
|
const realFetch = window.fetch.bind(window);
|
|
window.fetch = (input, init) => {
|
|
const requestUrl = typeof input === 'string' || input instanceof URL
|
|
? String(input)
|
|
: input?.url;
|
|
const url = new URL(requestUrl, window.location.href);
|
|
if (url.origin === appOrigin && url.pathname === '/api/openai/hud-summary') {
|
|
return Promise.resolve(new Response(JSON.stringify({ summary: 'QA globe ready' }), {
|
|
status: 200,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
}));
|
|
}
|
|
return realFetch(input, init);
|
|
};
|
|
}, STORAGE_KEY, APP_ORIGIN);
|
|
|
|
await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
|
await page.waitForFunction(
|
|
() => window.__godsEyeView?.viewer && window.__godsEyeView?.styleManager,
|
|
{ timeout: 60000, polling: 100 },
|
|
);
|
|
|
|
// flyToAustin schedules its 600 m arrival 500 ms after initialization.
|
|
// Let that callback start, then cancel it before establishing the global
|
|
// measurement camera so startup motion cannot invalidate the sample.
|
|
await new Promise((resolve) => setTimeout(resolve, 600));
|
|
await page.evaluate(() => window.__godsEyeView.viewer.camera.cancelFlight());
|
|
|
|
const stateChecks = await page.evaluate((storageKey) => {
|
|
const manager = window.__godsEyeView.styleManager;
|
|
const restored = manager.getDetectionState();
|
|
manager._syncDetectionUiFromEngine();
|
|
const storedAfterPassiveSync = localStorage.getItem(storageKey);
|
|
const off = manager.setDetection({ enabled: false, densityPct: 25 });
|
|
const restoredFromOff = manager.setDetection({ enabled: true });
|
|
return { restored, storedAfterPassiveSync, off, restoredFromOff };
|
|
}, STORAGE_KEY);
|
|
|
|
record(
|
|
'allocation preference survives passive UI sync',
|
|
stateChecks.storedAfterPassiveSync === 'WEIGHTED',
|
|
`share/engine=${stateChecks.restored.allocationStrategy}, stored=${stateChecks.storedAfterPassiveSync}`,
|
|
);
|
|
record(
|
|
'density update while OFF restores the matching profile',
|
|
stateChecks.off.detectionMode === 'OFF'
|
|
&& stateChecks.off.densityPct === 25
|
|
&& stateChecks.restoredFromOff.detectionMode === 'SPARSE'
|
|
&& stateChecks.restoredFromOff.densityPct === 25,
|
|
`OFF=${stateChecks.off.densityPct}%, restore=${stateChecks.restoredFromOff.detectionMode}/${stateChecks.restoredFromOff.densityPct}%`,
|
|
);
|
|
|
|
const injected = await page.evaluate(({ fieldCounts, normalCounts }) => {
|
|
const { viewer, dataManager, styleManager } = window.__godsEyeView;
|
|
viewer.camera.cancelFlight();
|
|
const Cartesian3 = viewer.camera.position.constructor;
|
|
const field = { flights: [], military: [], satellites: [] };
|
|
const layerIds = Object.keys(field);
|
|
const goldenAngle = Math.PI * (3 - Math.sqrt(5));
|
|
|
|
for (const layerId of layerIds) {
|
|
const count = fieldCounts[layerId];
|
|
for (let localIndex = 0; localIndex < count; localIndex++) {
|
|
const radius = 57 * Math.sqrt((localIndex + 0.5) / count);
|
|
const angle = localIndex * goldenAngle + layerIds.indexOf(layerId) * 0.7;
|
|
const lat = Math.max(-65, Math.min(75, 30 + radius * Math.sin(angle)));
|
|
const lon = -97 + radius * Math.cos(angle) / Math.max(0.4, Math.cos(lat * Math.PI / 180));
|
|
const sourceId = `${layerId.slice(0, 1)}-${String(localIndex).padStart(4, '0')}`;
|
|
field[layerId].push({
|
|
sourceId,
|
|
id: sourceId.toUpperCase(),
|
|
metric: layerId === 'satellites' ? 'LEO' : `${250 + (localIndex % 450)}KT`,
|
|
position: Cartesian3.fromDegrees(lon, lat, layerId === 'satellites' ? 550000 : 9000),
|
|
type: layerId === 'satellites' ? 'SAT' : 'AIR',
|
|
tier: layerId === 'military' ? 'military' : (layerId === 'satellites' ? 'space' : 'civil'),
|
|
});
|
|
}
|
|
}
|
|
|
|
window.__LABEL_QA_FIELD = field;
|
|
window.__LABEL_QA_NORMAL_COUNTS = normalCounts;
|
|
for (const layerId of layerIds) {
|
|
const entry = dataManager.layers.get(layerId);
|
|
entry.module.getDetectableObjects = () => window.__LABEL_QA_FIELD[layerId];
|
|
}
|
|
|
|
viewer.camera.setView({
|
|
destination: Cartesian3.fromDegrees(-97, 30, 25000000),
|
|
orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 },
|
|
});
|
|
styleManager.setDetection({
|
|
enabled: true,
|
|
densityPct: 100,
|
|
allocationStrategy: 'elastic',
|
|
});
|
|
|
|
window.__LABEL_QA = {
|
|
solve: [],
|
|
frames: [],
|
|
lastRevision: -1,
|
|
lastFrameNumber: -1,
|
|
startedAt: performance.now(),
|
|
};
|
|
let lastOrbitAt = performance.now();
|
|
window.__LABEL_QA_ORBIT = setInterval(() => {
|
|
const now = performance.now();
|
|
viewer.camera.rotateRight((now - lastOrbitAt) * 0.0000012);
|
|
lastOrbitAt = now;
|
|
viewer.scene.requestRender();
|
|
}, 100);
|
|
const sample = () => {
|
|
const diagnostics = styleManager.getDetectionDiagnostics();
|
|
const frameNumber = viewer.scene.frameState?.frameNumber ?? -1;
|
|
if (diagnostics && frameNumber !== window.__LABEL_QA.lastFrameNumber) {
|
|
window.__LABEL_QA.lastFrameNumber = frameNumber;
|
|
window.__LABEL_QA.frames.push({
|
|
didSolve: diagnostics.didSolve,
|
|
placementBuildCount: diagnostics.placementBuildCount,
|
|
selectedCount: diagnostics.selectedCount,
|
|
fadingCount: diagnostics.fadingCount,
|
|
cohortCount: diagnostics.cohortCount,
|
|
observationCount: diagnostics.observationCount,
|
|
frameTotalMs: diagnostics.frameTotalMs,
|
|
paintMs: diagnostics.paintMs,
|
|
throttleSkipCount: diagnostics.throttleSkipCount,
|
|
});
|
|
}
|
|
const dataset = document.querySelector('#world-overlay-canvas')?.dataset;
|
|
const revision = Number(dataset?.solveRevision || 0);
|
|
if (dataset && revision > 0 && revision !== window.__LABEL_QA.lastRevision) {
|
|
window.__LABEL_QA.lastRevision = revision;
|
|
window.__LABEL_QA.solve.push({
|
|
t: performance.now(),
|
|
revision,
|
|
solveMs: Number(dataset.solveMs),
|
|
frameTotalMs: Number(dataset.frameTotalMs),
|
|
paintMs: Number(dataset.paintMs),
|
|
throttleSkipCount: Number(dataset.throttleSkipCount),
|
|
placementBuildCount: Number(dataset.placementBuildCount),
|
|
observationCount: Number(dataset.observationCount),
|
|
selectedCount: Number(dataset.selectedCount),
|
|
fadingCount: Number(dataset.fadingCount),
|
|
cameraHeight: viewer.camera.positionCartographic.height,
|
|
demandByLayer: JSON.parse(dataset.demandByLayer || '{}'),
|
|
cohortByLayer: JSON.parse(dataset.cohortByLayer || '{}'),
|
|
entitlementByLayer: JSON.parse(dataset.entitlementByLayer || '{}'),
|
|
labeledKeys: JSON.parse(dataset.labeledKeys || '[]'),
|
|
});
|
|
}
|
|
requestAnimationFrame(sample);
|
|
};
|
|
requestAnimationFrame(sample);
|
|
return Object.fromEntries(layerIds.map((layerId) => [layerId, field[layerId].length]));
|
|
}, { fieldCounts: FIELD_COUNTS, normalCounts: NORMAL_COUNTS });
|
|
|
|
record(
|
|
'deterministic field contains exactly 12,000 observations',
|
|
Object.values(injected).reduce((sum, count) => sum + count, 0) === OBSERVATION_COUNT,
|
|
JSON.stringify(injected),
|
|
);
|
|
|
|
// Warm-up before steady-state p95: the first solves after field
|
|
// injection run on cold-tier code paths and skew p95 on a fresh server.
|
|
// (perf wave 2 added a 2.5 s sleep for this; main's settle+conclusive
|
|
// -samples gates supersede it — same fix, deterministic instead of timed.)
|
|
await waitForSettledSyntheticField(page);
|
|
await page.evaluate(() => {
|
|
const viewer = window.__godsEyeView.viewer;
|
|
const revision = Number(document.querySelector('#world-overlay-canvas')?.dataset?.solveRevision || 0);
|
|
window.__LABEL_QA.solve = [];
|
|
window.__LABEL_QA.frames = [];
|
|
window.__LABEL_QA.lastRevision = revision;
|
|
window.__LABEL_QA.lastFrameNumber = viewer.scene.frameState?.frameNumber ?? -1;
|
|
window.__LABEL_QA.startedAt = performance.now();
|
|
});
|
|
const syntheticSampling = await waitForConclusiveSamples(page, {
|
|
minimumElapsedMs: SYNTHETIC_SAMPLE_MS,
|
|
minimumSolves: MIN_SYNTHETIC_SOLVES,
|
|
minimumSolveSpanMs: MIN_SYNTHETIC_SPAN_MS,
|
|
});
|
|
|
|
await page.screenshot({
|
|
path: path.join(SHOT_DIR, 'deterministic-12000.png'),
|
|
fullPage: false,
|
|
});
|
|
|
|
const syntheticSamples = await page.evaluate(() => {
|
|
const completed = window.__LABEL_QA;
|
|
const counts = window.__LABEL_QA_NORMAL_COUNTS;
|
|
for (const layerId of Object.keys(window.__LABEL_QA_FIELD)) {
|
|
window.__LABEL_QA_FIELD[layerId] = window.__LABEL_QA_FIELD[layerId].slice(0, counts[layerId]);
|
|
}
|
|
window.__LABEL_QA = {
|
|
solve: [],
|
|
frames: [],
|
|
lastRevision: -1,
|
|
lastFrameNumber: -1,
|
|
startedAt: performance.now(),
|
|
};
|
|
return completed;
|
|
});
|
|
|
|
await page.waitForFunction(
|
|
() => window.__godsEyeView.styleManager.getDetectionDiagnostics()?.observationCount === 5200,
|
|
{ timeout: SAMPLE_TIMEOUT_MS, polling: 100 },
|
|
);
|
|
await page.evaluate(() => {
|
|
const viewer = window.__godsEyeView.viewer;
|
|
const revision = Number(document.querySelector('#world-overlay-canvas')?.dataset?.solveRevision || 0);
|
|
window.__LABEL_QA.solve = [];
|
|
window.__LABEL_QA.frames = [];
|
|
window.__LABEL_QA.lastRevision = revision;
|
|
window.__LABEL_QA.lastFrameNumber = viewer.scene.frameState?.frameNumber ?? -1;
|
|
window.__LABEL_QA.startedAt = performance.now();
|
|
});
|
|
const normalSampling = await waitForConclusiveSamples(page, {
|
|
minimumElapsedMs: NORMAL_SAMPLE_MS,
|
|
minimumNormalFrames: MIN_NORMAL_FRAMES,
|
|
});
|
|
const normalSamples = await page.evaluate(() => {
|
|
clearInterval(window.__LABEL_QA_ORBIT);
|
|
return window.__LABEL_QA;
|
|
});
|
|
|
|
await page.screenshot({
|
|
path: path.join(SHOT_DIR, 'normal-global-5200.png'),
|
|
fullPage: false,
|
|
});
|
|
|
|
const solveSamples = syntheticSamples.solve;
|
|
const steadySolveSamples = solveSamples.filter((sample) => (
|
|
sample.observationCount === OBSERVATION_COUNT
|
|
&& sample.labeledKeys.length > 0
|
|
&& sample.selectedCount === sample.labeledKeys.length
|
|
&& sample.cameraHeight >= GLOBAL_CAMERA_MIN_HEIGHT_M
|
|
&& sample.cameraHeight <= GLOBAL_CAMERA_MAX_HEIGHT_M
|
|
));
|
|
const normalFieldFrames = normalSamples.frames.filter(
|
|
(sample) => sample.observationCount === 5200,
|
|
);
|
|
const normalFrames = normalFieldFrames.filter((sample) => !sample.didSolve);
|
|
const solveP95 = percentile(solveSamples.map((sample) => sample.solveMs), 0.95);
|
|
const renderP95 = percentile(normalFrames.map((sample) => sample.frameTotalMs), 0.95);
|
|
// SwiftShader can render the complete Cesium scene below 8 FPS, making
|
|
// every sampled overlay frame eligible for the 125 ms solve tick. Paint is
|
|
// timed separately from solve, so it remains the deterministic frame-lane
|
|
// measurement even when there are no non-solve frames in this backend.
|
|
const paintP95 = percentile(
|
|
(normalFrames.length ? normalFrames : normalFieldFrames).map((sample) => sample.paintMs),
|
|
0.95,
|
|
);
|
|
const throttles = normalSamples.frames.map((sample) => sample.throttleSkipCount).filter(Number.isFinite);
|
|
const throttleDelta = throttles.length ? Math.max(...throttles) - Math.min(...throttles) : 0;
|
|
const normalPlacementOverflow = normalFieldFrames.filter(
|
|
(sample) => sample.placementBuildCount > sample.selectedCount + sample.fadingCount
|
|
+ (sample.didSolve ? sample.cohortCount : 0),
|
|
);
|
|
const solvePlacementOverflow = solveSamples.filter((sample) => {
|
|
const cohort = Object.values(sample.cohortByLayer).reduce((sum, count) => sum + count, 0);
|
|
return sample.placementBuildCount > cohort + sample.selectedCount + sample.fadingCount;
|
|
});
|
|
const cohortOverflow = solveSamples.filter((sample) => Object.entries(sample.cohortByLayer)
|
|
.some(([layerId, count]) => {
|
|
const quota = sample.entitlementByLayer[layerId] || 0;
|
|
return count > Math.min(256, Math.max(64, 4 * quota));
|
|
}));
|
|
const membership = analyzeMembership(solveSamples);
|
|
|
|
record(
|
|
'sampling is conclusive across the pathological and normal fields',
|
|
Boolean(syntheticSampling && normalSampling)
|
|
&& solveSamples.length >= MIN_SYNTHETIC_SOLVES
|
|
&& syntheticSampling.solveSpanMs >= MIN_SYNTHETIC_SPAN_MS
|
|
&& normalFieldFrames.length >= MIN_NORMAL_FRAMES,
|
|
`synthetic solves=${solveSamples.length}, span=${syntheticSampling?.solveSpanMs?.toFixed(0) || 0}ms; normal frames=${normalFieldFrames.length}`,
|
|
);
|
|
|
|
record(
|
|
'synthetic churn samples remain non-empty at the 12,000-contact global view',
|
|
steadySolveSamples.length === solveSamples.length,
|
|
`steady=${steadySolveSamples.length}/${solveSamples.length}; population=${membership.minimumPopulation}-${membership.maximumPopulation}`,
|
|
);
|
|
|
|
record(
|
|
'solve cohort remains bounded per layer',
|
|
cohortOverflow.length === 0,
|
|
`${solveSamples.length} solves checked`,
|
|
);
|
|
record(
|
|
'placement construction is bounded in both lanes',
|
|
normalPlacementOverflow.length === 0 && solvePlacementOverflow.length === 0,
|
|
`normal overflow=${normalPlacementOverflow.length}, solve overflow=${solvePlacementOverflow.length}`,
|
|
);
|
|
record(
|
|
'solve p95 is below approximately 4 ms',
|
|
solveP95 < 4.5,
|
|
`${solveP95.toFixed(2)} ms`,
|
|
);
|
|
record(
|
|
'normal-field overlay paint p95 is at or below 10 ms',
|
|
paintP95 <= 10,
|
|
`paint=${paintP95.toFixed(2)} ms${Number.isFinite(renderP95) ? `, non-solve total=${renderP95.toFixed(2)} ms` : ', every sampled frame included a solve'}`,
|
|
);
|
|
record(
|
|
'normal scene does not trigger adaptive throttling',
|
|
throttleDelta === 0,
|
|
`skip delta=${throttleDelta}`,
|
|
);
|
|
record(
|
|
'label churn remains below 8% per second',
|
|
membership.churnPctPerSec < 8,
|
|
`${membership.churnPctPerSec.toFixed(2)}%/s; replacements=${membership.replacements}, exposure=${membership.exposure.toFixed(1)} label-s, transitions=${membership.transitions}, population=${membership.minimumPopulation}-${membership.maximumPopulation}`,
|
|
);
|
|
record(
|
|
'no show-hide-show cycle occurs within two seconds',
|
|
membership.showHideShow === 0,
|
|
`cycles=${membership.showHideShow}`,
|
|
);
|
|
record(
|
|
'console remains clean',
|
|
consoleErrors.length === 0,
|
|
consoleErrors.slice(0, 3).join(' | ') || '0 errors',
|
|
);
|
|
record(
|
|
'no HTTP 5xx responses occur',
|
|
failedResponses.length === 0,
|
|
failedResponses.slice(0, 3).join(' | ') || '0 responses',
|
|
);
|
|
|
|
console.log(`\n Screenshot: ${path.join(SHOT_DIR, 'deterministic-12000.png')}`);
|
|
console.log(` Screenshot: ${path.join(SHOT_DIR, 'normal-global-5200.png')}`);
|
|
console.log(` Synthetic solves: ${solveSamples.length}; normal frames: ${normalFieldFrames.length} (${normalFrames.length} non-solve)`);
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
|
|
const failures = results.filter((result) => !result.ok);
|
|
console.log(`\n${results.length - failures.length}/${results.length} checks passed.`);
|
|
if (failures.length) process.exit(1);
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|