1126 lines
44 KiB
JavaScript
1126 lines
44 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import {
|
|
assertSanitizedEvidence,
|
|
extractProjectSummary,
|
|
parsePosthogLiveArguments,
|
|
parseSanitizedAgentProof,
|
|
PosthogLivePreflightError,
|
|
preflightFailureMessage,
|
|
preparePosthogLiveSmoke,
|
|
} from "./posthog-live-lib.mjs";
|
|
|
|
const TARGET_COMPANY_PREFIX = "PAP";
|
|
const TARGET_AGENT_NAME = "CodexCoderPro";
|
|
const PROJECT_GET = "project-get";
|
|
const PROJECT_SETTINGS_UPDATE = "project-settings-update";
|
|
const EXCLUDED_PROJECT_SWITCHERS = new Set(["switch-project", "switch-organization"]);
|
|
const DEFAULT_AGENT_TIMEOUT_MS = 15 * 60_000;
|
|
|
|
class SmokeFailure extends Error {
|
|
constructor(checkpoint, code, details = null) {
|
|
super(`${checkpoint}:${code}`);
|
|
this.name = "SmokeFailure";
|
|
this.checkpoint = checkpoint;
|
|
this.code = code;
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
function fail(checkpoint, code, details = null) {
|
|
throw new SmokeFailure(checkpoint, code, details);
|
|
}
|
|
|
|
function asArray(value, key) {
|
|
if (Array.isArray(value)) return value;
|
|
if (value && typeof value === "object" && Array.isArray(value[key])) return value[key];
|
|
return [];
|
|
}
|
|
|
|
function escapeRegex(value) {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|
|
|
|
function screenshotFile(outputDirectory, name) {
|
|
return path.join(outputDirectory, name);
|
|
}
|
|
|
|
function safeConnectionConfig(connection, expectedProjectId) {
|
|
const methodConfig = connection?.config?.methodConfig;
|
|
if (!methodConfig || typeof methodConfig !== "object" || Array.isArray(methodConfig)) {
|
|
fail("C.connection-detail", "method_config_missing");
|
|
}
|
|
if (connection.config?.connectionMethodKey !== "mcp-oauth") {
|
|
fail("C.connection-detail", "oauth_method_not_recorded");
|
|
}
|
|
if (String(methodConfig.projectId ?? "") !== expectedProjectId) {
|
|
fail("C.connection-detail", "wrong_project_pin");
|
|
}
|
|
if (methodConfig.readOnly !== false || methodConfig.mode !== "tools") {
|
|
fail("C.connection-detail", "unexpected_posthog_scope");
|
|
}
|
|
if ((methodConfig.features ?? "") !== "" || (methodConfig.tools ?? "") !== "") {
|
|
fail("C.connection-detail", "unexpected_tool_filter");
|
|
}
|
|
if (!connection.config?.oauth?.connectedAt) {
|
|
fail("B.oauth-callback", "oauth_not_connected");
|
|
}
|
|
}
|
|
|
|
function assertNoCredentialMaterial(value, secrets, checkpoint) {
|
|
const serialized = JSON.stringify(value);
|
|
for (const secret of secrets) {
|
|
if (secret && serialized.includes(secret)) fail(checkpoint, "credential_material_visible");
|
|
}
|
|
if (/[?&](?:code|state|access_token|refresh_token)=/i.test(serialized)) {
|
|
fail(checkpoint, "oauth_query_material_visible");
|
|
}
|
|
const forbiddenValueKey = /^(?:password|accessToken|access_token|refreshToken|refresh_token|oauthCode|oauth_code|clientSecret|client_secret|cookie|authorizationHeader)$/i;
|
|
const visit = (candidate, seen = new Set()) => {
|
|
if (!candidate || typeof candidate !== "object" || seen.has(candidate)) return;
|
|
seen.add(candidate);
|
|
for (const [key, child] of Object.entries(candidate)) {
|
|
if (forbiddenValueKey.test(key) && child !== null && child !== "") {
|
|
fail(checkpoint, "raw_credential_field_visible");
|
|
}
|
|
visit(child, seen);
|
|
}
|
|
};
|
|
visit(value);
|
|
}
|
|
|
|
async function apiJson(request, baseUrl, method, pathname, data, checkpoint, expectedStatuses = [200]) {
|
|
let response;
|
|
try {
|
|
const origin = new URL(baseUrl).origin;
|
|
response = await request.fetch(new URL(pathname, baseUrl).toString(), {
|
|
method,
|
|
...(data === undefined ? {} : { data }),
|
|
headers: {
|
|
accept: "application/json",
|
|
origin,
|
|
referer: `${origin}/`,
|
|
},
|
|
timeout: 30_000,
|
|
});
|
|
} catch {
|
|
fail(checkpoint, "request_failed");
|
|
}
|
|
if (!expectedStatuses.includes(response.status())) {
|
|
fail(checkpoint, `http_${response.status()}`);
|
|
}
|
|
try {
|
|
return await response.json();
|
|
} catch {
|
|
fail(checkpoint, "invalid_json");
|
|
}
|
|
}
|
|
|
|
async function waitFor(checkpoint, fn, { timeoutMs = 120_000, intervalMs = 2_000 } = {}) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let lastValue;
|
|
while (Date.now() < deadline) {
|
|
lastValue = await fn();
|
|
if (lastValue) return lastValue;
|
|
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
}
|
|
fail(checkpoint, "timed_out");
|
|
return lastValue;
|
|
}
|
|
|
|
async function expectVisible(locator, checkpoint, code, timeout = 30_000) {
|
|
try {
|
|
await locator.waitFor({ state: "visible", timeout });
|
|
} catch {
|
|
fail(checkpoint, code);
|
|
}
|
|
}
|
|
|
|
async function gotoPaperclipPage(
|
|
page,
|
|
url,
|
|
readyLocator,
|
|
checkpoint,
|
|
code,
|
|
{ attempts = 3, timeout = 15_000 } = {},
|
|
) {
|
|
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
try {
|
|
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30_000 });
|
|
await readyLocator.waitFor({ state: "visible", timeout });
|
|
return;
|
|
} catch {
|
|
if (attempt < attempts) await page.waitForTimeout(500);
|
|
}
|
|
}
|
|
fail(checkpoint, code);
|
|
}
|
|
|
|
async function stabilizeCompanyContext(page, config, companyId) {
|
|
const galleryPath = `/api/companies/${companyId}/tools/gallery`;
|
|
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
|
try {
|
|
const galleryResponsePromise = page.waitForResponse(
|
|
(response) => response.request().method() === "GET"
|
|
&& new URL(response.url()).pathname === galleryPath,
|
|
{ timeout: 30_000 },
|
|
);
|
|
await page.goto(new URL(`/${TARGET_COMPANY_PREFIX}/apps`, config.baseUrl).toString(), {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: 30_000,
|
|
});
|
|
const response = await galleryResponsePromise;
|
|
if (!response.ok()) continue;
|
|
const posthogAction = page.getByRole("button", {
|
|
name: /^(?:Connect for PostHog|Add another PostHog account)$/,
|
|
}).first();
|
|
await posthogAction.waitFor({ state: "visible", timeout: 30_000 });
|
|
// The company-prefixed route and selected-company provider settle in
|
|
// separate renders. Clicking the tile immediately can carry the prior
|
|
// company's gallery cache into the setup effect and redirect back out.
|
|
await page.waitForTimeout(2_000);
|
|
return;
|
|
} catch {
|
|
if (attempt < 3) await page.waitForTimeout(500);
|
|
}
|
|
}
|
|
fail("A.company-context", "posthog_gallery_context_missing");
|
|
}
|
|
|
|
async function openPosthogSetupFromGallery(page, config, companyId) {
|
|
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
|
await stabilizeCompanyContext(page, config, companyId);
|
|
const addAnother = page.getByRole("button", { name: /^Add another PostHog account$/ }).first();
|
|
const connect = page.getByRole("button", { name: /^Connect for PostHog$/ }).first();
|
|
const action = await addAnother.isVisible().catch(() => false) ? addAnother : connect;
|
|
try {
|
|
await action.click({ timeout: 5_000 });
|
|
await page.getByRole("button", { name: "Sign in with PostHog" }).waitFor({
|
|
state: "visible",
|
|
timeout: 30_000,
|
|
});
|
|
return;
|
|
} catch {
|
|
if (attempt < 3) await page.waitForTimeout(500);
|
|
}
|
|
}
|
|
fail("A.setup-route", "oauth_method_missing");
|
|
}
|
|
|
|
async function safePageState(page, resourceFailures, paperclipOrigin) {
|
|
let current;
|
|
try {
|
|
current = new URL(page.url());
|
|
} catch {
|
|
return { location: "invalid", resourceFailures };
|
|
}
|
|
const bodyText = await page.locator("body").innerText().catch(() => "");
|
|
return {
|
|
location: current.origin === paperclipOrigin
|
|
? `${current.hostname}${current.pathname}`
|
|
: current.hostname,
|
|
headingCount: await page.getByRole("heading").count().catch(() => 0),
|
|
buttonCount: await page.getByRole("button").count().catch(() => 0),
|
|
methodSignals: {
|
|
posthogSignIn: /sign in with posthog/i.test(bodyText),
|
|
personalApiKey: /personal api key/i.test(bodyText),
|
|
connectApp: /connect an app/i.test(bodyText),
|
|
},
|
|
resourceFailures,
|
|
};
|
|
}
|
|
|
|
async function clickVisibleButton(page, names) {
|
|
for (const name of names) {
|
|
for (const role of ["button", "link"]) {
|
|
const control = page.getByRole(role, { name, exact: false }).filter({ visible: true }).first();
|
|
if (await control.count()) {
|
|
try {
|
|
await control.click({ timeout: 2_000 });
|
|
return true;
|
|
} catch {
|
|
// Provider pages often replace their form between locator creation
|
|
// and click. The next loop re-reads the current DOM.
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
async function selectPosthogCloudRegion(page) {
|
|
const region = (process.env.POSTHOG_CLOUD_REGION || "us").trim().toLowerCase();
|
|
if (!new Set(["us", "eu"]).has(region)) {
|
|
fail("B.oauth-callback", "unsupported_cloud_region");
|
|
}
|
|
const expectedHost = `${region}.posthog.com`;
|
|
const links = page.getByRole("link");
|
|
for (let index = 0; index < await links.count(); index += 1) {
|
|
const link = links.nth(index);
|
|
const href = await link.getAttribute("href");
|
|
if (!href) continue;
|
|
try {
|
|
const target = new URL(href, page.url());
|
|
if (target.hostname !== expectedHost) continue;
|
|
await page.goto(target.toString(), { waitUntil: "domcontentloaded", timeout: 30_000 });
|
|
return true;
|
|
} catch {
|
|
// The provider can replace this chooser while the link is being read.
|
|
// The next authorization-loop iteration re-evaluates it.
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
async function completePosthogAuthorization(page, config) {
|
|
const paperclipOrigin = new URL(config.baseUrl).origin;
|
|
const providerTimeoutMs = Number(process.env.POSTHOG_PROVIDER_TIMEOUT_MS || 4 * 60_000);
|
|
const deadline = Date.now() + (Number.isFinite(providerTimeoutMs) && providerTimeoutMs > 0
|
|
? providerTimeoutMs
|
|
: 4 * 60_000);
|
|
let providerState = null;
|
|
let credentialFormSubmitted = false;
|
|
while (Date.now() < deadline) {
|
|
let current;
|
|
try {
|
|
current = new URL(page.url());
|
|
} catch {
|
|
fail("B.oauth-callback", "invalid_navigation_url");
|
|
}
|
|
if (current.origin === paperclipOrigin && current.pathname.includes("/apps/")) return;
|
|
|
|
const emailInput = page.locator('input[type="email"], input[name="email"], input[autocomplete="username"]').filter({ visible: true }).first();
|
|
const passwordInput = page.locator('input[type="password"], input[name="password"], input[autocomplete="current-password"]').filter({ visible: true }).first();
|
|
const identityFieldVisible = await emailInput.count() > 0;
|
|
const credentialFieldVisible = await passwordInput.count() > 0;
|
|
const credentialForm = page.locator("form").filter({ has: passwordInput }).first();
|
|
const submitControl = credentialForm.locator('button[type="submit"], input[type="submit"]').filter({ visible: true }).first();
|
|
const consentControlVisible = await page.getByRole("button", {
|
|
name: /^(?:authorize|allow|approve|grant access|accept)$/i,
|
|
}).first().isVisible().catch(() => false);
|
|
const bodyText = await page.locator("body").innerText().catch(() => "");
|
|
const linkHrefs = await page.getByRole("link").evaluateAll((links) =>
|
|
links.map((link) => link.getAttribute("href")).filter(Boolean)
|
|
).catch(() => []);
|
|
const linkTargets = Array.from(new Set(linkHrefs.map((href) => {
|
|
try {
|
|
const target = new URL(href, current.origin);
|
|
return `${target.hostname}${target.pathname}`;
|
|
} catch {
|
|
return "invalid";
|
|
}
|
|
})));
|
|
providerState = {
|
|
host: current.hostname,
|
|
path: current.pathname.slice(0, 200),
|
|
identityFieldVisible,
|
|
credentialFieldVisible,
|
|
credentialFormVisible: await credentialForm.count() > 0,
|
|
submitControlVisible: await submitControl.count() > 0,
|
|
submitControlDisabled: await submitControl.isDisabled().catch(() => false),
|
|
consentControlVisible,
|
|
alertCount: await page.getByRole("alert").count(),
|
|
headingCount: await page.getByRole("heading").count(),
|
|
buttonCount: await page.getByRole("button").count(),
|
|
linkCount: await page.getByRole("link").count(),
|
|
linkTargets,
|
|
frameCount: page.frames().length,
|
|
semanticSignals: {
|
|
signIn: /\b(?:sign in|log in)\b/i.test(bodyText),
|
|
continue: /\bcontinue\b/i.test(bodyText),
|
|
consent: /\b(?:authorize|allow|approve|grant access|accept)\b/i.test(bodyText),
|
|
loading: /\b(?:loading|preparing|opening|redirecting)\b/i.test(bodyText),
|
|
workspace: /\b(?:workspace|organization|project)\b/i.test(bodyText),
|
|
error: /\b(?:error|invalid|failed|problem|went wrong)\b/i.test(bodyText),
|
|
},
|
|
};
|
|
if (current.hostname === "oauth.posthog.com" && await selectPosthogCloudRegion(page)) {
|
|
await page.waitForTimeout(500);
|
|
continue;
|
|
}
|
|
if (await emailInput.count()) {
|
|
const currentValue = await emailInput.inputValue().catch(() => "");
|
|
if (!currentValue) await emailInput.fill(config.email);
|
|
}
|
|
|
|
if (await passwordInput.count()) {
|
|
const currentValue = await passwordInput.inputValue().catch(() => "");
|
|
if (!currentValue) await passwordInput.fill(config.password);
|
|
if (!credentialFormSubmitted) {
|
|
if (await submitControl.count() && !await submitControl.isDisabled().catch(() => true)) {
|
|
await submitControl.click({ noWaitAfter: true, timeout: 2_000 }).catch(() => {});
|
|
} else {
|
|
const submitted = await clickVisibleButton(page, [
|
|
/^sign in$/i,
|
|
/^log in$/i,
|
|
/^login$/i,
|
|
/^continue$/i,
|
|
/sign in with email/i,
|
|
/log in with email/i,
|
|
/login with email/i,
|
|
]);
|
|
if (!submitted) await passwordInput.press("Enter").catch(() => {});
|
|
}
|
|
credentialFormSubmitted = true;
|
|
}
|
|
} else if (await emailInput.count()) {
|
|
await clickVisibleButton(page, [/^continue$/i, /^next$/i, /continue with email/i, /sign in with email/i]);
|
|
} else {
|
|
await clickVisibleButton(page, [
|
|
/^authorize$/i,
|
|
/^allow$/i,
|
|
/^approve$/i,
|
|
/^grant access$/i,
|
|
/^accept$/i,
|
|
/^continue$/i,
|
|
/^sign in$/i,
|
|
/^log in$/i,
|
|
]);
|
|
}
|
|
await page.waitForTimeout(500);
|
|
}
|
|
fail("B.oauth-callback", "provider_authorization_timed_out", { providerState });
|
|
}
|
|
|
|
async function safeScreenshot(page, outputPath, config, checkpoint) {
|
|
const current = new URL(page.url());
|
|
if (current.origin !== new URL(config.baseUrl).origin) fail(checkpoint, "screenshot_not_on_paperclip");
|
|
for (const queryKey of ["code", "state", "token", "access_token", "refresh_token"]) {
|
|
if (current.searchParams.has(queryKey)) fail(checkpoint, "credential_query_in_screenshot_url");
|
|
}
|
|
const bodyText = await page.locator("body").innerText();
|
|
if (bodyText.includes(config.password) || /[?&](?:code|state|access_token|refresh_token)=/i.test(bodyText)) {
|
|
fail(checkpoint, "credential_material_in_screenshot");
|
|
}
|
|
await page.screenshot({
|
|
path: outputPath,
|
|
fullPage: true,
|
|
animations: "disabled",
|
|
mask: [page.getByText(config.email, { exact: false })],
|
|
});
|
|
}
|
|
|
|
function catalogFacts(catalog, checkpoint) {
|
|
const active = catalog.filter((entry) => entry.status !== "removed");
|
|
const projectGet = active.find((entry) => entry.toolName === PROJECT_GET);
|
|
const projectSettings = active.find((entry) => entry.toolName === PROJECT_SETTINGS_UPDATE);
|
|
if (!projectGet || !projectGet.isReadOnly) fail(checkpoint, "project_get_missing_or_not_read_only");
|
|
if (!projectSettings || projectSettings.isReadOnly) fail(checkpoint, "project_settings_update_missing_or_not_write");
|
|
for (const excluded of EXCLUDED_PROJECT_SWITCHERS) {
|
|
if (active.some((entry) => entry.toolName === excluded)) fail(checkpoint, `excluded_${excluded}_present`);
|
|
}
|
|
return { active, projectGet, projectSettings };
|
|
}
|
|
|
|
async function finishAgentOnlySetup(request, config, companyId, connectionId, catalog, agentId) {
|
|
const { active } = catalogFacts(catalog, "C.catalog-policy");
|
|
const enabledCatalogEntryIds = active.filter((entry) => entry.isReadOnly).map((entry) => entry.id);
|
|
await apiJson(
|
|
request,
|
|
config.baseUrl,
|
|
"POST",
|
|
`/api/companies/${companyId}/tools/apps/${connectionId}/finish`,
|
|
{
|
|
enabledCatalogEntryIds,
|
|
askFirstCatalogEntryIds: [],
|
|
reviewedCatalogEntryIds: active.filter((entry) => entry.status === "quarantined").map((entry) => entry.id),
|
|
access: { agentIds: [agentId] },
|
|
},
|
|
"C.catalog-policy",
|
|
);
|
|
await apiJson(
|
|
request,
|
|
config.baseUrl,
|
|
"PUT",
|
|
`/api/tool-connections/${connectionId}/installs`,
|
|
{ installs: [{ targetType: "agent", targetId: agentId }] },
|
|
"C.agent-install",
|
|
);
|
|
}
|
|
|
|
async function findConnectionIdByName(request, config, companyId, connectionName) {
|
|
const connectionsResponse = await apiJson(
|
|
request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/connections`,
|
|
undefined,
|
|
"F.cleanup-recovery",
|
|
);
|
|
const matching = asArray(connectionsResponse, "connections").filter(
|
|
(connection) => connection.name === connectionName && connection.status !== "archived",
|
|
);
|
|
if (matching.length > 1) fail("F.cleanup-recovery", "duplicate_test_connections");
|
|
return matching[0]?.id ?? null;
|
|
}
|
|
|
|
async function cleanupConnection(
|
|
request,
|
|
config,
|
|
companyId,
|
|
connectionId,
|
|
connectionName,
|
|
{ requireInstalledState = true } = {},
|
|
) {
|
|
const removed = await apiJson(
|
|
request,
|
|
config.baseUrl,
|
|
"DELETE",
|
|
`/api/tool-connections/${connectionId}`,
|
|
undefined,
|
|
"F.cleanup",
|
|
);
|
|
const receipt = removed.removal;
|
|
if (!receipt || (requireInstalledState && (
|
|
receipt.installsRemoved < 1
|
|
|| receipt.appProfileBindingsRemoved < 1
|
|
|| receipt.credentialRefsCleared + receipt.secretsRevoked < 1
|
|
|| !["deleted", "archived"].includes(receipt.appProfile)
|
|
))) {
|
|
fail("F.cleanup", "incomplete_removal_receipt");
|
|
}
|
|
|
|
const connectionsResponse = await apiJson(
|
|
request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/connections`,
|
|
undefined,
|
|
"F.cleanup-verification",
|
|
);
|
|
const remaining = asArray(connectionsResponse, "connections").filter(
|
|
(connection) => connection.name === connectionName && connection.status !== "archived",
|
|
);
|
|
if (remaining.length > 0) fail("F.cleanup-verification", "test_connection_remains");
|
|
|
|
const profilesResponse = await apiJson(
|
|
request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/profiles`,
|
|
undefined,
|
|
"F.cleanup-verification",
|
|
);
|
|
if (asArray(profilesResponse, "profiles").some((profile) => profile.profileKey === `app:${connectionId}` && profile.status === "active")) {
|
|
fail("F.cleanup-verification", "active_profile_remains");
|
|
}
|
|
const pendingResponse = await apiJson(
|
|
request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/action-requests?status=pending`,
|
|
undefined,
|
|
"F.cleanup-verification",
|
|
);
|
|
if (asArray(pendingResponse, "actionRequests").some((item) => (item.connectionId ?? item.request?.connectionId) === connectionId)) {
|
|
fail("F.cleanup-verification", "pending_action_remains");
|
|
}
|
|
return {
|
|
credentialsTornDown: receipt.credentialRefsCleared + receipt.secretsRevoked > 0,
|
|
accessBindingsRemoved: receipt.appProfileBindingsRemoved,
|
|
installsRemoved: receipt.installsRemoved,
|
|
appProfile: receipt.appProfile,
|
|
pendingActions: 0,
|
|
remainingConnections: 0,
|
|
};
|
|
}
|
|
|
|
async function runSmoke({ config, chromium }) {
|
|
const startedAt = new Date();
|
|
const runKey = startedAt.toISOString().replace(/[:.]/g, "-");
|
|
const connectionName = `PostHog live self-test ${startedAt.toISOString()}`;
|
|
const outputDirectory = process.env.POSTHOG_EVIDENCE_DIR
|
|
? path.resolve(process.env.POSTHOG_EVIDENCE_DIR)
|
|
: path.join(process.env.PAPERCLIP_RUN_SCRATCH_DIR || process.cwd(), `posthog-live-${runKey}`);
|
|
await mkdir(outputDirectory, { recursive: true });
|
|
|
|
const summary = {
|
|
schemaVersion: 1,
|
|
smoke: "posthog_mcp_live",
|
|
passed: false,
|
|
startedAt: startedAt.toISOString(),
|
|
completedAt: null,
|
|
target: { companyPrefix: TARGET_COMPANY_PREFIX, projectId: config.projectId },
|
|
connection: null,
|
|
catalog: null,
|
|
boardTest: null,
|
|
freshRun: null,
|
|
cleanup: null,
|
|
screenshots: [],
|
|
failure: null,
|
|
};
|
|
|
|
let browser;
|
|
let context;
|
|
let page;
|
|
let connectionId = null;
|
|
let companyId = null;
|
|
let cleanupComplete = false;
|
|
let caughtFailure = null;
|
|
let activeCheckpoint = "A.browser-launch";
|
|
const resourceFailures = [];
|
|
|
|
try {
|
|
browser = await chromium.launch({ headless: process.env.POSTHOG_SMOKE_HEADED !== "1" });
|
|
context = await browser.newContext({
|
|
viewport: { width: 1440, height: 1000 },
|
|
acceptDownloads: false,
|
|
serviceWorkers: "block",
|
|
});
|
|
page = await context.newPage();
|
|
page.on("requestfailed", (request) => {
|
|
if (!["document", "script", "stylesheet", "xhr", "fetch"].includes(request.resourceType())) return;
|
|
try {
|
|
const target = new URL(request.url());
|
|
if (target.origin !== new URL(config.baseUrl).origin) return;
|
|
resourceFailures.push({
|
|
target: `${target.hostname}${target.pathname}`,
|
|
resourceType: request.resourceType(),
|
|
error: request.failure()?.errorText ?? "unknown",
|
|
});
|
|
if (resourceFailures.length > 12) resourceFailures.shift();
|
|
} catch {
|
|
// Ignore malformed resource URLs rather than copying them into evidence.
|
|
}
|
|
});
|
|
|
|
activeCheckpoint = "A.paperclip-login";
|
|
await gotoPaperclipPage(
|
|
page,
|
|
new URL("/auth?next=/", config.baseUrl).toString(),
|
|
page.locator("#email"),
|
|
"A.paperclip-login",
|
|
"email_field_missing",
|
|
);
|
|
await page.locator("#email").fill(config.email);
|
|
await page.locator("#password").fill(config.password);
|
|
const loginResponsePromise = page.waitForResponse((response) =>
|
|
response.request().method() === "POST" && new URL(response.url()).pathname === "/api/auth/sign-in/email",
|
|
);
|
|
await page.getByRole("button", { name: /^sign in$/i }).click();
|
|
const loginResponse = await loginResponsePromise;
|
|
if (!loginResponse.ok()) fail("A.paperclip-login", `http_${loginResponse.status()}`);
|
|
await page.waitForURL((url) => url.pathname !== "/auth", { timeout: 30_000 }).catch(() => {
|
|
fail("A.paperclip-login", "login_redirect_missing");
|
|
});
|
|
|
|
activeCheckpoint = "A.company-selection";
|
|
const companiesResponse = await apiJson(context.request, config.baseUrl, "GET", "/api/companies", undefined, "A.company-selection");
|
|
const company = asArray(companiesResponse, "companies").find((candidate) => candidate.issuePrefix === TARGET_COMPANY_PREFIX);
|
|
if (!company) fail("A.company-selection", "pap_company_missing");
|
|
companyId = company.id;
|
|
|
|
activeCheckpoint = "C.agent-scope";
|
|
const agentsResponse = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/companies/${companyId}/agents`,
|
|
undefined,
|
|
"C.agent-scope",
|
|
);
|
|
const agent = asArray(agentsResponse, "agents").find((candidate) => candidate.name === TARGET_AGENT_NAME);
|
|
if (!agent) fail("C.agent-scope", "codex_coder_pro_missing");
|
|
|
|
activeCheckpoint = "A.setup-route";
|
|
await openPosthogSetupFromGallery(page, config, companyId);
|
|
await expectVisible(page.getByRole("button", { name: "Use a personal API key" }), "A.setup-route", "api_key_method_missing");
|
|
|
|
activeCheckpoint = "B.oauth-setup";
|
|
await page.getByRole("button", { name: "Sign in with PostHog" }).click();
|
|
const nameInput = page.locator('input[placeholder="My app"]');
|
|
await nameInput.fill(connectionName);
|
|
await page.locator('input[placeholder="12345"]').fill(config.projectId);
|
|
const readOnlyScope = page.locator("div").filter({ hasText: /^Read-only mode/ }).filter({ has: page.getByRole("switch") }).first();
|
|
const readOnlySwitch = readOnlyScope.getByRole("switch");
|
|
if (await readOnlySwitch.getAttribute("aria-checked") !== "false") fail("B.oauth-setup", "read_only_default_changed");
|
|
await page.getByText("Advanced", { exact: true }).click();
|
|
const featuresInput = page.locator("label", { hasText: "Feature groups" }).locator("..").locator("textarea");
|
|
const toolsInput = page.locator("label", { hasText: "Individual tools" }).locator("..").locator("textarea");
|
|
if (await featuresInput.inputValue() !== "" || await toolsInput.inputValue() !== "") {
|
|
fail("B.oauth-setup", "tool_filters_not_blank");
|
|
}
|
|
const responseMode = page.locator("label", { hasText: "Tool response mode" }).locator("..").locator("select");
|
|
if (await responseMode.inputValue() !== "tools") fail("B.oauth-setup", "individual_tools_mode_not_selected");
|
|
|
|
activeCheckpoint = "B.oauth-start";
|
|
const connectResponsePromise = page.waitForResponse(
|
|
(response) => {
|
|
const target = new URL(response.url());
|
|
return response.request().method() === "POST"
|
|
&& target.pathname === `/api/companies/${companyId}/tools/apps/connect`;
|
|
},
|
|
{ timeout: 120_000 },
|
|
);
|
|
await page.getByRole("button", { name: "Continue to sign in" }).click({ noWaitAfter: true });
|
|
const connectResponse = await connectResponsePromise;
|
|
if (!connectResponse.ok()) fail("B.oauth-start", `http_${connectResponse.status()}`);
|
|
let connectResult = null;
|
|
try {
|
|
connectResult = await connectResponse.json();
|
|
} catch {
|
|
// A successful create immediately redirects the page to PostHog. Chromium
|
|
// can discard that response body during the cross-origin navigation, so
|
|
// recover the uniquely named draft instead of orphaning it.
|
|
}
|
|
connectionId = connectResult?.connectionId ?? await waitFor(
|
|
"B.oauth-start",
|
|
() => findConnectionIdByName(context.request, config, companyId, connectionName),
|
|
{ timeoutMs: 15_000, intervalMs: 500 },
|
|
);
|
|
if (typeof connectionId !== "string" || !connectionId) fail("B.oauth-start", "connection_id_missing");
|
|
|
|
activeCheckpoint = "B.oauth-callback";
|
|
await completePosthogAuthorization(page, config);
|
|
const permissionsPath = `/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/permissions`;
|
|
await gotoPaperclipPage(
|
|
page,
|
|
new URL(permissionsPath, config.baseUrl).toString(),
|
|
page.getByText("PostHog connected", { exact: true }),
|
|
"B.oauth-callback",
|
|
"connected_state_missing",
|
|
{ attempts: 3, timeout: 30_000 },
|
|
);
|
|
|
|
activeCheckpoint = "C.connection-detail";
|
|
let connection = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}`,
|
|
undefined,
|
|
"C.connection-detail",
|
|
);
|
|
safeConnectionConfig(connection, config.projectId);
|
|
assertNoCredentialMaterial(connection, [config.password], "C.connection-detail");
|
|
summary.connection = {
|
|
id: connectionId,
|
|
name: connectionName,
|
|
authentication: "oauth",
|
|
projectId: config.projectId,
|
|
status: connection.status,
|
|
healthStatus: connection.healthStatus,
|
|
};
|
|
const connectedShot = "01-connected-setup.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, connectedShot), config, "F.connected-screenshot");
|
|
summary.screenshots.push(connectedShot);
|
|
|
|
activeCheckpoint = "C.catalog-policy";
|
|
let catalogResponse = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}/catalog`,
|
|
undefined,
|
|
"C.catalog-policy",
|
|
);
|
|
let catalog = asArray(catalogResponse, "catalog");
|
|
let facts = catalogFacts(catalog, "C.catalog-policy");
|
|
await finishAgentOnlySetup(context.request, config, companyId, connectionId, catalog, agent.id);
|
|
|
|
const health = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"POST",
|
|
`/api/tool-connections/${connectionId}/health-check`,
|
|
{},
|
|
"C.health-check",
|
|
);
|
|
if (health.connection?.healthStatus !== "healthy") fail("C.health-check", "connection_not_healthy");
|
|
const refreshed = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"POST",
|
|
`/api/tool-connections/${connectionId}/catalog/refresh`,
|
|
{},
|
|
"C.catalog-refresh",
|
|
);
|
|
catalog = asArray(refreshed, "catalog");
|
|
facts = catalogFacts(catalog, "C.catalog-refresh");
|
|
await finishAgentOnlySetup(context.request, config, companyId, connectionId, catalog, agent.id);
|
|
|
|
connection = await apiJson(context.request, config.baseUrl, "GET", `/api/tool-connections/${connectionId}`, undefined, "C.connection-detail");
|
|
safeConnectionConfig(connection, config.projectId);
|
|
if (connection.status !== "active" || connection.healthStatus !== "healthy") {
|
|
fail("C.connection-detail", "connection_not_active_and_healthy");
|
|
}
|
|
assertNoCredentialMaterial(connection, [config.password], "C.connection-detail");
|
|
summary.connection.status = connection.status;
|
|
summary.connection.healthStatus = connection.healthStatus;
|
|
|
|
const uniqueConnections = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/connections`,
|
|
undefined,
|
|
"C.connection-detail",
|
|
);
|
|
if (asArray(uniqueConnections, "connections").filter((candidate) => candidate.name === connectionName).length !== 1) {
|
|
fail("C.connection-detail", "duplicate_connection_detected");
|
|
}
|
|
|
|
const installs = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}/installs`,
|
|
undefined,
|
|
"C.agent-install",
|
|
);
|
|
const installRows = asArray(installs, "installs");
|
|
if (installRows.length !== 1 || installRows[0].targetType !== "agent" || installRows[0].targetId !== agent.id) {
|
|
fail("C.agent-install", "install_not_agent_only");
|
|
}
|
|
|
|
const testAgents = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}/test-agents`,
|
|
undefined,
|
|
"C.effective-policy",
|
|
);
|
|
const testAgent = asArray(testAgents, "agents").find((candidate) => candidate.id === agent.id);
|
|
const projectGetAccess = testAgent?.effectiveAccess?.tools?.find((tool) => tool.toolName === PROJECT_GET);
|
|
const projectSettingsAccess = testAgent?.effectiveAccess?.tools?.find((tool) => tool.toolName === PROJECT_SETTINGS_UPDATE);
|
|
if (projectGetAccess?.decision !== "allowed" || projectSettingsAccess?.decision !== "off") {
|
|
fail("C.effective-policy", "unexpected_effective_decision");
|
|
}
|
|
summary.catalog = {
|
|
discoveredCount: refreshed.discoveredCount,
|
|
projectGet: { catalogEntryId: facts.projectGet.id, toolName: PROJECT_GET, decision: "allowed" },
|
|
projectSettingsUpdate: { catalogEntryId: facts.projectSettings.id, toolName: PROJECT_SETTINGS_UPDATE, decision: "off" },
|
|
excludedToolsAbsent: [...EXCLUDED_PROJECT_SWITCHERS],
|
|
accessAgentId: agent.id,
|
|
installAgentId: agent.id,
|
|
healthCheck: "healthy",
|
|
catalogRefresh: "succeeded",
|
|
};
|
|
|
|
activeCheckpoint = "C.permissions-ui";
|
|
await gotoPaperclipPage(
|
|
page,
|
|
new URL(`/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/permissions`, config.baseUrl).toString(),
|
|
page.getByText("Who can use it", { exact: true }),
|
|
"C.permissions-ui",
|
|
"permissions_panel_missing",
|
|
);
|
|
const projectGetPermission = page.locator(`[data-action-id="${facts.projectGet.id}"] select`);
|
|
const projectSettingsPermission = page.locator(`[data-action-id="${facts.projectSettings.id}"] select`);
|
|
await expectVisible(projectGetPermission, "C.permissions-ui", "project_get_permission_missing");
|
|
await expectVisible(projectSettingsPermission, "C.permissions-ui", "project_settings_permission_missing");
|
|
if (await projectGetPermission.inputValue() !== "allowed" || await projectSettingsPermission.inputValue() !== "off") {
|
|
fail("C.permissions-ui", "permissions_ui_mismatch");
|
|
}
|
|
const permissionsShot = "02-scoped-permissions.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, permissionsShot), config, "F.permissions-screenshot");
|
|
summary.screenshots.push(permissionsShot);
|
|
|
|
activeCheckpoint = "D.test-panel";
|
|
await gotoPaperclipPage(
|
|
page,
|
|
new URL(`/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/test`, config.baseUrl).toString(),
|
|
page.getByLabel("Choose which agent to test as"),
|
|
"D.test-panel",
|
|
"agent_picker_missing",
|
|
);
|
|
await page.getByLabel("Choose which agent to test as").click();
|
|
await page.getByLabel("Search agents").fill(TARGET_AGENT_NAME);
|
|
await page.getByRole("button", { name: new RegExp(`^${escapeRegex(TARGET_AGENT_NAME)}`) }).click();
|
|
await page.getByLabel("Find an action").fill(PROJECT_GET);
|
|
const projectGetTitle = facts.projectGet.title ?? facts.projectGet.toolName;
|
|
const actionRow = page.locator("button").filter({ hasText: projectGetTitle }).filter({ hasText: "Allowed" }).first();
|
|
await expectVisible(actionRow, "D.test-panel", "project_get_allowed_row_missing");
|
|
await actionRow.click();
|
|
await expectVisible(page.getByText("This action takes no inputs."), "D.test-panel", "empty_input_form_missing");
|
|
|
|
activeCheckpoint = "D.project-get";
|
|
const boardTestStartedAt = Date.now();
|
|
const testCallResponsePromise = page.waitForResponse((response) =>
|
|
response.request().method() === "POST"
|
|
&& new URL(response.url()).pathname === `/api/tool-connections/${connectionId}/test-calls`,
|
|
);
|
|
await page.getByRole("button", { name: /^run$/i }).click();
|
|
const testCallResponse = await testCallResponsePromise;
|
|
if (!testCallResponse.ok()) fail("D.project-get", `http_${testCallResponse.status()}`);
|
|
let testCall;
|
|
try {
|
|
testCall = await testCallResponse.json();
|
|
} catch {
|
|
fail("D.project-get", "invalid_json");
|
|
}
|
|
let testCallInput;
|
|
try {
|
|
testCallInput = testCallResponse.request().postDataJSON();
|
|
} catch {
|
|
fail("D.project-get", "request_body_unavailable");
|
|
}
|
|
if (testCallInput?.agentId !== agent.id
|
|
|| testCallInput?.toolName !== PROJECT_GET
|
|
|| !testCallInput.parameters
|
|
|| typeof testCallInput.parameters !== "object"
|
|
|| Array.isArray(testCallInput.parameters)
|
|
|| Object.keys(testCallInput.parameters).length !== 0) {
|
|
fail("D.project-get", "nonempty_or_unexpected_input");
|
|
}
|
|
if (testCall.decision !== "allowed" || testCall.error || typeof testCall.invocationId !== "string") {
|
|
fail("D.project-get", "gateway_call_not_allowed");
|
|
}
|
|
const boardProject = extractProjectSummary(testCall.result, config.projectId);
|
|
if (!boardProject?.name) fail("D.project-get", "project_result_missing");
|
|
await expectVisible(page.getByText(/^Worked\./), "D.project-get", "success_result_missing");
|
|
const boardShot = "03-board-project-get.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, boardShot), config, "F.board-test-screenshot");
|
|
summary.screenshots.push(boardShot);
|
|
summary.boardTest = {
|
|
catalogEntryId: facts.projectGet.id,
|
|
toolName: PROJECT_GET,
|
|
invocationId: testCall.invocationId,
|
|
decision: testCall.decision,
|
|
httpStatus: testCallResponse.status(),
|
|
resultStatus: "succeeded",
|
|
project: boardProject,
|
|
durationMs: Date.now() - boardTestStartedAt,
|
|
};
|
|
|
|
activeCheckpoint = "E.create-proof-issue";
|
|
const parentIssueId = process.env.POSTHOG_PROOF_PARENT_ISSUE_ID || process.env.PAPERCLIP_TASK_ID;
|
|
if (!parentIssueId) fail("E.create-proof-issue", "parent_issue_id_missing");
|
|
const child = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"POST",
|
|
`/api/issues/${parentIssueId}/children`,
|
|
{
|
|
title: `PostHog installed-tool proof ${startedAt.toISOString()}`,
|
|
description: [
|
|
"Invoke exactly one installed PostHog action: the read-only upstream `project-get` tool, with an empty `{}` input.",
|
|
`Verify the returned project ID is exactly ${config.projectId} and make no PostHog mutations.`,
|
|
"Then post exactly one JSON object with keys `projectId`, `projectName`, and `invocationId` (the Paperclip invocation ID), and mark this issue done.",
|
|
"Do not report tokens, cookies, authorization data, request headers, raw tool payloads, or any other fields.",
|
|
].join("\n\n"),
|
|
status: "todo",
|
|
workMode: "standard",
|
|
priority: "medium",
|
|
assigneeAgentId: agent.id,
|
|
acceptanceCriteria: [
|
|
`The installed PostHog project-get action returns project ${config.projectId}.`,
|
|
"The comment contains only sanitized project ID/name and Paperclip invocation ID.",
|
|
"No mutation is attempted.",
|
|
],
|
|
},
|
|
"E.create-proof-issue",
|
|
[201],
|
|
);
|
|
if (child.status !== "todo") fail("E.create-proof-issue", "child_not_created_todo");
|
|
activeCheckpoint = "E.fresh-agent-run";
|
|
const observedStatuses = new Set(["todo"]);
|
|
const finishedChild = await waitFor("E.fresh-agent-run", async () => {
|
|
const issue = await apiJson(context.request, config.baseUrl, "GET", `/api/issues/${child.id}`, undefined, "E.fresh-agent-run");
|
|
observedStatuses.add(issue.status);
|
|
if (["blocked", "cancelled"].includes(issue.status)) fail("E.fresh-agent-run", `child_${issue.status}`);
|
|
return issue.status === "done" ? issue : null;
|
|
}, {
|
|
timeoutMs: Number(process.env.POSTHOG_AGENT_TIMEOUT_MS || DEFAULT_AGENT_TIMEOUT_MS),
|
|
intervalMs: 3_000,
|
|
});
|
|
if (!finishedChild.startedAt || !finishedChild.completedAt) fail("E.fresh-agent-run", "transition_timestamps_missing");
|
|
|
|
const commentsResponse = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/issues/${child.id}/comments`,
|
|
undefined,
|
|
"E.agent-proof-comment",
|
|
);
|
|
const comments = asArray(commentsResponse, "comments");
|
|
for (const comment of comments) assertNoCredentialMaterial(comment.body, [config.password], "E.agent-proof-comment");
|
|
const agentComments = comments.filter((comment) =>
|
|
comment.authorAgentId === agent.id || comment.derivedAuthorAgentId === agent.id,
|
|
);
|
|
const proofs = agentComments.map((comment) => parseSanitizedAgentProof(comment.body, config.projectId)).filter(Boolean);
|
|
if (agentComments.length !== 1 || proofs.length !== 1) {
|
|
fail("E.agent-proof-comment", "single_sanitized_proof_missing");
|
|
}
|
|
const [proof] = proofs;
|
|
|
|
let finalConnectionActivity;
|
|
const agentEvent = await waitFor("E.agent-audit", async () => {
|
|
const activity = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}/activity?limit=100`,
|
|
undefined,
|
|
"E.agent-audit",
|
|
);
|
|
assertNoCredentialMaterial(activity, [config.password], "E.agent-audit");
|
|
finalConnectionActivity = activity;
|
|
return asArray(activity, "events").find((event) =>
|
|
event.issueId === child.id
|
|
&& event.agentId === agent.id
|
|
&& event.toolName === PROJECT_GET
|
|
&& event.invocationId === proof.invocationId
|
|
&& event.outcome === "success",
|
|
) ?? null;
|
|
}, { timeoutMs: 60_000, intervalMs: 2_000 });
|
|
if (!agentEvent.runId) fail("E.agent-audit", "run_id_missing");
|
|
if (agentEvent.requestSummary?.summary !== "{}") fail("E.agent-audit", "project_get_input_not_empty");
|
|
const childToolEvents = asArray(finalConnectionActivity, "events").filter((event) => event.issueId === child.id && event.invocationId);
|
|
if (childToolEvents.length === 0 || childToolEvents.some((event) => event.toolName !== PROJECT_GET)) {
|
|
fail("E.agent-audit", "unexpected_upstream_action");
|
|
}
|
|
if (new Set(childToolEvents.map((event) => event.invocationId)).size !== 1) {
|
|
fail("E.agent-audit", "project_get_invoked_more_than_once");
|
|
}
|
|
if (proof.projectName !== boardProject.name) fail("E.agent-proof-comment", "project_name_mismatch");
|
|
|
|
const agentRun = await waitFor("E.agent-run-status", async () => {
|
|
const run = await apiJson(
|
|
context.request,
|
|
config.baseUrl,
|
|
"GET",
|
|
`/api/heartbeat-runs/${agentEvent.runId}`,
|
|
undefined,
|
|
"E.agent-run-status",
|
|
);
|
|
if (["failed", "cancelled", "timed_out"].includes(run.status)) fail("E.agent-run-status", `run_${run.status}`);
|
|
return run.status === "succeeded" ? run : null;
|
|
}, { timeoutMs: 60_000, intervalMs: 2_000 });
|
|
|
|
summary.freshRun = {
|
|
issueId: child.id,
|
|
issueIdentifier: child.identifier,
|
|
transition: {
|
|
created: "todo",
|
|
enteredInProgress: observedStatuses.has("in_progress") || Boolean(finishedChild.startedAt),
|
|
completed: "done",
|
|
},
|
|
runId: agentEvent.runId,
|
|
runStatus: agentRun.status,
|
|
invocationId: proof.invocationId,
|
|
project: { id: proof.projectId, name: proof.projectName },
|
|
auditOutcome: agentEvent.outcome,
|
|
durationMs: agentEvent.latencyMs,
|
|
};
|
|
|
|
activeCheckpoint = "F.evidence";
|
|
await gotoPaperclipPage(
|
|
page,
|
|
new URL(`/${TARGET_COMPANY_PREFIX}/issues/${child.identifier}`, config.baseUrl).toString(),
|
|
page.getByText(child.title, { exact: true }).first(),
|
|
"F.child-screenshot",
|
|
"child_issue_missing",
|
|
);
|
|
const childShot = "04-fresh-agent-proof.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, childShot), config, "F.child-screenshot");
|
|
summary.screenshots.push(childShot);
|
|
|
|
await gotoPaperclipPage(
|
|
page,
|
|
new URL(`/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/activity`, config.baseUrl).toString(),
|
|
page.getByText(PROJECT_GET, { exact: false }).first(),
|
|
"F.activity-screenshot",
|
|
"project_get_activity_missing",
|
|
);
|
|
const activityShot = "05-redacted-activity.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, activityShot), config, "F.activity-screenshot");
|
|
summary.screenshots.push(activityShot);
|
|
|
|
activeCheckpoint = "F.cleanup";
|
|
summary.cleanup = await cleanupConnection(context.request, config, companyId, connectionId, connectionName);
|
|
cleanupComplete = true;
|
|
summary.passed = true;
|
|
} catch (error) {
|
|
caughtFailure = error instanceof SmokeFailure ? error : new SmokeFailure(activeCheckpoint, "unexpected_error");
|
|
if (page) {
|
|
caughtFailure.details = {
|
|
...(caughtFailure.details ?? {}),
|
|
pageState: await safePageState(page, resourceFailures, new URL(config.baseUrl).origin),
|
|
};
|
|
}
|
|
} finally {
|
|
if (!connectionId && companyId && context) {
|
|
try {
|
|
connectionId = await findConnectionIdByName(
|
|
context.request,
|
|
config,
|
|
companyId,
|
|
connectionName,
|
|
);
|
|
} catch (error) {
|
|
summary.cleanup = {
|
|
completed: false,
|
|
code: error instanceof SmokeFailure ? error.code : "cleanup_recovery_failed",
|
|
};
|
|
}
|
|
}
|
|
if (connectionId && companyId && context && !cleanupComplete) {
|
|
try {
|
|
summary.cleanup = await cleanupConnection(
|
|
context.request,
|
|
config,
|
|
companyId,
|
|
connectionId,
|
|
connectionName,
|
|
{ requireInstalledState: false },
|
|
);
|
|
cleanupComplete = true;
|
|
} catch (error) {
|
|
summary.cleanup = {
|
|
completed: false,
|
|
code: error instanceof SmokeFailure ? error.code : "cleanup_failed",
|
|
};
|
|
if (!caughtFailure) caughtFailure = new SmokeFailure("F.cleanup", "cleanup_failed");
|
|
}
|
|
}
|
|
await context?.close().catch(() => {});
|
|
await browser?.close().catch(() => {});
|
|
}
|
|
|
|
summary.completedAt = new Date().toISOString();
|
|
if (caughtFailure) {
|
|
summary.failure = {
|
|
checkpoint: caughtFailure.checkpoint,
|
|
code: caughtFailure.code,
|
|
...(caughtFailure.details ? { details: caughtFailure.details } : {}),
|
|
};
|
|
}
|
|
assertSanitizedEvidence(summary);
|
|
const summaryPath = path.join(outputDirectory, "summary.json");
|
|
await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, { mode: 0o600 });
|
|
|
|
if (caughtFailure) throw caughtFailure;
|
|
return { outputDirectory, summaryPath, screenshots: summary.screenshots };
|
|
}
|
|
|
|
async function main() {
|
|
let prepared;
|
|
try {
|
|
const options = parsePosthogLiveArguments(process.argv.slice(2));
|
|
prepared = await preparePosthogLiveSmoke({
|
|
baseUrl: options.baseUrl,
|
|
loadBrowser: () => import("@playwright/test"),
|
|
});
|
|
} catch (error) {
|
|
process.stderr.write(`${preflightFailureMessage(error)}\n`);
|
|
process.exitCode = error instanceof PosthogLivePreflightError ? 2 : 1;
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const result = await runSmoke({ config: prepared.config, chromium: prepared.browserModule.chromium });
|
|
process.stdout.write(`PostHog live smoke passed. Sanitized evidence: ${result.outputDirectory}\n`);
|
|
} catch (error) {
|
|
const failure = error instanceof SmokeFailure ? error : new SmokeFailure("unexpected", "unexpected_error");
|
|
process.stderr.write(`PostHog live smoke failed at ${failure.checkpoint} (${failure.code}).\n`);
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
await main();
|