1180 lines
48 KiB
JavaScript
1180 lines
48 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import {
|
|
assertAutomaticRegistrationSource,
|
|
assertSanitizedEvidence,
|
|
connectionRemovalFacts,
|
|
extractNotionIdentity,
|
|
extractNotionVerificationCode,
|
|
fetchNotionTestCredentials,
|
|
inspectAuthorizationUrl,
|
|
isFreshNotionVerificationMessage,
|
|
NotionGenericLivePreflightError,
|
|
notionVerificationAuthenticationPassed,
|
|
parseRuntimeAbsenceProof,
|
|
parseSanitizedAgentProof,
|
|
persistedOAuthStartResult,
|
|
preflightFailureMessage,
|
|
prepareNotionGenericLiveSmoke,
|
|
safeEndpointSummary,
|
|
} from "./notion-generic-live-lib.mjs";
|
|
|
|
const TARGET_COMPANY_PREFIX = "PAP";
|
|
const TARGET_AGENT_NAME = "CodexCoderPro";
|
|
const NOTION_RESOURCE = "https://mcp.notion.com/mcp";
|
|
const NOTION_GET_SELF = "notion-get-self";
|
|
const NOTION_CREATE_PAGES = "notion-create-pages";
|
|
const DEFAULT_AGENT_TIMEOUT_MS = 15 * 60_000;
|
|
const DEFAULT_VERIFICATION_TIMEOUT_MS = 2 * 60_000;
|
|
|
|
class SmokeFailure extends Error {
|
|
constructor(checkpoint, code) {
|
|
super(`${checkpoint}:${code}`);
|
|
this.name = "SmokeFailure";
|
|
this.checkpoint = checkpoint;
|
|
this.code = code;
|
|
}
|
|
}
|
|
|
|
function fail(checkpoint, code) {
|
|
throw new SmokeFailure(checkpoint, code);
|
|
}
|
|
|
|
function asArray(value, key) {
|
|
if (Array.isArray(value)) return value;
|
|
if (value && typeof value === "object" && Array.isArray(value[key])) return value[key];
|
|
return [];
|
|
}
|
|
|
|
function escapeRegex(value) {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|
|
|
|
function screenshotFile(outputDirectory, name) {
|
|
return path.join(outputDirectory, name);
|
|
}
|
|
|
|
function assertNoCredentialMaterial(value, credentials, checkpoint) {
|
|
const serialized = JSON.stringify(value);
|
|
for (const credential of credentials) {
|
|
if (credential && serialized.includes(credential)) fail(checkpoint, "credential_material_visible");
|
|
}
|
|
if (/[?&](?:code|state|access_token|refresh_token)=/i.test(serialized)) {
|
|
fail(checkpoint, "oauth_query_material_visible");
|
|
}
|
|
const forbiddenValueKey = /^(?:password|accessToken|access_token|refreshToken|refresh_token|oauthCode|oauth_code|clientSecret|client_secret|cookie|authorizationHeader)$/i;
|
|
const visit = (candidate, seen = new Set()) => {
|
|
if (!candidate || typeof candidate !== "object" || seen.has(candidate)) return;
|
|
seen.add(candidate);
|
|
for (const [key, child] of Object.entries(candidate)) {
|
|
if (forbiddenValueKey.test(key) && child !== null && child !== "") {
|
|
fail(checkpoint, "raw_credential_field_visible");
|
|
}
|
|
visit(child, seen);
|
|
}
|
|
};
|
|
visit(value);
|
|
}
|
|
|
|
function issueMutationHeaders(config, method, pathname) {
|
|
if (method === "GET" || !pathname.startsWith("/api/issues/")) return {};
|
|
return { "X-Paperclip-Run-Id": config.runId };
|
|
}
|
|
|
|
async function apiJson(request, config, method, pathname, data, checkpoint, expectedStatuses = [200]) {
|
|
let response;
|
|
try {
|
|
response = await request.fetch(new URL(pathname, config.baseUrl).toString(), {
|
|
method,
|
|
...(data === undefined ? {} : { data }),
|
|
headers: {
|
|
accept: "application/json",
|
|
...(method === "GET" ? {} : { origin: config.baseUrl }),
|
|
...issueMutationHeaders(config, method, pathname),
|
|
},
|
|
timeout: 30_000,
|
|
});
|
|
} catch {
|
|
fail(checkpoint, "request_failed");
|
|
}
|
|
if (!expectedStatuses.includes(response.status())) {
|
|
fail(checkpoint, `http_${response.status()}`);
|
|
}
|
|
try {
|
|
return await response.json();
|
|
} catch {
|
|
fail(checkpoint, "invalid_json");
|
|
}
|
|
}
|
|
|
|
async function waitFor(checkpoint, fn, { timeoutMs = 120_000, intervalMs = 2_000 } = {}) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let lastValue;
|
|
while (Date.now() < deadline) {
|
|
lastValue = await fn();
|
|
if (lastValue) return lastValue;
|
|
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
}
|
|
fail(checkpoint, "timed_out");
|
|
return lastValue;
|
|
}
|
|
|
|
async function expectVisible(locator, checkpoint, code, timeout = 30_000) {
|
|
try {
|
|
await locator.waitFor({ state: "visible", timeout });
|
|
} catch {
|
|
fail(checkpoint, code);
|
|
}
|
|
}
|
|
|
|
async function gotoWithVisibleMarker(
|
|
page,
|
|
url,
|
|
marker,
|
|
checkpoint,
|
|
code,
|
|
{ attempts = 2, timeout = 15_000 } = {},
|
|
) {
|
|
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
try {
|
|
const response = await page.goto(url, { waitUntil: "domcontentloaded" });
|
|
if (response?.ok()) {
|
|
await marker().waitFor({ state: "visible", timeout });
|
|
return;
|
|
}
|
|
} catch {
|
|
// A credential-free Paperclip navigation is safe to repeat once. Do not
|
|
// retry provider pages or any mutation from this helper.
|
|
}
|
|
if (attempt < attempts) await page.waitForTimeout(500);
|
|
}
|
|
fail(checkpoint, code);
|
|
}
|
|
|
|
async function loginPaperclipBoard(page, context, config) {
|
|
const loginUrl = new URL("/auth?next=/", config.baseUrl).toString();
|
|
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
await gotoWithVisibleMarker(
|
|
page,
|
|
loginUrl,
|
|
() => page.getByLabel(/^email$/i),
|
|
"A.paperclip-login",
|
|
"email_field_missing",
|
|
);
|
|
const paperclipEmailInput = page.getByLabel(/^email$/i);
|
|
const paperclipPasswordInput = page.getByLabel(/^password$/i);
|
|
await expectVisible(paperclipPasswordInput, "A.paperclip-login", "password_field_missing");
|
|
await paperclipEmailInput.fill(config.paperclipEmail);
|
|
await paperclipPasswordInput.fill(config.paperclipPassword);
|
|
const loginResponsePromise = page.waitForResponse((response) =>
|
|
response.request().method() === "POST" && new URL(response.url()).pathname === "/api/auth/sign-in/email",
|
|
);
|
|
await page.getByRole("button", { name: /^sign in$/i }).click();
|
|
const loginResponse = await loginResponsePromise;
|
|
if (!loginResponse.ok()) fail("A.paperclip-login", `http_${loginResponse.status()}`);
|
|
|
|
// A redirect is a UI implementation detail; the authenticated session is
|
|
// the prerequisite the smoke actually needs. A successful sign-in response
|
|
// without a usable cookie has occurred intermittently through public dev
|
|
// proxies, so verify the cookie and repeat the idempotent sign-in once.
|
|
const session = await context.request.get(new URL("/api/auth/get-session", config.baseUrl).toString(), {
|
|
headers: { accept: "application/json" },
|
|
});
|
|
if (session.ok()) return;
|
|
if (attempt < 2) await page.waitForTimeout(500);
|
|
}
|
|
fail("A.paperclip-login", "session_cookie_missing");
|
|
}
|
|
|
|
async function clickVisibleButton(page, names) {
|
|
for (const name of names) {
|
|
const button = page.getByRole("button", { name, exact: false }).filter({ visible: true }).first();
|
|
if (!await button.count()) continue;
|
|
try {
|
|
await button.click({ timeout: 2_000 });
|
|
return true;
|
|
} catch {
|
|
// Provider pages replace controls while advancing between login steps.
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function verificationTimeoutMs() {
|
|
const configured = Number(process.env.NOTION_VERIFICATION_TIMEOUT_MS || DEFAULT_VERIFICATION_TIMEOUT_MS);
|
|
return Number.isFinite(configured)
|
|
? Math.min(Math.max(configured, 30_000), 5 * 60_000)
|
|
: DEFAULT_VERIFICATION_TIMEOUT_MS;
|
|
}
|
|
|
|
async function fetchNotionVerificationCodeFromAgentMail({ notBefore }) {
|
|
const apiKey = process.env.AGENTMAIL_API_KEY?.trim();
|
|
const inboxId = process.env.NOTION_AGENTMAIL_INBOX_ID?.trim();
|
|
if (!apiKey || !inboxId) fail("C.notion-login", "verification_inbox_unavailable");
|
|
|
|
let AgentMailClient;
|
|
try {
|
|
({ AgentMailClient } = await import("agentmail"));
|
|
} catch {
|
|
fail("C.notion-login", "agentmail_sdk_unavailable");
|
|
}
|
|
const client = new AgentMailClient({ apiKey });
|
|
try {
|
|
const inbox = await client.inboxes.get(inboxId);
|
|
if (inbox.inboxId !== inboxId && inbox.email !== inboxId) {
|
|
fail("C.notion-login", "verification_inbox_mismatch");
|
|
}
|
|
} catch (error) {
|
|
if (error instanceof SmokeFailure) throw error;
|
|
fail("C.notion-login", "verification_inbox_request_failed");
|
|
}
|
|
|
|
const deadline = Date.now() + verificationTimeoutMs();
|
|
const after = new Date(new Date(notBefore).getTime() - 5_000);
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
let pageToken;
|
|
let inspected = 0;
|
|
do {
|
|
const response = await client.inboxes.messages.list(inboxId, {
|
|
limit: 50,
|
|
after,
|
|
...(pageToken ? { pageToken } : {}),
|
|
});
|
|
for (const item of response.messages) {
|
|
inspected += 1;
|
|
if (!isFreshNotionVerificationMessage(item, { notBefore: after })) continue;
|
|
const message = await client.inboxes.messages.get(inboxId, item.messageId);
|
|
if (!notionVerificationAuthenticationPassed(message)) continue;
|
|
const code = extractNotionVerificationCode(message);
|
|
if (code) return code;
|
|
}
|
|
pageToken = response.nextPageToken;
|
|
} while (pageToken && inspected < 100);
|
|
} catch (error) {
|
|
if (error instanceof SmokeFailure) throw error;
|
|
fail("C.notion-login", "verification_inbox_request_failed");
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 2_000));
|
|
}
|
|
fail("C.notion-login", "verification_code_timed_out");
|
|
}
|
|
|
|
async function completeNotionAuthorization(page, config, credential, connectionId) {
|
|
const paperclipOrigin = new URL(config.baseUrl).origin;
|
|
const permissionsPath = `/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/permissions`;
|
|
const deadline = Date.now() + 6 * 60_000;
|
|
const verificationNotBefore = new Date();
|
|
let providerSeen = false;
|
|
let verificationCodeSubmitted = false;
|
|
while (Date.now() < deadline) {
|
|
let current;
|
|
try {
|
|
current = new URL(page.url());
|
|
} catch {
|
|
fail("C.oauth-callback", "invalid_navigation_url");
|
|
}
|
|
if (current.origin === paperclipOrigin) {
|
|
if (providerSeen && current.pathname === permissionsPath) return;
|
|
await page.waitForTimeout(300);
|
|
continue;
|
|
}
|
|
providerSeen = true;
|
|
|
|
const bodyText = await page.locator("body").innerText().catch(() => "");
|
|
if (/check your (?:email|inbox)|verification code|one-time code/i.test(bodyText)) {
|
|
if (verificationCodeSubmitted) fail("C.notion-login", "verification_code_rejected");
|
|
const codeInput = page.locator([
|
|
'input[autocomplete="one-time-code"]',
|
|
'input[name*="code" i]',
|
|
'input[inputmode="numeric"]',
|
|
].join(",")).filter({ visible: true }).first();
|
|
if (!await codeInput.count()) {
|
|
await page.waitForTimeout(300);
|
|
continue;
|
|
}
|
|
let code = await fetchNotionVerificationCodeFromAgentMail({ notBefore: verificationNotBefore });
|
|
await codeInput.fill(code);
|
|
code = "";
|
|
if (!await clickVisibleButton(page, [/^verify$/i, /^continue$/i, /^submit$/i, /^sign in$/i])) {
|
|
await codeInput.press("Enter");
|
|
}
|
|
verificationCodeSubmitted = true;
|
|
await page.waitForTimeout(600);
|
|
continue;
|
|
}
|
|
|
|
const usernameInput = page.locator([
|
|
'input[type="email"]',
|
|
'input[name="email"]',
|
|
'input[name="username"]',
|
|
'input[autocomplete="username"]',
|
|
].join(",")).filter({ visible: true }).first();
|
|
if (await usernameInput.count()) {
|
|
const currentValue = await usernameInput.inputValue().catch(() => "");
|
|
if (!currentValue) await usernameInput.fill(credential.username);
|
|
}
|
|
|
|
const passwordInput = page.locator([
|
|
'input[type="password"]',
|
|
'input[name="password"]',
|
|
'input[autocomplete="current-password"]',
|
|
].join(",")).filter({ visible: true }).first();
|
|
if (await passwordInput.count()) {
|
|
const currentValue = await passwordInput.inputValue().catch(() => "");
|
|
if (!currentValue) await passwordInput.fill(credential.password);
|
|
await clickVisibleButton(page, [/^sign in$/i, /^log in$/i, /^continue$/i]);
|
|
} else if (await usernameInput.count()) {
|
|
await clickVisibleButton(page, [/continue with email/i, /^continue$/i, /^next$/i, /^sign in$/i]);
|
|
} else {
|
|
await clickVisibleButton(page, [
|
|
/^authorize$/i,
|
|
/^allow$/i,
|
|
/allow access/i,
|
|
/^approve$/i,
|
|
/^grant access$/i,
|
|
/^accept$/i,
|
|
/^continue$/i,
|
|
/^select$/i,
|
|
]);
|
|
}
|
|
await page.waitForTimeout(600);
|
|
}
|
|
fail("C.oauth-callback", "provider_authorization_timed_out");
|
|
}
|
|
|
|
async function safeScreenshot(page, outputPath, config, credential, checkpoint) {
|
|
const current = new URL(page.url());
|
|
if (current.origin !== new URL(config.baseUrl).origin) fail(checkpoint, "screenshot_not_on_paperclip");
|
|
for (const queryKey of ["code", "state", "token", "access_token", "refresh_token"]) {
|
|
if (current.searchParams.has(queryKey)) fail(checkpoint, "credential_query_in_screenshot_url");
|
|
}
|
|
const bodyText = await page.locator("body").innerText();
|
|
if (bodyText.includes(config.paperclipPassword)
|
|
|| bodyText.includes(credential.password)
|
|
|| /[?&](?:code|state|access_token|refresh_token)=/i.test(bodyText)) {
|
|
fail(checkpoint, "credential_material_in_screenshot");
|
|
}
|
|
await page.screenshot({
|
|
path: outputPath,
|
|
fullPage: true,
|
|
animations: "disabled",
|
|
mask: [
|
|
page.getByText(config.paperclipEmail, { exact: false }),
|
|
page.getByText(credential.username, { exact: false }),
|
|
],
|
|
});
|
|
}
|
|
|
|
function assertGenericProvenance(connectResult, connectionName) {
|
|
if (connectResult?.application?.name !== connectionName) fail("B.generic-provenance", "application_name_mismatch");
|
|
if (!String(connectResult?.application?.applicationKey ?? "").startsWith("app-gallery:link:")) {
|
|
fail("B.generic-provenance", "application_key_not_generic");
|
|
}
|
|
if (connectResult?.application?.metadata?.source !== "link") fail("B.generic-provenance", "metadata_source_not_link");
|
|
const config = connectResult?.connection?.config;
|
|
if (connectResult?.connection?.transport !== "mcp_remote"
|
|
|| config?.url !== NOTION_RESOURCE
|
|
|| config?.unverifiedServer !== true
|
|
|| config?.galleryKey === "notion"
|
|
|| config?.sourceTemplateKey === "notion") {
|
|
fail("B.generic-provenance", "connection_not_unverified_generic");
|
|
}
|
|
}
|
|
|
|
function oauthEndpointProof(connection, startResult, config) {
|
|
const source = assertAutomaticRegistrationSource(startResult.registrationSource);
|
|
const oauth = connection?.config?.oauth;
|
|
if (!oauth || typeof oauth !== "object" || Array.isArray(oauth)) fail("C.oauth-proof", "oauth_config_missing");
|
|
if (connection.authKind !== "oauth" || oauth.clientRegistrationSource !== source) {
|
|
fail("C.oauth-proof", "registration_source_not_persisted");
|
|
}
|
|
if (oauth.resource !== NOTION_RESOURCE || startResult.resource !== NOTION_RESOURCE) {
|
|
fail("C.oauth-proof", "resource_mismatch");
|
|
}
|
|
if (oauth.clientRedirectUri !== config.callbackUrl) fail("C.oauth-proof", "client_callback_mismatch");
|
|
if (source === "cimd" && oauth.clientIdMetadataDocumentSupported !== true) {
|
|
fail("C.oauth-proof", "cimd_support_not_persisted");
|
|
}
|
|
const endpoints = {
|
|
issuer: safeEndpointSummary(oauth.issuer, "issuer"),
|
|
metadata: safeEndpointSummary(oauth.metadataUrl, "metadata"),
|
|
authorize: safeEndpointSummary(oauth.authorizationUrl, "authorize"),
|
|
exchange: safeEndpointSummary(oauth.tokenUrl, "exchange"),
|
|
...(oauth.registrationUrl ? { registration: safeEndpointSummary(oauth.registrationUrl, "registration") } : {}),
|
|
};
|
|
return { source, endpoints };
|
|
}
|
|
|
|
function catalogFacts(catalog, checkpoint) {
|
|
const active = catalog.filter((entry) => entry.status !== "removed");
|
|
const getSelf = active.find((entry) => entry.toolName === NOTION_GET_SELF);
|
|
const createPages = active.find((entry) => entry.toolName === NOTION_CREATE_PAGES);
|
|
if (!getSelf || !getSelf.isReadOnly) fail(checkpoint, "notion_get_self_missing_or_not_read_only");
|
|
if (!createPages || createPages.isReadOnly) fail(checkpoint, "notion_create_pages_missing_or_not_write");
|
|
return { active, getSelf, createPages };
|
|
}
|
|
|
|
async function finishAgentOnlySetup(request, config, companyId, connectionId, catalog, agentId) {
|
|
const facts = catalogFacts(catalog, "D.catalog-policy");
|
|
await apiJson(
|
|
request,
|
|
config,
|
|
"POST",
|
|
`/api/companies/${companyId}/tools/apps/${connectionId}/finish`,
|
|
{
|
|
enabledCatalogEntryIds: [facts.getSelf.id],
|
|
askFirstCatalogEntryIds: [],
|
|
reviewedCatalogEntryIds: facts.active
|
|
.filter((entry) => entry.status === "quarantined")
|
|
.map((entry) => entry.id),
|
|
access: { agentIds: [agentId] },
|
|
},
|
|
"D.catalog-policy",
|
|
);
|
|
await apiJson(
|
|
request,
|
|
config,
|
|
"PUT",
|
|
`/api/tool-connections/${connectionId}/installs`,
|
|
{ installs: [{ targetType: "agent", targetId: agentId }] },
|
|
"D.agent-install",
|
|
);
|
|
return facts;
|
|
}
|
|
|
|
async function createAndWaitForProofIssue(request, config, input, checkpoint) {
|
|
const child = await apiJson(
|
|
request,
|
|
config,
|
|
"POST",
|
|
`/api/issues/${config.taskId}/children`,
|
|
input,
|
|
`${checkpoint}.create`,
|
|
[201],
|
|
);
|
|
if (child.status !== "todo") fail(`${checkpoint}.create`, "child_not_created_todo");
|
|
const observedStatuses = new Set(["todo"]);
|
|
const finished = await waitFor(`${checkpoint}.run`, async () => {
|
|
const issue = await apiJson(request, config, "GET", `/api/issues/${child.id}`, undefined, `${checkpoint}.run`);
|
|
observedStatuses.add(issue.status);
|
|
if (["blocked", "cancelled"].includes(issue.status)) fail(`${checkpoint}.run`, `child_${issue.status}`);
|
|
return issue.status === "done" ? issue : null;
|
|
}, {
|
|
timeoutMs: Number(process.env.NOTION_AGENT_TIMEOUT_MS || DEFAULT_AGENT_TIMEOUT_MS),
|
|
intervalMs: 3_000,
|
|
});
|
|
if (!finished.startedAt || !finished.completedAt) fail(`${checkpoint}.run`, "transition_timestamps_missing");
|
|
const commentsResponse = await apiJson(
|
|
request,
|
|
config,
|
|
"GET",
|
|
`/api/issues/${child.id}/comments`,
|
|
undefined,
|
|
`${checkpoint}.comment`,
|
|
);
|
|
return { child, finished, observedStatuses, comments: asArray(commentsResponse, "comments") };
|
|
}
|
|
|
|
async function cleanupConnection(
|
|
request,
|
|
config,
|
|
companyId,
|
|
connectionId,
|
|
connectionName,
|
|
{ requireInstalled = false } = {},
|
|
) {
|
|
const removed = await apiJson(
|
|
request,
|
|
config,
|
|
"DELETE",
|
|
`/api/tool-connections/${connectionId}`,
|
|
undefined,
|
|
"F.cleanup",
|
|
);
|
|
const receipt = removed.removal;
|
|
const removalFacts = connectionRemovalFacts(receipt, { requireInstalled });
|
|
if (!removalFacts) fail("F.cleanup", "incomplete_removal_receipt");
|
|
|
|
const connections = await apiJson(
|
|
request,
|
|
config,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/connections`,
|
|
undefined,
|
|
"F.cleanup-verification",
|
|
);
|
|
if (asArray(connections, "connections").some((candidate) =>
|
|
candidate.id === connectionId
|
|
&& candidate.name === connectionName
|
|
&& candidate.status !== "archived")) {
|
|
fail("F.cleanup-verification", "test_connection_remains");
|
|
}
|
|
const profiles = await apiJson(
|
|
request,
|
|
config,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/profiles`,
|
|
undefined,
|
|
"F.cleanup-verification",
|
|
);
|
|
if (asArray(profiles, "profiles").some((profile) => profile.profileKey === `app:${connectionId}` && profile.status === "active")) {
|
|
fail("F.cleanup-verification", "active_profile_remains");
|
|
}
|
|
const pending = await apiJson(
|
|
request,
|
|
config,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/action-requests?status=pending`,
|
|
undefined,
|
|
"F.cleanup-verification",
|
|
);
|
|
if (asArray(pending, "actionRequests").some((item) => (item.connectionId ?? item.request?.connectionId) === connectionId)) {
|
|
fail("F.cleanup-verification", "pending_action_remains");
|
|
}
|
|
return {
|
|
completed: true,
|
|
...removalFacts,
|
|
pendingActions: 0,
|
|
remainingConnections: 0,
|
|
};
|
|
}
|
|
|
|
async function runSmoke({ config, chromium }) {
|
|
const startedAt = new Date();
|
|
const runKey = startedAt.toISOString().replace(/[:.]/g, "-");
|
|
const connectionName = `Notion generic self-test ${startedAt.toISOString()}`;
|
|
const outputDirectory = process.env.NOTION_EVIDENCE_DIR
|
|
? path.resolve(process.env.NOTION_EVIDENCE_DIR)
|
|
: path.join(process.env.PAPERCLIP_RUN_SCRATCH_DIR || process.cwd(), `notion-generic-live-${runKey}`);
|
|
await mkdir(outputDirectory, { recursive: true });
|
|
|
|
const summary = {
|
|
schemaVersion: 1,
|
|
smoke: "notion_generic_mcp_live",
|
|
passed: false,
|
|
startedAt: startedAt.toISOString(),
|
|
completedAt: null,
|
|
target: { companyPrefix: TARGET_COMPANY_PREFIX, resource: NOTION_RESOURCE },
|
|
importPreview: null,
|
|
connection: null,
|
|
oauthProof: null,
|
|
catalog: null,
|
|
boardTest: null,
|
|
freshRun: null,
|
|
cleanup: null,
|
|
postCleanupRun: null,
|
|
screenshots: [],
|
|
failure: null,
|
|
};
|
|
|
|
let browser;
|
|
let context;
|
|
let credential = null;
|
|
let connectionId = null;
|
|
let companyId = null;
|
|
let cleanupComplete = false;
|
|
let caughtFailure = null;
|
|
let activeCheckpoint = "A.paperclip-login";
|
|
|
|
try {
|
|
browser = await chromium.launch({ headless: process.env.NOTION_SMOKE_HEADED !== "1" });
|
|
context = await browser.newContext({
|
|
viewport: { width: 1440, height: 1000 },
|
|
acceptDownloads: false,
|
|
serviceWorkers: "block",
|
|
});
|
|
const page = await context.newPage();
|
|
|
|
await loginPaperclipBoard(page, context, config);
|
|
|
|
activeCheckpoint = "A.company-selection";
|
|
const companies = await apiJson(context.request, config, "GET", "/api/companies", undefined, "A.company-selection");
|
|
const company = asArray(companies, "companies").find((candidate) => candidate.issuePrefix === TARGET_COMPANY_PREFIX);
|
|
if (!company) fail("A.company-selection", "pap_company_missing");
|
|
companyId = company.id;
|
|
activeCheckpoint = "A.agent-selection";
|
|
const agents = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/companies/${companyId}/agents`,
|
|
undefined,
|
|
"A.agent-selection",
|
|
);
|
|
const agent = asArray(agents, "agents").find((candidate) => candidate.name === TARGET_AGENT_NAME);
|
|
if (!agent) fail("A.agent-selection", "codex_coder_pro_missing");
|
|
|
|
activeCheckpoint = "A.connection-isolation";
|
|
const existingConnections = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/connections`,
|
|
undefined,
|
|
"A.connection-isolation",
|
|
);
|
|
const existingIds = new Set(asArray(existingConnections, "connections").map((entry) => entry.id));
|
|
if (asArray(existingConnections, "connections").some((entry) => entry.name === connectionName && entry.status !== "archived")) {
|
|
fail("A.connection-isolation", "connection_name_collision");
|
|
}
|
|
|
|
// Fetch only after URL, health, Paperclip login, company, agent, and binding
|
|
// metadata have all passed. The value remains in this process and is never
|
|
// written to browser artifacts or command arguments.
|
|
activeCheckpoint = "A.secret-binding";
|
|
credential = await fetchNotionTestCredentials(config);
|
|
|
|
activeCheckpoint = "B.paste-config";
|
|
await gotoWithVisibleMarker(
|
|
page,
|
|
new URL(`/${TARGET_COMPANY_PREFIX}/apps/advanced/paste-config`, config.baseUrl).toString(),
|
|
() => page.getByRole("heading", { name: "Advanced setup", exact: true }),
|
|
"B.paste-config",
|
|
"advanced_setup_missing",
|
|
);
|
|
const configTextarea = page.locator("textarea").first();
|
|
await expectVisible(configTextarea, "B.paste-config", "config_textarea_missing");
|
|
const exactConfig = JSON.stringify({ mcpServers: { notion: { url: NOTION_RESOURCE } } }, null, 2);
|
|
await configTextarea.fill(exactConfig);
|
|
const importResponsePromise = page.waitForResponse((response) =>
|
|
response.request().method() === "POST"
|
|
&& new URL(response.url()).pathname === `/api/companies/${companyId}/tools/mcp/import-json`,
|
|
);
|
|
await page.getByRole("button", { name: /^check config$/i }).click();
|
|
const importResponse = await importResponsePromise;
|
|
if (!importResponse.ok()) fail("B.import-preview", `http_${importResponse.status()}`);
|
|
const preview = await importResponse.json().catch(() => fail("B.import-preview", "invalid_json"));
|
|
const drafts = asArray(preview, "drafts");
|
|
if (drafts.length !== 1
|
|
|| drafts[0].name !== "notion"
|
|
|| drafts[0].transport !== "mcp_remote"
|
|
|| drafts[0].config?.url !== NOTION_RESOURCE
|
|
|| drafts[0].credentialFields?.length !== 0
|
|
|| drafts[0].credentialRefs?.length !== 0) {
|
|
fail("B.import-preview", "unexpected_draft");
|
|
}
|
|
const afterPreview = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/connections`,
|
|
undefined,
|
|
"B.import-preview",
|
|
);
|
|
if (asArray(afterPreview, "connections").some((entry) => !existingIds.has(entry.id))) {
|
|
fail("B.import-preview", "preview_created_connection");
|
|
}
|
|
summary.importPreview = {
|
|
draftCount: 1,
|
|
name: "notion",
|
|
transport: "mcp_remote",
|
|
resource: NOTION_RESOURCE,
|
|
credentialFieldCount: 0,
|
|
credentialRefCount: 0,
|
|
};
|
|
|
|
activeCheckpoint = "B.generic-connect";
|
|
const connectionNameInput = page.getByText("Connection name", { exact: true }).locator("input");
|
|
await expectVisible(connectionNameInput, "B.generic-connect", "connection_name_input_missing");
|
|
await connectionNameInput.fill(connectionName);
|
|
const paperclipOrigin = new URL(config.baseUrl).origin;
|
|
const authorizationRequestPromise = page.waitForRequest((request) => {
|
|
if (!request.isNavigationRequest() || request.frame() !== page.mainFrame()) return false;
|
|
try {
|
|
const target = new URL(request.url());
|
|
return target.protocol === "https:" && target.origin !== paperclipOrigin;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}, { timeout: 90_000 });
|
|
await page.getByRole("button", { name: /^check actions$/i }).click({ noWaitAfter: true });
|
|
|
|
// `connect` returns the authorization URL inline, so the UI immediately
|
|
// replaces this page with the provider. That navigation can abort a
|
|
// Playwright response waiter even though the server committed successfully.
|
|
// Persisted connection state is the durable evidence and gives cleanup the
|
|
// ID before any provider credential entry.
|
|
const createdConnection = await waitFor("B.generic-connect", async () => {
|
|
const response = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/connections`,
|
|
undefined,
|
|
"B.generic-connect",
|
|
);
|
|
const matches = asArray(response, "connections").filter((candidate) =>
|
|
candidate.name === connectionName && !existingIds.has(candidate.id) && candidate.status !== "archived");
|
|
if (matches.length > 1) fail("B.generic-connect", "duplicate_connection_detected");
|
|
return matches[0] ?? null;
|
|
}, { timeoutMs: 90_000, intervalMs: 500 });
|
|
connectionId = createdConnection.id;
|
|
// Inspect the initial authorization request, not the provider's eventual
|
|
// login page after redirects (which legitimately omits OAuth parameters).
|
|
const authorizationRequest = await authorizationRequestPromise.catch(() =>
|
|
fail("C.oauth-navigation", "authorization_navigation_missing"));
|
|
const authorizationUrl = authorizationRequest.url();
|
|
let connection = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}`,
|
|
undefined,
|
|
"B.generic-connect",
|
|
);
|
|
const applications = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/applications`,
|
|
undefined,
|
|
"B.generic-connect",
|
|
);
|
|
const applicationMatches = asArray(applications, "applications").filter((candidate) =>
|
|
candidate.id === connection.applicationId && candidate.name === connectionName && candidate.status !== "archived");
|
|
if (applicationMatches.length !== 1) fail("B.generic-connect", "application_not_unique");
|
|
const application = applicationMatches[0];
|
|
assertGenericProvenance({ application, connection }, connectionName);
|
|
if (connection.authKind !== "oauth") fail("B.generic-connect", "oauth_challenge_missing");
|
|
summary.connection = {
|
|
id: connectionId,
|
|
applicationId: application.id,
|
|
name: connectionName,
|
|
applicationKey: application.applicationKey,
|
|
metadataSource: "link",
|
|
transport: "mcp_remote",
|
|
unverifiedServer: true,
|
|
status: connection.status,
|
|
healthStatus: connection.healthStatus,
|
|
};
|
|
|
|
const startResult = persistedOAuthStartResult(connection, authorizationUrl);
|
|
if (!startResult) fail("C.oauth-start", "persisted_start_missing");
|
|
if (startResult.connectionId !== connectionId) fail("C.oauth-start", "connection_id_changed");
|
|
activeCheckpoint = "C.oauth-proof";
|
|
const registrationSource = assertAutomaticRegistrationSource(startResult.registrationSource);
|
|
const navigationProof = inspectAuthorizationUrl(startResult.authorizationUrl, {
|
|
callbackUrl: config.callbackUrl,
|
|
resource: NOTION_RESOURCE,
|
|
registrationSource,
|
|
baseUrl: config.baseUrl,
|
|
});
|
|
summary.oauthProof = {
|
|
registrationSource,
|
|
resource: NOTION_RESOURCE,
|
|
callbackUriExact: true,
|
|
pkceS256: true,
|
|
statePresent: true,
|
|
endpoint: navigationProof.endpoint,
|
|
parameters: navigationProof.parameters,
|
|
discoveredEndpoints: null,
|
|
};
|
|
|
|
// Registration source and every URL/PKCE/state invariant are checked before
|
|
// the credential is entered into the provider page.
|
|
activeCheckpoint = "C.notion-login";
|
|
await completeNotionAuthorization(page, config, credential, connectionId);
|
|
activeCheckpoint = "C.oauth-callback";
|
|
const permissionsPath = `/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/permissions`;
|
|
await page.goto(new URL(permissionsPath, config.baseUrl).toString(), { waitUntil: "domcontentloaded" });
|
|
await expectVisible(page.getByText("OAuth connected", { exact: true }), "C.oauth-callback", "connected_state_missing", 45_000);
|
|
await expectVisible(page.getByText("Unverified server", { exact: true }), "C.oauth-callback", "unverified_badge_missing");
|
|
|
|
connection = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}`,
|
|
undefined,
|
|
"C.connection-detail",
|
|
);
|
|
if (connection.name !== connectionName
|
|
|| connection.config?.unverifiedServer !== true
|
|
|| connection.config?.sourceTemplateKey === "notion"
|
|
|| connection.authKind !== "oauth") {
|
|
fail("C.connection-detail", "generic_provenance_lost");
|
|
}
|
|
const endpointProof = oauthEndpointProof(connection, startResult, config);
|
|
assertNoCredentialMaterial(connection, [config.paperclipPassword, credential.username, credential.password], "C.connection-detail");
|
|
summary.connection = {
|
|
id: connectionId,
|
|
applicationId: connection.applicationId,
|
|
name: connectionName,
|
|
applicationKey: application.applicationKey,
|
|
metadataSource: "link",
|
|
transport: "mcp_remote",
|
|
unverifiedServer: true,
|
|
status: connection.status,
|
|
healthStatus: connection.healthStatus,
|
|
};
|
|
summary.oauthProof = {
|
|
registrationSource,
|
|
resource: NOTION_RESOURCE,
|
|
callbackUriExact: true,
|
|
pkceS256: true,
|
|
statePresent: true,
|
|
endpoint: navigationProof.endpoint,
|
|
parameters: navigationProof.parameters,
|
|
discoveredEndpoints: endpointProof.endpoints,
|
|
};
|
|
const connectedShot = "01-generic-connected.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, connectedShot), config, credential, "F.connected-screenshot");
|
|
summary.screenshots.push(connectedShot);
|
|
|
|
activeCheckpoint = "D.health-check";
|
|
const health = await apiJson(
|
|
context.request,
|
|
config,
|
|
"POST",
|
|
`/api/tool-connections/${connectionId}/health-check`,
|
|
{},
|
|
"D.health-check",
|
|
);
|
|
if (health.connection?.healthStatus !== "healthy") fail("D.health-check", "connection_not_healthy");
|
|
const refreshed = await apiJson(
|
|
context.request,
|
|
config,
|
|
"POST",
|
|
`/api/tool-connections/${connectionId}/catalog/refresh`,
|
|
{},
|
|
"D.catalog-refresh",
|
|
);
|
|
const catalog = asArray(refreshed, "catalog");
|
|
const facts = await finishAgentOnlySetup(context.request, config, companyId, connectionId, catalog, agent.id);
|
|
|
|
activeCheckpoint = "D.connection-active";
|
|
connection = await apiJson(context.request, config, "GET", `/api/tool-connections/${connectionId}`, undefined, "D.connection-active");
|
|
if (connection.status !== "active" || connection.healthStatus !== "healthy" || connection.config?.unverifiedServer !== true) {
|
|
fail("D.connection-active", "connection_not_active_healthy_generic");
|
|
}
|
|
const uniqueConnections = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/companies/${companyId}/tools/connections`,
|
|
undefined,
|
|
"D.connection-active",
|
|
);
|
|
if (asArray(uniqueConnections, "connections").filter((candidate) => candidate.name === connectionName).length !== 1) {
|
|
fail("D.connection-active", "duplicate_connection_detected");
|
|
}
|
|
const installs = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}/installs`,
|
|
undefined,
|
|
"D.agent-install",
|
|
);
|
|
const installRows = asArray(installs, "installs");
|
|
if (installRows.length !== 1 || installRows[0].targetType !== "agent" || installRows[0].targetId !== agent.id) {
|
|
fail("D.agent-install", "install_not_agent_only");
|
|
}
|
|
const testAgents = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}/test-agents`,
|
|
undefined,
|
|
"D.effective-policy",
|
|
);
|
|
const testAgent = asArray(testAgents, "agents").find((candidate) => candidate.id === agent.id);
|
|
const getSelfAccess = testAgent?.effectiveAccess?.tools?.find((tool) => tool.toolName === NOTION_GET_SELF);
|
|
const createPagesAccess = testAgent?.effectiveAccess?.tools?.find((tool) => tool.toolName === NOTION_CREATE_PAGES);
|
|
if (getSelfAccess?.decision !== "allowed" || createPagesAccess?.decision !== "off") {
|
|
fail("D.effective-policy", "unexpected_effective_decision");
|
|
}
|
|
const deniedWrite = await apiJson(
|
|
context.request,
|
|
config,
|
|
"POST",
|
|
`/api/tool-connections/${connectionId}/test-calls`,
|
|
{ agentId: agent.id, toolName: NOTION_CREATE_PAGES, parameters: {} },
|
|
"D.write-denial",
|
|
);
|
|
if (deniedWrite.decision !== "off" || typeof deniedWrite.invocationId !== "string" || !deniedWrite.error) {
|
|
fail("D.write-denial", "write_action_not_denied");
|
|
}
|
|
summary.catalog = {
|
|
discoveredCount: refreshed.discoveredCount,
|
|
advertisedReadTools: facts.active
|
|
.filter((entry) => [NOTION_GET_SELF, "notion-get-users", "notion-fetch"].includes(entry.toolName))
|
|
.map((entry) => entry.toolName),
|
|
getSelf: { catalogEntryId: facts.getSelf.id, decision: "allowed" },
|
|
createPages: {
|
|
catalogEntryId: facts.createPages.id,
|
|
decision: "off",
|
|
localDenialInvocationId: deniedWrite.invocationId,
|
|
upstreamSent: false,
|
|
},
|
|
accessAgentId: agent.id,
|
|
installAgentId: agent.id,
|
|
enabledCatalogEntryCount: 1,
|
|
healthCheck: "healthy",
|
|
catalogRefresh: "succeeded",
|
|
};
|
|
|
|
await page.goto(new URL(`/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/permissions`, config.baseUrl).toString(), { waitUntil: "domcontentloaded" });
|
|
await expectVisible(page.getByText("Who can use it", { exact: true }), "D.permissions-ui", "permissions_panel_missing");
|
|
const getSelfPermission = page.locator(`[data-action-id="${facts.getSelf.id}"] select`);
|
|
const createPagesPermission = page.locator(`[data-action-id="${facts.createPages.id}"] select`);
|
|
await expectVisible(getSelfPermission, "D.permissions-ui", "get_self_permission_missing");
|
|
await expectVisible(createPagesPermission, "D.permissions-ui", "create_pages_permission_missing");
|
|
if (await getSelfPermission.inputValue() !== "allowed" || await createPagesPermission.inputValue() !== "off") {
|
|
fail("D.permissions-ui", "permissions_ui_mismatch");
|
|
}
|
|
const permissionsShot = "02-read-only-policy.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, permissionsShot), config, credential, "F.permissions-screenshot");
|
|
summary.screenshots.push(permissionsShot);
|
|
|
|
activeCheckpoint = "D.test-panel";
|
|
await page.goto(new URL(`/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/test`, config.baseUrl).toString(), { waitUntil: "domcontentloaded" });
|
|
await expectVisible(page.getByLabel("Choose which agent to test as"), "D.test-panel", "agent_picker_missing");
|
|
await page.getByLabel("Choose which agent to test as").click();
|
|
await page.getByLabel("Search agents").fill(TARGET_AGENT_NAME);
|
|
await page.getByRole("button", { name: new RegExp(`^${escapeRegex(TARGET_AGENT_NAME)}`) }).click();
|
|
await page.getByLabel("Find an action").fill(NOTION_GET_SELF);
|
|
const getSelfTitle = facts.getSelf.title ?? facts.getSelf.toolName;
|
|
const actionRow = page.locator("button").filter({ hasText: getSelfTitle }).filter({ hasText: "Allowed" }).first();
|
|
await expectVisible(actionRow, "D.test-panel", "get_self_allowed_row_missing");
|
|
await actionRow.click();
|
|
await expectVisible(page.getByText("This action takes no inputs."), "D.test-panel", "empty_input_form_missing");
|
|
const boardStartedAt = Date.now();
|
|
const testCallResponsePromise = page.waitForResponse((response) =>
|
|
response.request().method() === "POST"
|
|
&& new URL(response.url()).pathname === `/api/tool-connections/${connectionId}/test-calls`,
|
|
);
|
|
await page.getByRole("button", { name: /^run$/i }).click();
|
|
const testCallResponse = await testCallResponsePromise;
|
|
if (!testCallResponse.ok()) fail("D.board-get-self", `http_${testCallResponse.status()}`);
|
|
const testCall = await testCallResponse.json().catch(() => fail("D.board-get-self", "invalid_json"));
|
|
const testCallInput = testCallResponse.request().postDataJSON();
|
|
if (testCallInput?.agentId !== agent.id
|
|
|| testCallInput?.toolName !== NOTION_GET_SELF
|
|
|| !testCallInput.parameters
|
|
|| Object.keys(testCallInput.parameters).length !== 0) {
|
|
fail("D.board-get-self", "nonempty_or_unexpected_input");
|
|
}
|
|
if (testCall.decision !== "allowed" || testCall.error || typeof testCall.invocationId !== "string") {
|
|
fail("D.board-get-self", "gateway_call_not_allowed");
|
|
}
|
|
const boardIdentity = extractNotionIdentity(testCall.result);
|
|
if (!boardIdentity) fail("D.board-get-self", "workspace_identity_missing");
|
|
await expectVisible(page.getByText(/^Worked\./), "D.board-get-self", "success_result_missing");
|
|
const boardShot = "03-board-notion-get-self.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, boardShot), config, credential, "F.board-test-screenshot");
|
|
summary.screenshots.push(boardShot);
|
|
summary.boardTest = {
|
|
catalogEntryId: facts.getSelf.id,
|
|
toolName: NOTION_GET_SELF,
|
|
invocationId: testCall.invocationId,
|
|
decision: "allowed",
|
|
resultStatus: "succeeded",
|
|
workspace: boardIdentity,
|
|
durationMs: Date.now() - boardStartedAt,
|
|
};
|
|
|
|
activeCheckpoint = "E.fresh-agent";
|
|
const proofIssue = await createAndWaitForProofIssue(context.request, config, {
|
|
title: `Notion generic installed-tool proof ${startedAt.toISOString()}`,
|
|
description: [
|
|
"Invoke exactly one installed action: the read-only `notion-get-self` tool with an empty `{}` input.",
|
|
"Make no Notion mutation and do not invoke any other Notion action.",
|
|
`Require workspace ID ${boardIdentity.workspaceId} and workspace name ${boardIdentity.workspaceName}.`,
|
|
"Post exactly one JSON object with keys `workspaceId`, `workspaceName`, and `invocationId` (the Paperclip invocation ID), then mark this issue done.",
|
|
"Do not report tokens, cookies, headers, authorization data, raw payloads, or any other fields.",
|
|
].join("\n\n"),
|
|
status: "todo",
|
|
workMode: "standard",
|
|
priority: "medium",
|
|
assigneeAgentId: agent.id,
|
|
acceptanceCriteria: [
|
|
"The installed notion-get-self action succeeds with empty input.",
|
|
"Only sanitized workspace ID/name and Paperclip invocation ID are reported.",
|
|
"No Notion mutation is attempted.",
|
|
],
|
|
}, "E.fresh-agent");
|
|
for (const comment of proofIssue.comments) {
|
|
assertNoCredentialMaterial(comment.body, [credential.username, credential.password], "E.agent-proof-comment");
|
|
}
|
|
const proof = proofIssue.comments
|
|
.filter((comment) => comment.authorAgentId === agent.id || comment.derivedAuthorAgentId === agent.id)
|
|
.map((comment) => parseSanitizedAgentProof(comment.body, boardIdentity))
|
|
.find(Boolean);
|
|
if (!proof) fail("E.agent-proof-comment", "sanitized_proof_missing");
|
|
|
|
let finalActivity;
|
|
const agentEvent = await waitFor("E.agent-audit", async () => {
|
|
const activity = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}/activity?limit=100`,
|
|
undefined,
|
|
"E.agent-audit",
|
|
);
|
|
assertNoCredentialMaterial(activity, [credential.username, credential.password], "E.agent-audit");
|
|
finalActivity = activity;
|
|
return asArray(activity, "events").find((event) =>
|
|
event.issueId === proofIssue.child.id
|
|
&& event.agentId === agent.id
|
|
&& event.toolName === NOTION_GET_SELF
|
|
&& event.invocationId === proof.invocationId
|
|
&& event.outcome === "success") ?? null;
|
|
}, { timeoutMs: 60_000, intervalMs: 2_000 });
|
|
if (!agentEvent.runId) fail("E.agent-audit", "run_id_missing");
|
|
if (agentEvent.requestSummary?.summary !== "{}") fail("E.agent-audit", "get_self_input_not_empty");
|
|
const childToolEvents = asArray(finalActivity, "events").filter((event) =>
|
|
event.issueId === proofIssue.child.id && event.invocationId);
|
|
if (childToolEvents.length !== 1 || childToolEvents[0].toolName !== NOTION_GET_SELF) {
|
|
fail("E.agent-audit", "unexpected_upstream_action");
|
|
}
|
|
const agentRun = await waitFor("E.agent-run-status", async () => {
|
|
const run = await apiJson(context.request, config, "GET", `/api/heartbeat-runs/${agentEvent.runId}`, undefined, "E.agent-run-status");
|
|
if (["failed", "cancelled", "timed_out"].includes(run.status)) fail("E.agent-run-status", `run_${run.status}`);
|
|
return run.status === "succeeded" ? run : null;
|
|
}, { timeoutMs: 60_000, intervalMs: 2_000 });
|
|
summary.freshRun = {
|
|
issueId: proofIssue.child.id,
|
|
issueIdentifier: proofIssue.child.identifier,
|
|
transition: {
|
|
created: "todo",
|
|
enteredInProgress: proofIssue.observedStatuses.has("in_progress") || Boolean(proofIssue.finished.startedAt),
|
|
completed: "done",
|
|
},
|
|
runId: agentEvent.runId,
|
|
runStatus: agentRun.status,
|
|
invocationId: proof.invocationId,
|
|
workspace: { id: proof.workspaceId, name: proof.workspaceName },
|
|
connectionId,
|
|
decision: agentEvent.decision ?? "allowed",
|
|
outcome: agentEvent.outcome,
|
|
durationMs: agentEvent.latencyMs,
|
|
};
|
|
|
|
await page.goto(new URL(`/${TARGET_COMPANY_PREFIX}/issues/${proofIssue.child.identifier}`, config.baseUrl).toString(), { waitUntil: "domcontentloaded" });
|
|
await expectVisible(page.getByText(proofIssue.child.title, { exact: true }).first(), "F.child-screenshot", "child_issue_missing");
|
|
const childShot = "04-fresh-agent-proof.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, childShot), config, credential, "F.child-screenshot");
|
|
summary.screenshots.push(childShot);
|
|
|
|
await page.goto(new URL(`/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/activity`, config.baseUrl).toString(), { waitUntil: "domcontentloaded" });
|
|
await expectVisible(page.getByText(NOTION_GET_SELF, { exact: false }).first(), "F.activity-screenshot", "get_self_activity_missing");
|
|
const activityShot = "05-redacted-audit.png";
|
|
await safeScreenshot(page, screenshotFile(outputDirectory, activityShot), config, credential, "F.activity-screenshot");
|
|
summary.screenshots.push(activityShot);
|
|
|
|
activeCheckpoint = "F.cleanup";
|
|
summary.cleanup = await cleanupConnection(
|
|
context.request,
|
|
config,
|
|
companyId,
|
|
connectionId,
|
|
connectionName,
|
|
{ requireInstalled: true },
|
|
);
|
|
cleanupComplete = true;
|
|
|
|
activeCheckpoint = "F.post-cleanup-runtime";
|
|
const absenceIssue = await createAndWaitForProofIssue(context.request, config, {
|
|
title: `Notion generic cleanup runtime proof ${startedAt.toISOString()}`,
|
|
description: [
|
|
`Inspect the tools available in this fresh runtime after generic connection ${connectionId} was removed.`,
|
|
"Do not call any Notion tool. Confirm `notion-get-self` from that removed connection is absent.",
|
|
`Post exactly {\"connectionId\":\"${connectionId}\",\"toolPresent\":false} and mark this issue done.`,
|
|
"Do not report any other fields or credential material.",
|
|
].join("\n\n"),
|
|
status: "todo",
|
|
workMode: "standard",
|
|
priority: "medium",
|
|
assigneeAgentId: agent.id,
|
|
acceptanceCriteria: [
|
|
"A fresh runtime does not expose the removed generic connection's notion-get-self tool.",
|
|
"No Notion action is invoked.",
|
|
],
|
|
}, "F.post-cleanup-runtime");
|
|
const absenceProof = absenceIssue.comments
|
|
.filter((comment) => comment.authorAgentId === agent.id || comment.derivedAuthorAgentId === agent.id)
|
|
.map((comment) => parseRuntimeAbsenceProof(comment.body, connectionId))
|
|
.find(Boolean);
|
|
if (!absenceProof) fail("F.post-cleanup-runtime", "absence_proof_missing");
|
|
const afterCleanupActivity = await apiJson(
|
|
context.request,
|
|
config,
|
|
"GET",
|
|
`/api/tool-connections/${connectionId}/activity?limit=100`,
|
|
undefined,
|
|
"F.post-cleanup-runtime",
|
|
);
|
|
if (asArray(afterCleanupActivity, "events").some((event) => event.issueId === absenceIssue.child.id && event.invocationId)) {
|
|
fail("F.post-cleanup-runtime", "post_cleanup_tool_invocation_detected");
|
|
}
|
|
summary.postCleanupRun = {
|
|
issueId: absenceIssue.child.id,
|
|
issueIdentifier: absenceIssue.child.identifier,
|
|
connectionId,
|
|
toolPresent: false,
|
|
invocationCount: 0,
|
|
completed: "done",
|
|
};
|
|
summary.passed = true;
|
|
} catch (error) {
|
|
caughtFailure = error instanceof SmokeFailure
|
|
? error
|
|
: error instanceof NotionGenericLivePreflightError
|
|
? new SmokeFailure(activeCheckpoint, error.code)
|
|
: new SmokeFailure(activeCheckpoint, "unexpected_error");
|
|
} finally {
|
|
if (connectionId && companyId && context && !cleanupComplete) {
|
|
try {
|
|
summary.cleanup = await cleanupConnection(context.request, config, companyId, connectionId, connectionName);
|
|
cleanupComplete = true;
|
|
} catch {
|
|
summary.cleanup = { completed: false, code: "cleanup_failed" };
|
|
if (!caughtFailure) caughtFailure = new SmokeFailure("F.cleanup", "cleanup_failed");
|
|
}
|
|
}
|
|
await context?.close().catch(() => {});
|
|
await browser?.close().catch(() => {});
|
|
}
|
|
|
|
summary.completedAt = new Date().toISOString();
|
|
if (caughtFailure) summary.failure = { checkpoint: caughtFailure.checkpoint, code: caughtFailure.code };
|
|
assertSanitizedEvidence(summary);
|
|
const summaryPath = path.join(outputDirectory, "summary.json");
|
|
await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, { mode: 0o600 });
|
|
if (caughtFailure) throw caughtFailure;
|
|
return { outputDirectory, summaryPath, screenshots: summary.screenshots };
|
|
}
|
|
|
|
async function main() {
|
|
const dryRun = process.argv.includes("--dry-run");
|
|
let prepared;
|
|
try {
|
|
prepared = await prepareNotionGenericLiveSmoke({
|
|
loadBrowser: dryRun ? async () => null : () => import("@playwright/test"),
|
|
});
|
|
} catch (error) {
|
|
process.stderr.write(`${preflightFailureMessage(error)}\n`);
|
|
process.exitCode = error instanceof NotionGenericLivePreflightError ? 2 : 1;
|
|
return;
|
|
}
|
|
if (dryRun) {
|
|
process.stdout.write("Notion generic live smoke dry-run passed; no credential value was fetched.\n");
|
|
return;
|
|
}
|
|
try {
|
|
const result = await runSmoke({ config: prepared.config, chromium: prepared.browserModule.chromium });
|
|
process.stdout.write(`Notion generic live smoke passed. Sanitized evidence: ${result.outputDirectory}\n`);
|
|
} catch (error) {
|
|
const failure = error instanceof SmokeFailure ? error : new SmokeFailure("unexpected", "unexpected_error");
|
|
process.stderr.write(`Notion generic live smoke failed at ${failure.checkpoint} (${failure.code}).\n`);
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
await main();
|