diff --git a/.github/scripts/check-pr-release-bootstrap.mjs b/.github/scripts/check-pr-release-bootstrap.mjs new file mode 100644 index 0000000000..215c3b3afe --- /dev/null +++ b/.github/scripts/check-pr-release-bootstrap.mjs @@ -0,0 +1,151 @@ +#!/usr/bin/env node +/** + * check-pr-release-bootstrap.mjs + * Detects release packages that this PR adds or newly release-enables whose + * names do not exist on npm yet, and emits an informational notice: the + * `policy` CI job will stay red until a maintainer bootstraps the name with + * `pnpm run release:bootstrap-package`. Contributors cannot fix that + * themselves, so the notice says so explicitly. + * + * Never fails (informational only) — outputs { passed: true, informational: string[] } + * + * Runs under pull_request_target from base-branch context: it only parses + * JSON and diff text fetched from the GitHub API and queries the npm registry + * with scope-validated names. It never executes PR code. + */ +import { fileURLToPath } from 'node:url'; +import { ghFetch } from './get-bot-token.mjs'; +import { resolveBaseRef } from './check-pr-dependencies.mjs'; + +const MANIFEST_PATH = 'scripts/release-package-manifest.json'; + +// Manifest content comes from the PR head (fork-controlled), so only names +// matching our scope are ever looked up on the registry. +const SCOPE_RE = /^@paperclipai\/[a-z0-9][a-z0-9._-]*$/; + +const MAX_REGISTRY_LOOKUPS = 5; + +function buildContentsPath(repo, filename, ref) { + return `/repos/${repo}/contents/${filename}?${new URLSearchParams({ ref }).toString()}`; +} + +async function fetchManifestEntries(fetchFromGitHub, token, repo, ref) { + try { + const res = await fetchFromGitHub(buildContentsPath(repo, MANIFEST_PATH, ref), token); + const parsed = JSON.parse(Buffer.from(res.content, 'base64').toString()); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; // manifest missing or unreadable on this ref + } +} + +export async function fetchRegistryPackageExists(packageName) { + const res = await fetch(`https://registry.npmjs.org/${encodeURIComponent(packageName)}`, { + method: 'HEAD', + }); + if (res.status === 404) return false; + if (res.ok) return true; + throw new Error(`npm registry returned ${res.status} for ${packageName}`); +} + +// Names this PR newly declares a workspace dependency on, per the diff of any +// changed package.json. If one of them is an unpublished manifest entry that +// is not publishFromCi:true, the release manifest validator rejects the PR. +export function addedWorkspaceDependencyNames(files) { + const names = new Set(); + for (const file of files) { + if (!file.filename.endsWith('package.json')) continue; + if (file.filename.includes('node_modules')) continue; + for (const line of (file.patch ?? '').split('\n')) { + if (!line.startsWith('+')) continue; + const match = line.match(/"(@paperclipai\/[a-z0-9][a-z0-9._-]*)"\s*:\s*"workspace:/); + if (match) names.add(match[1]); + } + } + return names; +} + +function buildNotice({ name, reason }) { + const bootstrap = + `a **maintainer** must run \`pnpm run release:bootstrap-package -- ${name} --publish\` ` + + 'and configure npm trusted publishing (see `doc/PUBLISHING.md`)'; + + if (reason === 'depended') { + return ( + `🚀 New release package \`${name}\` is not on npm yet, and published packages in this PR ` + + `depend on it, so the \`policy\` check will stay red: ${bootstrap}, then set its manifest ` + + `entry to \`"publishFromCi": true\` — or drop the workspace dependency. ` + + 'No contributor action is needed for the bootstrap itself.' + ); + } + + return ( + `🚀 New release package \`${name}\` is not on npm yet, so the \`policy\` check will stay ` + + `red: ${bootstrap}. No contributor action is needed for this.` + ); +} + +export async function checkReleaseBootstrap(files, token, repo, prNumber, baseRef, deps = {}) { + const { fetchFromGitHub = ghFetch, registryPackageExists = fetchRegistryPackageExists } = deps; + + const manifestChanged = files.some( + f => f.filename === MANIFEST_PATH && f.status !== 'removed' + ); + // A PR can hit the manifest edge validator without touching the manifest: + // adding a workspace:* dependency on an existing unpublished + // publishFromCi:false package. Patch parsing is free, so compute the added + // dependencies first and keep the zero-API fast path only for PRs that + // neither touch the manifest nor add a workspace dependency. + const dependedOn = addedWorkspaceDependencyNames(files); + if (!manifestChanged && dependedOn.size === 0) return { passed: true, informational: [] }; + + const resolvedBaseRef = await resolveBaseRef(fetchFromGitHub, token, repo, prNumber, baseRef); + const [baseEntries, headEntries] = await Promise.all([ + fetchManifestEntries(fetchFromGitHub, token, repo, resolvedBaseRef), + fetchManifestEntries(fetchFromGitHub, token, repo, `refs/pull/${prNumber}/head`), + ]); + + const basePublishFromCiByName = new Map( + baseEntries + .filter(e => e && typeof e.name === 'string') + .map(e => [e.name, e.publishFromCi === true]) + ); + + const candidates = []; + for (const entry of headEntries) { + if (!entry || typeof entry.name !== 'string') continue; + const name = entry.name; + if (!SCOPE_RE.test(name)) continue; + + const enabled = entry.publishFromCi === true; + const baseEnabled = basePublishFromCiByName.get(name); + + if (enabled && baseEnabled !== true) { + // Newly release-enabled (added as true, or flipped false -> true): the + // bootstrap gate itself will fail if the name is missing from npm. + candidates.push({ name, reason: 'enabled' }); + } else if (!enabled && dependedOn.has(name)) { + // Not release-enabled but this PR makes published packages depend on + // it: the manifest edge validator will fail if it stays unpublished. + candidates.push({ name, reason: 'depended' }); + } + } + + const informational = []; + for (const candidate of candidates.slice(0, MAX_REGISTRY_LOOKUPS)) { + let exists; + try { + exists = await registryPackageExists(candidate.name); + } catch { + continue; // registry hiccup: stay quiet, the policy job is the enforcer + } + if (!exists) informational.push(buildNotice(candidate)); + } + + return { passed: true, informational }; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + console.error('check-pr-release-bootstrap.mjs is a library used by run-quality-gates.mjs'); + process.exit(1); +} diff --git a/.github/scripts/run-quality-gates.mjs b/.github/scripts/run-quality-gates.mjs index a99786fbf2..fde29a1a33 100644 --- a/.github/scripts/run-quality-gates.mjs +++ b/.github/scripts/run-quality-gates.mjs @@ -16,6 +16,7 @@ import { checkDedupSearch } from './check-pr-dedup-search.mjs'; import { checkTestCoverage } from './check-pr-test-coverage.mjs'; import { checkLockfile } from './check-pr-lockfile.mjs'; import { checkDependencies } from './check-pr-dependencies.mjs'; +import { checkReleaseBootstrap } from './check-pr-release-bootstrap.mjs'; const COMMENT_SIGNATURE = '— commitperclip'; @@ -111,7 +112,7 @@ async function main() { // Run all quality gates (pure functions run sync, deps check is async) const prTitle = pr.title ?? ''; - const [templateResult, issueResult, dedupResult, testResult, lockfileResult, depsResult] = + const [templateResult, issueResult, dedupResult, testResult, lockfileResult, depsResult, bootstrapResult] = await Promise.all([ Promise.resolve(checkTemplate(prBody)), Promise.resolve(checkLinkedIssue(prBody, prTitle)), @@ -119,6 +120,7 @@ async function main() { Promise.resolve(checkTestCoverage(files, prTitle)), Promise.resolve(checkLockfile(files, author, branch)), checkDependencies(files, GH_TOKEN, GH_REPO, prNumber, pr.base?.ref), + checkReleaseBootstrap(files, GH_TOKEN, GH_REPO, prNumber, pr.base?.ref), ]); const allFailures = [ @@ -128,7 +130,10 @@ async function main() { ...testResult.failures, ...lockfileResult.failures, ]; - const informational = depsResult.informational ?? []; + const informational = [ + ...(depsResult.informational ?? []), + ...(bootstrapResult.informational ?? []), + ]; const allPassed = allFailures.length === 0; const commentBody = buildComment(author, allFailures, informational); diff --git a/.github/scripts/tests/check-pr-release-bootstrap.test.mjs b/.github/scripts/tests/check-pr-release-bootstrap.test.mjs new file mode 100644 index 0000000000..ad631c61ff --- /dev/null +++ b/.github/scripts/tests/check-pr-release-bootstrap.test.mjs @@ -0,0 +1,265 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + addedWorkspaceDependencyNames, + checkReleaseBootstrap, +} from '../check-pr-release-bootstrap.mjs'; + +const MANIFEST_PATH = 'scripts/release-package-manifest.json'; + +function encodeManifest(entries) { + return { content: Buffer.from(JSON.stringify(entries)).toString('base64') }; +} + +function stubGitHub({ base = [], head = [] }) { + return async (path) => { + if (path.includes('ref=refs%2Fpull%2F')) return encodeManifest(head); + if (path.includes(`/contents/`)) return encodeManifest(base); + throw new Error(`unexpected fetch: ${path}`); + }; +} + +const manifestChangedFile = { filename: MANIFEST_PATH, status: 'modified' }; + +test('does nothing (and fetches nothing) when the manifest is untouched', async () => { + const result = await checkReleaseBootstrap( + [{ filename: 'server/src/index.ts', status: 'modified' }], + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: async () => { throw new Error('should not fetch'); }, + registryPackageExists: async () => { throw new Error('should not look up'); }, + } + ); + + assert.deepEqual(result, { passed: true, informational: [] }); +}); + +test('notices a new publishFromCi:true package that is missing from npm', async () => { + const result = await checkReleaseBootstrap( + [manifestChangedFile], + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: stubGitHub({ + base: [{ dir: 'a', name: '@paperclipai/existing', publishFromCi: true }], + head: [ + { dir: 'a', name: '@paperclipai/existing', publishFromCi: true }, + { dir: 'b', name: '@paperclipai/brand-new', publishFromCi: true }, + ], + }), + registryPackageExists: async (name) => name !== '@paperclipai/brand-new', + } + ); + + assert.equal(result.informational.length, 1); + assert.match(result.informational[0], /@paperclipai\/brand-new/); + assert.match(result.informational[0], /release:bootstrap-package -- @paperclipai\/brand-new --publish/); + assert.match(result.informational[0], /No contributor action/); +}); + +test('stays quiet when the new package already exists on npm', async () => { + const result = await checkReleaseBootstrap( + [manifestChangedFile], + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: stubGitHub({ + base: [], + head: [{ dir: 'b', name: '@paperclipai/already-bootstrapped', publishFromCi: true }], + }), + registryPackageExists: async () => true, + } + ); + + assert.deepEqual(result.informational, []); +}); + +test('notices a publishFromCi flip from false to true on a missing package', async () => { + const result = await checkReleaseBootstrap( + [manifestChangedFile], + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: stubGitHub({ + base: [{ dir: 'b', name: '@paperclipai/flipped', publishFromCi: false }], + head: [{ dir: 'b', name: '@paperclipai/flipped', publishFromCi: true }], + }), + registryPackageExists: async () => false, + } + ); + + assert.equal(result.informational.length, 1); + assert.match(result.informational[0], /@paperclipai\/flipped/); +}); + +test('notices a publishFromCi:false package that published packages newly depend on', async () => { + const files = [ + manifestChangedFile, + { + filename: 'server/package.json', + status: 'modified', + patch: '@@ -1 +1 @@\n+ "@paperclipai/adapter-kimi-local": "workspace:*",', + }, + ]; + + const result = await checkReleaseBootstrap( + files, + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: stubGitHub({ + base: [], + head: [{ dir: 'b', name: '@paperclipai/adapter-kimi-local', publishFromCi: false }], + }), + registryPackageExists: async () => false, + } + ); + + assert.equal(result.informational.length, 1); + assert.match(result.informational[0], /depend on it/); + assert.match(result.informational[0], /"publishFromCi": true/); + assert.match(result.informational[0], /drop the workspace dependency/); +}); + +test('notices a newly added dependency on an existing unpublished package even when the manifest is untouched', async () => { + const files = [ + { + filename: 'server/package.json', + status: 'modified', + patch: '@@ -1 +1 @@\n+ "@paperclipai/adapter-hermes-gateway": "workspace:*",', + }, + ]; + + const manifest = [{ dir: 'g', name: '@paperclipai/adapter-hermes-gateway', publishFromCi: false }]; + const result = await checkReleaseBootstrap( + files, + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: stubGitHub({ base: manifest, head: manifest }), + registryPackageExists: async () => false, + } + ); + + assert.equal(result.informational.length, 1); + assert.match(result.informational[0], /@paperclipai\/adapter-hermes-gateway/); + assert.match(result.informational[0], /depend on it/); +}); + +test('stays quiet for a publishFromCi:false package nothing depends on', async () => { + const result = await checkReleaseBootstrap( + [manifestChangedFile], + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: stubGitHub({ + base: [], + head: [{ dir: 'b', name: '@paperclipai/deliberately-private', publishFromCi: false }], + }), + registryPackageExists: async () => { throw new Error('should not look up'); }, + } + ); + + assert.deepEqual(result.informational, []); +}); + +test('never looks up names outside the @paperclipai scope', async () => { + const lookedUp = []; + const result = await checkReleaseBootstrap( + [manifestChangedFile], + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: stubGitHub({ + base: [], + head: [ + { dir: 'x', name: '@evil/probe', publishFromCi: true }, + { dir: 'y', name: 'unscoped-name', publishFromCi: true }, + { dir: 'z', name: '@paperclipai/UPPER', publishFromCi: true }, + ], + }), + registryPackageExists: async (name) => { + lookedUp.push(name); + return false; + }, + } + ); + + assert.deepEqual(lookedUp, []); + assert.deepEqual(result.informational, []); +}); + +test('stays quiet when the registry lookup fails', async () => { + const result = await checkReleaseBootstrap( + [manifestChangedFile], + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: stubGitHub({ + base: [], + head: [{ dir: 'b', name: '@paperclipai/brand-new', publishFromCi: true }], + }), + registryPackageExists: async () => { throw new Error('registry down'); }, + } + ); + + assert.deepEqual(result, { passed: true, informational: [] }); +}); + +test('treats a missing base manifest as empty (every head entry is new)', async () => { + const result = await checkReleaseBootstrap( + [manifestChangedFile], + 'token', + 'paperclipai/paperclip', + 9967, + 'master', + { + fetchFromGitHub: async (path) => { + if (path.includes('ref=refs%2Fpull%2F')) { + return encodeManifest([{ dir: 'b', name: '@paperclipai/brand-new', publishFromCi: true }]); + } + throw new Error('404 base manifest'); + }, + registryPackageExists: async () => false, + } + ); + + assert.equal(result.informational.length, 1); +}); + +test('addedWorkspaceDependencyNames reads only added lines of package.json patches', () => { + const names = addedWorkspaceDependencyNames([ + { + filename: 'server/package.json', + patch: [ + '@@ -1,3 +1,4 @@', + ' "@paperclipai/context-line": "workspace:*",', + '- "@paperclipai/removed-dep": "workspace:*",', + '+ "@paperclipai/added-dep": "workspace:*",', + ].join('\n'), + }, + { filename: 'ui/src/index.ts', patch: '+ "@paperclipai/not-a-pkg-json": "workspace:*",' }, + { filename: 'node_modules/x/package.json', patch: '+ "@paperclipai/vendored": "workspace:*",' }, + ]); + + assert.deepEqual([...names], ['@paperclipai/added-dep']); +}); diff --git a/doc/PUBLISHING.md b/doc/PUBLISHING.md index 451853c352..433860153f 100644 --- a/doc/PUBLISHING.md +++ b/doc/PUBLISHING.md @@ -184,53 +184,65 @@ CI publishing is controlled by [`scripts/release-package-manifest.json`](../scri When you add a new public package: 1. add it to the manifest and decide whether CI should publish it immediately -2. if CI should publish it, bootstrap the package on npm before merge +2. if CI should publish it, reserve the name on npm with the placeholder bootstrap before merge 3. if CI should not publish it yet, keep `"publishFromCi": false` 4. only enable `"publishFromCi": true` after npm trusted publishing is configured for that package -PR CI now checks changed release-enabled package manifests against npm. That catches a missing first-publish bootstrap before the change reaches `master`. +PR CI now checks changed release-enabled package manifests against npm. That catches a missing first-publish bootstrap before the change reaches `master`. When a PR needs this bootstrap, commitperclip also posts an informational notice on the PR naming the exact command, so contributors know a maintainer action is pending rather than something they can fix. ### One-time bootstrap sequence for a new package -The first publish of a brand-new package still needs one human maintainer with npm write access. -After that, trusted publishing can take over. +Creating a brand-new package name on npm still needs one human maintainer with npm write access. +After that, trusted publishing takes over — and CI publishes the only real content the package ever gets. + +The bootstrap intentionally does **not** publish the package's real build output. +It publishes a tiny placeholder at version `0.0.0` (a manifest, a README, and an +`index.js` that throws a descriptive error), because: + +- real package content should only ever reach npm from CI, after the PR that adds the package has been reviewed and merged +- the PR CI gate only requires the name to resolve on the registry +- the trusted publisher rule can only be configured once the package page exists +- the placeholder needs no local build and no workspace state, so it can be published from any checkout (including `master`, before the new package's PR merges) Example for a newly added public package from the repo root: ```bash -# safe preview +# safe preview (stages the placeholder and runs npm publish --dry-run) pnpm run release:bootstrap-package -- @paperclipai/new-package -# one-time first publish from an authenticated maintainer machine -pnpm run release:bootstrap-package -- @paperclipai/new-package --publish --otp 123456 +# one-time placeholder publish from an authenticated maintainer machine +# (prompts for npm one-time passwords; they are never passed as arguments) +pnpm run release:bootstrap-package -- @paperclipai/new-package --publish ``` The helper script: +- refuses names outside the `@paperclipai/` scope - checks that the package does not already exist on npm -- builds the target package unless `--skip-build` is passed -- runs `pnpm publish --dry-run --no-git-checks --access public` from the repo root -- only runs the real `pnpm publish --no-git-checks --access public` when `--publish --otp ` is provided +- stages the placeholder in a temporary directory and previews it with `npm publish --dry-run --access public` +- with `--publish`, prompts for a one-time password and publishes. Codes are entered interactively and handed to npm through its environment (`npm_config_otp`), so they never appear on a command line, in shell history, or in a process listing; a rejected or expired code re-prompts +- then waits for the registry to show the package (a first publish can take a few minutes to become visible on the read/write endpoints) and prompts for a second code to deprecate the placeholder, so accidental installs warn loudly. If the wait times out or the deprecation fails, it prints the exact `npm deprecate` command to run manually -The helper intentionally uses `pnpm publish` instead of `npm publish` so workspace -dependencies and `publishConfig` export fields are normalized before the package -is sent to the registry. +Until the first stable release supersedes it, the `latest` dist-tag points at the +deprecated placeholder. Internal consumers are unaffected: release version +rewrites pin exact calver versions, so nothing inside the release package set +resolves through `latest`. For the real `--publish` step, the maintainer machine must already be authenticated to npm. If `npm whoami` returns `401`, first run `npm logout --registry=https://registry.npmjs.org/` to clear any stale local auth, then run `npm login` or `npm adduser` locally as an npm org member, and finally rerun the helper. That local human auth is fine for the one-time bootstrap publish; we just do not want the same auth model inside CI. -The helper now requires `--otp ` up front for `--publish`, so it fails before the real publish attempt if the one-time password is missing. +`--publish` requires an interactive terminal: the helper prompts for the one-time password right before the publish and again before the deprecation, handing each code to npm through its environment (`npm_config_otp`), so codes never appear in command arguments, shell history, or process listings. -After that first publish succeeds: +After the placeholder publish succeeds: 1. open `https://www.npmjs.com/package/@paperclipai/new-package` 2. go to `Settings` → `Trusted publishing` 3. add repository `paperclipai/paperclip` 4. set workflow filename to `release.yml` 5. optionally go to `Settings` → `Publishing access` and enable `Require two-factor authentication and disallow tokens` -6. keep `publishFromCi: true` in [`scripts/release-package-manifest.json`](../scripts/release-package-manifest.json) +6. only then set `"publishFromCi": true` in [`scripts/release-package-manifest.json`](../scripts/release-package-manifest.json) -Once those steps are done, future canary and stable publishes for that package are automated through GitHub OIDC. The manual step is only the first package creation on npm. +Once those steps are done, future canary and stable publishes for that package are automated through GitHub OIDC. The manual step only reserves the name on npm; every real version ships from CI. ## Rollback model diff --git a/scripts/bootstrap-npm-package.mjs b/scripts/bootstrap-npm-package.mjs index d1b0fa0de6..5d774f52ec 100644 --- a/scripts/bootstrap-npm-package.mjs +++ b/scripts/bootstrap-npm-package.mjs @@ -1,87 +1,166 @@ #!/usr/bin/env node +// One-time npm bootstrap for a brand-new release package. Publishes a minimal +// placeholder at version 0.0.0 — never the package's real build output — so: +// +// - the PR CI gate (scripts/check-release-package-bootstrap.mjs) passes, since +// it only requires the name to resolve on the registry +// - trusted publishing can be configured on npmjs.com (the package page must +// exist before a trusted publisher rule can be added) +// - real package content only ever reaches npm from CI, after the PR that adds +// the package has been reviewed and merged +// +// The first real calver release supersedes the placeholder, and a stable +// release moves `latest` off it. The placeholder needs no local build and no +// workspace state, so it can run from any checkout (including master, before +// the package's PR merges). +// +// npm one-time passwords are single-use and time-limited, so the helper +// prompts for them interactively (publish and deprecate each need their own +// code) and hands them to npm through its environment (npm_config_otp) — +// codes never appear on a command line, in shell history, or in a process +// listing. It also waits for the registry to show the package before +// deprecating — a first publish can take a few minutes to become visible on +// the read/write endpoints. + import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { createInterface } from "node:readline/promises"; +import { setTimeout as delay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; -import { dirname, resolve } from "node:path"; -import { buildReleasePackagePlan } from "./release-package-map.mjs"; +export const PLACEHOLDER_VERSION = "0.0.0"; -const __dirname = dirname(fileURLToPath(import.meta.url)); -const repoRoot = resolve(__dirname, ".."); +const SCOPE_RE = /^@paperclipai\/[a-z0-9][a-z0-9._-]*$/; -function normalizePath(filePath) { - return filePath.replace(/\\/g, "/").replace(/^\.\//, ""); -} +const REGISTRY_POLL_INTERVAL_MS = 15_000; +const REGISTRY_POLL_ATTEMPTS = 40; // ~10 minutes +// Require back-to-back sightings: the write endpoint used by `npm deprecate` +// can trail the read endpoint, so one extra interval is cheap insurance. +const REGISTRY_POLL_CONSECUTIVE = 2; + +const OTP_ATTEMPTS = 3; function usage() { process.stderr.write( [ "Usage:", - " node scripts/bootstrap-npm-package.mjs [--publish --otp ] [--skip-build]", + " node scripts/bootstrap-npm-package.mjs [--publish]", + "", + "Publishes an empty placeholder at version 0.0.0 that reserves on npm", + "so the release-bootstrap CI gate passes and trusted publishing can be configured.", + "Real package content is only ever published by CI. Without --publish this is a dry run.", + "", + "With --publish the helper prompts for npm one-time passwords interactively", + "(publish and deprecate each need their own code) and hands them to npm via its", + "environment, so codes never appear on a command line.", "", "Examples:", - " node scripts/bootstrap-npm-package.mjs @paperclipai/plugin-workspace-diff", - " node scripts/bootstrap-npm-package.mjs packages/plugins/plugin-workspace-diff --publish", + " node scripts/bootstrap-npm-package.mjs @paperclipai/new-package", + " node scripts/bootstrap-npm-package.mjs @paperclipai/new-package --publish", "", ].join("\n"), ); } -function parseArgs(argv) { +export function parseArgs(argv) { const flags = new Set(); - let selector = null; - let otp = null; + let packageName = null; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; + for (const arg of argv) { if (arg === "--") { continue; } - if (arg === "--publish" || arg === "--skip-build") { + if (arg === "--publish") { flags.add(arg); continue; } - if (arg === "--otp") { - const value = argv[index + 1]; - if (!value || value.startsWith("--")) { - throw new Error("expected a one-time password after --otp"); - } - otp = value; - index += 1; - continue; - } - if (arg === "--help" || arg === "-h") { - return { help: true, selector: null, publish: false, skipBuild: false, otp: null }; + return { help: true, packageName: null, publish: false }; } if (arg.startsWith("--")) { throw new Error(`unknown option: ${arg}`); } - if (selector) { - throw new Error("expected exactly one package selector"); + if (packageName) { + throw new Error("expected exactly one package name"); } - selector = arg; + packageName = arg; } return { help: false, - selector, + packageName, publish: flags.has("--publish"), - skipBuild: flags.has("--skip-build"), - otp, }; } -function runCommand(command, args, options = {}) { - const result = spawnSync(command, args, { - cwd: repoRoot, +export function validatePackageName(packageName) { + if (!SCOPE_RE.test(packageName)) { + throw new Error( + `refusing to publish a placeholder for ${JSON.stringify(packageName)}: ` + + "the name must be a lowercase package inside the @paperclipai scope " + + "(this guard prevents accidental publishes to names we do not own).", + ); + } +} + +export function buildPlaceholderFiles(packageName) { + const deprecationNote = + `${packageName}@${PLACEHOLDER_VERSION} is a placeholder that reserves the package name ` + + "for Paperclip's release pipeline. It contains no functionality; the first real release " + + "supersedes it. See https://github.com/paperclipai/paperclip"; + + const packageJson = { + name: packageName, + version: PLACEHOLDER_VERSION, + description: + "Placeholder publish reserving this name for Paperclip's release pipeline. Do not install this version.", + license: "MIT", + main: "index.js", + files: ["index.js"], + repository: { + type: "git", + url: "git+https://github.com/paperclipai/paperclip.git", + }, + homepage: "https://github.com/paperclipai/paperclip", + publishConfig: { + access: "public", + }, + }; + + const indexJs = `throw new Error(${JSON.stringify(deprecationNote)});\n`; + + const readme = [ + `# ${packageName}`, + "", + `Version ${PLACEHOLDER_VERSION} is a **placeholder publish**. It reserves this package name so`, + "Paperclip's release-bootstrap CI gate can pass before the package's first real", + "release ships from CI. It intentionally contains no functionality.", + "", + "Real versions are published by the release workflow of", + "[paperclipai/paperclip](https://github.com/paperclipai/paperclip).", + "", + ].join("\n"); + + return { + "package.json": `${JSON.stringify(packageJson, null, 2)}\n`, + "index.js": indexJs, + "README.md": readme, + deprecationNote, + }; +} + +function runNpm(args, options = {}) { + const result = spawnSync("npm", args, { encoding: "utf8", - stdio: ["inherit", "pipe", "pipe"], + stdio: ["ignore", "pipe", "pipe"], ...options, }); @@ -89,39 +168,22 @@ function runCommand(command, args, options = {}) { throw result.error; } + const stdout = result.stdout ?? ""; + const stderr = result.stderr ?? ""; + if (stdout) process.stdout.write(stdout); + if (stderr) process.stderr.write(stderr); + return result; } -function runChecked(command, args, options = {}) { - const result = runCommand(command, args, options); - const stdout = result.stdout ?? ""; - const stderr = result.stderr ?? ""; - - if (stdout) process.stdout.write(stdout); - if (stderr) process.stderr.write(stderr); - - if (result.status !== 0) { - throw new Error(`${command} ${args.join(" ")} failed with status ${result.status ?? "unknown"}`); - } -} - -function formatCommand(command, args) { - return `${command} ${args.join(" ")}`; -} - -function ensureNpmAuth() { - const result = runCommand("npm", ["whoami"]); - const stdout = result.stdout ?? ""; - const stderr = result.stderr ?? ""; - - if (stdout) process.stdout.write(stdout); - if (stderr) process.stderr.write(stderr); +export function ensureNpmAuth() { + const result = runNpm(["whoami"]); if (result.status === 0) { return; } - const output = `${stdout}\n${stderr}`.trim(); + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim(); if (/\bE401\b|401 Unauthorized/i.test(output)) { throw new Error( [ @@ -136,8 +198,18 @@ function ensureNpmAuth() { throw new Error("npm whoami failed"); } -function inspectNpmPackage(packageName) { - const result = runCommand("npm", ["view", packageName, "version", "--json"]); +export function inspectNpmPackage(packageName) { + // Deliberately quiet: for a fresh bootstrap the expected outcome is E404 + // ("the name is free"), and npm's error dump for that reads like a failure. + // Output is only surfaced when the query fails for an unexpected reason. + const result = spawnSync("npm", ["view", packageName, "version", "--json"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + + if (result.error) { + throw result.error; + } if (result.status === 0) { const version = JSON.parse((result.stdout ?? "").trim()); @@ -149,158 +221,228 @@ function inspectNpmPackage(packageName) { return { exists: false }; } - process.stderr.write(output ? `${output}\n` : ""); + if (output) process.stderr.write(`${output}\n`); throw new Error(`failed to query npm for ${packageName}`); } -function resolveTargetPackage(selector, packages = buildReleasePackagePlan()) { - const normalizedSelector = normalizePath(selector); - const matches = packages.filter( - (pkg) => pkg.name === selector || normalizePath(pkg.dir) === normalizedSelector, - ); - - if (matches.length === 1) { - return matches[0]; +export async function promptOtp(rl, purpose) { + for (;;) { + const answer = (await rl.question(`Enter the npm one-time password to ${purpose}: `)).trim(); + if (answer) return answer; + process.stdout.write("A one-time password is required.\n"); } - - if (matches.length > 1) { - throw new Error(`package selector is ambiguous: ${selector}`); - } - - throw new Error( - `unknown package selector: ${selector}\nKnown packages:\n- ${packages.map((pkg) => `${pkg.name} (${pkg.dir})`).join("\n- ")}`, - ); } -function printNextSteps(pkg) { +export async function waitForPackageVisible( + packageName, + { + attempts = REGISTRY_POLL_ATTEMPTS, + intervalMs = REGISTRY_POLL_INTERVAL_MS, + consecutive = REGISTRY_POLL_CONSECUTIVE, + inspect = inspectNpmPackage, + sleep = delay, + } = {}, +) { + let seen = 0; + for (let attempt = 0; attempt < attempts; attempt += 1) { + if (attempt > 0) await sleep(intervalMs); + + let state = null; + try { + state = inspect(packageName); + } catch { + state = null; // transient registry error: keep polling + } + + if (state?.exists) { + seen += 1; + if (seen >= consecutive) return true; + } else { + seen = 0; + } + } + return false; +} + +async function publishPlaceholder(packageName, stageDir, rl) { + for (let attempt = 1; attempt <= OTP_ATTEMPTS; attempt += 1) { + const otp = await promptOtp(rl, `publish ${packageName}@${PLACEHOLDER_VERSION}`); + // Hand the code to npm through its environment (npm_config_otp), not argv, + // so it never appears in a process listing. + const result = runNpm(["publish", "--access", "public"], { + cwd: stageDir, + env: { ...process.env, npm_config_otp: otp }, + }); + if (result.status === 0) return; + + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim(); + if (/\bEOTP\b|one-time password/i.test(output) && attempt < OTP_ATTEMPTS) { + process.stdout.write("The code was rejected or expired. Try a fresh one.\n"); + continue; + } + throw new Error(`npm publish failed with status ${result.status ?? "unknown"}`); + } + throw new Error("npm publish failed: too many rejected one-time passwords"); +} + +async function deprecatePlaceholder(packageName, deprecationNote, rl) { + const spec = `${packageName}@${PLACEHOLDER_VERSION}`; + for (let attempt = 1; attempt <= OTP_ATTEMPTS; attempt += 1) { + const otp = await promptOtp(rl, `deprecate ${spec}`); + const result = runNpm(["deprecate", spec, deprecationNote], { + env: { ...process.env, npm_config_otp: otp }, + }); + if (result.status === 0) return true; + + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim(); + if (/\bEOTP\b|one-time password/i.test(output)) { + process.stdout.write("The code was rejected or expired. Try a fresh one.\n"); + continue; + } + if (/\bE404\b|404 Not Found/i.test(output)) { + process.stdout.write( + "The registry's write endpoint has not caught up yet; waiting 30s before retrying...\n", + ); + await delay(30_000); + continue; + } + break; + } + return false; +} + +function printManualDeprecateFallback(packageName, deprecationNote) { process.stdout.write( [ "", - "Publish succeeded. Next:", - `1. Open https://www.npmjs.com/package/${pkg.name}`, - "2. Go to Settings -> Trusted publishing", - "3. Add repository paperclipai/paperclip", - "4. Set workflow filename to release.yml", - "5. Optionally enable Settings -> Publishing access -> Require two-factor authentication and disallow tokens", + "The placeholder could not be deprecated automatically. Once `npm view` resolves the package, run:", + `npm deprecate ${packageName}@${PLACEHOLDER_VERSION} ${JSON.stringify(deprecationNote)} --otp `, "", ].join("\n"), ); } -function buildPublishArgs(pkg, { dryRun = false, otp = null } = {}) { - const args = ["publish", pkg.dir, "--no-git-checks", "--access", "public"]; - - if (dryRun) { - args.push("--dry-run"); - } - - if (otp) { - args.push("--otp", otp); - } - - return args; +function printNextSteps(packageName) { + process.stdout.write( + [ + "", + "Next:", + `1. Open https://www.npmjs.com/package/${packageName}`, + "2. Go to Settings -> Trusted publishing", + "3. Add repository paperclipai/paperclip", + "4. Set workflow filename to release.yml", + "5. Optionally enable Settings -> Publishing access -> Require two-factor authentication and disallow tokens", + `6. Only then flip the package to "publishFromCi": true in scripts/release-package-manifest.json`, + "", + ].join("\n"), + ); } -function publishPackage(pkg, otp) { - const publishArgs = buildPublishArgs(pkg, { otp }); +async function stageAndPublish(packageName, { publish }) { + const files = buildPlaceholderFiles(packageName); + const stageDir = mkdtempSync(join(tmpdir(), "paperclip-npm-placeholder-")); - const result = runCommand("pnpm", publishArgs); - const stdout = result.stdout ?? ""; - const stderr = result.stderr ?? ""; - const output = `${stdout}\n${stderr}`.trim(); + try { + for (const fileName of ["package.json", "index.js", "README.md"]) { + writeFileSync(join(stageDir, fileName), files[fileName]); + } - if (stdout) process.stdout.write(stdout); - if (stderr) process.stderr.write(stderr); + process.stdout.write(`Staged placeholder for ${packageName} in ${stageDir}\n`); + process.stdout.write(`Previewing publish payload (npm publish --dry-run)...\n`); + const dryRun = runNpm(["publish", "--dry-run", "--access", "public"], { cwd: stageDir }); + if (dryRun.status !== 0) { + throw new Error(`npm publish --dry-run failed with status ${dryRun.status ?? "unknown"}`); + } - if (result.status === 0) { - return; + if (!publish) { + process.stdout.write( + [ + "", + "Dry run complete. To publish the placeholder from an authenticated maintainer machine, run:", + `node scripts/bootstrap-npm-package.mjs ${packageName} --publish`, + "", + ].join("\n"), + ); + return; + } + + const rl = createInterface({ input: process.stdin, output: process.stdout }); + try { + await publishPlaceholder(packageName, stageDir, rl); + process.stdout.write(`Placeholder ${packageName}@${PLACEHOLDER_VERSION} published.\n`); + + process.stdout.write( + "Waiting for the registry to show the package before deprecating (a first publish can take a few minutes)...\n", + ); + const visible = await waitForPackageVisible(packageName); + + let deprecated = false; + if (visible) { + deprecated = await deprecatePlaceholder(packageName, files.deprecationNote, rl); + } else { + process.stdout.write("Timed out waiting for the registry to show the package.\n"); + } + + if (deprecated) { + process.stdout.write(`Deprecated ${packageName}@${PLACEHOLDER_VERSION}.\n`); + } else { + printManualDeprecateFallback(packageName, files.deprecationNote); + } + + printNextSteps(packageName); + } finally { + rl.close(); + } + } finally { + rmSync(stageDir, { recursive: true, force: true }); } - - if (/\bEOTP\b|one-time password/i.test(output)) { - throw new Error( - [ - "npm publish reached the publish-time 2FA check.", - "Complete the browser auth URL printed by npm and rerun the helper, or rerun with `--otp ` if your npm account uses authenticator-app codes.", - ].join(" "), - ); - } - - throw new Error(`${formatCommand("pnpm", publishArgs)} failed with status ${result.status ?? "unknown"}`); } -function main(argv) { - const { help, selector, publish, skipBuild, otp } = parseArgs(argv); +async function main(argv) { + const { help, packageName, publish } = parseArgs(argv); if (help) { usage(); return; } - if (!selector) { + if (!packageName) { usage(); - throw new Error("missing package selector"); + throw new Error("missing package name"); } - const pkg = resolveTargetPackage(selector); - process.stdout.write(`Selected ${pkg.name} (${pkg.dir})\n`); + validatePackageName(packageName); - if (publish && !otp) { - throw new Error("`--publish` requires `--otp `. Generate a fresh npm one-time password and rerun."); + if (publish && !process.stdin.isTTY) { + throw new Error( + "--publish needs an interactive terminal: the helper prompts for npm one-time passwords instead of taking them as arguments.", + ); } - const npmState = inspectNpmPackage(pkg.name); + const npmState = inspectNpmPackage(packageName); if (npmState.exists) { - throw new Error(`${pkg.name} already exists on npm at version ${npmState.version}; bootstrap is only for first publish`); + throw new Error( + `${packageName} already exists on npm at version ${npmState.version}; the bootstrap flow is only for names that have never been published`, + ); } - process.stdout.write(`${pkg.name} is not on npm yet; continuing with bootstrap flow.\n`); + process.stdout.write(`${packageName} is not on npm yet; continuing with placeholder bootstrap.\n`); if (publish) { process.stdout.write("Checking npm auth with npm whoami...\n"); ensureNpmAuth(); } - if (!skipBuild && typeof pkg.pkg?.scripts?.build === "string") { - process.stdout.write(`Building ${pkg.name}...\n`); - runChecked("pnpm", ["--filter", pkg.name, "build"]); - } - - process.stdout.write(`Previewing publish payload for ${pkg.name}...\n`); - runChecked("pnpm", buildPublishArgs(pkg, { dryRun: true })); - - if (!publish) { - process.stdout.write( - [ - "", - "Dry run complete. To perform the first publish from an authenticated maintainer machine, run:", - `node scripts/bootstrap-npm-package.mjs ${pkg.name} --publish --otp `, - "", - ].join("\n"), - ); - return; - } - - process.stdout.write(`Publishing ${pkg.name}...\n`); - publishPackage(pkg, otp); - printNextSteps(pkg); + await stageAndPublish(packageName, { publish }); } const isDirectRun = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url); if (isDirectRun) { try { - main(process.argv.slice(2)); + await main(process.argv.slice(2)); } catch (error) { process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); process.exit(1); } } - -export { - buildPublishArgs, - ensureNpmAuth, - inspectNpmPackage, - parseArgs, - publishPackage, - resolveTargetPackage, -}; diff --git a/scripts/bootstrap-npm-package.test.mjs b/scripts/bootstrap-npm-package.test.mjs index 2780eb5a0f..b8541a04f3 100644 --- a/scripts/bootstrap-npm-package.test.mjs +++ b/scripts/bootstrap-npm-package.test.mjs @@ -1,87 +1,129 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { buildPublishArgs, parseArgs, resolveTargetPackage } from "./bootstrap-npm-package.mjs"; +import { + PLACEHOLDER_VERSION, + buildPlaceholderFiles, + parseArgs, + promptOtp, + validatePackageName, + waitForPackageVisible, +} from "./bootstrap-npm-package.mjs"; -test("parseArgs recognizes publish and skip-build flags", () => { - assert.deepEqual(parseArgs(["@paperclipai/plugin-workspace-diff", "--publish", "--skip-build"]), { +test("parseArgs recognizes the publish flag", () => { + assert.deepEqual(parseArgs(["@paperclipai/adapter-kimi-local", "--publish"]), { help: false, - selector: "@paperclipai/plugin-workspace-diff", + packageName: "@paperclipai/adapter-kimi-local", publish: true, - skipBuild: true, - otp: null, }); }); -test("parseArgs accepts an explicit otp value", () => { - assert.deepEqual(parseArgs(["packages/plugins/plugin-workspace-diff", "--publish", "--otp", "123456"]), { +test("parseArgs defaults to a dry run", () => { + assert.deepEqual(parseArgs(["@paperclipai/adapter-kimi-local"]), { help: false, - selector: "packages/plugins/plugin-workspace-diff", - publish: true, - skipBuild: false, - otp: "123456", - }); -}); - -test("parseArgs leaves otp null when omitted", () => { - assert.deepEqual(parseArgs(["packages/plugins/plugin-workspace-diff", "--publish"]), { - help: false, - selector: "packages/plugins/plugin-workspace-diff", - publish: true, - skipBuild: false, - otp: null, - }); -}); - -test("parseArgs returns help mode", () => { - assert.deepEqual(parseArgs(["--help"]), { - help: true, - selector: null, + packageName: "@paperclipai/adapter-kimi-local", publish: false, - skipBuild: false, - otp: null, }); }); -test("resolveTargetPackage matches by package name or dir", () => { - const packages = [ - { dir: "packages/a", name: "@paperclipai/a", pkg: {} }, - { dir: "packages/b", name: "@paperclipai/b", pkg: {} }, - ]; - - assert.equal(resolveTargetPackage("@paperclipai/a", packages).dir, "packages/a"); - assert.equal(resolveTargetPackage("./packages/b", packages).name, "@paperclipai/b"); +test("parseArgs rejects a second package name", () => { + assert.throws(() => parseArgs(["@paperclipai/a", "@paperclipai/b"]), /exactly one package name/); }); -test("resolveTargetPackage includes the workspace diff plugin bootstrap package", () => { - const pkg = resolveTargetPackage("@paperclipai/plugin-workspace-diff"); - - assert.equal(pkg.dir, "packages/plugins/plugin-workspace-diff"); +test("parseArgs rejects unknown options", () => { + assert.throws(() => parseArgs(["@paperclipai/a", "--skip-build"]), /unknown option/); + assert.throws(() => parseArgs(["@paperclipai/a", "--otp", "123456"]), /unknown option/); }); -test("buildPublishArgs publishes from the repo root through pnpm", () => { - const pkg = { dir: "packages/adapters/hermes", name: "@paperclipai/hermes-paperclip-adapter" }; - - assert.deepEqual(buildPublishArgs(pkg), [ - "publish", - "packages/adapters/hermes", - "--no-git-checks", - "--access", - "public", - ]); +test("validatePackageName accepts @paperclipai scoped names", () => { + validatePackageName("@paperclipai/adapter-kimi-local"); + validatePackageName("@paperclipai/plugin-workspace-diff"); }); -test("buildPublishArgs includes dry-run and otp flags when requested", () => { - const pkg = { dir: "packages/adapters/hermes", name: "@paperclipai/hermes-paperclip-adapter" }; - - assert.deepEqual(buildPublishArgs(pkg, { dryRun: true, otp: "123456" }), [ - "publish", - "packages/adapters/hermes", - "--no-git-checks", - "--access", - "public", - "--dry-run", - "--otp", - "123456", - ]); +test("validatePackageName rejects names outside the @paperclipai scope", () => { + assert.throws(() => validatePackageName("left-pad"), /@paperclipai scope/); + assert.throws(() => validatePackageName("@evil/adapter-kimi-local"), /@paperclipai scope/); + assert.throws(() => validatePackageName("@paperclipai/UPPER"), /@paperclipai scope/); +}); + +test("buildPlaceholderFiles produces a publishable manifest at the placeholder version", () => { + const files = buildPlaceholderFiles("@paperclipai/adapter-kimi-local"); + const manifest = JSON.parse(files["package.json"]); + + assert.equal(manifest.name, "@paperclipai/adapter-kimi-local"); + assert.equal(manifest.version, PLACEHOLDER_VERSION); + assert.equal(manifest.publishConfig.access, "public"); + assert.deepEqual(manifest.files, ["index.js"]); + assert.match(manifest.description, /[Pp]laceholder/); +}); + +test("buildPlaceholderFiles entry point throws with a pointer to the repo", () => { + const files = buildPlaceholderFiles("@paperclipai/adapter-kimi-local"); + + assert.match(files["index.js"], /^throw new Error\(/); + assert.match(files["index.js"], /placeholder/); + assert.match(files["index.js"], /github\.com\/paperclipai\/paperclip/); + // The entry point must be valid JS: evaluating it should throw our message, + // not a SyntaxError. + assert.throws(() => new Function(files["index.js"])(), /placeholder that reserves/); +}); + +test("buildPlaceholderFiles README explains the placeholder", () => { + const files = buildPlaceholderFiles("@paperclipai/adapter-kimi-local"); + assert.match(files["README.md"], /placeholder publish/); + assert.match(files["README.md"], /release-bootstrap CI gate/); +}); + +test("promptOtp re-prompts until a non-empty code is entered", async () => { + const answers = ["", " ", " 123456 "]; + const rl = { question: async () => answers.shift() ?? "" }; + + assert.equal(await promptOtp(rl, "publish"), "123456"); + assert.equal(answers.length, 0); +}); + +test("waitForPackageVisible requires consecutive sightings before reporting success", async () => { + const states = [{ exists: false }, { exists: true }, { exists: false }, { exists: true }, { exists: true }]; + let sleeps = 0; + + const visible = await waitForPackageVisible("@paperclipai/x", { + attempts: 10, + consecutive: 2, + inspect: () => states.shift() ?? { exists: true }, + sleep: async () => { + sleeps += 1; + }, + }); + + assert.equal(visible, true); + // Five inspections happened; sleeps run between attempts, not before the first. + assert.equal(sleeps, 4); +}); + +test("waitForPackageVisible times out when the package never appears", async () => { + const visible = await waitForPackageVisible("@paperclipai/x", { + attempts: 3, + consecutive: 2, + inspect: () => ({ exists: false }), + sleep: async () => {}, + }); + + assert.equal(visible, false); +}); + +test("waitForPackageVisible treats registry errors as misses and keeps polling", async () => { + let calls = 0; + const visible = await waitForPackageVisible("@paperclipai/x", { + attempts: 6, + consecutive: 2, + inspect: () => { + calls += 1; + if (calls <= 2) throw new Error("transient registry error"); + return { exists: true }; + }, + sleep: async () => {}, + }); + + assert.equal(visible, true); + assert.equal(calls, 4); });