diff --git a/.github/scripts/tests/pr-dependency-cache.test.mjs b/.github/scripts/tests/pr-dependency-cache.test.mjs new file mode 100644 index 0000000000..c4803e0108 --- /dev/null +++ b/.github/scripts/tests/pr-dependency-cache.test.mjs @@ -0,0 +1,39 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +const workflow = readFileSync(new URL("../../workflows/pr-trusted.yml", import.meta.url), "utf8"); +const jobs = [...workflow.matchAll(/^ ([a-z_][a-z_0-9]*):\n([\s\S]*?)(?=^ [a-z_][a-z_0-9]*:\n|$(?![\s\S]))/gm)]; +const installers = jobs.filter(([, , body]) => body.includes("run: pnpm install --frozen-lockfile")); + +test("PR workflows restore dependency stores without creating branch copies", () => { + assert.equal(installers.length, 7); + assert.doesNotMatch(workflow, /^ +cache: pnpm$/m); + assert.doesNotMatch(workflow, /uses: actions\/cache(?:@|\/save@)/); + for (const [, job, body] of jobs) { + for (const step of body.split(" - name:").filter((step) => step.includes("uses: actions/setup-node@"))) { + assert.match(step, /package-manager-cache: false/, job); + } + } + const policy = jobs.find(([, name]) => name === "policy")[2]; + assert.doesNotMatch(policy, /uses: actions\/cache|cache: pnpm/); +}); + +for (const [, job, body] of installers) { + test(`${job}: reuse master keys before restoring the resolved PR lockfile`, () => { + const locate = body.indexOf(" - name: Locate pnpm store"); + const restore = body.indexOf(" - name: Restore pnpm store (read only)"); + const artifact = body.indexOf(" - name: Restore regenerated PR lockfile"); + const install = body.indexOf("run: pnpm install --frozen-lockfile"); + assert.ok(locate >= 0 && locate < restore && restore < artifact && artifact < install); + const cache = body.slice(restore, artifact); + assert.match(body.slice(locate, restore), /pnpm store path --silent/); + assert.match(body.slice(locate, restore), /node -p 'process.arch'/); + assert.match(cache, /uses: actions\/cache\/restore@[a-f0-9]{40}/); + assert.ok(cache.includes("key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}")); + assert.ok(cache.includes("restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-")); + assert.match(body.slice(artifact, install), /if: needs.policy.outputs.lockfile_regenerated == '1'/); + assert.match(body.slice(artifact, install), /name: pr-lockfile/); + assert.doesNotMatch(body.slice(artifact, install), /continue-on-error/); + }); +} diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 93815b1296..f8b68cac61 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -284,6 +284,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 + package-manager-cache: false - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 @@ -295,12 +296,6 @@ jobs: version: 9.15.4 run_install: false - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - cache: pnpm - - name: Validate migration ordering against target branch run: >- node .github/scripts/check-pr-migration-order.mjs @@ -389,6 +384,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 + package-manager-cache: false - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 @@ -399,6 +395,21 @@ jobs: with: version: 9.15.4 + # Share the checked-in lockfile key with master. PR merge refs must not + # save full copies of the store or evict the post-merge build caches. + - name: Locate pnpm store + id: pnpm_store + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" + + - name: Restore pnpm store (read only) + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm_store.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- + - name: Restore regenerated PR lockfile (if policy uploaded one) if: needs.policy.outputs.lockfile_regenerated == '1' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -406,12 +417,6 @@ jobs: name: pr-lockfile path: . - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - cache: pnpm - - name: Install dependencies run: pnpm install --frozen-lockfile @@ -485,6 +490,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 + package-manager-cache: false - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 @@ -495,6 +501,21 @@ jobs: with: version: 9.15.4 + # Share the checked-in lockfile key with master. PR merge refs must not + # save full copies of the store or evict the post-merge build caches. + - name: Locate pnpm store + id: pnpm_store + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" + + - name: Restore pnpm store (read only) + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm_store.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- + - name: Restore regenerated PR lockfile (if policy uploaded one) if: needs.policy.outputs.lockfile_regenerated == '1' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -502,12 +523,6 @@ jobs: name: pr-lockfile path: . - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - cache: pnpm - - name: Install dependencies run: pnpm install --frozen-lockfile @@ -607,6 +622,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 + package-manager-cache: false - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 @@ -617,6 +633,21 @@ jobs: with: version: 9.15.4 + # Share the checked-in lockfile key with master. PR merge refs must not + # save full copies of the store or evict the post-merge build caches. + - name: Locate pnpm store + id: pnpm_store + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" + + - name: Restore pnpm store (read only) + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm_store.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- + - name: Restore regenerated PR lockfile (if policy uploaded one) if: needs.policy.outputs.lockfile_regenerated == '1' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -624,12 +655,6 @@ jobs: name: pr-lockfile path: . - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - cache: pnpm - - name: Install dependencies run: pnpm install --frozen-lockfile @@ -653,6 +678,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 + package-manager-cache: false - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 @@ -663,6 +689,21 @@ jobs: with: version: 9.15.4 + # Share the checked-in lockfile key with master. PR merge refs must not + # save full copies of the store or evict the post-merge build caches. + - name: Locate pnpm store + id: pnpm_store + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" + + - name: Restore pnpm store (read only) + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm_store.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- + - name: Restore regenerated PR lockfile (if policy uploaded one) if: needs.policy.outputs.lockfile_regenerated == '1' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -670,12 +711,6 @@ jobs: name: pr-lockfile path: . - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - cache: pnpm - - name: Install dependencies run: pnpm install --frozen-lockfile @@ -727,6 +762,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 + package-manager-cache: false - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 @@ -737,6 +773,21 @@ jobs: with: version: 9.15.4 + # Share the checked-in lockfile key with master. PR merge refs must not + # save full copies of the store or evict the post-merge build caches. + - name: Locate pnpm store + id: pnpm_store + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" + + - name: Restore pnpm store (read only) + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm_store.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- + - name: Restore regenerated PR lockfile (if policy uploaded one) if: needs.policy.outputs.lockfile_regenerated == '1' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -744,12 +795,6 @@ jobs: name: pr-lockfile path: . - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - cache: pnpm - - name: Install dependencies run: pnpm install --frozen-lockfile @@ -773,6 +818,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 + package-manager-cache: false - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 @@ -783,6 +829,21 @@ jobs: with: version: 9.15.4 + # Share the checked-in lockfile key with master. PR merge refs must not + # save full copies of the store or evict the post-merge build caches. + - name: Locate pnpm store + id: pnpm_store + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" + + - name: Restore pnpm store (read only) + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm_store.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- + - name: Restore regenerated PR lockfile (if policy uploaded one) if: needs.policy.outputs.lockfile_regenerated == '1' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -790,12 +851,6 @@ jobs: name: pr-lockfile path: . - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - cache: pnpm - - name: Install dependencies run: pnpm install --frozen-lockfile @@ -860,6 +915,7 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 + package-manager-cache: false - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 @@ -870,6 +926,21 @@ jobs: with: version: 9.15.4 + # Share the checked-in lockfile key with master. PR merge refs must not + # save full copies of the store or evict the post-merge build caches. + - name: Locate pnpm store + id: pnpm_store + run: | + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + echo "arch=$(node -p 'process.arch')" >> "$GITHUB_OUTPUT" + + - name: Restore pnpm store (read only) + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm_store.outputs.path }} + key: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: node-cache-${{ runner.os }}-${{ steps.pnpm_store.outputs.arch }}-pnpm- + - name: Restore regenerated PR lockfile (if policy uploaded one) if: needs.policy.outputs.lockfile_regenerated == '1' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -877,12 +948,6 @@ jobs: name: pr-lockfile path: . - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - cache: pnpm - - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md index 9bb0bee1d4..b687197182 100644 --- a/doc/cloud-build-readiness.md +++ b/doc/cloud-build-readiness.md @@ -198,7 +198,37 @@ all typechecks still execute. A missing or invalidated cache triggers compilatio The Refresh Lockfile workflow does not cache the pnpm store. Its resolution-only command does not download packages and can save an empty default-branch cache -before full install jobs finish. Jobs that install dependencies retain caching. +before full install jobs finish. The PR policy job also leaves store caching off. + +PR install jobs restore the pnpm store without saving it. They hash the checked-in +lockfile before downloading the policy job's regenerated lockfile, matching the +key format used by master install jobs. A same-OS, same-architecture pnpm fallback +can reuse older package downloads when the exact key is absent. Each job still +installs with `--frozen-lockfile` against the policy artifact when one exists; +cache contents do not select dependency versions. A cache miss downloads packages +normally. New PR-only dependencies may be downloaded again on each PR run until +master populates a cache that contains them. + +This avoids storing a full dependency archive under every PR merge ref. Those +copies competed with the Rust caches for the repository's storage limit. Keep +master cache writes enabled so trusted post-merge installs refresh shared stores. +After activating the new trusted workflow pin, verify cache restores and package +reuse in an allowlisted PR, and verify that no new `node-cache-` entries appear +under its `refs/pull//merge` ref. Existing copies can expire normally. + +The repository cache storage ceiling is managed in GitHub Settings, separately +from this workflow. Check it with: + +```sh +gh api repos/paperclipai/paperclip/actions/cache/storage-limit +``` + +Increasing the repository limit above 10 GB can require an organization owner to +raise the maximum in organization Settings → Actions → General first. Repository +administration access alone cannot override that maximum. Paid cache storage also +requires a payment method and sufficient Actions Cache Storage budget; see the +[GitHub cache storage documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#increasing-cache-size). +Preserve populated master pnpm and Rust caches when inspecting pressure. After deploying this correction, remove any existing empty default-branch entry for the current lockfile key. List cache IDs, branches, and archive sizes first: diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs index d860e0353e..2fa156fc2f 100644 --- a/scripts/__tests__/e2e-shard.test.mjs +++ b/scripts/__tests__/e2e-shard.test.mjs @@ -316,10 +316,11 @@ test("the trusted PR workflow regenerates stale stacked lockfiles", () => { /policy:\n needs: \[gate\][\s\S]{0,160}timeout-minutes: 10/, "the unconditional resolution step needs the same timeout headroom as the lockfile refresh workflow", ); - assert.match( - workflow, - /- name: Setup Node\.js\n uses: actions\/setup-node@[0-9a-f]+[^\n]*\n with:\n node-version: 24\n cache: pnpm/, - "the policy job must restore the pnpm cache before dependency resolution", + const policy = workflow.split(" policy:\n")[1].split(" typecheck_release_registry:\n")[0]; + assert.doesNotMatch( + policy, + /cache: pnpm|uses: actions\/cache/, + "resolution-only policy must not restore or save a dependency store", ); assert.match( workflow,