From 916c13501f80c7d1d659b89e4a06dc47366aa80f Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 30 Jul 2026 11:37:00 -0700 Subject: [PATCH] Replace host-to-host Cloud Sync with full-fidelity company Import/Export (#10507) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A company accumulates real state — issues, labels, blockers, documents, work products, monitors, attachments, agents, routines — and people need to move that state between instances: self-hosted to cloud, cloud back to self-hosted, or plain backups > - The experimental, flag-gated Cloud Sync transport (#6548) tried to solve this host-to-host: the source pushed into a receiver over HTTPS with a cross-instance consent/token handshake, which required the destination to be publicly reachable and broke for common self-hosted topologies (plain-HTTP LAN/VPN origins); the receiver half never landed upstream at all > - Meanwhile the portability bundle and the existing export/import pages already move companies offline with none of those networking constraints — but silently dropped labels, blockers, issue documents, work products, monitors, and every attachment > - This pull request removes the host-to-host transport and makes Import/Export the single data-movement path: the pages become first-class company-settings destinations, exports declare exactly what they do not carry, and bundle schemaVersion 6 now carries all of the above, with attachments as content-addressed sha256 blobs verified before a single row is written > - The benefit is a migration and backup flow that works between any two instances with no reachability requirements, no cross-instance auth, and no silent data loss ## Linked Issues or Issue Description - Refs #6548 — the original Cloud Sync sender this PR supersedes and removes. - Related, not duplicates: #1697 (goals in the portability manifest — orthogonal field addition), #954 (an earlier import/export + skill-visibility proposal predating the current portability bundle). - No open issue describes this directly, so in brief (feature-request shape): **Problem** — moving a company between instances silently lost labels (imports with label references actually hard-failed), blocker relations, issue documents, work products, monitor state, and all attachments, and the alternative Cloud Sync transport required the destination to be publicly reachable over HTTPS plus a consent handshake, which failed for typical self-hosted setups. **Desired behavior** — one Import/Export flow in company settings that produces a portable bundle carrying all of that data, tells the operator up front what it cannot carry, imports with automations paused, and offers real one-click activation afterwards. ## What Changed - New export fidelity report (`GET /api/companies/:companyId/export/fidelity`) + an "Export fidelity" panel on the Export page listing anything a bundle will not include (now only: approvals, cost history, activity history) - Imports accept `pauseAutomations`; imported agents and routines land paused, the import result reports created routines, and the Import page ends in an activation panel that actually resumes selected agents/activates routines - Export and Import pages promoted into the company-settings nav; the Cloud Upstream wizard, ux-lab page, and API client removed; the old settings route redirects to Export - Host-to-host transport removed: upstream-sync/receiver-client routes and services, CLI `cloud connect`/`cloud push` + keypair store, the shared upstream transfer contract, and the `enableCloudSync` flag; migration `0196` drops the two experimental `cloud_upstream_*` sender tables - Bundle schemaVersion 6: labels (definitions + per-task names, remapped by name on import), blocker relations (`blockedBy` slugs, cycle-tolerant), issue documents (`tasks//documents/.md`), work products (system refs nulled), monitors (notes/scheduledBy restored, imported un-armed) - Attachments travel as content-addressed `blobs/` entries (deduped; comment-scoped attachments re-link via comment index); every blob is hash-verified **before any write**, so a corrupted bundle cannot leave a partially imported company; both zip codecs now round-trip extensionless/binary entries byte-exactly; the Import page preflights the inline body limit and offers continue-without-attachments - v5 (and older) bundles still import, with an informational warning; bundles newer than v6 are rejected cleanly - Docs: board-operator import/export guide, CLI README, README/ROADMAP updated ## Verification - `pnpm -r` typechecks (shared, db incl. migration numbering/safety checks, server, ui, cli) and `pnpm check:token-gates` — clean - Vitest: full server + shared sweep 4,888 passed / 1 skipped, with the only 3 failures being pre-existing on `master` (2× heartbeat-workspace-branch-containment, 1× workspace-runtime auto-port; reproduced identically with this change stashed); ui + cli suites green; the embedded-Postgres export-fidelity suite applies the full migration chain including the new `0196` against a fresh database - Live end-to-end on a scratch instance: seeded a company with labels, a blocker pair, an issue document, a work product, a monitor, an agent, a routine, and two binary attachments (one comment-scoped) → export → import into a fresh company → labels remapped to new ids, blocker edge and document restored, monitor un-armed with notes intact, attachments byte-identical (sha256-compared through the API), agents/routines paused → activation panel resumed them; a v5-shaped bundle imported with only the info warning; flipping one byte in a blob made the import 422 with **zero** rows created - Reviewer repro: create a company with a labeled issue + attachment → Settings → Export → download → Settings → Import on another company/instance → watch the preview, apply with "start paused", then activate ## Risks - Migration `0196` drops `cloud_upstream_connections`/`cloud_upstream_runs` — experimental tables behind a default-off flag; their connection/run history is intentionally discarded - Breaking removals are all of experimental, flag-gated surface: `/api/upstream-sync/*` + `/api/cloud-upstreams/*` routes, `paperclipai cloud connect|push`, and the `enableCloudSync` flag (stale keys in stored instance settings parse harmlessly) - Import remains non-atomic on mid-apply errors generally (pre-existing behavior); the new blob verification specifically moved ahead of all writes so tampered bundles cannot create partial state - GitHub-sourced imports do not fetch `blobs/*` and skip attachments with a warning ## Model Used - Claude Fable 5 (`claude-fable-5`, Anthropic), via Claude Code CLI with extended thinking, tool use, and subagent orchestration; implementation and review split across Fable 5 subagents, with live end-to-end verification against a running instance ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- README.md | 2 +- ROADMAP.md | 4 +- cli/README.md | 15 +- cli/src/__tests__/cloud.test.ts | 243 --- cli/src/__tests__/company-import-zip.test.ts | 7 + cli/src/__tests__/company.test.ts | 3 + cli/src/__tests__/helpers/zip.ts | 4 +- cli/src/__tests__/zip-codec.test.ts | 69 + cli/src/commands/client/cloud-store.ts | 177 --- cli/src/commands/client/cloud-transfer.ts | 297 ---- cli/src/commands/client/cloud.ts | 722 --------- cli/src/commands/client/company.ts | 16 +- cli/src/commands/client/zip.ts | 37 +- cli/src/index.ts | 2 - .../board-operator/importing-and-exporting.md | 13 + .../0196_drop_cloud_upstream_tables.sql | 9 + packages/db/src/migrations/meta/_journal.json | 7 + packages/db/src/schema/cloud_upstreams.ts | 75 - packages/db/src/schema/index.ts | 1 - packages/db/src/table-size-estimates.ts | 2 - packages/shared/src/feature-catalog.ts | 8 - packages/shared/src/index.ts | 23 +- .../shared/src/portability-fidelity.test.ts | 76 + packages/shared/src/portability-fidelity.ts | 64 + packages/shared/src/portability-hash.ts | 25 + packages/shared/src/types/cloud-upstream.ts | 110 -- .../shared/src/types/company-portability.ts | 85 ++ packages/shared/src/types/index.ts | 7 + packages/shared/src/types/instance.ts | 1 - .../src/validators/company-portability.ts | 55 + packages/shared/src/validators/instance.ts | 1 - server/src/__tests__/cloud-upstreams.test.ts | 334 ----- ...ompanies-route-cross-company-authz.test.ts | 4 + .../companies-route-path-guard.test.ts | 3 + .../__tests__/company-branding-route.test.ts | 3 + .../company-portability-routes.test.ts | 49 +- .../src/__tests__/company-portability.test.ts | 1313 ++++++++++++++++- server/src/__tests__/export-fidelity.test.ts | 127 ++ .../instance-settings-routes.test.ts | 5 - .../instance-settings-service.test.ts | 2 - server/src/__tests__/openapi-routes.test.ts | 1 - .../server-startup-feedback-export.test.ts | 1 - server/src/index.ts | 14 - server/src/routes/cloud-upstreams.ts | 118 -- server/src/routes/companies.ts | 18 +- server/src/routes/index.ts | 1 - server/src/routes/openapi.ts | 97 +- server/src/services/cloud-upstreams.ts | 1309 ---------------- server/src/services/company-portability.ts | 965 +++++++++++- server/src/services/export-fidelity.ts | 83 ++ server/src/services/index.ts | 2 +- server/src/services/instance-settings.ts | 2 - ui/src/App.tsx | 5 +- ui/src/api/cloudUpstreams.ts | 40 - ui/src/api/companies.ts | 3 + .../CompanySettingsSidebar.test.tsx | 94 +- ui/src/components/CompanySettingsSidebar.tsx | 19 +- ui/src/components/Layout.test.tsx | 3 +- .../access/CompanySettingsNav.test.tsx | 8 +- .../components/access/CompanySettingsNav.tsx | 11 +- ui/src/index.css | 5 +- ui/src/lib/company-export-selection.test.ts | 331 ++++- ui/src/lib/company-export-selection.ts | 260 +++- ui/src/lib/company-routes.test.ts | 3 - ui/src/lib/import-preflight.test.ts | 92 ++ ui/src/lib/import-preflight.ts | 80 + ui/src/lib/queryKeys.ts | 2 +- ui/src/lib/zip.test.ts | 104 +- ui/src/lib/zip.ts | 104 +- ui/src/pages/Agents.test.tsx | 1 - ui/src/pages/CloudUpstream.test.tsx | 413 ------ ui/src/pages/CloudUpstream.tsx | 649 -------- ui/src/pages/CloudUpstreamUxLab.tsx | 822 ----------- ui/src/pages/CompanyExport.test.tsx | 353 +++++ ui/src/pages/CompanyExport.tsx | 248 +++- ui/src/pages/CompanyImport.test.tsx | 321 ++++ ui/src/pages/CompanyImport.tsx | 236 ++- ui/src/pages/CompanySettings.tsx | 24 +- .../InstanceExperimentalSettings.test.tsx | 1 - ui/src/pages/InstanceExperimentalSettings.tsx | 11 - ui/storybook/stories/team-catalog.stories.tsx | 1 + 81 files changed, 5015 insertions(+), 5840 deletions(-) delete mode 100644 cli/src/__tests__/cloud.test.ts create mode 100644 cli/src/__tests__/zip-codec.test.ts delete mode 100644 cli/src/commands/client/cloud-store.ts delete mode 100644 cli/src/commands/client/cloud-transfer.ts delete mode 100644 cli/src/commands/client/cloud.ts create mode 100644 packages/db/src/migrations/0196_drop_cloud_upstream_tables.sql delete mode 100644 packages/db/src/schema/cloud_upstreams.ts create mode 100644 packages/shared/src/portability-fidelity.test.ts create mode 100644 packages/shared/src/portability-fidelity.ts create mode 100644 packages/shared/src/portability-hash.ts delete mode 100644 packages/shared/src/types/cloud-upstream.ts delete mode 100644 server/src/__tests__/cloud-upstreams.test.ts create mode 100644 server/src/__tests__/export-fidelity.test.ts delete mode 100644 server/src/routes/cloud-upstreams.ts delete mode 100644 server/src/services/cloud-upstreams.ts create mode 100644 server/src/services/export-fidelity.ts delete mode 100644 ui/src/api/cloudUpstreams.ts create mode 100644 ui/src/lib/import-preflight.test.ts create mode 100644 ui/src/lib/import-preflight.ts delete mode 100644 ui/src/pages/CloudUpstream.test.tsx delete mode 100644 ui/src/pages/CloudUpstream.tsx delete mode 100644 ui/src/pages/CloudUpstreamUxLab.tsx create mode 100644 ui/src/pages/CompanyExport.test.tsx create mode 100644 ui/src/pages/CompanyImport.test.tsx diff --git a/README.md b/README.md index 2d02971edc..0d94189718 100644 --- a/README.md +++ b/README.md @@ -415,7 +415,7 @@ See [doc/DEVELOPING.md](doc/DEVELOPING.md) for the full development guide. - ⚪ Self-Organization - ⚪ Automatic Organizational Learning - ⚪ CEO Chat -- 🟡 Cloud deployments (multi-tenant isolation & local→cloud sync shipped) +- 🟡 Cloud deployments (multi-tenant isolation & company Import/Export shipped) - ⚪ Desktop App - ⚪ Bring-your-own-ticket-system (Asana / Linear / Jira as on-ramps) - ⚪ Connected Apps (one-click integrations, e.g. Vercel) diff --git a/ROADMAP.md b/ROADMAP.md index fd10044308..fd245a3e88 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -108,11 +108,11 @@ Paperclip should get better at turning completed work into reusable organization We want a lighter-weight way to talk to leadership agents, but those conversations should still resolve to real work objects like plans, issues, approvals, or decisions. This should improve interaction without changing the core task-and-comments model. -### 🟡 Cloud deployments (multi-tenant isolation & local→cloud sync shipped) +### 🟡 Cloud deployments (multi-tenant isolation & company Import/Export shipped) Local-first remains important, but Paperclip also needs a cleaner shared deployment story. Teams should be able to run the same product in hosted or semi-hosted environments without changing the mental model. -Shipped so far: multi-tenant isolation with per-company JWT keys and company-scoped cloud tenants, local→cloud upstream sync, and cloud-managed instance bootstrap. +Shipped so far: multi-tenant isolation with per-company JWT keys and company-scoped cloud tenants, portable company Import/Export (zip bundles that move a company between instances, local or cloud), and cloud-managed instance bootstrap. Next: a blob-store relay so large instances can move without a hand-carried bundle. ### ⚪ Desktop App diff --git a/cli/README.md b/cli/README.md index 97277ddd2c..a32a6c5611 100644 --- a/cli/README.md +++ b/cli/README.md @@ -332,20 +332,17 @@ By default, agents run on scheduled heartbeats and event-based triggers (task as
-## Paperclip Cloud Sync +## Importing & Exporting Companies -Cloud upstream sync is behind the `Cloud Sync` experimental setting. Enable it in Instance Settings before pushing. +Export a company to a portable package and import it into any other instance — local or cloud — from a local path or GitHub: ```bash -paperclipai cloud connect https://your-stack.paperclip.app -paperclipai cloud connect https://your-stack.paperclip.app --no-browser -paperclipai cloud push --company --dry-run -paperclipai cloud push --company +paperclipai company export --out ./my-export +paperclipai company import ./my-export --dry-run +paperclipai company import org/repo --target new ``` -`cloud connect` authorizes the local instance against the target stack and stores the upstream token in the local instance secret store. The default path opens a browser for consent; `--no-browser` uses the device-code flow and prints the verification URL and user code. - -`cloud push --dry-run` exports the selected local company, sends a preview bundle to the connected Cloud stack, and exits with code `2` when conflicts need user resolution. A schema mismatch exits with code `3`. Running without `--dry-run` stages chunks idempotently, applies the run, and prints the final summary and recent progress events. +The board UI has matching Export and Import pages in company settings: the Export page shows a fidelity panel listing what the bundle will not carry, and the Import page starts imported agents and routines paused by default, with a post-import activation step. See the [Importing & Exporting guide](https://github.com/paperclipai/paperclip/blob/master/docs/guides/board-operator/importing-and-exporting.md) for details. ## Development diff --git a/cli/src/__tests__/cloud.test.ts b/cli/src/__tests__/cloud.test.ts deleted file mode 100644 index c66fd33f46..0000000000 --- a/cli/src/__tests__/cloud.test.ts +++ /dev/null @@ -1,243 +0,0 @@ -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { CompanyPortabilityExportResult } from "@paperclipai/shared"; -import { - assertDiscoveryCompatible, - buildBundleFromLocalCompany, - cloudCommandExitCodes, - connectCloud, - resolveDeviceCodeExpiresAt, -} from "../commands/client/cloud.js"; -import { - LocalUpstreamPushCoordinator, - normalizedContentHash, - type LocalUpstreamExportBundle, -} from "../commands/client/cloud-transfer.js"; -import { getCloudConnection } from "../commands/client/cloud-store.js"; - -const originalEnv = { ...process.env }; -const originalFetch = globalThis.fetch; - -describe("cloud CLI helpers", () => { - let tempHome: string; - - beforeEach(() => { - tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-cloud-cli-")); - process.env = { ...originalEnv, PAPERCLIP_HOME: tempHome }; - }); - - afterEach(() => { - process.env = { ...originalEnv }; - globalThis.fetch = originalFetch; - vi.restoreAllMocks(); - fs.rmSync(tempHome, { recursive: true, force: true }); - }); - - it("connects with the device-code flow and stores the resulting cloud connection", async () => { - globalThis.fetch = vi.fn(async (url, init) => { - const requestUrl = String(url); - if (requestUrl.endsWith("/.well-known/paperclip-upstream")) { - return jsonResponse(discovery()); - } - if (requestUrl.endsWith("/api/upstream-sync/device-code")) { - expect(JSON.parse(String(init?.body))).toMatchObject({ - stackId: "stack-1", - scopes: ["upstream_import:preview", "upstream_import:write", "upstream_import:read"], - }); - return jsonResponse({ - deviceCode: "device-1", - userCode: "ABCD-EFGH", - verificationUri: "https://cloud.example.test/api/upstream-sync/device-code/approve", - expiresAt: new Date(Date.now() + 60_000).toISOString(), - intervalSeconds: 0, - }); - } - if (requestUrl.endsWith("/api/upstream-sync/token")) { - return jsonResponse({ - accessToken: "upt_test", - scopes: ["upstream_import:preview"], - token: { - id: "token-1", - companyStackId: "stack-1", - targetOrigin: "https://cloud.example.test", - sourceInstanceId: "paperclip-local-default", - sourceInstanceFingerprint: "sha256:test", - scopes: ["upstream_import:preview"], - expiresAt: new Date(Date.now() + 60_000).toISOString(), - }, - }); - } - return jsonResponse({ error: "not_found" }, 404); - }) as typeof fetch; - - const connection = await connectCloud("https://cloud.example.test", { noBrowser: true, json: true }); - - expect(connection.accessToken).toBe("upt_test"); - expect(getCloudConnection("https://cloud.example.test")?.token.id).toBe("token-1"); - }); - - it("hard-blocks incompatible transfer schema versions with the stable schema exit code", () => { - expect(() => assertDiscoveryCompatible(discovery({ supportedSchemaMajor: 99 }))).toThrow(/schema mismatch/i); - expect(cloudCommandExitCodes.schemaMismatch).toBe(3); - }); - - it("falls back to a bounded device-code expiry when the cloud omits or malforms expiresAt", () => { - const now = Date.UTC(2026, 4, 22, 13, 0, 0); - const validExpiry = "2026-05-22T13:05:00.000Z"; - - expect(resolveDeviceCodeExpiresAt(validExpiry, now)).toBe(Date.parse(validExpiry)); - expect(resolveDeviceCodeExpiresAt(undefined, now)).toBe(now + 15 * 60_000); - expect(resolveDeviceCodeExpiresAt("not-a-date", now)).toBe(now + 15 * 60_000); - }); - - it("builds deterministic chunks with validated payload hashes", async () => { - const bundle = await buildTestBundle(); - - expect(bundle.chunks).toHaveLength(2); - expect(bundle.chunks[0]?.sha256).toBe(normalizedContentHash(bundle.chunks[0]?.payload)); - expect(bundle.manifest.chunks[0]?.manifestHash).toBe(bundle.manifest.manifestHash); - expect(bundle.manifest.idempotencyKey).toBe((await buildTestBundle()).manifest.idempotencyKey); - }); - - it("reuses the same manifest and chunk identity when an interrupted apply is retried", async () => { - const bundle = await buildTestBundle(); - const calls: Array<{ path: string; body: unknown }> = []; - const coordinator = new LocalUpstreamPushCoordinator({ - targetOrigin: "https://cloud.example.test", - paperclipCompanyId: "target-company-1", - fetch: async (url, init) => { - const parsed = new URL(String(url)); - const body = init?.body ? JSON.parse(String(init.body)) as unknown : {}; - calls.push({ path: parsed.pathname, body }); - if (parsed.pathname.endsWith("/runs")) return jsonResponse({ run: { id: "run-1" } }); - return jsonResponse({ run: { id: "run-1" }, summary: { create: 0, update: 0, adopt: 0, skip: 2, conflict: 0, staleMapping: 0 } }); - }, - }); - - await coordinator.apply(bundle); - await coordinator.apply(bundle); - - const runBodies = calls.filter((call) => call.path.endsWith("/runs")).map((call) => call.body as { manifest: { idempotencyKey: string } }); - const chunkBodies = calls.filter((call) => call.path.endsWith("/chunks")).map((call) => call.body as { chunkIndex: number; sha256: string }); - expect(runBodies).toHaveLength(2); - expect(runBodies[0]?.manifest.idempotencyKey).toBe(runBodies[1]?.manifest.idempotencyKey); - expect(chunkBodies[0]).toEqual(chunkBodies[2]); - expect(chunkBodies[1]).toEqual(chunkBodies[3]); - }); -}); - -async function buildTestBundle(): Promise { - return buildBundleFromLocalCompany({ - localCompanyId: "local-company-1", - connection: { - id: "conn-1", - remoteUrl: "https://cloud.example.test", - targetOrigin: "https://cloud.example.test", - targetHost: "cloud.example.test", - stackId: "stack-1", - targetCompanyId: "target-company-1", - accessToken: "upt_test", - token: { - id: "token-1", - companyStackId: "stack-1", - targetOrigin: "https://cloud.example.test", - sourceInstanceId: "paperclip-local-default", - sourceInstanceFingerprint: "sha256:test", - scopes: ["upstream_import:preview"], - expiresAt: new Date(Date.now() + 60_000).toISOString(), - }, - privateKeyPem: "unused", - sourcePublicKey: "unused", - sourceInstanceId: "paperclip-local-default", - sourceInstanceFingerprint: "sha256:test", - scopes: ["upstream_import:preview"], - createdAt: "2026-05-18T00:00:00.000Z", - updatedAt: "2026-05-18T00:00:00.000Z", - }, - discovery: discovery(), - localApi: { - post: async () => portabilityExport() as T, - }, - maxEntitiesPerChunk: 1, - mode: "apply", - }); -} - -function discovery(overrides: Partial<{ supportedSchemaMajor: number }> = {}) { - return { - schema: "paperclip-upstream-discovery-v1", - stack: { - id: "stack-1", - slug: "cloud-test", - displayName: "Cloud Test", - companyId: "target-company-1", - origin: "https://cloud.example.test", - }, - auth: { - deviceCode: { - deviceCodeUrl: "https://cloud.example.test/api/upstream-sync/device-code", - verificationUrl: "https://cloud.example.test/api/upstream-sync/device-code/approve", - tokenUrl: "https://cloud.example.test/api/upstream-sync/token", - }, - scopes: ["upstream_import:preview", "upstream_import:write", "upstream_import:read"], - }, - transfer: { - supportedSchemaMajor: overrides.supportedSchemaMajor ?? 1, - featureFlags: ["cloud_sync"], - }, - }; -} - -function portabilityExport(): CompanyPortabilityExportResult { - return { - rootPath: ".", - paperclipExtensionPath: ".paperclip.yaml", - manifest: { - schemaVersion: 1, - generatedAt: "2026-05-18T00:00:00.000Z", - source: { - companyId: "local-company-1", - companyName: "Local Company", - }, - includes: { - company: true, - agents: true, - projects: true, - issues: true, - skills: true, - }, - company: { - path: "company.json", - name: "Local Company", - description: null, - brandColor: null, - logoPath: null, - attachmentMaxBytes: null, - requireBoardApprovalForNewAgents: false, - feedbackDataSharingEnabled: false, - feedbackDataSharingConsentAt: null, - feedbackDataSharingConsentByUserId: null, - feedbackDataSharingTermsVersion: null, - }, - sidebar: null, - agents: [], - skills: [], - projects: [], - issues: [], - envInputs: [], - }, - files: { - "README.md": "Local Company", - }, - warnings: [], - }; -} - -function jsonResponse(body: unknown, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: { "Content-Type": "application/json" }, - }); -} diff --git a/cli/src/__tests__/company-import-zip.test.ts b/cli/src/__tests__/company-import-zip.test.ts index e2983e9a3a..db01253c22 100644 --- a/cli/src/__tests__/company-import-zip.test.ts +++ b/cli/src/__tests__/company-import-zip.test.ts @@ -18,12 +18,14 @@ describe("resolveInlineSourceFromPath", () => { const tempDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-company-import-zip-")); tempDirs.push(tempDir); + const blobBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x00, 0xff]); const archivePath = path.join(tempDir, "paperclip-demo.zip"); const archive = createStoredZipArchive( { "COMPANY.md": "# Company\n", ".paperclip.yaml": "schema: paperclip/v1\n", "agents/ceo/AGENT.md": "# CEO\n", + "blobs/4f2d1c9a": blobBytes, "notes/todo.txt": "ignore me\n", }, "paperclip-demo", @@ -38,6 +40,11 @@ describe("resolveInlineSourceFromPath", () => { "COMPANY.md": "# Company\n", ".paperclip.yaml": "schema: paperclip/v1\n", "agents/ceo/AGENT.md": "# CEO\n", + "blobs/4f2d1c9a": { + encoding: "base64", + data: Buffer.from(blobBytes).toString("base64"), + contentType: "application/octet-stream", + }, }, }); }); diff --git a/cli/src/__tests__/company.test.ts b/cli/src/__tests__/company.test.ts index 3cac3ec72c..1532047f6f 100644 --- a/cli/src/__tests__/company.test.ts +++ b/cli/src/__tests__/company.test.ts @@ -509,6 +509,9 @@ describe("renderCompanyImportResult", () => { { slug: "ops", id: "project-2", action: "updated", name: "Operations", reason: "replace strategy" }, { slug: "archive", id: null, action: "skipped", name: "Archive", reason: "skip strategy" }, ], + routines: [ + { slug: "weekly-report", id: "routine-1", action: "created", title: "Weekly report", status: "paused" }, + ], envInputs: [], warnings: ["Review API keys"], }, diff --git a/cli/src/__tests__/helpers/zip.ts b/cli/src/__tests__/helpers/zip.ts index ef79b5beda..0bfd26e64b 100644 --- a/cli/src/__tests__/helpers/zip.ts +++ b/cli/src/__tests__/helpers/zip.ts @@ -21,7 +21,7 @@ function crc32(bytes: Uint8Array) { return (crc ^ 0xffffffff) >>> 0; } -export function createStoredZipArchive(files: Record, rootPath: string) { +export function createStoredZipArchive(files: Record, rootPath: string) { const encoder = new TextEncoder(); const localChunks: Uint8Array[] = []; const centralChunks: Uint8Array[] = []; @@ -30,7 +30,7 @@ export function createStoredZipArchive(files: Record, rootPath: for (const [relativePath, content] of Object.entries(files).sort(([left], [right]) => left.localeCompare(right))) { const fileName = encoder.encode(`${rootPath}/${relativePath}`); - const body = encoder.encode(content); + const body = typeof content === "string" ? encoder.encode(content) : content; const checksum = crc32(body); const localHeader = new Uint8Array(30 + fileName.length); diff --git a/cli/src/__tests__/zip-codec.test.ts b/cli/src/__tests__/zip-codec.test.ts new file mode 100644 index 0000000000..03788e8807 --- /dev/null +++ b/cli/src/__tests__/zip-codec.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { bytesToPortableFileEntry, isBlobStorePath, readZipArchive } from "../commands/client/zip.js"; +import { createStoredZipArchive } from "./helpers/zip.js"; + +describe("isBlobStorePath", () => { + it("matches blobs/ entries at the archive root and under a package root", () => { + expect(isBlobStorePath("blobs/4f2d1c9a")).toBe(true); + expect(isBlobStorePath("paperclip-demo/blobs/4f2d1c9a")).toBe(true); + expect(isBlobStorePath("tasks/pap-1/TASK.md")).toBe(false); + expect(isBlobStorePath("blobs/nested/file")).toBe(false); + }); +}); + +describe("bytesToPortableFileEntry", () => { + it("keeps blobs/ entries as base64 octet streams regardless of extension", () => { + const bytes = new Uint8Array([0x00, 0x01, 0x80, 0xfe, 0xff]); + expect(bytesToPortableFileEntry("blobs/4f2d1c9a", bytes)).toEqual({ + encoding: "base64", + data: Buffer.from(bytes).toString("base64"), + contentType: "application/octet-stream", + }); + }); + + it("falls back to base64 when bytes are not valid UTF-8", () => { + const invalidUtf8 = new Uint8Array([0x68, 0x69, 0xff, 0xfe, 0xc0]); + expect(bytesToPortableFileEntry("tasks/pap-1/raw-notes", invalidUtf8)).toEqual({ + encoding: "base64", + data: Buffer.from(invalidUtf8).toString("base64"), + contentType: "application/octet-stream", + }); + }); + + it("decodes valid UTF-8 entries to text", () => { + const bytes = new TextEncoder().encode("# Notes\n\ncafé ✅\n"); + expect(bytesToPortableFileEntry("tasks/pap-1/TASK.md", bytes)).toBe("# Notes\n\ncafé ✅\n"); + }); +}); + +describe("readZipArchive", () => { + it("round-trips blob and invalid UTF-8 entries byte-exactly", async () => { + const blobBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x00, 0xff]); + const invalidUtf8 = new Uint8Array([0x68, 0x69, 0xff, 0xfe, 0xc0]); + const archive = createStoredZipArchive( + { + "COMPANY.md": "# Company\n", + "blobs/4f2d1c9a": blobBytes, + "notes/raw": invalidUtf8, + }, + "paperclip-demo", + ); + + await expect(readZipArchive(archive)).resolves.toEqual({ + rootPath: "paperclip-demo", + files: { + "COMPANY.md": "# Company\n", + "blobs/4f2d1c9a": { + encoding: "base64", + data: Buffer.from(blobBytes).toString("base64"), + contentType: "application/octet-stream", + }, + "notes/raw": { + encoding: "base64", + data: Buffer.from(invalidUtf8).toString("base64"), + contentType: "application/octet-stream", + }, + }, + }); + }); +}); diff --git a/cli/src/commands/client/cloud-store.ts b/cli/src/commands/client/cloud-store.ts deleted file mode 100644 index fa63c7133b..0000000000 --- a/cli/src/commands/client/cloud-store.ts +++ /dev/null @@ -1,177 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import { resolvePaperclipInstanceRoot } from "../../config/home.js"; - -export interface CloudConnectionTokenRecord { - id: string; - companyStackId: string; - targetOrigin: string; - sourceInstanceId: string; - sourceInstanceFingerprint: string; - scopes: string[]; - expiresAt: string; - [key: string]: unknown; -} - -export interface CloudConnection { - id: string; - remoteUrl: string; - targetOrigin: string; - targetHost: string; - stackId: string; - stackSlug?: string | null; - stackDisplayName?: string | null; - targetCompanyId: string; - accessToken: string; - token: CloudConnectionTokenRecord; - privateKeyPem: string; - sourcePublicKey: string; - sourceInstanceId: string; - sourceInstanceFingerprint: string; - scopes: string[]; - createdAt: string; - updatedAt: string; -} - -interface CloudConnectionStore { - version: 1; - connections: Record; - currentConnectionId?: string; -} - -function defaultStore(): CloudConnectionStore { - return { - version: 1, - connections: {}, - }; -} - -export function resolveCloudConnectionStorePath(): string { - return path.resolve(resolvePaperclipInstanceRoot(), "secrets", "cloud-upstream-connections.json"); -} - -export function readCloudConnectionStore(storePath = resolveCloudConnectionStorePath()): CloudConnectionStore { - if (!fs.existsSync(storePath)) return defaultStore(); - const raw = JSON.parse(fs.readFileSync(storePath, "utf8")) as Partial | null; - const connections: Record = {}; - if (raw?.connections && typeof raw.connections === "object") { - for (const [id, value] of Object.entries(raw.connections)) { - const normalized = normalizeConnection(value); - if (normalized) connections[id] = normalized; - } - } - const currentConnectionId = - typeof raw?.currentConnectionId === "string" && connections[raw.currentConnectionId] - ? raw.currentConnectionId - : Object.values(connections).sort((left, right) => right.updatedAt.localeCompare(left.updatedAt))[0]?.id; - return { - version: 1, - connections, - currentConnectionId, - }; -} - -export function writeCloudConnectionStore( - store: CloudConnectionStore, - storePath = resolveCloudConnectionStorePath(), -): void { - fs.mkdirSync(path.dirname(storePath), { recursive: true }); - fs.writeFileSync(storePath, `${JSON.stringify(store, null, 2)}\n`, { mode: 0o600 }); -} - -export function upsertCloudConnection( - connection: CloudConnection, - storePath = resolveCloudConnectionStorePath(), -): CloudConnection { - const store = readCloudConnectionStore(storePath); - const existing = store.connections[connection.id]; - const now = new Date().toISOString(); - const next = { - ...connection, - createdAt: existing?.createdAt ?? connection.createdAt ?? now, - updatedAt: now, - }; - store.connections[next.id] = next; - store.currentConnectionId = next.id; - writeCloudConnectionStore(store, storePath); - return next; -} - -export function getCloudConnection( - remoteUrlOrOrigin?: string, - storePath = resolveCloudConnectionStorePath(), -): CloudConnection | null { - const store = readCloudConnectionStore(storePath); - if (remoteUrlOrOrigin?.trim()) { - const needle = normalizeRemoteLookup(remoteUrlOrOrigin); - return Object.values(store.connections).find((connection) => - normalizeRemoteLookup(connection.remoteUrl) === needle || - normalizeRemoteLookup(connection.targetOrigin) === needle - ) ?? null; - } - return store.currentConnectionId ? store.connections[store.currentConnectionId] ?? null : null; -} - -function normalizeRemoteLookup(value: string): string { - try { - const url = new URL(value); - return url.origin.replace(/\/+$/u, ""); - } catch { - return value.trim().replace(/\/+$/u, ""); - } -} - -function normalizeConnection(value: unknown): CloudConnection | null { - if (typeof value !== "object" || value === null || Array.isArray(value)) return null; - const record = value as Record; - const id = stringValue(record.id); - const remoteUrl = stringValue(record.remoteUrl); - const targetOrigin = stringValue(record.targetOrigin); - const targetHost = stringValue(record.targetHost); - const stackId = stringValue(record.stackId); - const targetCompanyId = stringValue(record.targetCompanyId); - const accessToken = stringValue(record.accessToken); - const token = typeof record.token === "object" && record.token !== null && !Array.isArray(record.token) - ? record.token as CloudConnectionTokenRecord - : null; - const privateKeyPem = stringValue(record.privateKeyPem); - const sourcePublicKey = stringValue(record.sourcePublicKey); - const sourceInstanceId = stringValue(record.sourceInstanceId); - const sourceInstanceFingerprint = stringValue(record.sourceInstanceFingerprint); - const createdAt = stringValue(record.createdAt); - const updatedAt = stringValue(record.updatedAt); - if ( - !id || !remoteUrl || !targetOrigin || !targetHost || !stackId || !targetCompanyId || - !accessToken || !token || !privateKeyPem || !sourcePublicKey || !sourceInstanceId || - !sourceInstanceFingerprint || !createdAt || !updatedAt - ) { - return null; - } - return { - id, - remoteUrl, - targetOrigin, - targetHost, - stackId, - stackSlug: stringValue(record.stackSlug), - stackDisplayName: stringValue(record.stackDisplayName), - targetCompanyId, - accessToken, - token, - privateKeyPem, - sourcePublicKey, - sourceInstanceId, - sourceInstanceFingerprint, - scopes: stringArray(record.scopes), - createdAt, - updatedAt, - }; -} - -function stringValue(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; -} - -function stringArray(value: unknown): string[] { - return Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === "string") : []; -} diff --git a/cli/src/commands/client/cloud-transfer.ts b/cli/src/commands/client/cloud-transfer.ts deleted file mode 100644 index 9cd1ccbe74..0000000000 --- a/cli/src/commands/client/cloud-transfer.ts +++ /dev/null @@ -1,297 +0,0 @@ -import { createHash } from "node:crypto"; - -export const upstreamTransferSchema = { - family: "paperclip-upstream-transfer", - version: "1.0.0", - major: 1, - minor: 0, -} as const; - -export type NormalizedSha256 = `sha256:${string}`; - -export interface SourceEntityKey { - sourceInstanceId: string; - sourceCompanyId: string; - sourceEntityType: string; - sourceEntityId: string; - sourceNaturalKey?: string; -} - -export interface UpstreamTransferWarning { - code: string; - severity: "info" | "warning" | "blocker"; - message: string; - entity?: SourceEntityKey; -} - -export interface UpstreamTransferEntityRecord { - key: SourceEntityKey; - contentHash: NormalizedSha256; - dependencies: SourceEntityKey[]; - warnings: UpstreamTransferWarning[]; -} - -export interface UpstreamTransferManifestSource { - sourceInstanceId: string; - sourceCompanyId: string; - sourceInstanceKeyFingerprint: string; - exporterVersion: string; - sourceSchemaVersion: string; -} - -export interface UpstreamTransferManifestTarget { - targetStackId: string; - targetCompanyId: string; - targetOrigin: string; - supportedSchemaMajor: number; -} - -export interface UpstreamTransferChunk { - chunkIndex: number; - totalChunks: number; - byteLength: number; - sha256: NormalizedSha256; - manifestHash: NormalizedSha256; -} - -export interface UpstreamTransferManifest { - schema: typeof upstreamTransferSchema; - source: UpstreamTransferManifestSource; - target: UpstreamTransferManifestTarget; - runId: string; - idempotencyKey: string; - generatedAt: string; - entityCount: number; - entities: UpstreamTransferEntityRecord[]; - chunks: UpstreamTransferChunk[]; - warnings: UpstreamTransferWarning[]; - featureFlags: string[]; - manifestHash: NormalizedSha256; -} - -export interface LocalUpstreamExportEntityInput { - key: SourceEntityKey; - body: Record; - dependencies?: SourceEntityKey[]; - warnings?: UpstreamTransferWarning[]; - conflictKeys?: string[]; -} - -export interface LocalUpstreamExportEntity { - record: UpstreamTransferEntityRecord; - body: Record; - conflictKeys?: string[]; -} - -export interface LocalUpstreamExportChunk { - chunkIndex: number; - totalChunks: number; - byteLength: number; - sha256: NormalizedSha256; - payload: { - entityKeys: SourceEntityKey[]; - }; -} - -export interface LocalUpstreamExportBundle { - manifest: UpstreamTransferManifest; - entities: LocalUpstreamExportEntity[]; - chunks: LocalUpstreamExportChunk[]; -} - -export interface BuildLocalUpstreamExportBundleInput { - source: UpstreamTransferManifestSource; - target: UpstreamTransferManifestTarget; - runId: string; - idempotencyKey: string; - entities: LocalUpstreamExportEntityInput[]; - warnings?: UpstreamTransferWarning[]; - featureFlags?: string[]; - maxEntitiesPerChunk?: number; -} - -export interface LocalUpstreamPushCoordinatorOptions { - targetOrigin: string; - paperclipCompanyId: string; - fetch?: typeof fetch; - headers?: (input: { method: string; path: string }) => HeadersInit | Promise; -} - -export class UpstreamImportRequestError extends Error { - readonly status: number; - readonly body: unknown; - - constructor(status: number, message: string, body: unknown) { - super(message); - this.status = status; - this.body = body; - } -} - -export class LocalUpstreamPushCoordinator { - readonly #targetOrigin: string; - readonly #paperclipCompanyId: string; - readonly #fetch: typeof fetch; - readonly #headers: NonNullable; - - constructor(options: LocalUpstreamPushCoordinatorOptions) { - this.#targetOrigin = options.targetOrigin.replace(/\/+$/u, ""); - this.#paperclipCompanyId = options.paperclipCompanyId; - this.#fetch = options.fetch ?? fetch; - this.#headers = options.headers ?? (() => ({})); - } - - async preview(bundle: LocalUpstreamExportBundle): Promise { - return this.post(`/api/companies/${encodeURIComponent(this.#paperclipCompanyId)}/upstream-imports/preview`, { - manifest: bundle.manifest, - entities: bundle.entities, - }); - } - - async apply(bundle: LocalUpstreamExportBundle): Promise { - const run = await this.post(`/api/companies/${encodeURIComponent(this.#paperclipCompanyId)}/upstream-imports/runs`, { - mode: "apply", - manifest: bundle.manifest, - entities: bundle.entities, - }) as { run?: { id?: unknown } }; - const runId = typeof run.run?.id === "string" ? run.run.id : undefined; - if (!runId) { - throw new Error("Remote upstream importer did not return a run id"); - } - - for (const chunk of bundle.chunks) { - await this.post(`/api/upstream-import-runs/${encodeURIComponent(runId)}/chunks`, chunk); - } - - return this.post(`/api/upstream-import-runs/${encodeURIComponent(runId)}/apply`, {}); - } - - async events(runId: string): Promise { - return this.get(`/api/upstream-import-runs/${encodeURIComponent(runId)}/events`); - } - - private async get(path: string): Promise { - const response = await this.#fetch(`${this.#targetOrigin}${path}`, { - method: "GET", - headers: await this.#headers({ method: "GET", path }), - }); - return parseCoordinatorResponse(response); - } - - private async post(path: string, body: unknown): Promise { - const response = await this.#fetch(`${this.#targetOrigin}${path}`, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...(await this.#headers({ method: "POST", path })), - }, - body: JSON.stringify(body), - }); - return parseCoordinatorResponse(response); - } -} - -export function buildLocalUpstreamExportBundle( - input: BuildLocalUpstreamExportBundleInput, -): LocalUpstreamExportBundle { - const entities = input.entities.map((entity) => ({ - record: { - key: entity.key, - contentHash: normalizedContentHash(entity.body), - dependencies: entity.dependencies ?? [], - warnings: entity.warnings ?? [], - }, - body: entity.body, - conflictKeys: entity.conflictKeys, - })); - const chunks = buildLocalChunks(entities, input.maxEntitiesPerChunk ?? 100); - const manifestWithoutHash = { - schema: upstreamTransferSchema, - source: input.source, - target: input.target, - runId: input.runId, - idempotencyKey: input.idempotencyKey, - generatedAt: new Date(0).toISOString(), - entityCount: entities.length, - entities: entities.map((entity) => entity.record), - chunks: chunks.map(({ payload: _payload, ...chunk }) => chunk), - warnings: input.warnings ?? [], - featureFlags: (input.featureFlags ?? ["cloud_sync"]).slice().sort(), - }; - const manifestHash = normalizedContentHash(manifestWithoutHash); - return { - manifest: { - ...manifestWithoutHash, - chunks: manifestWithoutHash.chunks.map((chunk) => ({ ...chunk, manifestHash })), - manifestHash, - }, - entities, - chunks, - }; -} - -export function normalizedContentHash(value: unknown): NormalizedSha256 { - return `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`; -} - -export function canonicalJson(value: unknown): string { - return JSON.stringify(sortJson(value)); -} - -function buildLocalChunks( - entities: LocalUpstreamExportEntity[], - maxEntitiesPerChunk: number, -): LocalUpstreamExportChunk[] { - if (!Number.isInteger(maxEntitiesPerChunk) || maxEntitiesPerChunk < 1) { - throw new Error("maxEntitiesPerChunk must be a positive integer"); - } - if (entities.length === 0) return []; - - const groups: LocalUpstreamExportEntity[][] = []; - for (let index = 0; index < entities.length; index += maxEntitiesPerChunk) { - groups.push(entities.slice(index, index + maxEntitiesPerChunk)); - } - - return groups.map((group, index) => { - const payload = { - entityKeys: group.map((entity) => entity.record.key), - }; - return { - chunkIndex: index, - totalChunks: groups.length, - byteLength: Buffer.byteLength(canonicalJson(payload)), - sha256: normalizedContentHash(payload), - payload, - }; - }); -} - -function sortJson(value: unknown): unknown { - if (Array.isArray(value)) return value.map(sortJson); - if (typeof value !== "object" || value === null) return value; - return Object.fromEntries( - Object.entries(value as Record) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, entry]) => [key, sortJson(entry)]), - ); -} - -async function parseCoordinatorResponse(response: Response): Promise { - const text = await response.text(); - const parsed = text.trim() ? safeParseJson(text) : {}; - if (!response.ok) { - const message = typeof parsed === "object" && parsed !== null && "error" in parsed - ? String((parsed as { error: unknown }).error) - : `Upstream importer request failed with ${response.status}`; - throw new UpstreamImportRequestError(response.status, message, parsed); - } - return parsed; -} - -function safeParseJson(text: string): unknown { - try { - return JSON.parse(text); - } catch { - return text; - } -} diff --git a/cli/src/commands/client/cloud.ts b/cli/src/commands/client/cloud.ts deleted file mode 100644 index 1e3046f14c..0000000000 --- a/cli/src/commands/client/cloud.ts +++ /dev/null @@ -1,722 +0,0 @@ -import { createHash, generateKeyPairSync, randomBytes, randomUUID, sign } from "node:crypto"; -import { createServer, type Server } from "node:http"; -import { URL } from "node:url"; -import { Command } from "commander"; -import pc from "picocolors"; -import type { - CompanyPortabilityExportResult, - CompanyPortabilityFileEntry, - InstanceExperimentalSettings, -} from "@paperclipai/shared"; -import { openUrl } from "../../client/board-auth.js"; -import { resolvePaperclipInstanceId } from "../../config/home.js"; -import { - addCommonClientOptions, - apiPath, - handleCommandError, - printOutput, - resolveCommandContext, - type BaseClientOptions, -} from "./common.js"; -import { - buildLocalUpstreamExportBundle, - LocalUpstreamPushCoordinator, - normalizedContentHash, - upstreamTransferSchema, - UpstreamImportRequestError, - type LocalUpstreamExportBundle, - type LocalUpstreamExportEntityInput, - type SourceEntityKey, - type UpstreamTransferManifestSource, - type UpstreamTransferManifestTarget, - type UpstreamTransferWarning, -} from "./cloud-transfer.js"; -import { - getCloudConnection, - upsertCloudConnection, - type CloudConnection, - type CloudConnectionTokenRecord, -} from "./cloud-store.js"; - -const CLOUD_SYNC_CONFLICT_EXIT_CODE = 2; -const CLOUD_SYNC_SCHEMA_MISMATCH_EXIT_CODE = 3; -const CLOUD_SYNC_SCOPES = ["upstream_import:preview", "upstream_import:write", "upstream_import:read"]; -const DEVICE_CODE_FALLBACK_EXPIRES_MS = 15 * 60_000; - -interface CloudConnectOptions extends BaseClientOptions { - noBrowser?: boolean; -} - -interface CloudPushOptions extends BaseClientOptions { - company?: string; - remoteUrl?: string; - dryRun?: boolean; - maxEntitiesPerChunk?: number; -} - -interface UpstreamDiscovery { - schema: string; - stack: { - id: string; - slug?: string; - displayName?: string; - companyId: string; - origin: string; - }; - auth: { - pkce?: { - authorizeUrl: string; - tokenUrl: string; - codeChallengeMethod: string; - }; - deviceCode?: { - deviceCodeUrl: string; - verificationUrl: string; - tokenUrl: string; - }; - scopes?: string[]; - }; - transfer: { - supportedSchemaMajor: number; - featureFlags?: string[]; - }; -} - -interface TokenResponse { - accessToken: string; - token: CloudConnectionTokenRecord; - scopes?: string[]; - expiresAt?: string; -} - -class CloudAuthRequestError extends Error { - readonly status: number; - readonly body: unknown; - - constructor(status: number, message: string, body: unknown) { - super(message); - this.status = status; - this.body = body; - } -} - -export function registerCloudCommands(program: Command): void { - const cloud = program.command("cloud").description("Paperclip Cloud upstream sync commands"); - - addCommonClientOptions( - cloud - .command("connect") - .description("Authorize this local instance to push into a Paperclip Cloud stack") - .argument("", "Paperclip Cloud stack URL") - .option("--no-browser", "Use the device-code flow instead of opening a browser", false) - .action(async (remoteUrl: string, opts: CloudConnectOptions) => { - try { - await connectCloud(remoteUrl, opts); - } catch (err) { - handleCommandError(err); - } - }), - ); - - addCommonClientOptions( - cloud - .command("push") - .description("Preview or apply a local company push into the connected Paperclip Cloud stack") - .requiredOption("--company ", "Local company ID to export") - .option("--remote-url ", "Use a specific stored cloud connection") - .option("--dry-run", "Preview without applying", false) - .option("--max-entities-per-chunk ", "Chunk size for upstream uploads", (value) => Number(value), 100) - .action(async (opts: CloudPushOptions) => { - try { - await pushCloud(opts); - } catch (err) { - if (isSchemaMismatchError(err)) { - console.error(pc.red(err instanceof Error ? err.message : String(err))); - process.exitCode = CLOUD_SYNC_SCHEMA_MISMATCH_EXIT_CODE; - return; - } - handleCommandError(err); - } - }), - ); -} - -export async function connectCloud(remoteUrl: string, opts: CloudConnectOptions = {}): Promise { - const ctx = resolveCommandContext(opts); - const discovery = await discoverUpstream(remoteUrl); - assertDiscoveryCompatible(discovery); - const source = createSourceIdentity(); - const token = await authorizeConnection(discovery, source, { - noBrowser: Boolean(opts.noBrowser), - }); - const targetOrigin = discovery.stack.origin.replace(/\/+$/u, ""); - const targetHost = new URL(targetOrigin).host; - const now = new Date().toISOString(); - const connection = upsertCloudConnection({ - id: connectionId(targetOrigin), - remoteUrl, - targetOrigin, - targetHost, - stackId: discovery.stack.id, - stackSlug: discovery.stack.slug ?? null, - stackDisplayName: discovery.stack.displayName ?? null, - targetCompanyId: discovery.stack.companyId, - accessToken: token.accessToken, - token: token.token, - privateKeyPem: source.privateKeyPem, - sourcePublicKey: source.sourcePublicKey, - sourceInstanceId: source.sourceInstanceId, - sourceInstanceFingerprint: source.sourceInstanceFingerprint, - scopes: token.scopes ?? token.token.scopes ?? CLOUD_SYNC_SCOPES, - createdAt: now, - updatedAt: now, - }); - - if (ctx.json) { - printOutput(redactConnection(connection), { json: true }); - } else { - console.log(pc.bold("Connected to Paperclip Cloud")); - console.log(`stack=${connection.stackDisplayName ?? connection.stackSlug ?? connection.stackId}`); - console.log(`origin=${connection.targetOrigin}`); - console.log(`company=${connection.targetCompanyId}`); - } - return connection; -} - -export async function pushCloud(opts: CloudPushOptions): Promise { - const ctx = resolveCommandContext(opts, { requireCompany: false }); - const localCompanyId = requiredString(opts.company, "--company"); - await assertCloudSyncEnabled(ctx.api.get("/api/instance/settings/experimental")); - const connection = getCloudConnection(opts.remoteUrl); - if (!connection) { - throw new Error("No cloud connection found. Run `paperclipai cloud connect ` first."); - } - - const discovery = await discoverUpstream(connection.targetOrigin); - assertDiscoveryCompatible(discovery); - const bundle = await buildBundleFromLocalCompany({ - localCompanyId, - connection, - discovery, - localApi: ctx.api, - maxEntitiesPerChunk: opts.maxEntitiesPerChunk, - mode: opts.dryRun ? "preview" : "apply", - }); - const coordinator = new LocalUpstreamPushCoordinator({ - targetOrigin: connection.targetOrigin, - paperclipCompanyId: connection.targetCompanyId, - headers: ({ method, path }) => cloudProofHeaders(connection, method, path), - }); - - const result = opts.dryRun ? await coordinator.preview(bundle) : await coordinator.apply(bundle); - const runId = getRunId(result); - const events = !opts.dryRun && runId ? await coordinator.events(runId).catch(() => null) : null; - const summary = summarizeResult(result); - const conflictCount = summary.conflict + summary.staleMapping; - - if (ctx.json) { - printOutput({ result, events }, { json: true }); - } else { - console.log(pc.bold(opts.dryRun ? "Cloud Push Preview" : "Cloud Push Applied")); - console.log(`run=${runId ?? "-"}`); - console.log(`manifest=${bundle.manifest.manifestHash}`); - console.log( - `create=${summary.create} update=${summary.update} adopt=${summary.adopt} ` + - `skip=${summary.skip} conflict=${summary.conflict} staleMapping=${summary.staleMapping}`, - ); - printWarnings(result); - printConflicts(result); - printEvents(events); - } - - if (conflictCount > 0) { - process.exitCode = CLOUD_SYNC_CONFLICT_EXIT_CODE; - } - return result; -} - -export async function discoverUpstream(remoteUrl: string): Promise { - const base = new URL(remoteUrl); - const discoveryUrl = new URL("/.well-known/paperclip-upstream", base); - return requestCloudJson(discoveryUrl.toString(), { method: "GET" }); -} - -export function assertDiscoveryCompatible(discovery: UpstreamDiscovery): void { - if (discovery.schema !== "paperclip-upstream-discovery-v1") { - throw new Error("Remote URL is not a Paperclip Cloud upstream target."); - } - if (discovery.transfer.supportedSchemaMajor !== upstreamTransferSchema.major) { - throw new Error( - `Cloud upstream schema mismatch: local major ${upstreamTransferSchema.major}, remote supports ${discovery.transfer.supportedSchemaMajor}.`, - ); - } - if (!discovery.transfer.featureFlags?.includes("cloud_sync")) { - throw new Error("Remote Paperclip Cloud stack does not advertise the cloud_sync transfer flag."); - } -} - -export function resolveDeviceCodeExpiresAt(expiresAt: string | undefined, nowMs = Date.now()): number { - const parsed = typeof expiresAt === "string" ? Date.parse(expiresAt) : NaN; - return Number.isFinite(parsed) ? parsed : nowMs + DEVICE_CODE_FALLBACK_EXPIRES_MS; -} - -export async function buildBundleFromLocalCompany(input: { - localCompanyId: string; - connection: CloudConnection; - discovery: UpstreamDiscovery; - localApi: { - post(path: string, body?: unknown): Promise; - }; - maxEntitiesPerChunk?: number; - mode: "preview" | "apply"; -}): Promise { - const exported = await input.localApi.post( - apiPath`/api/companies/${input.localCompanyId}/export`, - { - include: { - company: true, - agents: true, - projects: true, - issues: true, - skills: true, - }, - expandReferencedSkills: true, - }, - ); - if (!exported) throw new Error("Local company export returned no data."); - - const sourceHash = normalizedContentHash({ - manifest: exported.manifest, - files: exported.files, - }); - const source: UpstreamTransferManifestSource = { - sourceInstanceId: input.connection.sourceInstanceId, - sourceCompanyId: input.localCompanyId, - sourceInstanceKeyFingerprint: input.connection.sourceInstanceFingerprint, - exporterVersion: "paperclipai-cli-cloud-v1", - sourceSchemaVersion: "paperclip-local-portability-v1", - }; - const target: UpstreamTransferManifestTarget = { - targetStackId: input.discovery.stack.id, - targetCompanyId: input.discovery.stack.companyId, - targetOrigin: input.discovery.stack.origin, - supportedSchemaMajor: input.discovery.transfer.supportedSchemaMajor, - }; - const entities = buildEntitiesFromPortableExport(input.localCompanyId, input.connection.sourceInstanceId, exported); - const idempotencyKey = [ - input.mode, - input.connection.sourceInstanceId, - input.localCompanyId, - input.discovery.stack.id, - sourceHash, - ].join(":"); - return buildLocalUpstreamExportBundle({ - source, - target, - runId: `local-${input.mode}-${shortHash(idempotencyKey)}`, - idempotencyKey, - entities, - warnings: exported.warnings.map((message): UpstreamTransferWarning => ({ - code: "local_company_export_warning", - severity: "warning", - message, - })), - featureFlags: ["cloud_sync"], - maxEntitiesPerChunk: input.maxEntitiesPerChunk, - }); -} - -async function authorizeConnection( - discovery: UpstreamDiscovery, - source: ReturnType, - opts: { noBrowser: boolean }, -): Promise { - if (!opts.noBrowser && canOpenBrowser() && discovery.auth.pkce) { - try { - return await authorizeWithBrowser(discovery, source); - } catch (error) { - console.error(pc.yellow(`Browser authorization failed; falling back to device-code flow. ${errorMessage(error)}`)); - } - } - if (!discovery.auth.deviceCode) { - throw new Error("Remote Paperclip Cloud stack does not support device-code authorization."); - } - return authorizeWithDeviceCode(discovery, source, { openBrowser: !opts.noBrowser && canOpenBrowser() }); -} - -async function authorizeWithBrowser( - discovery: UpstreamDiscovery, - source: ReturnType, -): Promise { - const pkce = discovery.auth.pkce; - if (!pkce) throw new Error("Remote did not advertise PKCE authorization."); - const callback = await startPkceCallbackServer(); - const verifier = randomBytes(32).toString("base64url"); - const challenge = createHash("sha256").update(verifier).digest("base64url"); - const state = randomUUID(); - const authorizeUrl = new URL(pkce.authorizeUrl); - authorizeUrl.searchParams.set("redirectUri", callback.redirectUri); - authorizeUrl.searchParams.set("state", state); - authorizeUrl.searchParams.set("codeChallenge", challenge); - authorizeUrl.searchParams.set("codeChallengeMethod", "S256"); - authorizeUrl.searchParams.set("sourceInstanceId", source.sourceInstanceId); - authorizeUrl.searchParams.set("sourceInstanceFingerprint", source.sourceInstanceFingerprint); - authorizeUrl.searchParams.set("sourcePublicKey", source.sourcePublicKey); - authorizeUrl.searchParams.set("scopes", CLOUD_SYNC_SCOPES.join(" ")); - - try { - console.error(`Open this URL to approve cloud sync:\n${authorizeUrl.toString()}`); - if (!(await openUrl(authorizeUrl.toString()))) { - throw new Error("Could not open a browser."); - } - const code = await callback.waitForCode(state); - return requestCloudJson(pkce.tokenUrl, { - method: "POST", - body: JSON.stringify({ - grantType: "authorization_code", - code, - redirectUri: callback.redirectUri, - codeVerifier: verifier, - }), - }); - } finally { - await callback.close(); - } -} - -async function authorizeWithDeviceCode( - discovery: UpstreamDiscovery, - source: ReturnType, - opts: { openBrowser: boolean }, -): Promise { - const device = discovery.auth.deviceCode; - if (!device) throw new Error("Remote did not advertise device-code authorization."); - const response = await requestCloudJson<{ - deviceCode: string; - userCode: string; - verificationUri: string; - expiresAt?: string; - intervalSeconds?: number; - }>(device.deviceCodeUrl, { - method: "POST", - body: JSON.stringify({ - stackId: discovery.stack.id, - sourceInstanceId: source.sourceInstanceId, - sourceInstanceFingerprint: source.sourceInstanceFingerprint, - sourcePublicKey: source.sourcePublicKey, - scopes: CLOUD_SYNC_SCOPES, - }), - }); - console.error(pc.bold("Cloud device authorization required")); - console.error(`Open: ${response.verificationUri}`); - console.error(`Code: ${response.userCode}`); - if (opts.openBrowser) await openUrl(response.verificationUri); - - const expiresAt = resolveDeviceCodeExpiresAt(response.expiresAt); - const intervalMs = Math.max(500, (response.intervalSeconds ?? 5) * 1000); - while (Date.now() < expiresAt) { - await sleep(intervalMs); - try { - return await requestCloudJson(device.tokenUrl, { - method: "POST", - body: JSON.stringify({ - grantType: "device_code", - deviceCode: response.deviceCode, - }), - }); - } catch (error) { - if (error instanceof CloudAuthRequestError && error.body && typeof error.body === "object") { - const code = (error.body as { error?: unknown }).error; - if (code === "authorization_pending") continue; - } - throw error; - } - } - throw new Error("Device-code authorization expired before it was approved."); -} - -function buildEntitiesFromPortableExport( - localCompanyId: string, - sourceInstanceId: string, - exported: CompanyPortabilityExportResult, -): LocalUpstreamExportEntityInput[] { - const companyKey: SourceEntityKey = { - sourceInstanceId, - sourceCompanyId: localCompanyId, - sourceEntityType: "company", - sourceEntityId: localCompanyId, - sourceNaturalKey: exported.manifest.company?.name ?? localCompanyId, - }; - const entities: LocalUpstreamExportEntityInput[] = [ - { - key: companyKey, - body: { - kind: "paperclip_company_portability_manifest", - manifest: exported.manifest, - rootPath: exported.rootPath, - paperclipExtensionPath: exported.paperclipExtensionPath, - fileCount: Object.keys(exported.files).length, - }, - conflictKeys: [`company:${companyKey.sourceNaturalKey ?? localCompanyId}`], - }, - ]; - - for (const [filePath, entry] of Object.entries(exported.files).sort(([left], [right]) => left.localeCompare(right))) { - entities.push({ - key: { - sourceInstanceId, - sourceCompanyId: localCompanyId, - sourceEntityType: "company_setting", - sourceEntityId: shortHash(filePath), - sourceNaturalKey: filePath, - }, - body: { - kind: "paperclip_portable_file", - path: filePath, - entry: normalizePortableFileEntry(entry), - }, - dependencies: [companyKey], - conflictKeys: [`portable_file:${filePath}`], - }); - } - return entities; -} - -function normalizePortableFileEntry(entry: CompanyPortabilityFileEntry): Record { - if (typeof entry === "string") { - return { encoding: "utf8", data: entry }; - } - return { ...entry }; -} - -async function assertCloudSyncEnabled(settingsPromise: Promise): Promise { - const settings = await settingsPromise; - if (settings?.enableCloudSync !== true) { - throw new Error( - "Cloud sync is disabled. Enable the cloud sync experimental setting before running `paperclipai cloud push`.", - ); - } -} - -function cloudProofHeaders(connection: CloudConnection, method: string, pathAndSearch: string): Record { - const timestamp = new Date().toISOString(); - const nonce = randomUUID(); - const payload = [ - method, - connection.targetHost.toLowerCase(), - pathAndSearch, - connection.token.id, - connection.sourceInstanceId, - timestamp, - nonce, - ].join("\n"); - return { - Authorization: `Bearer ${connection.accessToken}`, - "X-Paperclip-Upstream-Source-Instance-Id": connection.sourceInstanceId, - "X-Paperclip-Upstream-Proof-Timestamp": timestamp, - "X-Paperclip-Upstream-Proof-Nonce": nonce, - "X-Paperclip-Upstream-Proof-Signature": sign( - null, - Buffer.from(payload, "utf8"), - connection.privateKeyPem, - ).toString("base64url"), - }; -} - -async function requestCloudJson(url: string, init: RequestInit): Promise { - const headers = new Headers(init.headers); - headers.set("accept", "application/json"); - if (init.body !== undefined && !headers.has("content-type")) { - headers.set("content-type", "application/json"); - } - const response = await fetch(url, { ...init, headers }); - const text = await response.text(); - const parsed = text.trim() ? JSON.parse(text) as unknown : {}; - if (!response.ok) { - const message = typeof parsed === "object" && parsed !== null && "error" in parsed - ? String((parsed as { error: unknown }).error) - : `Cloud request failed with ${response.status}`; - throw new CloudAuthRequestError(response.status, message, parsed); - } - return parsed as T; -} - -function createSourceIdentity() { - const { publicKey, privateKey } = generateKeyPairSync("ed25519"); - const sourcePublicKey = publicKey.export({ type: "spki", format: "pem" }).toString(); - const sourceInstanceFingerprint = `sha256:${createHash("sha256") - .update(publicKey.export({ type: "spki", format: "der" })) - .digest("hex")}`; - return { - sourceInstanceId: `paperclip-local-${resolvePaperclipInstanceId()}`, - sourceInstanceFingerprint, - sourcePublicKey, - privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(), - }; -} - -async function startPkceCallbackServer(): Promise<{ - redirectUri: string; - waitForCode: (state: string) => Promise; - close: () => Promise; -}> { - let resolveCode: ((code: string) => void) | null = null; - let rejectCode: ((error: Error) => void) | null = null; - let expectedState = ""; - const codePromise = new Promise((resolve, reject) => { - resolveCode = resolve; - rejectCode = reject; - }); - const server = createServer((req, res) => { - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - const code = url.searchParams.get("code"); - const state = url.searchParams.get("state"); - if (!code || state !== expectedState) { - res.writeHead(400, { "Content-Type": "text/plain" }); - res.end("Paperclip Cloud authorization failed. You can close this tab."); - rejectCode?.(new Error("Authorization callback was missing a valid code or state.")); - return; - } - res.writeHead(200, { "Content-Type": "text/plain" }); - res.end("Paperclip Cloud authorization complete. You can close this tab."); - resolveCode?.(code); - }); - await listenOnLoopback(server); - const address = server.address(); - if (typeof address !== "object" || !address?.port) { - throw new Error("Failed to start local authorization callback server."); - } - return { - redirectUri: `http://127.0.0.1:${address.port}/cloud/callback`, - waitForCode: (state: string) => { - expectedState = state; - return codePromise; - }, - close: () => closeServer(server), - }; -} - -function listenOnLoopback(server: Server): Promise { - return new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(0, "127.0.0.1", () => { - server.off("error", reject); - resolve(); - }); - }); -} - -function closeServer(server: Server): Promise { - return new Promise((resolve, reject) => { - server.close((error) => error ? reject(error) : resolve()); - }); -} - -function canOpenBrowser(): boolean { - if (process.platform === "darwin" || process.platform === "win32") return true; - return Boolean(process.env.DISPLAY || process.env.WAYLAND_DISPLAY); -} - -function summarizeResult(result: unknown): { - create: number; - update: number; - adopt: number; - skip: number; - conflict: number; - staleMapping: number; -} { - const summary = asRecord(asRecord(result)?.summary); - return { - create: numberValue(summary?.create), - update: numberValue(summary?.update), - adopt: numberValue(summary?.adopt), - skip: numberValue(summary?.skip), - conflict: numberValue(summary?.conflict), - staleMapping: numberValue(summary?.staleMapping), - }; -} - -function printWarnings(result: unknown): void { - const warnings = Array.isArray(asRecord(result)?.warnings) ? asRecord(result)?.warnings as unknown[] : []; - for (const warning of warnings) { - const record = asRecord(warning); - console.log(pc.yellow(`warning=${record?.code ?? "warning"} ${record?.message ?? ""}`.trim())); - } -} - -function printConflicts(result: unknown): void { - const conflicts = Array.isArray(asRecord(result)?.conflicts) ? asRecord(result)?.conflicts as unknown[] : []; - for (const conflict of conflicts.slice(0, 10)) { - const record = asRecord(conflict); - console.log(pc.red(`conflict=${record?.conflictKind ?? "target_conflict"} target=${record?.targetEntityId ?? "-"}`)); - } - if (conflicts.length > 10) console.log(pc.red(`conflicts_truncated=${conflicts.length - 10}`)); -} - -function printEvents(events: unknown): void { - const rows = Array.isArray(asRecord(events)?.events) ? asRecord(events)?.events as unknown[] : []; - for (const row of rows.slice(-10)) { - const event = asRecord(row); - console.log(pc.dim(`event=${event?.action ?? "-"} target=${event?.targetEntityId ?? "-"}`)); - } -} - -function getRunId(result: unknown): string | null { - const run = asRecord(asRecord(result)?.run); - return typeof run?.id === "string" ? run.id : null; -} - -function redactConnection(connection: CloudConnection): Record { - return { - id: connection.id, - remoteUrl: connection.remoteUrl, - targetOrigin: connection.targetOrigin, - stackId: connection.stackId, - targetCompanyId: connection.targetCompanyId, - scopes: connection.scopes, - expiresAt: connection.token.expiresAt, - }; -} - -function connectionId(targetOrigin: string): string { - return `cloud-${shortHash(targetOrigin)}`; -} - -function shortHash(value: string): string { - return createHash("sha256").update(value).digest("hex").slice(0, 16); -} - -function requiredString(value: unknown, label: string): string { - if (typeof value === "string" && value.trim()) return value.trim(); - throw new Error(`${label} is required.`); -} - -function numberValue(value: unknown): number { - return typeof value === "number" && Number.isFinite(value) ? value : 0; -} - -function asRecord(value: unknown): Record | null { - return typeof value === "object" && value !== null && !Array.isArray(value) - ? value as Record - : null; -} - -function isSchemaMismatchError(error: unknown): boolean { - if (error instanceof UpstreamImportRequestError) { - return JSON.stringify(error.body).toLowerCase().includes("schema"); - } - return error instanceof Error && error.message.toLowerCase().includes("schema mismatch"); -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -export const cloudCommandExitCodes = { - conflict: CLOUD_SYNC_CONFLICT_EXIT_CODE, - schemaMismatch: CLOUD_SYNC_SCHEMA_MISMATCH_EXIT_CODE, -} as const; diff --git a/cli/src/commands/client/company.ts b/cli/src/commands/client/company.ts index e8368ccf9d..bd0d162a50 100644 --- a/cli/src/commands/client/company.ts +++ b/cli/src/commands/client/company.ts @@ -15,7 +15,7 @@ import type { import { getTelemetryClient, trackCompanyImported } from "../../telemetry.js"; import { ApiRequestError } from "../../client/http.js"; import { openUrl } from "../../client/board-auth.js"; -import { binaryContentTypeByExtension, readZipArchive } from "./zip.js"; +import { binaryContentTypeByExtension, bytesToPortableFileEntry, isBlobStorePath, readZipArchive } from "./zip.js"; import { addCommonClientOptions, apiPath, @@ -140,16 +140,6 @@ type ImportSelectionState = { skills: Set; }; -function readPortableFileEntry(filePath: string, contents: Buffer): CompanyPortabilityFileEntry { - const contentType = binaryContentTypeByExtension[path.extname(filePath).toLowerCase()]; - if (!contentType) return contents.toString("utf8"); - return { - encoding: "base64", - data: contents.toString("base64"), - contentType, - }; -} - function portableFileEntryToWriteValue(entry: CompanyPortabilityFileEntry): string | Uint8Array { if (typeof entry === "string") return entry; return Buffer.from(entry.data, "base64"); @@ -213,7 +203,7 @@ function shouldIncludePortableFile(filePath: string): boolean { const isMarkdown = baseName.endsWith(".md"); const isPaperclipYaml = baseName === ".paperclip.yaml" || baseName === ".paperclip.yml"; const contentType = binaryContentTypeByExtension[path.extname(baseName).toLowerCase()]; - return isMarkdown || isPaperclipYaml || Boolean(contentType); + return isMarkdown || isPaperclipYaml || Boolean(contentType) || isBlobStorePath(filePath); } function findPortableExtensionPath(files: Record): string | null { @@ -932,7 +922,7 @@ async function collectPackageFiles( if (!entry.isFile()) continue; const relativePath = path.relative(root, absolutePath).replace(/\\/g, "/"); if (!shouldIncludePortableFile(relativePath)) continue; - files[relativePath] = readPortableFileEntry(relativePath, await readFile(absolutePath)); + files[relativePath] = bytesToPortableFileEntry(relativePath, await readFile(absolutePath)); } } diff --git a/cli/src/commands/client/zip.ts b/cli/src/commands/client/zip.ts index b75935e953..18265fcdb8 100644 --- a/cli/src/commands/client/zip.ts +++ b/cli/src/commands/client/zip.ts @@ -3,6 +3,9 @@ import path from "node:path"; import type { CompanyPortabilityFileEntry } from "@paperclipai/shared"; const textDecoder = new TextDecoder(); +// ignoreBOM keeps a leading BOM in the decoded text so text entries +// re-encode to their original bytes; fatal surfaces invalid UTF-8. +const strictTextDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); export const binaryContentTypeByExtension: Record = { ".gif": "image/gif", @@ -46,14 +49,34 @@ function sharedArchiveRoot(paths: string[]) { : null; } -function bytesToPortableFileEntry(pathValue: string, bytes: Uint8Array): CompanyPortabilityFileEntry { +export function isBlobStorePath(pathValue: string) { + return /(^|\/)blobs\/[^/]+$/.test(normalizeArchivePath(pathValue)); +} + +function decodeStrictUtf8(bytes: Uint8Array): string | null { + let text: string; + try { + text = strictTextDecoder.decode(bytes); + } catch { + return null; + } + return Buffer.from(text, "utf8").equals(Buffer.from(bytes)) ? text : null; +} + +export function bytesToPortableFileEntry(pathValue: string, bytes: Uint8Array): CompanyPortabilityFileEntry { + // Content-addressed blob entries are opaque bytes regardless of extension. + if (isBlobStorePath(pathValue)) { + return { encoding: "base64", data: Buffer.from(bytes).toString("base64"), contentType: "application/octet-stream" }; + } const contentType = binaryContentTypeByExtension[path.extname(pathValue).toLowerCase()]; - if (!contentType) return textDecoder.decode(bytes); - return { - encoding: "base64", - data: Buffer.from(bytes).toString("base64"), - contentType, - }; + if (contentType) { + return { encoding: "base64", data: Buffer.from(bytes).toString("base64"), contentType }; + } + const text = decodeStrictUtf8(bytes); + if (text !== null) return text; + // Bytes that are not valid UTF-8 must not be decoded lossily; fall back + // to base64 so they round-trip exactly. + return { encoding: "base64", data: Buffer.from(bytes).toString("base64"), contentType: "application/octet-stream" }; } async function inflateZipEntry(compressionMethod: number, bytes: Uint8Array) { diff --git a/cli/src/index.ts b/cli/src/index.ts index 2adb131a2b..c53454636e 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -22,7 +22,6 @@ import { registerRoutineCommands } from "./commands/routines.js"; import { registerPipelineCommands } from "./commands/pipelines.js"; import { registerFeedbackCommands } from "./commands/client/feedback.js"; import { registerSecretCommands } from "./commands/client/secrets.js"; -import { registerCloudCommands } from "./commands/client/cloud.js"; import { registerSkillsCommands } from "./commands/client/skills.js"; import { registerTeamCommands } from "./commands/client/teams.js"; import { applyDataDirOverride, type DataDirOptionLike } from "./config/data-dir.js"; @@ -181,7 +180,6 @@ registerRoutineCommands(program); registerPipelineCommands(program); registerFeedbackCommands(program); registerSecretCommands(program); -registerCloudCommands(program); registerSkillsCommands(program); registerTeamCommands(program); registerWorktreeCommands(program); diff --git a/docs/guides/board-operator/importing-and-exporting.md b/docs/guides/board-operator/importing-and-exporting.md index 02c8cc132f..1d8e40d241 100644 --- a/docs/guides/board-operator/importing-and-exporting.md +++ b/docs/guides/board-operator/importing-and-exporting.md @@ -29,6 +29,14 @@ my-company/ - **SKILL.md** files are compatible with the Agent Skills ecosystem. - **.paperclip.yaml** holds Paperclip-specific config (adapter types, env inputs, budgets) as an optional sidecar. +## Export & Import in the App + +Both flows are also available in the web UI as company settings pages: **Export** and **Import** appear in the company settings navigation. + +The **Export** page lets you pick exactly which files go into the bundle before downloading it. Above the file tree it shows a **"Not included in this export"** panel — the export fidelity report — listing data the bundle will not carry (for example attachments, approvals, cost history, or activity log entries), with blocking issues highlighted. + +The **Import** page previews the package, lets you resolve name collisions and adapter assignments, and applies the import. A **"Start imported agents and routines paused"** checkbox (on by default) makes imported agents and routines land paused instead of live. After the import finishes, an **"Activate imported agents and routines"** panel lists everything that was imported paused so you can resume the agents and activate the routines you select — nothing starts running until you say so. + ## Exporting a Company Export a company into a portable folder: @@ -142,6 +150,8 @@ The preview shows: Imported agents always land with timer heartbeats disabled. Assignment/on-demand wake behavior from the package is preserved, but scheduled runs stay off until a board operator re-enables them. +Imports can additionally request `pauseAutomations` (the default in the app's Import page) so imported agents and routines land fully paused. Use the post-import activation panel — or resume the agents and activate the routines individually — when you are ready for them to run. + ### Common Workflows **Clone a company template from GitHub:** @@ -184,11 +194,14 @@ The CLI commands use these API endpoints under the hood: | Action | Endpoint | |--------|----------| | Export company | `POST /api/companies/{companyId}/export` | +| Export fidelity report | `GET /api/companies/{companyId}/export/fidelity` | | Preview import (existing company) | `POST /api/companies/{companyId}/imports/preview` | | Apply import (existing company) | `POST /api/companies/{companyId}/imports/apply` | | Preview import (new company) | `POST /api/companies/import/preview` | | Apply import (new company) | `POST /api/companies/import` | +Import apply requests accept `pauseAutomations: true` to create imported agents and routines in a paused state. + CEO agents can also use the safe import routes (`/imports/preview` and `/imports/apply`) which enforce non-destructive rules: `replace` is rejected, collisions resolve with `rename` or `skip`, and issues are always created as new. ## GitHub Sources diff --git a/packages/db/src/migrations/0196_drop_cloud_upstream_tables.sql b/packages/db/src/migrations/0196_drop_cloud_upstream_tables.sql new file mode 100644 index 0000000000..f90bb718f1 --- /dev/null +++ b/packages/db/src/migrations/0196_drop_cloud_upstream_tables.sql @@ -0,0 +1,9 @@ +-- Drop the cloud sync (cloud upstream) tables. The host-to-host transport +-- has been removed in favor of the Import/Export flow, and nothing reads or +-- writes these tables any more. The sender tables (connections, runs) came +-- from 0089; the receiver-side tables never shipped, so these two are all +-- that exist. The feature was experimental and flag-gated off by default, +-- so its run history is intentionally discarded. +DROP TABLE IF EXISTS "cloud_upstream_runs"; +--> statement-breakpoint +DROP TABLE IF EXISTS "cloud_upstream_connections"; diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index a62b0c9359..21e680bc2d 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -1359,6 +1359,13 @@ "when": 1785170000001, "tag": "0195_built_in_agent_unique_marker", "breakpoints": true + }, + { + "idx": 196, + "version": "7", + "when": 1785170001001, + "tag": "0196_drop_cloud_upstream_tables", + "breakpoints": true } ] } diff --git a/packages/db/src/schema/cloud_upstreams.ts b/packages/db/src/schema/cloud_upstreams.ts deleted file mode 100644 index 93b4341f7e..0000000000 --- a/packages/db/src/schema/cloud_upstreams.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { boolean, index, integer, jsonb, pgTable, text, timestamp, uuid } from "drizzle-orm/pg-core"; -import { companies } from "./companies.js"; - -export const cloudUpstreamConnections = pgTable( - "cloud_upstream_connections", - { - id: uuid("id").primaryKey().defaultRandom(), - companyId: uuid("company_id").notNull().references(() => companies.id, { onDelete: "cascade" }), - remoteUrl: text("remote_url").notNull(), - sourceInstanceId: text("source_instance_id").notNull(), - sourceInstanceFingerprint: text("source_instance_fingerprint").notNull(), - sourcePublicKey: text("source_public_key").notNull(), - // Stored through the Cloud Upstream service as an encrypted credential envelope. - privateKeyPem: text("private_key_pem").notNull(), - tokenStatus: text("token_status").notNull(), - scopes: text("scopes").array().notNull().default([]), - authorizedGlobalUserId: text("authorized_global_user_id"), - // Stored through the Cloud Upstream service as an encrypted credential envelope. - accessToken: text("access_token"), - tokenId: text("token_id"), - tokenExpiresAt: timestamp("token_expires_at", { withTimezone: true }), - - targetStackId: text("target_stack_id").notNull(), - targetStackSlug: text("target_stack_slug"), - targetStackDisplayName: text("target_stack_display_name"), - targetCompanyId: text("target_company_id").notNull(), - targetOrigin: text("target_origin").notNull(), - targetPrimaryHost: text("target_primary_host").notNull(), - targetProduct: text("target_product").notNull(), - targetSchemaMajor: integer("target_schema_major").notNull(), - targetMaxChunkBytes: integer("target_max_chunk_bytes").notNull(), - - pendingState: text("pending_state"), - pendingCodeVerifier: text("pending_code_verifier"), - pendingRedirectUri: text("pending_redirect_uri"), - pendingTokenUrl: text("pending_token_url"), - - lastRunId: uuid("last_run_id"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [ - index("cloud_upstream_connections_company_idx").on(table.companyId), - ], -); - -export const cloudUpstreamRuns = pgTable( - "cloud_upstream_runs", - { - id: uuid("id").primaryKey().defaultRandom(), - connectionId: uuid("connection_id").notNull().references(() => cloudUpstreamConnections.id, { onDelete: "cascade" }), - companyId: uuid("company_id").notNull().references(() => companies.id, { onDelete: "cascade" }), - remoteRunId: text("remote_run_id"), - status: text("status").notNull(), - activeStep: text("active_step").notNull(), - progressPercent: integer("progress_percent").notNull().default(0), - dryRun: boolean("dry_run").notNull().default(false), - retryOfRunId: uuid("retry_of_run_id"), - summary: jsonb("summary").$type().notNull().default([]), - warnings: jsonb("warnings").$type().notNull().default([]), - conflicts: jsonb("conflicts").$type().notNull().default([]), - events: jsonb("events").$type().notNull().default([]), - report: jsonb("report").$type>().notNull().default({}), - idempotencyKey: text("idempotency_key").notNull(), - manifestHash: text("manifest_hash").notNull(), - targetUrl: text("target_url"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), - completedAt: timestamp("completed_at", { withTimezone: true }), - }, - (table) => [ - index("cloud_upstream_runs_company_created_idx").on(table.companyId, table.createdAt), - index("cloud_upstream_runs_connection_idx").on(table.connectionId), - ], -); diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 8e558866b7..3589c16ab0 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -2,7 +2,6 @@ export { companies } from "./companies.js"; export { companyLogos } from "./company_logos.js"; export { authUsers, authSessions, authAccounts, authVerifications } from "./auth.js"; export { instanceSettings } from "./instance_settings.js"; -export { cloudUpstreamConnections, cloudUpstreamRuns } from "./cloud_upstreams.js"; export { instanceUserRoles } from "./instance_user_roles.js"; export { userSidebarPreferences } from "./user_sidebar_preferences.js"; export { agents } from "./agents.js"; diff --git a/packages/db/src/table-size-estimates.ts b/packages/db/src/table-size-estimates.ts index 8a6d680af4..f701fc5ea2 100644 --- a/packages/db/src/table-size-estimates.ts +++ b/packages/db/src/table-size-estimates.ts @@ -92,8 +92,6 @@ export const LOCAL_TABLE_ROW_COUNTS = [ { table: "budget_incidents", localRows: 0 }, { table: "budget_policies", localRows: 0 }, { table: "cli_auth_challenges", localRows: 0 }, - { table: "cloud_upstream_connections", localRows: 0 }, - { table: "cloud_upstream_runs", localRows: 0 }, { table: "company_logos", localRows: 0 }, { table: "company_secret_provider_configs", localRows: 0 }, { table: "company_skill_comments", localRows: 0 }, diff --git a/packages/shared/src/feature-catalog.ts b/packages/shared/src/feature-catalog.ts index 510679acac..21c93ebe57 100644 --- a/packages/shared/src/feature-catalog.ts +++ b/packages/shared/src/feature-catalog.ts @@ -127,14 +127,6 @@ export const INSTANCE_FEATURE_CATALOG: Record { + it("returns no warnings when the export carries everything", () => { + expect(buildExportFidelityWarnings(zeroCounts)).toEqual([]); + }); + + it("emits no warnings for data the bundle now carries", () => { + expect(buildExportFidelityWarnings({ + ...zeroCounts, + labelDefinitions: 2, + issueLabelReferences: 3, + issueBlockerRelations: 2, + issueDocuments: 1, + issueWorkProducts: 3, + issueAttachments: 4, + issueMonitors: 8, + })).toEqual([]); + }); + + it("emits one warning per unsupported data category with counts", () => { + const warnings = buildExportFidelityWarnings({ + ...zeroCounts, + approvals: 5, + costEvents: 6, + activityLogEntries: 7, + }); + expect(warnings.map((warning) => warning.code)).toEqual([ + "approvals_not_exported", + "cost_history_not_exported", + "activity_history_not_exported", + ]); + expect(warnings.every((warning) => warning.severity === "warning")).toBe(true); + expect(warnings[0]?.message).toBe("5 approvals are not included in the export bundle."); + expect(warnings[1]?.message).toBe("6 cost events are not included in the export bundle."); + }); +}); + +describe("normalizeExportFidelityCounts", () => { + it("round-trips a valid counts object", () => { + const counts = { ...zeroCounts, issueAttachments: 12 }; + expect(normalizeExportFidelityCounts(counts)).toEqual(counts); + }); + + it("rejects non-objects, arrays, and missing keys", () => { + expect(normalizeExportFidelityCounts(null)).toBeNull(); + expect(normalizeExportFidelityCounts([])).toBeNull(); + expect(normalizeExportFidelityCounts("counts")).toBeNull(); + const { issueMonitors: _dropped, ...partial } = zeroCounts; + expect(normalizeExportFidelityCounts(partial)).toBeNull(); + }); + + it("rejects negative and non-finite values", () => { + expect(normalizeExportFidelityCounts({ ...zeroCounts, approvals: -1 })).toBeNull(); + expect(normalizeExportFidelityCounts({ ...zeroCounts, approvals: Number.NaN })).toBeNull(); + expect(normalizeExportFidelityCounts({ ...zeroCounts, approvals: Number.POSITIVE_INFINITY })).toBeNull(); + }); +}); diff --git a/packages/shared/src/portability-fidelity.ts b/packages/shared/src/portability-fidelity.ts new file mode 100644 index 0000000000..3465dc5777 --- /dev/null +++ b/packages/shared/src/portability-fidelity.ts @@ -0,0 +1,64 @@ +export type PortabilityFidelitySeverity = "info" | "warning" | "blocker"; + +export interface PortabilityFidelityWarning { + code: string; + severity: PortabilityFidelitySeverity; + message: string; +} + +export const EXPORT_FIDELITY_REPORT_SCHEMA = "paperclip-export-fidelity-v1"; + +export const EXPORT_FIDELITY_COUNT_KEYS = [ + "labelDefinitions", + "issueLabelReferences", + "issueBlockerRelations", + "issueDocuments", + "issueWorkProducts", + "issueAttachments", + "approvals", + "costEvents", + "activityLogEntries", + "issueMonitors", +] as const; + +export type ExportFidelityCounts = Record<(typeof EXPORT_FIDELITY_COUNT_KEYS)[number], number>; + +export interface ExportFidelityReport { + schema: typeof EXPORT_FIDELITY_REPORT_SCHEMA; + companyId: string; + counts: ExportFidelityCounts; + warnings: PortabilityFidelityWarning[]; + generatedAt: string; +} + +const UNSUPPORTED_DATA_WARNINGS: ReadonlyArray<[code: string, countKey: keyof ExportFidelityCounts, singular: string, plural: string]> = [ + ["approvals_not_exported", "approvals", "approval", "approvals"], + ["cost_history_not_exported", "costEvents", "cost event", "cost events"], + ["activity_history_not_exported", "activityLogEntries", "activity log entry", "activity log entries"], +]; + +export function buildExportFidelityWarnings(counts: ExportFidelityCounts): PortabilityFidelityWarning[] { + const warnings: PortabilityFidelityWarning[] = []; + for (const [code, countKey, singular, plural] of UNSUPPORTED_DATA_WARNINGS) { + const rowCount = counts[countKey]; + if (rowCount <= 0) continue; + warnings.push({ + code, + severity: "warning", + message: `${rowCount} ${rowCount === 1 ? `${singular} is` : `${plural} are`} not included in the export bundle.`, + }); + } + return warnings; +} + +export function normalizeExportFidelityCounts(value: unknown): ExportFidelityCounts | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const record = value as Record; + const counts = {} as Record<(typeof EXPORT_FIDELITY_COUNT_KEYS)[number], number>; + for (const key of EXPORT_FIDELITY_COUNT_KEYS) { + const raw = record[key]; + if (typeof raw !== "number" || !Number.isFinite(raw) || raw < 0) return null; + counts[key] = raw; + } + return counts; +} diff --git a/packages/shared/src/portability-hash.ts b/packages/shared/src/portability-hash.ts new file mode 100644 index 0000000000..4465c125a0 --- /dev/null +++ b/packages/shared/src/portability-hash.ts @@ -0,0 +1,25 @@ +import { createHash } from "node:crypto"; + +export type NormalizedSha256 = `sha256:${string}`; + +export function normalizedContentHash(value: unknown): NormalizedSha256 { + return `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`; +} + +export function canonicalJson(value: unknown): string { + return JSON.stringify(sortJson(value)); +} + +export function sha256HexOfBytes(data: Uint8Array): string { + return createHash("sha256").update(data).digest("hex"); +} + +function sortJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortJson); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, sortJson(entry)]), + ); +} diff --git a/packages/shared/src/types/cloud-upstream.ts b/packages/shared/src/types/cloud-upstream.ts deleted file mode 100644 index 211fa25d53..0000000000 --- a/packages/shared/src/types/cloud-upstream.ts +++ /dev/null @@ -1,110 +0,0 @@ -export type CloudUpstreamStep = "connect" | "scan" | "preview" | "push" | "verify" | "activate"; - -export type CloudUpstreamRunStatus = "previewed" | "running" | "succeeded" | "failed" | "cancelled"; - -export type CloudUpstreamActivationEntityType = "agents" | "routines" | "monitors"; - -export interface CloudUpstreamActivationDecision { - entityType: CloudUpstreamActivationEntityType; - count: number; - status: "paused" | "activated"; - activatedAt: string | null; -} - -export interface CloudUpstreamTarget { - stackId: string; - stackSlug: string | null; - stackDisplayName: string | null; - companyId: string; - primaryHost: string; - origin: string; - product: string; - schemaMajor: number; - maxChunkBytes: number; -} - -export interface CloudUpstreamConnection { - id: string; - companyId: string; - remoteUrl: string; - target: CloudUpstreamTarget; - tokenStatus: "pending" | "connected" | "expired" | "revoked"; - scopes: string[]; - authorizedGlobalUserId: string | null; - expiresAt: string | null; - createdAt: string; - updatedAt: string; - lastRunId: string | null; -} - -export interface CloudUpstreamSummaryCount { - key: string; - label: string; - count: number; -} - -export interface CloudUpstreamWarning { - code: string; - severity: "warning" | "blocker"; - title: string; - detail: string; -} - -export interface CloudUpstreamConflict { - id: string; - entityType: string; - sourceLabel: string; - targetLabel: string; - plannedAction: "create" | "update" | "skip" | "blocked"; - reason: string; -} - -export interface CloudUpstreamPreview { - connectionId: string; - sourceCompanyId: string; - target: CloudUpstreamTarget; - schemaCompatible: boolean; - summary: CloudUpstreamSummaryCount[]; - warnings: CloudUpstreamWarning[]; - conflicts: CloudUpstreamConflict[]; - generatedAt: string; -} - -export interface CloudUpstreamRunEvent { - id: string; - at: string; - phase: CloudUpstreamStep; - type: "created" | "updated" | "skipped" | "conflict" | "retrying" | "failed" | "completed"; - message: string; -} - -export interface CloudUpstreamRun { - id: string; - connectionId: string; - companyId: string; - status: CloudUpstreamRunStatus; - activeStep: CloudUpstreamStep; - progressPercent: number; - dryRun: boolean; - summary: CloudUpstreamSummaryCount[]; - warnings: CloudUpstreamWarning[]; - conflicts: CloudUpstreamConflict[]; - events: CloudUpstreamRunEvent[]; - targetUrl: string | null; - report: Record; - retryOfRunId: string | null; - createdAt: string; - updatedAt: string; - completedAt: string | null; -} - -export interface CloudUpstreamsState { - connections: CloudUpstreamConnection[]; - runs: CloudUpstreamRun[]; -} - -export interface CloudUpstreamConnectStartResponse { - pendingConnectionId: string; - authorizationUrl: string; - connection: CloudUpstreamConnection; -} diff --git a/packages/shared/src/types/company-portability.ts b/packages/shared/src/types/company-portability.ts index 1434ab733a..21134f3815 100644 --- a/packages/shared/src/types/company-portability.ts +++ b/packages/shared/src/types/company-portability.ts @@ -49,6 +49,39 @@ export interface CompanyPortabilitySidebarOrder { projects: string[]; } +export interface CompanyPortabilityLabelManifestEntry { + name: string; + color: string; +} + +export interface CompanyPortabilityBlobManifestEntry { + sha256: string; + byteSize: number; + contentType: string; +} + +export interface CompanyPortabilityEmbeddedAssetManifestEntry { + /** + * The asset id on the SOURCE board. Source ids are only meaningful as the + * rewrite key for the /api/assets//content references embedded in + * the bundle's markdown; the importer mints fresh asset ids and rewrites + * every reference to them. + */ + assetId: string; + /** Content address of the image bytes in the bundle's blobs/ store. */ + sha256: string; + contentType: string; + originalFilename: string | null; + /** + * Export categories whose files reference this asset ("agents", + * "projects", "skills", "tasks", "routines"), or "always" when an + * always-exported root file references it. Lets export selection follow + * the toggles of the referencing files. Absent entries are treated as + * always included. + */ + ownedBy?: string[]; +} + export interface CompanyPortabilityProjectManifestEntry { slug: string; name: string; @@ -107,6 +140,41 @@ export interface CompanyPortabilityIssueCommentManifestEntry { createdAt: string | null; } +export interface CompanyPortabilityIssueDocumentManifestEntry { + key: string; + title: string | null; + format: string; + path: string; +} + +export interface CompanyPortabilityIssueWorkProductManifestEntry { + type: string; + provider: string; + externalId: string | null; + title: string; + url: string | null; + status: string; + reviewState: string; + isPrimary: boolean; + healthStatus: string; + summary: string | null; + metadata: Record | null; +} + +export interface CompanyPortabilityIssueMonitorManifestEntry { + notes: string | null; + scheduledBy: string | null; + hadSchedule: boolean; +} + +export interface CompanyPortabilityIssueAttachmentManifestEntry { + sha256: string; + contentType: string; + originalFilename: string | null; + byteSize: number; + commentIndex: number | null; +} + export interface CompanyPortabilityIssueManifestEntry { slug: string; identifier: string | null; @@ -122,10 +190,16 @@ export interface CompanyPortabilityIssueManifestEntry { status: string | null; priority: string | null; labelIds: string[]; + labelNames?: string[]; billingCode: string | null; executionWorkspaceSettings: Record | null; assigneeAdapterOverrides: Record | null; comments: CompanyPortabilityIssueCommentManifestEntry[]; + blockedBy?: string[]; + documents?: CompanyPortabilityIssueDocumentManifestEntry[]; + workProducts?: CompanyPortabilityIssueWorkProductManifestEntry[]; + monitor?: CompanyPortabilityIssueMonitorManifestEntry | null; + attachments?: CompanyPortabilityIssueAttachmentManifestEntry[]; metadata: Record | null; } @@ -181,6 +255,9 @@ export interface CompanyPortabilityManifest { includes: CompanyPortabilityInclude; company: CompanyPortabilityCompanyManifestEntry | null; sidebar: CompanyPortabilitySidebarOrder | null; + labels?: CompanyPortabilityLabelManifestEntry[]; + blobs?: CompanyPortabilityBlobManifestEntry[]; + embeddedAssets?: CompanyPortabilityEmbeddedAssetManifestEntry[]; agents: CompanyPortabilityAgentManifestEntry[]; skills: CompanyPortabilitySkillManifestEntry[]; projects: CompanyPortabilityProjectManifestEntry[]; @@ -302,6 +379,7 @@ export interface CompanyPortabilityAdapterOverride { export interface CompanyPortabilityImportRequest extends CompanyPortabilityPreviewRequest { adapterOverrides?: Record; secretValues?: Record; + pauseAutomations?: boolean; } export interface CompanyPortabilityImportResult { @@ -324,6 +402,13 @@ export interface CompanyPortabilityImportResult { name: string; reason: string | null; }[]; + routines: { + slug: string; + id: string | null; + action: "created"; + title: string; + status: string; + }[]; envInputs: CompanyPortabilityEnvInput[]; warnings: string[]; } diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index f35cc14894..95ef134711 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -814,6 +814,9 @@ export type { CompanyPortabilityFileEntry, CompanyPortabilityCompanyManifestEntry, CompanyPortabilitySidebarOrder, + CompanyPortabilityLabelManifestEntry, + CompanyPortabilityBlobManifestEntry, + CompanyPortabilityEmbeddedAssetManifestEntry, CompanyPortabilityAgentManifestEntry, CompanyPortabilitySkillManifestEntry, CompanyPortabilityProjectManifestEntry, @@ -821,6 +824,10 @@ export type { CompanyPortabilityIssueRoutineTriggerManifestEntry, CompanyPortabilityIssueRoutineManifestEntry, CompanyPortabilityIssueCommentManifestEntry, + CompanyPortabilityIssueDocumentManifestEntry, + CompanyPortabilityIssueWorkProductManifestEntry, + CompanyPortabilityIssueMonitorManifestEntry, + CompanyPortabilityIssueAttachmentManifestEntry, CompanyPortabilityIssueManifestEntry, CompanyPortabilityManifest, CompanyPortabilityExportResult, diff --git a/packages/shared/src/types/instance.ts b/packages/shared/src/types/instance.ts index 7ca3d07f11..a92eee8e3f 100644 --- a/packages/shared/src/types/instance.ts +++ b/packages/shared/src/types/instance.ts @@ -55,7 +55,6 @@ export interface InstanceExperimentalSettings { enableTaskWatchdogs: boolean; enableIssuePlanDecompositions: boolean; enableExperimentalFileViewer: boolean; - enableCloudSync: boolean; enableExternalObjects: boolean; enableSmokeLab: boolean; enableBuiltInAgents: boolean; diff --git a/packages/shared/src/validators/company-portability.ts b/packages/shared/src/validators/company-portability.ts index 8a1e97a6cc..8a86eb593f 100644 --- a/packages/shared/src/validators/company-portability.ts +++ b/packages/shared/src/validators/company-portability.ts @@ -57,6 +57,17 @@ export const portabilitySidebarOrderSchema = z.object({ projects: z.array(z.string().min(1)).default([]), }); +export const portabilityLabelManifestEntrySchema = z.object({ + name: z.string().min(1), + color: z.string().min(1), +}); + +export const portabilityBlobManifestEntrySchema = z.object({ + sha256: z.string().min(1), + byteSize: z.number().int().nonnegative(), + contentType: z.string().min(1), +}); + export const portabilityAgentManifestEntrySchema = z.object({ slug: z.string().min(1), name: z.string().min(1), @@ -151,6 +162,41 @@ export const portabilityIssueCommentManifestEntrySchema = z.object({ createdAt: z.string().datetime().nullable(), }); +export const portabilityIssueDocumentManifestEntrySchema = z.object({ + key: z.string().min(1), + title: z.string().nullable(), + format: z.string().min(1), + path: z.string().min(1), +}); + +export const portabilityIssueWorkProductManifestEntrySchema = z.object({ + type: z.string().min(1), + provider: z.string().min(1), + externalId: z.string().nullable(), + title: z.string().min(1), + url: z.string().nullable(), + status: z.string().min(1), + reviewState: z.string().min(1), + isPrimary: z.boolean().default(false), + healthStatus: z.string().min(1), + summary: z.string().nullable(), + metadata: z.record(z.string(), z.unknown()).nullable(), +}); + +export const portabilityIssueMonitorManifestEntrySchema = z.object({ + notes: z.string().nullable(), + scheduledBy: z.string().nullable(), + hadSchedule: z.boolean().default(false), +}); + +export const portabilityIssueAttachmentManifestEntrySchema = z.object({ + sha256: z.string().min(1), + contentType: z.string().min(1), + originalFilename: z.string().nullable(), + byteSize: z.number().int().nonnegative(), + commentIndex: z.number().int().nonnegative().nullable().default(null), +}); + export const portabilityIssueManifestEntrySchema = z.object({ slug: z.string().min(1), identifier: z.string().min(1).nullable(), @@ -166,10 +212,16 @@ export const portabilityIssueManifestEntrySchema = z.object({ status: z.string().nullable(), priority: z.string().nullable(), labelIds: z.array(z.string().min(1)).default([]), + labelNames: z.array(z.string().min(1)).default([]), billingCode: z.string().nullable(), executionWorkspaceSettings: z.record(z.string(), z.unknown()).nullable(), assigneeAdapterOverrides: z.record(z.string(), z.unknown()).nullable(), comments: z.array(portabilityIssueCommentManifestEntrySchema).default([]), + blockedBy: z.array(z.string().min(1)).default([]), + documents: z.array(portabilityIssueDocumentManifestEntrySchema).default([]), + workProducts: z.array(portabilityIssueWorkProductManifestEntrySchema).default([]), + monitor: portabilityIssueMonitorManifestEntrySchema.nullable().default(null), + attachments: z.array(portabilityIssueAttachmentManifestEntrySchema).default([]), metadata: z.record(z.string(), z.unknown()).nullable(), }); @@ -191,6 +243,8 @@ export const portabilityManifestSchema = z.object({ }), company: portabilityCompanyManifestEntrySchema.nullable(), sidebar: portabilitySidebarOrderSchema.nullable(), + labels: z.array(portabilityLabelManifestEntrySchema).default([]), + blobs: z.array(portabilityBlobManifestEntrySchema).default([]), agents: z.array(portabilityAgentManifestEntrySchema), skills: z.array(portabilitySkillManifestEntrySchema).default([]), projects: z.array(portabilityProjectManifestEntrySchema).default([]), @@ -262,6 +316,7 @@ export const portabilityAdapterOverrideSchema = z.object({ export const companyPortabilityImportSchema = companyPortabilityPreviewSchema.extend({ adapterOverrides: z.record(z.string().min(1), portabilityAdapterOverrideSchema).optional(), secretValues: z.record(z.string().min(1), z.string()).optional(), + pauseAutomations: z.boolean().optional(), }); export type CompanyPortabilityImport = z.infer; diff --git a/packages/shared/src/validators/instance.ts b/packages/shared/src/validators/instance.ts index 72c8725daa..52434de5bf 100644 --- a/packages/shared/src/validators/instance.ts +++ b/packages/shared/src/validators/instance.ts @@ -49,7 +49,6 @@ export const instanceExperimentalSettingsSchema = z.object({ enableTaskWatchdogs: z.boolean().default(false), enableIssuePlanDecompositions: z.boolean().default(false), enableExperimentalFileViewer: z.boolean().default(false), - enableCloudSync: z.boolean().default(false), enableExternalObjects: z.boolean().default(false), enableSmokeLab: z.boolean().default(false), enableBuiltInAgents: z.boolean().default(false), diff --git a/server/src/__tests__/cloud-upstreams.test.ts b/server/src/__tests__/cloud-upstreams.test.ts deleted file mode 100644 index fce78dfd1e..0000000000 --- a/server/src/__tests__/cloud-upstreams.test.ts +++ /dev/null @@ -1,334 +0,0 @@ -import { generateKeyPairSync, randomUUID } from "node:crypto"; -import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { companies, cloudUpstreamConnections, cloudUpstreamRuns, companySkills, createDb } from "@paperclipai/db"; - -import { HttpError } from "../errors.js"; -import { - cloudUpstreamRemoteFailureReport, - cloudUpstreamService, - reconcileCloudUpstreamRunsOnStartup, - sealCloudUpstreamCredential, - unsealCloudUpstreamCredential, -} from "../services/cloud-upstreams.js"; -import { - getEmbeddedPostgresTestSupport, - startEmbeddedPostgresTestDatabase, -} from "./helpers/embedded-postgres.js"; - -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); -const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; - -if (!embeddedPostgresSupport.supported) { - console.warn( - `Skipping embedded Postgres cloud upstream tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, - ); -} - -describe("cloud upstream remote failures", () => { - it("preserves the cloud response body and message on run reports", () => { - const body = { - error: "bad_request", - message: "entities[42].body must be an object", - errors: [{ path: "entities[42].body" }], - }; - - expect(cloudUpstreamRemoteFailureReport(new HttpError(400, "bad_request", body))).toEqual({ - error: "bad_request", - errorMessage: "entities[42].body must be an object", - details: body, - }); - }); - - it("falls back to the thrown error message for non-remote failures", () => { - expect(cloudUpstreamRemoteFailureReport(new Error("network failed"))).toEqual({ - error: "network failed", - }); - }); -}); - -describe("cloud upstream credential storage", () => { - const previousMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; - - afterEach(() => { - if (previousMasterKey === undefined) { - delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; - } else { - process.env.PAPERCLIP_SECRETS_MASTER_KEY = previousMasterKey; - } - }); - - it("stores new credentials as encrypted envelopes and preserves legacy plaintext reads", async () => { - process.env.PAPERCLIP_SECRETS_MASTER_KEY = "12345678901234567890123456789012"; - const sealed = await sealCloudUpstreamCredential("cloud-access-token"); - - expect(sealed).toMatch(/^paperclip-cloud-credential:/); - expect(sealed).not.toContain("cloud-access-token"); - await expect(unsealCloudUpstreamCredential(sealed)).resolves.toBe("cloud-access-token"); - await expect(unsealCloudUpstreamCredential("legacy-plaintext-token")).resolves.toBe("legacy-plaintext-token"); - }); -}); - -describeEmbeddedPostgres("cloud upstream persistence", () => { - let db!: ReturnType; - let tempDb: Awaited> | null = null; - const previousMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; - - beforeAll(async () => { - process.env.PAPERCLIP_SECRETS_MASTER_KEY = "12345678901234567890123456789012"; - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-cloud-upstreams-"); - db = createDb(tempDb.connectionString); - }, 20_000); - - afterEach(async () => { - vi.restoreAllMocks(); - await db.delete(cloudUpstreamRuns); - await db.delete(cloudUpstreamConnections); - await db.delete(companySkills); - await db.delete(companies); - }); - - afterAll(async () => { - if (previousMasterKey === undefined) { - delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; - } else { - process.env.PAPERCLIP_SECRETS_MASTER_KEY = previousMasterKey; - } - await tempDb?.cleanup(); - }); - - it("encrypts stored upstream credentials while keeping connection flows usable", async () => { - const companyId = randomUUID(); - await seedCompany(companyId); - const tokenUrl = "https://cloud.example.test/oauth/token"; - vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { - const url = String(input); - if (url.startsWith("https://cloud.example.test/.well-known/paperclip-upstream")) { - return jsonResponse({ - product: "Paperclip Cloud", - stack: { - id: "stack-1", - companyId: "cloud-company-1", - origin: "https://cloud.example.test", - primaryHost: "cloud.example.test", - }, - transfer: { - supportedSchemaMajor: 1, - maxChunkBytes: 8192, - }, - auth: { - scopes: ["upstream_import:write"], - pkce: { - authorizeUrl: "https://cloud.example.test/oauth/authorize", - tokenUrl, - }, - }, - }); - } - if (url === tokenUrl && init?.method === "POST") { - const payload = JSON.parse(String(init.body)); - expect(payload.codeVerifier).toEqual(expect.any(String)); - expect(payload.codeVerifier).not.toContain("paperclip-cloud-credential:"); - return jsonResponse({ - accessToken: "cloud-access-token", - token: { - id: "token-1", - expiresAt: "2026-05-22T13:00:00.000Z", - globalUserId: "user-1", - }, - }); - } - throw new Error(`Unexpected fetch: ${url}`); - }); - - const service = cloudUpstreamService(db, { instanceId: "test" }); - const started = await service.startConnect({ - companyId, - remoteUrl: "https://cloud.example.test", - redirectUri: "http://localhost:3100/callback", - }); - await service.finishConnect({ - pendingConnectionId: started.pendingConnectionId, - code: "auth-code", - state: new URL(started.authorizationUrl).searchParams.get("state") ?? "", - }); - - const [row] = await db.select().from(cloudUpstreamConnections); - expect(row.privateKeyPem).toMatch(/^paperclip-cloud-credential:/); - expect(row.privateKeyPem).not.toContain("BEGIN PRIVATE KEY"); - expect(row.accessToken).toMatch(/^paperclip-cloud-credential:/); - expect(row.accessToken).not.toContain("cloud-access-token"); - }); - - it("marks orphaned running runs failed during startup reconciliation", async () => { - const companyId = randomUUID(); - const connectionId = randomUUID(); - const runningRunId = randomUUID(); - const succeededRunId = randomUUID(); - const reconciledAt = new Date("2026-05-22T13:00:00.000Z"); - await seedCompany(companyId); - await db.insert(cloudUpstreamConnections).values({ - id: connectionId, - companyId, - remoteUrl: "https://cloud.example.test", - sourceInstanceId: "source-1", - sourceInstanceFingerprint: "sha256:test", - sourcePublicKey: "public-key", - privateKeyPem: "legacy-private-key", - tokenStatus: "connected", - scopes: ["upstream_import:write"], - authorizedGlobalUserId: "user-1", - accessToken: "legacy-token", - tokenId: "token-1", - targetStackId: "stack-1", - targetCompanyId: "cloud-company-1", - targetOrigin: "https://cloud.example.test", - targetPrimaryHost: "cloud.example.test", - targetProduct: "Paperclip Cloud", - targetSchemaMajor: 1, - targetMaxChunkBytes: 8192, - }); - await db.insert(cloudUpstreamRuns).values([ - cloudRunRow({ id: runningRunId, connectionId, companyId, status: "running" }), - cloudRunRow({ id: succeededRunId, connectionId, companyId, status: "succeeded", completedAt: reconciledAt }), - ]); - - await expect(reconcileCloudUpstreamRunsOnStartup(db, reconciledAt)).resolves.toEqual({ reconciled: 1 }); - - const rows = await db.select().from(cloudUpstreamRuns); - const running = rows.find((row) => row.id === runningRunId); - const succeeded = rows.find((row) => row.id === succeededRunId); - expect(running?.status).toBe("failed"); - expect(running?.completedAt?.toISOString()).toBe(reconciledAt.toISOString()); - expect(running?.events.at(-1)?.message).toContain("server startup"); - expect(running?.report).toMatchObject({ - error: "orphaned_running_run", - reconciledAt: reconciledAt.toISOString(), - }); - expect(succeeded?.status).toBe("succeeded"); - }); - - it("rejects a new run when the connection already has a running run", async () => { - const companyId = randomUUID(); - const connectionId = randomUUID(); - const runningRunId = randomUUID(); - await seedCompany(companyId); - await db.insert(cloudUpstreamConnections).values(cloudConnectionRow({ id: connectionId, companyId })); - await db.insert(cloudUpstreamRuns).values( - cloudRunRow({ id: runningRunId, connectionId, companyId, status: "running" }), - ); - - await expect(cloudUpstreamService(db).createRun({ connectionId, companyId })).rejects.toMatchObject({ - status: 409, - details: { runId: runningRunId }, - }); - }); - - it("preserves a cancelled run when an in-flight createRun tries to finish", async () => { - const companyId = randomUUID(); - const connectionId = randomUUID(); - await seedCompany(companyId); - await db.insert(cloudUpstreamConnections).values(cloudConnectionRow({ id: connectionId, companyId })); - - const service = cloudUpstreamService(db); - const remoteCalls: string[] = []; - globalThis.fetch = vi.fn(async (input) => { - const path = new URL(String(input)).pathname; - remoteCalls.push(path); - if (path.endsWith("/upstream-imports/runs")) { - return jsonResponse({ run: { id: "remote-run-1" } }); - } - if (path.endsWith("/chunks")) { - const run = await db.select().from(cloudUpstreamRuns).then((rows) => rows[0]); - expect(run?.status).toBe("running"); - await service.cancelRun(connectionId, run.id, companyId); - return jsonResponse({ ok: true }); - } - if (path.endsWith("/cancel")) { - return jsonResponse({ ok: true }); - } - if (path.endsWith("/apply")) { - return jsonResponse({ ok: true }); - } - if (path.endsWith("/events")) { - return jsonResponse({ events: [] }); - } - return jsonResponse({ error: "not_found" }, 404); - }) as typeof fetch; - - const result = await service.createRun({ connectionId, companyId }); - - expect(result.status).toBe("cancelled"); - expect(remoteCalls.some((path) => path.endsWith("/apply"))).toBe(false); - const rows = await db.select().from(cloudUpstreamRuns); - expect(rows).toHaveLength(1); - expect(rows[0]?.status).toBe("cancelled"); - }); - - async function seedCompany(companyId: string) { - await db.insert(companies).values({ - id: companyId, - name: "Paperclip", - issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, - requireBoardApprovalForNewAgents: false, - }); - } -}); - -function jsonResponse(body: unknown): Response { - return new Response(JSON.stringify(body), { - status: 200, - headers: { "Content-Type": "application/json" }, - }); -} - -function cloudConnectionRow(input: { id: string; companyId: string }) { - const { privateKey } = generateKeyPairSync("ed25519"); - return { - id: input.id, - companyId: input.companyId, - remoteUrl: "https://cloud.example.test", - sourceInstanceId: "source-1", - sourceInstanceFingerprint: "sha256:test", - sourcePublicKey: "public-key", - privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(), - tokenStatus: "connected", - scopes: ["upstream_import:write"], - authorizedGlobalUserId: "user-1", - accessToken: "legacy-token", - tokenId: "token-1", - targetStackId: "stack-1", - targetCompanyId: "cloud-company-1", - targetOrigin: "https://cloud.example.test", - targetPrimaryHost: "cloud.example.test", - targetProduct: "Paperclip Cloud", - targetSchemaMajor: 1, - targetMaxChunkBytes: 8192, - }; -} - -function cloudRunRow(input: { - id: string; - connectionId: string; - companyId: string; - status: string; - completedAt?: Date; -}) { - return { - id: input.id, - connectionId: input.connectionId, - companyId: input.companyId, - status: input.status, - activeStep: "push", - progressPercent: input.status === "running" ? 45 : 100, - dryRun: false, - summary: [], - warnings: [], - conflicts: [], - events: [], - report: {}, - idempotencyKey: `key-${input.id}`, - manifestHash: `sha256:${input.id.replace(/-/g, "")}`, - targetUrl: "https://cloud.example.test", - completedAt: input.completedAt, - }; -} diff --git a/server/src/__tests__/companies-route-cross-company-authz.test.ts b/server/src/__tests__/companies-route-cross-company-authz.test.ts index d6aee5b30c..31f5f73cfc 100644 --- a/server/src/__tests__/companies-route-cross-company-authz.test.ts +++ b/server/src/__tests__/companies-route-cross-company-authz.test.ts @@ -253,6 +253,10 @@ describe.sequential("company route cross-company authorization", () => { label: "POST /api/companies/:companyId/exports/preview", request: (app: express.Express) => request(app).post(`/api/companies/${companyBId}/exports/preview`).send(exportRequest), }, + { + label: "GET /api/companies/:companyId/export/fidelity", + request: (app: express.Express) => request(app).get(`/api/companies/${companyBId}/export/fidelity`), + }, { label: "POST /api/companies/:companyId/imports/preview", request: (app: express.Express) => request(app).post(`/api/companies/${companyBId}/imports/preview`).send(importRequest()), diff --git a/server/src/__tests__/companies-route-path-guard.test.ts b/server/src/__tests__/companies-route-path-guard.test.ts index 6ae4391f83..83f1d70c92 100644 --- a/server/src/__tests__/companies-route-path-guard.test.ts +++ b/server/src/__tests__/companies-route-path-guard.test.ts @@ -38,6 +38,9 @@ vi.mock("../services/index.js", () => ({ getFeedbackTraceById: vi.fn(), saveIssueVote: vi.fn(), }), + instanceSettingsService: () => ({ + getExperimental: vi.fn(), + }), logActivity: vi.fn(), })); diff --git a/server/src/__tests__/company-branding-route.test.ts b/server/src/__tests__/company-branding-route.test.ts index 40a9a692ee..6c73987595 100644 --- a/server/src/__tests__/company-branding-route.test.ts +++ b/server/src/__tests__/company-branding-route.test.ts @@ -51,6 +51,9 @@ vi.mock("../services/index.js", () => ({ companyPortabilityService: () => mockCompanyPortabilityService, companyService: () => mockCompanyService, feedbackService: () => mockFeedbackService, + instanceSettingsService: () => ({ + getExperimental: vi.fn(), + }), logActivity: mockLogActivity, })); diff --git a/server/src/__tests__/company-portability-routes.test.ts b/server/src/__tests__/company-portability-routes.test.ts index 4198c10091..088d38b1d4 100644 --- a/server/src/__tests__/company-portability-routes.test.ts +++ b/server/src/__tests__/company-portability-routes.test.ts @@ -43,6 +43,10 @@ const mockFeedbackService = vi.hoisted(() => ({ saveIssueVote: vi.fn(), })); +const mockInstanceSettingsService = vi.hoisted(() => ({ + getExperimental: vi.fn(), +})); + vi.mock("../services/access.js", () => ({ accessService: () => mockAccessService, })); @@ -79,6 +83,7 @@ vi.mock("../services/index.js", () => ({ companyPortabilityService: () => mockCompanyPortabilityService, companyService: () => mockCompanyService, feedbackService: () => mockFeedbackService, + instanceSettingsService: () => mockInstanceSettingsService, logActivity: mockLogActivity, })); @@ -91,6 +96,7 @@ function registerCompanyRouteMocks() { companyPortabilityService: () => mockCompanyPortabilityService, companyService: () => mockCompanyService, feedbackService: () => mockFeedbackService, + instanceSettingsService: () => mockInstanceSettingsService, logActivity: mockLogActivity, })); } @@ -635,7 +641,7 @@ describe.sequential("company portability routes", () => { expect(accepted.body.statusUrl).toMatch(/^\/api\/companies\/import\/jobs\/tenant-import-/); expect(accepted.body.retryAfterMs).toBe(1000); await waitForCondition(() => mockCompanyPortabilityService.importBundle.mock.calls.length === 1, "import job start"); - expect(mockCompanyPortabilityService.importBundle).toHaveBeenCalledWith(importRequest, "cloud-user-1"); + expect(mockCompanyPortabilityService.importBundle).toHaveBeenCalledWith(importRequest, "cloud-user-1", { pauseAutomations: false }); expect(mockLogActivity).not.toHaveBeenCalled(); resolveImport(createImportResult("updated")); @@ -721,10 +727,49 @@ describe.sequential("company portability routes", () => { expect(res.body.company.id).toBe(companyId); expect(res.body.company.action).toBe("created"); expect(res.body.job).toBeUndefined(); - expect(mockCompanyPortabilityService.importBundle).toHaveBeenCalledWith(importRequest, "cloud-user-1"); + expect(mockCompanyPortabilityService.importBundle).toHaveBeenCalledWith(importRequest, "cloud-user-1", { pauseAutomations: false }); expect(mockLogActivity).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ action: "company.imported", companyId, })); }); + + it.sequential("forwards pauseAutomations from the global import body to the portability service", async () => { + mockCompanyPortabilityService.importBundle.mockResolvedValueOnce(createImportResult("created")); + const app = await createApp(cloudTenantActor()); + + const res = await request(app) + .post("/api/companies/import") + .set(cloudHeaders) + .send({ ...importRequest, pauseAutomations: true }); + + expect(res.status).toBe(200); + expect(mockCompanyPortabilityService.importBundle).toHaveBeenCalledWith( + { ...importRequest, pauseAutomations: true }, + "cloud-user-1", + { pauseAutomations: true }, + ); + }); + + it.sequential("forwards pauseAutomations from CEO-safe import apply bodies to the portability service", async () => { + mockCompanyPortabilityService.importBundle.mockResolvedValueOnce(createImportResult("created")); + const app = await createApp({ + type: "agent", + agentId: ceoAgentId, + companyId, + source: "agent_key", + runId: "run-1", + }); + + const res = await request(app) + .post(`/api/companies/${companyId}/imports/apply`) + .send({ ...importRequest, pauseAutomations: true }); + + expect(res.status).toBe(200); + expect(mockCompanyPortabilityService.importBundle).toHaveBeenCalledWith( + { ...importRequest, pauseAutomations: true }, + null, + { mode: "agent_safe", sourceCompanyId: companyId, pauseAutomations: true }, + ); + }); }); diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index b7f76b7ff8..3dd5d0110d 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -1,4 +1,5 @@ import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { promises as fs } from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -41,6 +42,21 @@ const issueSvc = { getByIdentifier: vi.fn(), create: vi.fn(), addComment: vi.fn(), + listLabels: vi.fn(), + createLabel: vi.fn(), + getRelationSummaries: vi.fn(), + listAttachments: vi.fn(), + createAttachment: vi.fn(), +}; + +const documentSvc = { + listIssueDocuments: vi.fn(), + upsertIssueDocument: vi.fn(), +}; + +const workProductSvc = { + listForIssue: vi.fn(), + createForIssue: vi.fn(), }; const routineSvc = { @@ -96,6 +112,18 @@ vi.mock("../services/issues.js", () => ({ issueService: () => issueSvc, })); +vi.mock("../services/documents.js", () => ({ + documentService: () => documentSvc, + extractLegacyPlanBody: () => null, + mapIssueDocumentRow: (row: unknown) => row, + issueDocumentSelect: {}, +})); + +vi.mock("../services/work-products.js", () => ({ + workProductService: () => workProductSvc, + toIssueWorkProduct: (row: unknown) => row, +})); + vi.mock("../services/routines.js", () => ({ routineService: () => routineSvc, })); @@ -244,6 +272,20 @@ describe("company portability", () => { issueSvc.list.mockResolvedValue([]); issueSvc.getById.mockResolvedValue(null); issueSvc.getByIdentifier.mockResolvedValue(null); + issueSvc.listLabels.mockResolvedValue([]); + issueSvc.createLabel.mockImplementation(async (_companyId: string, data: { name: string; color: string }) => ({ + id: `label-created-${data.name}`, + companyId: "company-imported", + name: data.name, + color: data.color, + })); + issueSvc.getRelationSummaries.mockResolvedValue({ blockedBy: [], blocks: [] }); + issueSvc.listAttachments.mockResolvedValue([]); + issueSvc.createAttachment.mockResolvedValue({ id: "attachment-imported" }); + documentSvc.listIssueDocuments.mockResolvedValue([]); + documentSvc.upsertIssueDocument.mockResolvedValue({ created: true }); + workProductSvc.listForIssue.mockResolvedValue([]); + workProductSvc.createForIssue.mockResolvedValue({ id: "work-product-imported" }); routineSvc.list.mockResolvedValue([]); routineSvc.getDetail.mockImplementation(async (id: string) => { const rows = await routineSvc.list(); @@ -2331,6 +2373,148 @@ describe("company portability", () => { replayWindowSec: 120, }), expect.any(Object)); expect(issueSvc.create).not.toHaveBeenCalled(); + expect(result.routines).toEqual([ + { slug: "monday-review", id: "routine-created", action: "created", title: "Monday Review", status: "paused" }, + ]); + }); + + it("pauses imported agents and routines when pauseAutomations is requested", async () => { + const portability = companyPortabilityService({} as any); + + companySvc.create.mockResolvedValue({ + id: "company-imported", + name: "Imported Paperclip", + }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.create.mockResolvedValue({ + id: "agent-created", + name: "ClaudeCoder", + status: "paused", + }); + projectSvc.create.mockResolvedValue({ + id: "project-created", + name: "Launch", + urlKey: "launch", + }); + agentSvc.list.mockResolvedValue([]); + projectSvc.list.mockResolvedValue([]); + + const files = { + "COMPANY.md": ['---', 'schema: "agentcompanies/v1"', 'name: "Imported Paperclip"', "---", ""].join("\n"), + "agents/claudecoder/AGENTS.md": ['---', 'name: "ClaudeCoder"', "---", "", "You write code.", ""].join("\n"), + "projects/launch/PROJECT.md": ['---', 'name: "Launch"', "---", ""].join("\n"), + "tasks/monday-review/TASK.md": [ + "---", + 'name: "Monday Review"', + 'project: "launch"', + 'assignee: "claudecoder"', + "recurring: true", + "---", + "", + "Review pipeline health.", + "", + ].join("\n"), + ".paperclip.yaml": [ + 'schema: "paperclip/v1"', + "routines:", + " monday-review:", + " triggers:", + " - kind: schedule", + ' cronExpression: "0 9 * * 1"', + ' timezone: "America/Chicago"', + "", + ].join("\n"), + }; + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: "paperclip-demo", files }, + include: { company: true, agents: true, projects: true, issues: true, skills: false }, + target: { mode: "new_company", newCompanyName: "Imported Paperclip" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1", { pauseAutomations: true }); + + expect(agentSvc.create).toHaveBeenCalledWith("company-imported", expect.objectContaining({ + status: "paused", + pauseReason: "system", + pausedAt: expect.any(Date), + })); + expect(routineSvc.create).toHaveBeenCalledWith("company-imported", expect.objectContaining({ + title: "Monday Review", + status: "paused", + }), expect.any(Object)); + expect(result.routines).toEqual([ + { slug: "monday-review", id: "routine-created", action: "created", title: "Monday Review", status: "paused" }, + ]); + }); + + it("leaves imported agents and routines active when pauseAutomations is absent", async () => { + const portability = companyPortabilityService({} as any); + + companySvc.create.mockResolvedValue({ + id: "company-imported", + name: "Imported Paperclip", + }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.create.mockResolvedValue({ + id: "agent-created", + name: "ClaudeCoder", + }); + projectSvc.create.mockResolvedValue({ + id: "project-created", + name: "Launch", + urlKey: "launch", + }); + agentSvc.list.mockResolvedValue([]); + projectSvc.list.mockResolvedValue([]); + + const files = { + "COMPANY.md": ['---', 'schema: "agentcompanies/v1"', 'name: "Imported Paperclip"', "---", ""].join("\n"), + "agents/claudecoder/AGENTS.md": ['---', 'name: "ClaudeCoder"', "---", "", "You write code.", ""].join("\n"), + "projects/launch/PROJECT.md": ['---', 'name: "Launch"', "---", ""].join("\n"), + "tasks/monday-review/TASK.md": [ + "---", + 'name: "Monday Review"', + 'project: "launch"', + 'assignee: "claudecoder"', + "recurring: true", + "---", + "", + "Review pipeline health.", + "", + ].join("\n"), + ".paperclip.yaml": [ + 'schema: "paperclip/v1"', + "routines:", + " monday-review:", + " triggers:", + " - kind: schedule", + ' cronExpression: "0 9 * * 1"', + ' timezone: "America/Chicago"', + "", + ].join("\n"), + }; + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: "paperclip-demo", files }, + include: { company: true, agents: true, projects: true, issues: true, skills: false }, + target: { mode: "new_company", newCompanyName: "Imported Paperclip" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + expect(agentSvc.create).toHaveBeenCalledTimes(1); + const [, createdAgentInput] = agentSvc.create.mock.calls[0]!; + expect(createdAgentInput.status).toBe("idle"); + expect(createdAgentInput.pauseReason).toBeUndefined(); + expect(createdAgentInput.pausedAt).toBeUndefined(); + expect(routineSvc.create).toHaveBeenCalledWith("company-imported", expect.objectContaining({ + title: "Monday Review", + status: "active", + }), expect.any(Object)); + expect(result.routines).toEqual([ + { slug: "monday-review", id: "routine-created", action: "created", title: "Monday Review", status: "active" }, + ]); }); it("migrates legacy schedule.recurrence imports into routine triggers", async () => { @@ -2407,33 +2591,51 @@ describe("company portability", () => { expect(issueSvc.create).not.toHaveBeenCalled(); }); - it("flags recurring task imports that are missing routine-required fields", async () => { + it("imports recurring tasks without a project or assignee as paused routines", async () => { const portability = companyPortabilityService({} as any); - const preview = await portability.previewImport({ - source: { - type: "inline", - rootPath: "paperclip-demo", - files: { - "COMPANY.md": ['---', 'schema: "agentcompanies/v1"', 'name: "Imported Paperclip"', "---", ""].join("\n"), - "tasks/monday-review/TASK.md": [ - "---", - 'name: "Monday Review"', - "recurring: true", - "---", - "", - "Review pipeline health.", - "", - ].join("\n"), - }, - }, - include: { company: true, agents: false, projects: false, issues: true, skills: false }, - target: { mode: "new_company", newCompanyName: "Imported Paperclip" }, - collisionStrategy: "rename", + companySvc.create.mockResolvedValue({ + id: "company-imported", + name: "Imported Paperclip", }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + projectSvc.list.mockResolvedValue([]); - expect(preview.errors).toContain("Recurring task monday-review must declare a project to import as a routine."); - expect(preview.errors).toContain("Recurring task monday-review must declare an assignee to import as a routine."); + const files = { + "COMPANY.md": ['---', 'schema: "agentcompanies/v1"', 'name: "Imported Paperclip"', "---", ""].join("\n"), + "tasks/monday-review/TASK.md": [ + "---", + 'name: "Monday Review"', + "recurring: true", + "---", + "", + "Review pipeline health.", + "", + ].join("\n"), + }; + const request = { + source: { type: "inline" as const, rootPath: "paperclip-demo", files }, + include: { company: true, agents: false, projects: false, issues: true, skills: false }, + target: { mode: "new_company" as const, newCompanyName: "Imported Paperclip" }, + collisionStrategy: "rename" as const, + }; + + const preview = await portability.previewImport(request); + expect(preview.errors).toEqual([]); + expect(preview.warnings).toContain( + "Recurring task monday-review has no assignee; the routine will stay paused until one is set.", + ); + + const result = await portability.importBundle(request, "user-1"); + expect(routineSvc.create).toHaveBeenCalledWith("company-imported", expect.objectContaining({ + projectId: null, + assigneeAgentId: null, + title: "Monday Review", + }), expect.any(Object)); + expect(result.warnings).toContain( + "Routine monday-review was imported without an assignee and will stay paused until one is set.", + ); }); it("imports a vendor-neutral package without .paperclip.yaml", async () => { @@ -3169,7 +3371,7 @@ describe("company portability", () => { expect(lastCreateInput.adapterConfig?.dangerouslyBypassApprovalsAndSandbox).toBeUndefined(); }); - it("preserves issue labelIds through export and import round-trip", async () => { + it("carries labels by name through export and import round-trip", async () => { const portability = companyPortabilityService({} as any); projectSvc.list.mockResolvedValue([ @@ -3202,15 +3404,31 @@ describe("company portability", () => { assigneeAdapterOverrides: null, }, ]); + issueSvc.listLabels.mockResolvedValueOnce([ + { id: "label-a", companyId: "company-1", name: "bug", color: "#ff0000" }, + { id: "label-b", companyId: "company-1", name: "urgent", color: "#00ff00" }, + ]); const exported = await portability.exportBundle("company-1", { include: { company: true, agents: false, projects: true, issues: true }, }); const extension = asTextFile(exported.files[".paperclip.yaml"]); - expect(extension).toContain("labelIds:"); - expect(extension).toContain("label-a"); - expect(extension).toContain("label-b"); + expect(extension).toContain("labels:"); + expect(extension).toContain('"bug"'); + expect(extension).toContain('"urgent"'); + expect(extension).toContain('"#ff0000"'); + expect(extension).toContain('"#00ff00"'); + expect(extension).not.toContain("labelIds"); + expect(extension).not.toContain("label-a"); + // Fresh exports declare the current bundle shape end-to-end. + expect(extension).toContain("schemaVersion: 6"); + expect(exported.manifest.schemaVersion).toBe(6); + expect(exported.manifest.labels).toEqual([ + { name: "bug", color: "#ff0000" }, + { name: "urgent", color: "#00ff00" }, + ]); + expect(exported.manifest.issues[0]?.labelNames).toEqual(["bug", "urgent"]); companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported" }); accessSvc.ensureMembership.mockResolvedValue(undefined); @@ -3218,8 +3436,9 @@ describe("company portability", () => { projectSvc.list.mockResolvedValue([]); projectSvc.create.mockResolvedValue({ id: "project-imported", name: "Launch", urlKey: "launch" }); issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Labelled task" }); + issueSvc.listLabels.mockResolvedValueOnce([]); - await portability.importBundle({ + const result = await portability.importBundle({ source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, include: { company: true, agents: false, projects: true, issues: true }, target: { mode: "new_company", newCompanyName: "Imported" }, @@ -3227,14 +3446,1050 @@ describe("company portability", () => { collisionStrategy: "rename", }, "user-1"); + expect(result.warnings.some((warning) => warning.includes("predates"))).toBe(false); + expect(issueSvc.createLabel).toHaveBeenCalledWith("company-imported", { name: "bug", color: "#ff0000" }); + expect(issueSvc.createLabel).toHaveBeenCalledWith("company-imported", { name: "urgent", color: "#00ff00" }); + expect(issueSvc.createLabel).toHaveBeenCalledTimes(2); expect(issueSvc.create).toHaveBeenCalledWith( "company-imported", expect.objectContaining({ - labelIds: ["label-a", "label-b"], + labelIds: ["label-created-bug", "label-created-urgent"], }), ); }); + it("reuses existing target labels on name collision and keeps the target color", async () => { + const portability = companyPortabilityService({} as any); + + projectSvc.list.mockResolvedValue([]); + projectSvc.listWorkspaces.mockResolvedValue([]); + issueSvc.list.mockResolvedValue([ + { + id: "issue-1", + identifier: "PAP-1", + title: "Labelled task", + description: null, + projectId: null, + projectWorkspaceId: null, + assigneeAgentId: null, + status: "todo", + priority: "medium", + labelIds: ["label-a", "label-b"], + billingCode: null, + executionWorkspaceSettings: null, + assigneeAdapterOverrides: null, + }, + ]); + issueSvc.listLabels.mockResolvedValueOnce([ + { id: "label-a", companyId: "company-1", name: "bug", color: "#ff0000" }, + { id: "label-b", companyId: "company-1", name: "urgent", color: "#00ff00" }, + ]); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Labelled task" }); + // Import target already has a "bug" label with a different color. + issueSvc.listLabels.mockResolvedValueOnce([ + { id: "target-bug", companyId: "company-1", name: "bug", color: "#123456" }, + ]); + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: false, agents: false, projects: false, issues: true }, + target: { mode: "existing_company", companyId: "company-1" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + expect(issueSvc.createLabel).toHaveBeenCalledTimes(1); + expect(issueSvc.createLabel).toHaveBeenCalledWith("company-1", { name: "urgent", color: "#00ff00" }); + expect(issueSvc.create).toHaveBeenCalledWith( + "company-1", + expect.objectContaining({ + labelIds: ["target-bug", "label-created-urgent"], + }), + ); + expect(result.warnings).toContain( + "Existing label color was kept for bug; the imported bundle used different colors.", + ); + }); + + it("drops unresolvable raw labelIds from old bundles with a warning instead of failing", async () => { + const portability = companyPortabilityService({} as any); + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Legacy Import" }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Kickoff" }); + + const result = await portability.importBundle({ + source: { + type: "inline", + rootPath: "legacy-package", + files: { + "COMPANY.md": [ + "---", + 'schema: "agentcompanies/v1"', + 'name: "Legacy Import"', + "---", + "", + ].join("\n"), + "tasks/kickoff/TASK.md": [ + "---", + 'name: "Kickoff"', + "---", + "", + "Legacy labelled task.", + "", + ].join("\n"), + ".paperclip.yaml": [ + 'schema: "paperclip/v1"', + "tasks:", + " kickoff:", + ' status: "todo"', + " labelIds:", + ' - "0a45b7de-9fb1-4c94-9c9d-3f61c2ab0001"', + ' - "0a45b7de-9fb1-4c94-9c9d-3f61c2ab0002"', + "", + ].join("\n"), + }, + }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Legacy Import" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + expect(issueSvc.createLabel).not.toHaveBeenCalled(); + expect(issueSvc.create).toHaveBeenCalledWith( + "company-imported", + expect.objectContaining({ labelIds: [] }), + ); + expect(result.warnings).toContain( + "Task kickoff dropped 2 label references because the bundle carries raw label ids that do not exist in the target company.", + ); + }); + + function mockTaskFidelityExportSources() { + projectSvc.list.mockResolvedValue([]); + projectSvc.listWorkspaces.mockResolvedValue([]); + issueSvc.list.mockResolvedValue([ + { + id: "issue-1", + identifier: "PAP-1", + title: "Alpha task", + description: "Carries documents, work products, and a monitor", + projectId: null, + projectWorkspaceId: null, + assigneeAgentId: null, + status: "todo", + priority: "high", + labelIds: [], + billingCode: null, + executionWorkspaceSettings: null, + assigneeAdapterOverrides: null, + monitorNotes: "Check deploy daily", + monitorScheduledBy: "agent", + monitorNextCheckAt: new Date("2026-07-01T00:00:00.000Z"), + }, + { + id: "issue-2", + identifier: "PAP-2", + title: "Beta task", + description: "Blocked by Alpha", + projectId: null, + projectWorkspaceId: null, + assigneeAgentId: null, + status: "todo", + priority: "medium", + labelIds: [], + billingCode: null, + executionWorkspaceSettings: null, + assigneeAdapterOverrides: null, + }, + ]); + const relationSummary = (id: string, identifier: string, title: string) => ({ + id, + identifier, + title, + status: "todo", + priority: "medium", + assigneeAgentId: null, + assigneeUserId: null, + }); + issueSvc.getRelationSummaries.mockImplementation(async (issueId: string) => { + if (issueId === "issue-1") { + return { + blockedBy: [relationSummary("issue-outside", "PAP-9", "Outside task")], + blocks: [relationSummary("issue-2", "PAP-2", "Beta task")], + }; + } + if (issueId === "issue-2") { + return { + blockedBy: [relationSummary("issue-1", "PAP-1", "Alpha task")], + blocks: [], + }; + } + return { blockedBy: [], blocks: [] }; + }); + documentSvc.listIssueDocuments.mockImplementation(async (issueId: string) => issueId === "issue-1" + ? [ + { + id: "document-1", + companyId: "company-1", + issueId: "issue-1", + key: "spec", + title: "Spec", + format: "markdown", + body: "# Spec\n\nDetails.", + latestRevisionId: "revision-1", + latestRevisionNumber: 1, + }, + ] + : []); + workProductSvc.listForIssue.mockImplementation(async (issueId: string) => issueId === "issue-1" + ? [ + { + id: "work-product-1", + companyId: "company-1", + projectId: null, + issueId: "issue-1", + executionWorkspaceId: "ws-1", + runtimeServiceId: null, + type: "pull_request", + provider: "github", + externalId: "42", + title: "Fix bug", + url: "https://github.com/example/repo/pull/42", + status: "merged", + reviewState: "approved", + isPrimary: true, + healthStatus: "healthy", + summary: "Fixes the bug", + metadata: { repo: "example/repo" }, + sourceTrust: null, + createdByRunId: "run-1", + createdAt: new Date("2026-06-01T00:00:00.000Z"), + updatedAt: new Date("2026-06-01T00:00:00.000Z"), + }, + ] + : []); + } + + function fakeImportDb() { + const insertedRelationValues: Array> = []; + const monitorUpdates: Array> = []; + const db = { + insert: () => ({ + values: (rows: Array>) => ({ + onConflictDoNothing: async () => { + insertedRelationValues.push(...rows); + }, + }), + }), + update: () => ({ + set: (patch: Record) => ({ + where: async () => { + monitorUpdates.push(patch); + }, + }), + }), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + return { db, insertedRelationValues, monitorUpdates }; + } + + it("carries blockers, documents, work products, and monitors through export and import", async () => { + const { db, insertedRelationValues, monitorUpdates } = fakeImportDb(); + const portability = companyPortabilityService(db); + mockTaskFidelityExportSources(); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + + expect(asTextFile(exported.files["tasks/pap-1/documents/spec.md"])).toBe("# Spec\n\nDetails."); + const extension = asTextFile(exported.files[".paperclip.yaml"]); + expect(extension).toContain("blockedBy:"); + expect(extension).toContain('"pap-1"'); + expect(extension).toContain("workProducts:"); + expect(extension).toContain("monitor:"); + expect(extension).toContain('"Check deploy daily"'); + expect(extension).not.toContain("ws-1"); + expect(extension).not.toContain("run-1"); + expect(exported.warnings).toContain( + "1 blocker relation references a task outside this export and was not included.", + ); + expect(exported.warnings).toContain( + "1 work product references execution workspaces or runs that are not portable; those references were omitted from the export.", + ); + const alphaEntry = exported.manifest.issues.find((issue) => issue.slug === "pap-1"); + const betaEntry = exported.manifest.issues.find((issue) => issue.slug === "pap-2"); + expect(alphaEntry?.documents).toEqual([ + { key: "spec", title: "Spec", format: "markdown", path: "tasks/pap-1/documents/spec.md" }, + ]); + expect(alphaEntry?.workProducts).toEqual([ + expect.objectContaining({ + type: "pull_request", + provider: "github", + externalId: "42", + title: "Fix bug", + status: "merged", + reviewState: "approved", + isPrimary: true, + healthStatus: "healthy", + }), + ]); + expect(alphaEntry?.monitor).toEqual({ + notes: "Check deploy daily", + scheduledBy: "agent", + hadSchedule: true, + }); + expect(alphaEntry?.blockedBy).toEqual([]); + expect(betaEntry?.blockedBy).toEqual(["pap-1"]); + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported" }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockImplementation(async (_companyId: string, input: Record) => ({ + id: input.title === "Alpha task" ? "issue-imported-1" : "issue-imported-2", + title: input.title, + projectId: null, + })); + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Imported" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + expect(documentSvc.upsertIssueDocument).toHaveBeenCalledWith({ + issueId: "issue-imported-1", + key: "spec", + title: "Spec", + format: "markdown", + body: "# Spec\n\nDetails.", + createdByUserId: "user-1", + }); + expect(workProductSvc.createForIssue).toHaveBeenCalledWith( + "issue-imported-1", + "company-imported", + expect.objectContaining({ + type: "pull_request", + provider: "github", + externalId: "42", + title: "Fix bug", + status: "merged", + reviewState: "approved", + isPrimary: true, + healthStatus: "healthy", + executionWorkspaceId: null, + runtimeServiceId: null, + createdByRunId: null, + sourceTrust: null, + }), + ); + expect(insertedRelationValues).toEqual([ + { + companyId: "company-imported", + issueId: "issue-imported-1", + relatedIssueId: "issue-imported-2", + type: "blocks", + createdByAgentId: null, + createdByUserId: "user-1", + }, + ]); + expect(monitorUpdates).toEqual([ + { monitorNotes: "Check deploy daily", monitorScheduledBy: "agent" }, + ]); + expect(result.warnings).toContain( + "1 monitor was imported un-armed; re-arm it from the task page to resume checks.", + ); + }); + + it("skips blockers and documents of tasks excluded from the import selection", async () => { + const { db, insertedRelationValues } = fakeImportDb(); + const portability = companyPortabilityService(db); + mockTaskFidelityExportSources(); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported" }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported-2", title: "Beta task", projectId: null }); + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Imported" }, + agents: "all", + collisionStrategy: "rename", + selectedFiles: ["COMPANY.md", ".paperclip.yaml", "tasks/pap-2/TASK.md"], + }, "user-1"); + + expect(issueSvc.create).toHaveBeenCalledTimes(1); + expect(issueSvc.create).toHaveBeenCalledWith( + "company-imported", + expect.objectContaining({ title: "Beta task" }), + ); + expect(documentSvc.upsertIssueDocument).not.toHaveBeenCalled(); + expect(workProductSvc.createForIssue).not.toHaveBeenCalled(); + expect(insertedRelationValues).toEqual([]); + expect(result.warnings).toContain( + "Task pap-2 blocker pap-1 was skipped because that task was not imported.", + ); + }); + + const attachmentBytesByObjectKey: Record = { + "issues/issue-1/notes.bin": "png-bytes", + "issues/issue-1/screenshot.png": "png-bytes", + "issues/issue-1/big.bin": "twenty-byte-payload!", + "assets/general/embed.png": "embedded-image-bytes", + }; + + function sha256Of(content: string) { + return createHash("sha256").update(content).digest("hex"); + } + + function fakeAttachmentStorage() { + return { + getObject: vi.fn().mockImplementation(async (_companyId: string, objectKey: string) => { + const content = attachmentBytesByObjectKey[objectKey]; + if (content === undefined) throw new Error(`missing object ${objectKey}`); + return { stream: Readable.from([Buffer.from(content)]) }; + }), + putFile: vi.fn().mockImplementation(async (input: { + originalFilename: string | null; + contentType: string; + body: Buffer; + }) => ({ + provider: "local_disk", + objectKey: `stored/${input.originalFilename ?? "blob"}`, + contentType: input.contentType, + byteSize: input.body.length, + sha256: sha256Of(input.body.toString()), + originalFilename: input.originalFilename, + })), + }; + } + + function mockAttachmentExportSources(extraRows: Array> = []) { + projectSvc.list.mockResolvedValue([]); + projectSvc.listWorkspaces.mockResolvedValue([]); + issueSvc.list.mockResolvedValue([ + { + id: "issue-1", + identifier: "PAP-1", + title: "Attachment task", + description: null, + projectId: null, + projectWorkspaceId: null, + assigneeAgentId: null, + status: "todo", + priority: "medium", + labelIds: [], + billingCode: null, + executionWorkspaceSettings: null, + assigneeAdapterOverrides: null, + }, + ]); + issueSvc.listComments.mockResolvedValue([ + { + id: "comment-1", + body: "First comment", + authorType: "system", + authorAgentId: null, + presentation: null, + metadata: null, + createdAt: new Date("2026-06-01T00:00:00.000Z"), + }, + { + id: "comment-2", + body: "Screenshot attached", + authorType: "system", + authorAgentId: null, + presentation: null, + metadata: null, + createdAt: new Date("2026-06-02T00:00:00.000Z"), + }, + ]); + // listAttachments returns newest-first like the real service; the export + // re-sorts chronologically. + issueSvc.listAttachments.mockResolvedValue([ + { + id: "attachment-2", + issueId: "issue-1", + issueCommentId: "comment-2", + provider: "local_disk", + objectKey: "issues/issue-1/screenshot.png", + contentType: "image/png", + byteSize: 9, + sha256: sha256Of("png-bytes"), + originalFilename: "screenshot.png", + createdAt: new Date("2026-06-03T00:00:00.000Z"), + }, + { + id: "attachment-1", + issueId: "issue-1", + issueCommentId: null, + provider: "local_disk", + objectKey: "issues/issue-1/notes.bin", + contentType: "application/octet-stream", + byteSize: 9, + sha256: "stale-asset-row-hash", + originalFilename: "notes.bin", + createdAt: new Date("2026-06-02T12:00:00.000Z"), + }, + ...extraRows, + ]); + } + + it("carries issue attachments as content-addressed blobs through export and import", async () => { + const storage = fakeAttachmentStorage(); + const portability = companyPortabilityService({} as any, storage as any); + mockAttachmentExportSources(); + const sha = sha256Of("png-bytes"); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + + // Both attachments share the same bytes, so the bundle holds one blob. + expect(Object.keys(exported.files).filter((filePath) => filePath.startsWith("blobs/"))).toEqual([ + `blobs/${sha}`, + ]); + expect(exported.files[`blobs/${sha}`]).toEqual({ + encoding: "base64", + data: Buffer.from("png-bytes").toString("base64"), + contentType: "application/octet-stream", + }); + expect(exported.manifest.blobs).toEqual([ + { sha256: sha, byteSize: 9, contentType: "application/octet-stream" }, + ]); + const taskEntry = exported.manifest.issues.find((issue) => issue.slug === "pap-1"); + expect(taskEntry?.attachments).toEqual([ + { + sha256: sha, + contentType: "application/octet-stream", + originalFilename: "notes.bin", + byteSize: 9, + commentIndex: null, + }, + { + sha256: sha, + contentType: "image/png", + originalFilename: "screenshot.png", + byteSize: 9, + commentIndex: 1, + }, + ]); + expect(exported.warnings).toContain( + "Attachment notes.bin on task pap-1 was exported under its recomputed content hash because the stored hash did not match.", + ); + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported", attachmentMaxBytes: null }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Attachment task", projectId: null }); + issueSvc.addComment + .mockResolvedValueOnce({ id: "comment-imported-1" }) + .mockResolvedValueOnce({ id: "comment-imported-2" }); + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Imported" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + expect(storage.putFile).toHaveBeenCalledTimes(2); + expect(storage.putFile).toHaveBeenCalledWith(expect.objectContaining({ + companyId: "company-imported", + namespace: "issues/issue-imported", + originalFilename: "screenshot.png", + contentType: "image/png", + body: Buffer.from("png-bytes"), + })); + expect(issueSvc.createAttachment).toHaveBeenCalledTimes(2); + expect(issueSvc.createAttachment).toHaveBeenCalledWith(expect.objectContaining({ + issueId: "issue-imported", + issueCommentId: null, + originalFilename: "notes.bin", + contentType: "application/octet-stream", + sha256: sha, + byteSize: 9, + createdByUserId: "user-1", + })); + expect(issueSvc.createAttachment).toHaveBeenCalledWith(expect.objectContaining({ + issueId: "issue-imported", + issueCommentId: "comment-imported-2", + originalFilename: "screenshot.png", + contentType: "image/png", + })); + expect(result.warnings.filter((warning) => warning.includes("attachment"))).toEqual([]); + }); + + it("skips attachment export with a per-task warning when storage is unavailable", async () => { + const portability = companyPortabilityService({} as any); + mockAttachmentExportSources(); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + + expect(exported.warnings).toContain( + "Skipped 2 attachments on task pap-1 because storage is unavailable.", + ); + expect(Object.keys(exported.files).some((filePath) => filePath.startsWith("blobs/"))).toBe(false); + expect(asTextFile(exported.files[".paperclip.yaml"])).not.toContain("attachments:"); + }); + + it("skips all attachment imports with one warning when the target has no storage", async () => { + const storage = fakeAttachmentStorage(); + const exporting = companyPortabilityService({} as any, storage as any); + mockAttachmentExportSources(); + const exported = await exporting.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + + const importing = companyPortabilityService({} as any); + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported" }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Attachment task", projectId: null }); + + const result = await importing.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Imported" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + expect(issueSvc.createAttachment).not.toHaveBeenCalled(); + expect(result.warnings).toContain("Skipped 2 attachments because storage is unavailable."); + }); + + it("fails closed when a bundle blob does not match its declared sha256", async () => { + const storage = fakeAttachmentStorage(); + const portability = companyPortabilityService({} as any, storage as any); + mockAttachmentExportSources(); + const sha = sha256Of("png-bytes"); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + exported.files[`blobs/${sha}`] = { + encoding: "base64", + data: Buffer.from("tampered-bytes").toString("base64"), + contentType: "application/octet-stream", + }; + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported" }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Attachment task", projectId: null }); + + await expect(portability.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Imported" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1")).rejects.toThrow(/does not match its declared sha256/); + expect(issueSvc.createAttachment).not.toHaveBeenCalled(); + // Blob verification runs before any write, so a tampered package cannot + // leave a partially imported company behind. + expect(companySvc.create).not.toHaveBeenCalled(); + expect(issueSvc.create).not.toHaveBeenCalled(); + }); + + it("skips oversized and missing-blob attachments with warnings instead of failing", async () => { + const storage = fakeAttachmentStorage(); + const portability = companyPortabilityService({} as any, storage as any); + mockAttachmentExportSources([ + { + id: "attachment-3", + issueId: "issue-1", + issueCommentId: null, + provider: "local_disk", + objectKey: "issues/issue-1/big.bin", + contentType: "application/octet-stream", + byteSize: 20, + sha256: sha256Of("twenty-byte-payload!"), + originalFilename: "big.bin", + createdAt: new Date("2026-06-04T00:00:00.000Z"), + }, + ]); + const sha = sha256Of("png-bytes"); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + delete exported.files[`blobs/${sha}`]; + + // The target company only accepts attachments up to 10 bytes. + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported", attachmentMaxBytes: 10 }); + companySvc.update.mockResolvedValue({ id: "company-imported", name: "Imported", attachmentMaxBytes: 10 }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Attachment task", projectId: null }); + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Imported" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + expect(issueSvc.createAttachment).not.toHaveBeenCalled(); + expect(result.warnings).toContain( + `Task pap-1 attachment notes.bin was skipped because its blob is missing from the package: blobs/${sha}`, + ); + expect(result.warnings).toContain( + `Task pap-1 attachment screenshot.png was skipped because its blob is missing from the package: blobs/${sha}`, + ); + expect(result.warnings).toContain( + "Task pap-1 attachment big.bin was skipped because it exceeds this board's attachment size limit of 10 bytes.", + ); + }); + + const EMBEDDED_ASSET_ID = "0f9a4c9e-1b2d-4e3f-8a5b-6c7d8e9f0a1b"; + const embeddedAssetUrl = (assetId: string) => `/api/assets/${assetId}/content`; + + function mockEmbeddedAssetExportSources(description?: string) { + projectSvc.list.mockResolvedValue([]); + projectSvc.listWorkspaces.mockResolvedValue([]); + issueSvc.list.mockResolvedValue([ + { + id: "issue-1", + identifier: "PAP-1", + title: "Embedded image task", + description: description ?? `Intro\n\n![shot](${embeddedAssetUrl(EMBEDDED_ASSET_ID)})`, + projectId: null, + projectWorkspaceId: null, + assigneeAgentId: null, + status: "todo", + priority: "medium", + labelIds: [], + billingCode: null, + executionWorkspaceSettings: null, + assigneeAdapterOverrides: null, + }, + ]); + issueSvc.listComments.mockResolvedValue([ + { + id: "comment-1", + body: `Inline too: ![inline](${embeddedAssetUrl(EMBEDDED_ASSET_ID)})`, + authorType: "system", + authorAgentId: null, + presentation: null, + metadata: null, + createdAt: new Date("2026-06-01T00:00:00.000Z"), + }, + ]); + documentSvc.listIssueDocuments.mockResolvedValue([ + { + id: "document-1", + key: "spec", + title: "Spec", + format: "markdown", + body: `# Spec\n\n![shot](${embeddedAssetUrl(EMBEDDED_ASSET_ID)})`, + }, + ]); + assetSvc.getById.mockImplementation(async (assetId: string) => ( + assetId === EMBEDDED_ASSET_ID + ? { + id: EMBEDDED_ASSET_ID, + companyId: "company-1", + provider: "local_disk", + objectKey: "assets/general/embed.png", + contentType: "image/png", + byteSize: 20, + sha256: "stale-asset-row-hash", + originalFilename: "embed.png", + } + : null + )); + } + + it("carries embedded asset images through export and import with rewritten references", async () => { + const storage = fakeAttachmentStorage(); + const portability = companyPortabilityService({} as any, storage as any); + mockEmbeddedAssetExportSources(); + const sha = sha256Of("embedded-image-bytes"); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + + // The description, a comment, and a document all reference the same + // asset, so the bundle holds one blob and one embeddedAssets entry. + expect(Object.keys(exported.files).filter((filePath) => filePath.startsWith("blobs/"))).toEqual([ + `blobs/${sha}`, + ]); + expect(exported.files[`blobs/${sha}`]).toEqual({ + encoding: "base64", + data: Buffer.from("embedded-image-bytes").toString("base64"), + contentType: "application/octet-stream", + }); + expect(exported.manifest.blobs).toEqual([ + { sha256: sha, byteSize: 20, contentType: "application/octet-stream" }, + ]); + expect(exported.manifest.embeddedAssets).toEqual([ + { + assetId: EMBEDDED_ASSET_ID, + sha256: sha, + contentType: "image/png", + originalFilename: "embed.png", + ownedBy: ["tasks"], + }, + ]); + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported", attachmentMaxBytes: null }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Embedded image task", projectId: null }); + assetSvc.create.mockResolvedValue({ id: "asset-imported-1" }); + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Imported" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + // One asset row is recreated for the shared reference, from bytes that + // hash to the exported blob's address. + expect(storage.putFile).toHaveBeenCalledTimes(1); + expect(storage.putFile).toHaveBeenCalledWith(expect.objectContaining({ + companyId: "company-imported", + namespace: "assets/general", + originalFilename: "embed.png", + contentType: "image/png", + body: Buffer.from("embedded-image-bytes"), + })); + expect(assetSvc.create).toHaveBeenCalledTimes(1); + expect(assetSvc.create).toHaveBeenCalledWith("company-imported", expect.objectContaining({ + contentType: "image/png", + originalFilename: "embed.png", + sha256: sha, + createdByAgentId: null, + createdByUserId: "user-1", + })); + + // Every reference now points at the minted asset id, not the source id. + const importedDescription = issueSvc.create.mock.calls[0]![1].description as string; + expect(importedDescription).toContain(embeddedAssetUrl("asset-imported-1")); + expect(importedDescription).not.toContain(EMBEDDED_ASSET_ID); + const importedCommentBody = issueSvc.addComment.mock.calls[0]![1] as string; + expect(importedCommentBody).toContain(embeddedAssetUrl("asset-imported-1")); + expect(importedCommentBody).not.toContain(EMBEDDED_ASSET_ID); + expect(documentSvc.upsertIssueDocument).toHaveBeenCalledWith(expect.objectContaining({ + key: "spec", + body: expect.stringContaining(embeddedAssetUrl("asset-imported-1")), + })); + const importedDocumentBody = documentSvc.upsertIssueDocument.mock.calls[0]![0].body as string; + expect(importedDocumentBody).not.toContain(EMBEDDED_ASSET_ID); + expect(result.warnings.filter((warning) => warning.includes("embedded"))).toEqual([]); + }); + + it("skips embedded image references that are foreign or dangling with one aggregate warning", async () => { + const storage = fakeAttachmentStorage(); + const portability = companyPortabilityService({} as any, storage as any); + const foreignAssetId = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"; + const missingAssetId = "12345678-1234-4123-8123-123456789abc"; + mockEmbeddedAssetExportSources( + `![theirs](${embeddedAssetUrl(foreignAssetId)})\n\n![gone](${embeddedAssetUrl(missingAssetId)})`, + ); + issueSvc.listComments.mockResolvedValue([]); + documentSvc.listIssueDocuments.mockResolvedValue([]); + // A crafted reference naming another company's asset id must not pull + // that asset's bytes into the bundle. + assetSvc.getById.mockImplementation(async (assetId: string) => ( + assetId === foreignAssetId + ? { + id: foreignAssetId, + companyId: "company-2", + provider: "local_disk", + objectKey: "assets/general/secret.png", + contentType: "image/png", + byteSize: 6, + sha256: sha256Of("secret"), + originalFilename: "secret.png", + } + : null + )); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + + expect(Object.keys(exported.files).some((filePath) => filePath.startsWith("blobs/"))).toBe(false); + expect(storage.getObject).not.toHaveBeenCalled(); + expect(asTextFile(exported.files[".paperclip.yaml"])).not.toContain("embeddedAssets:"); + expect(exported.manifest.embeddedAssets).toEqual([]); + expect(exported.warnings).toContain( + "2 embedded image references point at assets that do not belong to this company or no longer exist; their images were not exported.", + ); + }); + + it("leaves embedded image references untouched when their blob is missing at import", async () => { + const storage = fakeAttachmentStorage(); + const portability = companyPortabilityService({} as any, storage as any); + mockEmbeddedAssetExportSources(); + const sha = sha256Of("embedded-image-bytes"); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + delete exported.files[`blobs/${sha}`]; + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported", attachmentMaxBytes: null }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Embedded image task", projectId: null }); + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Imported" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + expect(assetSvc.create).not.toHaveBeenCalled(); + expect(result.warnings).toContain( + `Embedded image asset embed.png was skipped because its blob is missing from the package: blobs/${sha}; its references were left unchanged.`, + ); + const importedDescription = issueSvc.create.mock.calls[0]![1].description as string; + expect(importedDescription).toContain(embeddedAssetUrl(EMBEDDED_ASSET_ID)); + const importedCommentBody = issueSvc.addComment.mock.calls[0]![1] as string; + expect(importedCommentBody).toContain(embeddedAssetUrl(EMBEDDED_ASSET_ID)); + }); + + it("prunes embedded asset entries and blobs when the referencing files are excluded from the export selection", async () => { + const storage = fakeAttachmentStorage(); + const portability = companyPortabilityService({} as any, storage as any); + mockEmbeddedAssetExportSources(); + const sha = sha256Of("embedded-image-bytes"); + + const kept = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + selectedFiles: ["COMPANY.md", ".paperclip.yaml", "tasks/pap-1/TASK.md", "tasks/pap-1/documents/spec.md", `blobs/${sha}`], + }); + expect(asTextFile(kept.files[".paperclip.yaml"])).toContain("embeddedAssets:"); + expect(kept.files[`blobs/${sha}`]).toBeDefined(); + + const pruned = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + selectedFiles: ["COMPANY.md", ".paperclip.yaml"], + }); + expect(Object.keys(pruned.files).some((filePath) => filePath.startsWith("blobs/"))).toBe(false); + const prunedYaml = asTextFile(pruned.files[".paperclip.yaml"]); + expect(prunedYaml).not.toContain("embeddedAssets:"); + expect(prunedYaml).not.toContain("blobs:"); + expect(pruned.manifest.embeddedAssets).toEqual([]); + }); + + function legacyPackageFiles(extensionLines: string[]) { + return { + "COMPANY.md": [ + "---", + 'schema: "agentcompanies/v1"', + 'name: "Legacy Import"', + "---", + "", + ].join("\n"), + "tasks/kickoff/TASK.md": [ + "---", + 'name: "Kickoff"', + "---", + "", + "Legacy task.", + "", + ].join("\n"), + ".paperclip.yaml": [ + 'schema: "paperclip/v1"', + ...extensionLines, + "tasks:", + " kickoff:", + ' status: "todo"', + "", + ].join("\n"), + }; + } + + it("imports unstamped v5 packages with an info warning about task data they predate", async () => { + const portability = companyPortabilityService({} as any); + const v5Warning = + "This package declares schemaVersion 5 and predates label, blocker, document, work product, monitor, attachment, and embedded image transfer; that task data imports only if the bundle carries it."; + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Legacy Import" }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + issueSvc.create.mockResolvedValue({ id: "issue-imported", title: "Kickoff", projectId: null }); + + const request = { + source: { type: "inline" as const, rootPath: "legacy-package", files: legacyPackageFiles([]) }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company" as const, newCompanyName: "Legacy Import" }, + agents: "all" as const, + collisionStrategy: "rename" as const, + }; + + const preview = await portability.previewImport(request); + expect(preview.manifest.schemaVersion).toBe(5); + expect(preview.warnings).toContain(v5Warning); + + const result = await portability.importBundle(request, "user-1"); + expect(issueSvc.create).toHaveBeenCalledWith( + "company-imported", + expect.objectContaining({ title: "Kickoff" }), + ); + expect(result.warnings).toContain(v5Warning); + }); + + it("keeps packages declaring schemaVersions below 5 importable", async () => { + const portability = companyPortabilityService({} as any); + + const preview = await portability.previewImport({ + source: { type: "inline", rootPath: "legacy-package", files: legacyPackageFiles(["schemaVersion: 1"]) }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Legacy Import" }, + agents: "all", + collisionStrategy: "rename", + }); + + expect(preview.errors).toEqual([]); + expect(preview.manifest.schemaVersion).toBe(1); + expect(preview.warnings.some((warning) => warning.startsWith("This package declares schemaVersion 1"))).toBe(true); + }); + + it("rejects packages produced by a newer Paperclip", async () => { + const portability = companyPortabilityService({} as any); + + await expect(portability.importBundle({ + source: { type: "inline", rootPath: "future-package", files: legacyPackageFiles(["schemaVersion: 7"]) }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Future Import" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1")).rejects.toThrow(/newer Paperclip/); + expect(issueSvc.create).not.toHaveBeenCalled(); + }); + it("preserves issue comment presentation fields through export and import", async () => { const portability = companyPortabilityService({} as any); const presentation = { kind: "system_notice", tone: "warning", detailsDefaultOpen: false }; diff --git a/server/src/__tests__/export-fidelity.test.ts b/server/src/__tests__/export-fidelity.test.ts new file mode 100644 index 0000000000..79b93ce35c --- /dev/null +++ b/server/src/__tests__/export-fidelity.test.ts @@ -0,0 +1,127 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { companies, createDb, issueLabels, issueRelations, issues, labels } from "@paperclipai/db"; + +import { buildExportFidelityReport, collectExportFidelityCounts } from "../services/export-fidelity.js"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres export fidelity tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +describeEmbeddedPostgres("export fidelity counts", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-export-fidelity-"); + db = createDb(tempDb.connectionString); + }, 20_000); + + afterEach(async () => { + await db.delete(issueLabels); + await db.delete(issueRelations); + await db.delete(labels); + await db.delete(issues); + await db.delete(companies); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + it("counts labels, label references, blocker relations, and monitors scoped to the company", async () => { + const companyId = randomUUID(); + const otherCompanyId = randomUUID(); + await seedCompany(companyId); + await seedCompany(otherCompanyId); + + const issueA = randomUUID(); + const issueB = randomUUID(); + const otherIssue = randomUUID(); + await db.insert(issues).values([ + issueRow({ id: issueA, companyId, identifier: "LOCAL-1" }), + issueRow({ id: issueB, companyId, identifier: "LOCAL-2", monitorScheduledBy: "agent" }), + issueRow({ id: otherIssue, companyId: otherCompanyId, identifier: "OTHER-1", monitorScheduledBy: "agent" }), + ]); + + const labelId = randomUUID(); + const otherLabelId = randomUUID(); + await db.insert(labels).values([ + { id: labelId, companyId, name: "bug", color: "#ff0000" }, + { id: otherLabelId, companyId: otherCompanyId, name: "bug", color: "#00ff00" }, + ]); + await db.insert(issueLabels).values([ + { issueId: issueA, labelId, companyId }, + { issueId: otherIssue, labelId: otherLabelId, companyId: otherCompanyId }, + ]); + await db.insert(issueRelations).values([ + { companyId, issueId: issueA, relatedIssueId: issueB, type: "blocks" as const }, + ]); + + const counts = await collectExportFidelityCounts(db, companyId); + expect(counts).toEqual({ + labelDefinitions: 1, + issueLabelReferences: 1, + issueBlockerRelations: 1, + issueDocuments: 0, + issueWorkProducts: 0, + issueAttachments: 0, + approvals: 0, + costEvents: 0, + activityLogEntries: 0, + issueMonitors: 1, + }); + + const otherCounts = await collectExportFidelityCounts(db, otherCompanyId); + expect(otherCounts.labelDefinitions).toBe(1); + expect(otherCounts.issueBlockerRelations).toBe(0); + expect(otherCounts.issueMonitors).toBe(1); + }); + + it("builds a report whose warnings reflect the collected counts", async () => { + const companyId = randomUUID(); + await seedCompany(companyId); + const issueId = randomUUID(); + await db.insert(issues).values([issueRow({ id: issueId, companyId, identifier: "LOCAL-1" })]); + const labelId = randomUUID(); + await db.insert(labels).values([{ id: labelId, companyId, name: "bug", color: "#ff0000" }]); + await db.insert(issueLabels).values([{ issueId, labelId, companyId }]); + + const report = buildExportFidelityReport(companyId, await collectExportFidelityCounts(db, companyId)); + expect(report.schema).toBe("paperclip-export-fidelity-v1"); + expect(report.companyId).toBe(companyId); + // Labels travel in the bundle now, so their counts stay informational. + expect(report.counts.labelDefinitions).toBe(1); + expect(report.counts.issueLabelReferences).toBe(1); + expect(report.warnings).toEqual([]); + expect(Date.parse(report.generatedAt)).not.toBeNaN(); + }); + + async function seedCompany(companyId: string) { + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + } + + function issueRow(overrides: { id: string; companyId: string; identifier: string; monitorScheduledBy?: string }) { + return { + description: "", + status: "todo", + priority: "medium", + title: overrides.identifier, + ...overrides, + }; + } +}); diff --git a/server/src/__tests__/instance-settings-routes.test.ts b/server/src/__tests__/instance-settings-routes.test.ts index a0adb781c6..042ec129d3 100644 --- a/server/src/__tests__/instance-settings-routes.test.ts +++ b/server/src/__tests__/instance-settings-routes.test.ts @@ -80,7 +80,6 @@ describe("instance settings routes", () => { enableIsolatedWorkspaces: false, enableIssuePlanDecompositions: false, enableExperimentalFileViewer: false, - enableCloudSync: false, enableExternalObjects: false, enableBuiltInAgents: false, enableBetaSkills: false, @@ -109,7 +108,6 @@ describe("instance settings routes", () => { enableIssuePlanDecompositions: false, enableExperimentalFileViewer: false, enableTaskWatchdogs: false, - enableCloudSync: false, enableExternalObjects: false, enableBuiltInAgents: false, enableBetaSkills: false, @@ -137,7 +135,6 @@ describe("instance settings routes", () => { enableIsolatedWorkspaces: true, enableIssuePlanDecompositions: true, enableExperimentalFileViewer: true, - enableCloudSync: true, enableExternalObjects: false, enableBuiltInAgents: false, enableBetaSkills: false, @@ -171,7 +168,6 @@ describe("instance settings routes", () => { enableIssuePlanDecompositions: true, enableExperimentalFileViewer: true, enableTaskWatchdogs: true, - enableCloudSync: true, enableExternalObjects: false, enableBuiltInAgents: true, enableGoalsSidebarLink: false, @@ -232,7 +228,6 @@ describe("instance settings routes", () => { enableIssuePlanDecompositions: false, enableExperimentalFileViewer: false, enableTaskWatchdogs: false, - enableCloudSync: false, enableExternalObjects: false, enableBuiltInAgents: false, enableBetaSkills: false, diff --git a/server/src/__tests__/instance-settings-service.test.ts b/server/src/__tests__/instance-settings-service.test.ts index 8a38433227..f641eb6a5a 100644 --- a/server/src/__tests__/instance-settings-service.test.ts +++ b/server/src/__tests__/instance-settings-service.test.ts @@ -14,7 +14,6 @@ describe("instance settings service", () => { enableIssuePlanDecompositions: true, enableExperimentalFileViewer: true, enableTaskWatchdogs: true, - enableCloudSync: true, enableBuiltInAgents: true, enableGoalsSidebarLink: true, enableServerInfoDebugView: true, @@ -37,7 +36,6 @@ describe("instance settings service", () => { enableIssuePlanDecompositions: true, enableExperimentalFileViewer: true, enableTaskWatchdogs: true, - enableCloudSync: true, enableBuiltInAgents: true, enableBetaSkills: false, enableSummaries: false, diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 04d87aba27..0780f02c17 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -21,7 +21,6 @@ const apiPrefixes: Record = { "auth.ts": "/api/auth", "board-chat.ts": "/api", "built-in-agents.ts": "/api", - "cloud-upstreams.ts": "/api", "companies.ts": "/api/companies", "company-skills.ts": "/api", "company-skill-policy.ts": "/api", diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index d309533dde..5a7b5ca4cc 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -219,7 +219,6 @@ vi.mock("../services/index.js", () => ({ }, })), })), - reconcileCloudUpstreamRunsOnStartup: vi.fn(async () => ({ reconciled: 0 })), reconcileCodexLocalManagedHomesOnStartup: vi.fn(async () => ({ scanned: 0, seeded: 0, diff --git a/server/src/index.ts b/server/src/index.ts index a21de6b67c..50a5278091 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -50,7 +50,6 @@ import { issueService, instanceSettingsService, reconcileBuiltInAgentsOnStartup, - reconcileCloudUpstreamRunsOnStartup, reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup, routineService, @@ -792,19 +791,6 @@ export async function startServer(): Promise { logger.error({ err }, "startup reconciliation of persisted runtime services failed"); }); - void reconcileCloudUpstreamRunsOnStartup(db as any) - .then((result) => { - if (result.reconciled > 0) { - logger.warn( - { reconciled: result.reconciled }, - "reconciled cloud upstream runs from a previous server process", - ); - } - }) - .catch((err) => { - logger.error({ err }, "startup reconciliation of cloud upstream runs failed"); - }); - // Backfill auth.json into any already-isolated codex_local managed home that // was created by the #8272 isolation guard before the Phase 1 seeding fix. // Idempotent; the Phase 1 execute-time seeding covers new strandings. diff --git a/server/src/routes/cloud-upstreams.ts b/server/src/routes/cloud-upstreams.ts deleted file mode 100644 index 085afe470d..0000000000 --- a/server/src/routes/cloud-upstreams.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { Router } from "express"; -import type { Db } from "@paperclipai/db"; -import { badRequest, notFound } from "../errors.js"; -import { assertBoardOrgAccess } from "./authz.js"; -import { cloudUpstreamService, instanceSettingsService } from "../services/index.js"; - -export function cloudUpstreamRoutes(db: Db, options: { instanceId?: string } = {}) { - const router = Router(); - const service = cloudUpstreamService(db, options); - const settings = instanceSettingsService(db); - - async function assertEnabled() { - const experimental = await settings.getExperimental(); - if (experimental.enableCloudSync !== true) { - throw notFound("Cloud sync is not enabled"); - } - } - - router.get("/cloud-upstreams", async (req, res) => { - assertBoardOrgAccess(req); - await assertEnabled(); - const companyId = stringQuery(req.query.companyId, "companyId"); - res.json(await service.list(companyId)); - }); - - router.post("/cloud-upstreams/connect/start", async (req, res) => { - assertBoardOrgAccess(req); - await assertEnabled(); - const companyId = stringBody(req.body, "companyId"); - const remoteUrl = stringBody(req.body, "remoteUrl"); - const redirectUri = stringBody(req.body, "redirectUri"); - res.json(await service.startConnect({ companyId, remoteUrl, redirectUri })); - }); - - router.post("/cloud-upstreams/connect/finish", async (req, res) => { - assertBoardOrgAccess(req); - await assertEnabled(); - res.json(await service.finishConnect({ - pendingConnectionId: stringBody(req.body, "pendingConnectionId"), - code: stringBody(req.body, "code"), - state: stringBody(req.body, "state"), - })); - }); - - router.post("/cloud-upstreams/:connectionId/push-runs/preview", async (req, res) => { - assertBoardOrgAccess(req); - await assertEnabled(); - res.json(await service.preview(req.params.connectionId, stringBody(req.body, "companyId"))); - }); - - router.post("/cloud-upstreams/:connectionId/push-runs", async (req, res) => { - assertBoardOrgAccess(req); - await assertEnabled(); - res.json(await service.createRun({ - connectionId: req.params.connectionId, - companyId: stringBody(req.body, "companyId"), - retryOfRunId: optionalString(req.body?.retryOfRunId), - })); - }); - - router.get("/cloud-upstreams/:connectionId/push-runs/:runId", async (req, res) => { - assertBoardOrgAccess(req); - await assertEnabled(); - res.json(await service.readRun(req.params.connectionId, req.params.runId, stringQuery(req.query.companyId, "companyId"))); - }); - - router.post("/cloud-upstreams/:connectionId/push-runs/:runId/cancel", async (req, res) => { - assertBoardOrgAccess(req); - await assertEnabled(); - res.json(await service.cancelRun(req.params.connectionId, req.params.runId, stringBody(req.body, "companyId"))); - }); - - router.post("/cloud-upstreams/:connectionId/push-runs/:runId/activation", async (req, res) => { - assertBoardOrgAccess(req); - await assertEnabled(); - res.json(await service.activateRunEntities({ - connectionId: req.params.connectionId, - runId: req.params.runId, - companyId: stringBody(req.body, "companyId"), - entityType: activationEntityTypeBody(req.body), - })); - }); - - return router; -} - -function stringQuery(value: unknown, label: string): string { - if (typeof value !== "string" || value.trim().length === 0) { - throw badRequest(`${label} is required`); - } - return value; -} - -function stringBody(body: unknown, key: string): string { - if (!body || typeof body !== "object" || Array.isArray(body)) { - throw badRequest(`${key} is required`); - } - const value = (body as Record)[key]; - if (typeof value !== "string" || value.trim().length === 0) { - throw badRequest(`${key} is required`); - } - return value; -} - -function optionalString(value: unknown): string | null { - return typeof value === "string" && value.length > 0 ? value : null; -} - -function activationEntityTypeBody(body: unknown): "agents" | "routines" | "monitors" { - if (!body || typeof body !== "object" || Array.isArray(body)) { - throw badRequest("entityType is required"); - } - const value = (body as Record).entityType; - if (value !== "agents" && value !== "routines" && value !== "monitors") { - throw badRequest("entityType must be agents, routines, or monitors"); - } - return value; -} diff --git a/server/src/routes/companies.ts b/server/src/routes/companies.ts index ae660703a9..4ae00a5b4c 100644 --- a/server/src/routes/companies.ts +++ b/server/src/routes/companies.ts @@ -23,6 +23,8 @@ import { accessService, agentService, budgetService, + buildExportFidelityReport, + collectExportFidelityCounts, companyArtifactsService, companyPortabilityService, companyService, @@ -252,6 +254,13 @@ export function companyRoutes(db: Db, storage?: StorageService) { res.json(result); }); + router.get("/:companyId/export/fidelity", async (req, res) => { + const companyId = req.params.companyId as string; + await assertSameCompanyCeoAgentOrBoard(req, companyId, "company export fidelity"); + const counts = await collectExportFidelityCounts(db, companyId); + res.json(buildExportFidelityReport(companyId, counts)); + }); + router.post("/import/preview", async (req, res) => { assertBoard(req); const body = companyPortabilityPreviewSchema.parse(req.body); @@ -285,7 +294,9 @@ export function companyRoutes(db: Db, storage?: StorageService) { const importBody = companyPortabilityImportSchema.parse(rawImportBody); assertImportTargetAccess(req, importBody.target); const activity = importedCompanyActivityContext(actor, importBody.include ?? null); - const result = await portability.importBundle(importBody, boardUserId); + const result = await portability.importBundle(importBody, boardUserId, { + pauseAutomations: importBody.pauseAutomations === true, + }); await logImportedCompanyActivity(db, activity, result); return result; }; @@ -299,7 +310,9 @@ export function companyRoutes(db: Db, storage?: StorageService) { const importBody = companyPortabilityImportSchema.parse(rawImportBody); assertImportTargetAccess(req, importBody.target); const activity = importedCompanyActivityContext(actor, importBody.include ?? null); - const result = await portability.importBundle(importBody, boardUserId); + const result = await portability.importBundle(importBody, boardUserId, { + pauseAutomations: importBody.pauseAutomations === true, + }); await logImportedCompanyActivity(db, activity, result); res.json(result); }); @@ -351,6 +364,7 @@ export function companyRoutes(db: Db, storage?: StorageService) { const result = await portability.importBundle(body, req.actor.type === "board" ? req.actor.userId : null, { mode: "agent_safe", sourceCompanyId: companyId, + pauseAutomations: body.pauseAutomations === true, }); await logActivity(db, { companyId: result.company.id, diff --git a/server/src/routes/index.ts b/server/src/routes/index.ts index 0d96dde775..78f1c3f4f4 100644 --- a/server/src/routes/index.ts +++ b/server/src/routes/index.ts @@ -31,4 +31,3 @@ export { llmRoutes } from "./llms.js"; export { accessRoutes } from "./access.js"; export { instanceSettingsRoutes } from "./instance-settings.js"; export { instanceDatabaseBackupRoutes } from "./instance-database-backups.js"; -export { cloudUpstreamRoutes } from "./cloud-upstreams.js"; diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 35b3a3aec3..e619f5f3d2 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -704,7 +704,6 @@ const PUBLIC_OPERATIONS = new Set([ const BOARD_ONLY_PREFIXES = [ "/api/auth/", "/api/admin/", - "/api/cloud-upstreams", "/api/plugins", "/api/instance/", ]; @@ -5203,6 +5202,15 @@ registry.registerPath({ responses: { 200: r.ok(), 401: r.unauthorized }, }); +registry.registerPath({ + method: "get", + path: "/api/companies/{companyId}/export/fidelity", + tags: ["companies"], + summary: "Report company data that an export bundle does not include", + request: { params: z.object({ companyId: z.string() }) }, + responses: { 200: r.ok(), 401: r.unauthorized }, +}); + registry.registerPath({ method: "post", path: "/api/companies/import/preview", @@ -5563,93 +5571,6 @@ for (const route of [ }); } -const cloudCompanyQuerySchema = z.object({ - companyId: z.string().min(1), -}); -const cloudCompanyBodySchema = z.object({ - companyId: z.string().min(1), -}); -const cloudConnectStartSchema = z.object({ - companyId: z.string().min(1), - remoteUrl: z.string().min(1), - redirectUri: z.string().min(1), -}); -const cloudConnectFinishSchema = z.object({ - pendingConnectionId: z.string().min(1), - code: z.string().min(1), - state: z.string().min(1), -}); -const cloudPushRunSchema = cloudCompanyBodySchema.extend({ - retryOfRunId: z.string().optional(), -}); -const cloudPushRunActivationSchema = cloudCompanyBodySchema.extend({ - entityType: z.enum(["agents", "routines", "monitors"]), -}); - -registerCurrentRoute({ - method: "get", - path: "/api/cloud-upstreams", - tags: ["cloud-upstreams"], - summary: "List cloud upstream connections", - query: cloudCompanyQuerySchema, -}); - -registerCurrentRoute({ - method: "post", - path: "/api/cloud-upstreams/connect/start", - tags: ["cloud-upstreams"], - summary: "Start a cloud upstream connection", - body: cloudConnectStartSchema, -}); - -registerCurrentRoute({ - method: "post", - path: "/api/cloud-upstreams/connect/finish", - tags: ["cloud-upstreams"], - summary: "Finish a cloud upstream connection", - body: cloudConnectFinishSchema, -}); - -registerCurrentRoute({ - method: "post", - path: "/api/cloud-upstreams/{connectionId}/push-runs/preview", - tags: ["cloud-upstreams"], - summary: "Preview a cloud upstream push run", - body: cloudCompanyBodySchema, -}); - -registerCurrentRoute({ - method: "post", - path: "/api/cloud-upstreams/{connectionId}/push-runs", - tags: ["cloud-upstreams"], - summary: "Create a cloud upstream push run", - body: cloudPushRunSchema, -}); - -registerCurrentRoute({ - method: "get", - path: "/api/cloud-upstreams/{connectionId}/push-runs/{runId}", - tags: ["cloud-upstreams"], - summary: "Get a cloud upstream push run", - query: cloudCompanyQuerySchema, -}); - -registerCurrentRoute({ - method: "post", - path: "/api/cloud-upstreams/{connectionId}/push-runs/{runId}/cancel", - tags: ["cloud-upstreams"], - summary: "Cancel a cloud upstream push run", - body: cloudCompanyBodySchema, -}); - -registerCurrentRoute({ - method: "post", - path: "/api/cloud-upstreams/{connectionId}/push-runs/{runId}/activation", - tags: ["cloud-upstreams"], - summary: "Activate cloud upstream push run entities", - body: cloudPushRunActivationSchema, -}); - for (const route of [ ["get", "/api/companies/{companyId}/secret-providers/health", "Check configured secret providers"], ["get", "/api/companies/{companyId}/secret-provider-configs", "List secret provider configurations"], diff --git a/server/src/services/cloud-upstreams.ts b/server/src/services/cloud-upstreams.ts deleted file mode 100644 index 576d0dafae..0000000000 --- a/server/src/services/cloud-upstreams.ts +++ /dev/null @@ -1,1309 +0,0 @@ -import crypto, { sign } from "node:crypto"; -import { and, count, desc, eq, sql } from "drizzle-orm"; -import type { - CloudUpstreamConnectStartResponse, - CloudUpstreamActivationDecision, - CloudUpstreamActivationEntityType, - CloudUpstreamConnection, - CloudUpstreamConflict, - CloudUpstreamPreview, - CloudUpstreamRun, - CloudUpstreamRunEvent, - CloudUpstreamsState, - CloudUpstreamSummaryCount, - CloudUpstreamTarget, - CloudUpstreamWarning, - CompanyPortabilityExportResult, - CompanyPortabilityFileEntry, -} from "@paperclipai/shared"; -import type { Db } from "@paperclipai/db"; -import { - agents, - cloudUpstreamConnections, - cloudUpstreamRuns, - companies, - goals, - issueComments, - issues, - projects, - routines, -} from "@paperclipai/db"; -import { badRequest, conflict, HttpError, notFound } from "../errors.js"; -import { companyPortabilityService } from "./company-portability.js"; -import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js"; - -const DEFAULT_SCOPES = ["upstream_import:preview", "upstream_import:write", "upstream_import:read"]; -const TRANSFER_SCHEMA = { - family: "paperclip-upstream-transfer", - version: "1.0.0", - major: 1, - minor: 0, -} as const; -const DEFAULT_MAX_ENTITIES_PER_CHUNK = 100; -const DISCOVERY_FETCH_TIMEOUT_MS = 30_000; -const REMOTE_FETCH_TIMEOUT_MS = 120_000; -const CLOUD_CREDENTIAL_PREFIX = "paperclip-cloud-credential:"; - -type NormalizedSha256 = `sha256:${string}`; - -type SourceEntityKey = { - sourceInstanceId: string; - sourceCompanyId: string; - sourceEntityType: string; - sourceEntityId: string; - sourceNaturalKey?: string; -}; - -type UpstreamTransferWarning = { - code: string; - severity: "info" | "warning" | "blocker"; - message: string; - entity?: SourceEntityKey; -}; - -type UpstreamTransferEntityRecord = { - key: SourceEntityKey; - contentHash: NormalizedSha256; - dependencies: SourceEntityKey[]; - warnings: UpstreamTransferWarning[]; -}; - -type LocalUpstreamExportEntity = { - record: UpstreamTransferEntityRecord; - body: Record; - conflictKeys?: string[]; -}; - -type LocalUpstreamExportChunk = { - chunkIndex: number; - totalChunks: number; - byteLength: number; - sha256: NormalizedSha256; - payload: { - entityKeys: SourceEntityKey[]; - }; -}; - -type UpstreamTransferManifest = { - schema: typeof TRANSFER_SCHEMA; - source: { - sourceInstanceId: string; - sourceCompanyId: string; - sourceInstanceKeyFingerprint: string; - exporterVersion: string; - sourceSchemaVersion: string; - }; - target: { - targetStackId: string; - targetCompanyId: string; - targetOrigin: string; - supportedSchemaMajor: number; - }; - runId: string; - idempotencyKey: string; - generatedAt: string; - entityCount: number; - perEntityTypeCounts: Record; - entities: UpstreamTransferEntityRecord[]; - chunks: Array & { manifestHash: NormalizedSha256 }>; - warnings: UpstreamTransferWarning[]; - featureFlags: string[]; - manifestHash: NormalizedSha256; -}; - -type LocalUpstreamExportBundle = { - manifest: UpstreamTransferManifest; - entities: LocalUpstreamExportEntity[]; - chunks: LocalUpstreamExportChunk[]; -}; - -type ConnectionRow = typeof cloudUpstreamConnections.$inferSelect; -type RunRow = typeof cloudUpstreamRuns.$inferSelect; - -export function cloudUpstreamService(db: Db, options: { instanceId?: string } = {}) { - const sourceInstanceId = `paperclip-local-${options.instanceId ?? "default"}`; - const portability = companyPortabilityService(db); - - return { - list: async (companyId: string): Promise => { - const [connectionRows, runRows] = await Promise.all([ - db - .select() - .from(cloudUpstreamConnections) - .where(eq(cloudUpstreamConnections.companyId, companyId)) - .orderBy(desc(cloudUpstreamConnections.updatedAt)), - db - .select() - .from(cloudUpstreamRuns) - .where(eq(cloudUpstreamRuns.companyId, companyId)) - .orderBy(desc(cloudUpstreamRuns.createdAt)) - .limit(50), - ]); - return { - connections: connectionRows.map(connectionFromRow), - runs: runRows.map(runFromRow), - }; - }, - - startConnect: async (input: { - companyId: string; - remoteUrl: string; - redirectUri: string; - }): Promise => { - await requireCompany(input.companyId); - const remoteUrl = input.remoteUrl.trim(); - if (!remoteUrl) throw badRequest("Remote URL is required"); - - const discovery = await fetchDiscovery(remoteUrl); - const target = targetFromDiscovery(discovery); - const connectionId = crypto.randomUUID(); - const state = crypto.randomBytes(24).toString("base64url"); - const codeVerifier = crypto.randomBytes(32).toString("base64url"); - const codeChallenge = crypto.createHash("sha256").update(codeVerifier, "utf8").digest("base64url"); - const { publicKey, privateKey } = crypto.generateKeyPairSync("ed25519"); - const sourcePublicKey = publicKey.export({ type: "spki", format: "pem" }).toString(); - const sourceInstanceFingerprint = `sha256:${crypto - .createHash("sha256") - .update(publicKey.export({ type: "spki", format: "der" })) - .digest("hex")}`; - - const [row] = await db.insert(cloudUpstreamConnections).values({ - id: connectionId, - companyId: input.companyId, - remoteUrl, - sourceInstanceId, - sourceInstanceFingerprint, - sourcePublicKey, - privateKeyPem: await sealCloudUpstreamCredential(privateKey.export({ type: "pkcs8", format: "pem" }).toString()), - tokenStatus: "pending", - scopes: scopesFromDiscovery(discovery), - targetStackId: target.stackId, - targetStackSlug: target.stackSlug, - targetStackDisplayName: target.stackDisplayName, - targetCompanyId: target.companyId, - targetOrigin: target.origin, - targetPrimaryHost: target.primaryHost, - targetProduct: target.product, - targetSchemaMajor: target.schemaMajor, - targetMaxChunkBytes: target.maxChunkBytes, - pendingState: state, - pendingCodeVerifier: await sealCloudUpstreamCredential(codeVerifier), - pendingRedirectUri: input.redirectUri, - pendingTokenUrl: tokenUrlFromDiscovery(discovery), - }).returning(); - if (!row) throw badRequest("Failed to create cloud upstream connection"); - - const authorizationUrl = new URL(consentUrlFromDiscovery(discovery)); - authorizationUrl.searchParams.set("stackId", target.stackId); - authorizationUrl.searchParams.set("redirectUri", input.redirectUri); - authorizationUrl.searchParams.set("state", state); - authorizationUrl.searchParams.set("codeChallenge", codeChallenge); - authorizationUrl.searchParams.set("codeChallengeMethod", "S256"); - authorizationUrl.searchParams.set("sourceInstanceId", sourceInstanceId); - authorizationUrl.searchParams.set("sourceInstanceFingerprint", sourceInstanceFingerprint); - authorizationUrl.searchParams.set("sourcePublicKey", sourcePublicKey); - authorizationUrl.searchParams.set("scopes", row.scopes.join(" ")); - - return { - pendingConnectionId: row.id, - authorizationUrl: authorizationUrl.toString(), - connection: connectionFromRow(row), - }; - }, - - finishConnect: async (input: { - pendingConnectionId: string; - code: string; - state: string; - }): Promise => { - const pending = await getConnectionRow(input.pendingConnectionId); - if (!pending.pendingState || !pending.pendingCodeVerifier || !pending.pendingRedirectUri || !pending.pendingTokenUrl) { - throw notFound("Pending cloud upstream connection was not found"); - } - if (input.state !== pending.pendingState) throw badRequest("Cloud upstream state did not match"); - const tokenResponse = await postJson>(pending.pendingTokenUrl, { - grantType: "authorization_code", - code: input.code, - redirectUri: pending.pendingRedirectUri, - codeVerifier: await unsealCloudUpstreamCredential(pending.pendingCodeVerifier), - }); - const accessToken = stringField(tokenResponse, "accessToken"); - const token = objectField(tokenResponse, "token"); - const expiresAt = optionalString(token.expiresAt) ?? optionalString(tokenResponse.expiresAt); - const [updated] = await db - .update(cloudUpstreamConnections) - .set({ - tokenStatus: "connected", - authorizedGlobalUserId: optionalString(token.globalUserId), - accessToken: await sealCloudUpstreamCredential(accessToken), - tokenId: optionalString(token.id), - tokenExpiresAt: expiresAt ? new Date(expiresAt) : null, - pendingState: null, - pendingCodeVerifier: null, - pendingRedirectUri: null, - pendingTokenUrl: null, - updatedAt: new Date(), - }) - .where(eq(cloudUpstreamConnections.id, pending.id)) - .returning(); - if (!updated) throw notFound("Cloud upstream connection was not found"); - return connectionFromRow(updated); - }, - - preview: async (connectionId: string, companyId: string): Promise => { - const connection = await getConnectionRow(connectionId, companyId); - const basePreview = await localPreview(connection); - if (!basePreview.schemaCompatible || connection.tokenStatus !== "connected") { - return basePreview; - } - - const bundle = await buildBundle(connection, "preview"); - const conflictKeysBySource: Record = {}; - for (const entity of bundle.entities) { - if (!entity.conflictKeys || entity.conflictKeys.length === 0) continue; - conflictKeysBySource[sourceEntityKeyString(entity.record.key)] = [...entity.conflictKeys]; - } - const remotePreview = await remotePost(connection, `/api/companies/${encodeURIComponent(connection.targetCompanyId)}/upstream-imports/preview`, { - manifest: bundle.manifest, - previewShape: "manifest_only", - conflictKeysBySource, - }); - return { - ...basePreview, - warnings: mergeWarnings(basePreview.warnings, warningsFromRemote(remotePreview)), - conflicts: conflictsFromRemote(remotePreview), - }; - }, - - createRun: async (input: { connectionId: string; companyId: string; retryOfRunId?: string | null }): Promise => { - const connection = await getConnectionRow(input.connectionId, input.companyId); - if (connection.tokenStatus !== "connected") { - throw badRequest("Cloud upstream connection is not connected"); - } - await assertNoRunningRun(input.connectionId, input.companyId, db); - const preview = await localPreview(connection); - if (!preview.schemaCompatible) { - throw badRequest("Cloud stack schema is not compatible with this local Paperclip version"); - } - - const bundle = await buildBundle(connection, "apply"); - const runId = crypto.randomUUID(); - const now = new Date(); - const initialEvents = [ - event(now.toISOString(), "connect", "completed", "Connected to the target Paperclip Cloud stack."), - event(now.toISOString(), "scan", "completed", "Scanned the local company inventory."), - event(now.toISOString(), "preview", "completed", "Generated the transfer manifest."), - ...(input.retryOfRunId - ? [event(now.toISOString(), "push", "retrying", `Retrying run ${input.retryOfRunId} with the same import ledger idempotency key.`)] - : []), - ]; - const created = await db.transaction(async (tx) => { - await tx.execute( - sql`select ${cloudUpstreamConnections.id} from ${cloudUpstreamConnections} where ${cloudUpstreamConnections.id} = ${connection.id} and ${cloudUpstreamConnections.companyId} = ${connection.companyId} for update`, - ); - await assertNoRunningRun(input.connectionId, input.companyId, tx); - const [row] = await tx.insert(cloudUpstreamRuns).values({ - id: runId, - connectionId: connection.id, - companyId: connection.companyId, - status: "running", - activeStep: "push", - progressPercent: 45, - dryRun: false, - retryOfRunId: input.retryOfRunId ?? null, - summary: preview.summary, - warnings: preview.warnings, - conflicts: preview.conflicts, - events: initialEvents, - report: {}, - idempotencyKey: bundle.manifest.idempotencyKey, - manifestHash: bundle.manifest.manifestHash, - targetUrl: connection.targetOrigin, - createdAt: now, - updatedAt: now, - }).returning(); - return row; - }); - if (!created) throw badRequest("Failed to create cloud upstream run"); - - try { - const remoteRun = await remotePost(connection, `/api/companies/${encodeURIComponent(connection.targetCompanyId)}/upstream-imports/runs`, { - mode: "apply", - manifest: bundle.manifest, - entities: bundle.entities, - }); - const remoteRunId = remoteRunIdFromResponse(remoteRun); - const pushedRun = await updateRunIfRunning(runId, { - remoteRunId, - activeStep: "push", - progressPercent: 60, - events: [ - ...initialEvents, - event(new Date().toISOString(), "push", "updated", "Created or resumed the cloud import ledger run."), - ], - }); - if (pushedRun.status !== "running") return pushedRun; - - for (const chunk of bundle.chunks) { - await remotePost(connection, `/api/upstream-import-runs/${encodeURIComponent(remoteRunId)}/chunks`, chunk); - } - const verifiedRun = await updateRunIfRunning(runId, { - activeStep: "verify", - progressPercent: 82, - events: [ - ...initialEvents, - event(new Date().toISOString(), "push", "completed", `Uploaded ${bundle.chunks.length} manifest chunk${bundle.chunks.length === 1 ? "" : "s"}.`), - ], - }); - if (verifiedRun.status !== "running") return verifiedRun; - - const applied = await remotePost(connection, `/api/upstream-import-runs/${encodeURIComponent(remoteRunId)}/apply`, {}); - const remoteEvents = await remoteGet(connection, `/api/upstream-import-runs/${encodeURIComponent(remoteRunId)}/events`).catch(() => null); - const completedAt = new Date(); - const finalEvents = [ - ...initialEvents, - event(completedAt.toISOString(), "push", "completed", "Pushed mapped objects without duplicate creation."), - event(completedAt.toISOString(), "verify", "completed", "Verified the cloud import ledger and generated a run report."), - event(completedAt.toISOString(), "activate", "completed", "Activation checklist is ready for manual unpause decisions."), - ...eventsFromRemote(remoteEvents), - ]; - const finalRun = await updateRunIfRunning(runId, { - remoteRunId, - status: "succeeded", - activeStep: "activate", - progressPercent: 100, - warnings: mergeWarnings(preview.warnings, warningsFromRemote(applied)), - conflicts: conflictsFromRemote(applied), - events: finalEvents, - report: { - runId, - remoteRunId, - target: targetFromConnectionRow(connection), - manifestHash: bundle.manifest.manifestHash, - idempotencyKey: bundle.manifest.idempotencyKey, - retryOfRunId: input.retryOfRunId ?? null, - result: applied, - events: remoteEvents, - }, - completedAt, - }); - if (finalRun.status === "succeeded") { - await db - .update(cloudUpstreamConnections) - .set({ lastRunId: finalRun.id, updatedAt: new Date() }) - .where(eq(cloudUpstreamConnections.id, connection.id)); - } - return finalRun; - } catch (error) { - const failedAt = new Date(); - const failure = cloudUpstreamRemoteFailureReport(error); - return updateRunIfRunning(runId, { - status: "failed", - activeStep: "push", - progressPercent: 100, - events: [ - ...initialEvents, - event(failedAt.toISOString(), "push", "failed", failure.errorMessage ?? failure.error), - ], - report: { - runId, - target: targetFromConnectionRow(connection), - manifestHash: bundle.manifest.manifestHash, - idempotencyKey: bundle.manifest.idempotencyKey, - retryOfRunId: input.retryOfRunId ?? null, - ...failure, - }, - completedAt: failedAt, - }); - } - }, - - readRun: async (connectionId: string, runId: string, companyId: string): Promise => { - const row = await getRunRow(connectionId, runId, companyId); - return runFromRow(row); - }, - - cancelRun: async (connectionId: string, runId: string, companyId: string): Promise => { - const row = await getRunRow(connectionId, runId, companyId); - if (row.status !== "running") return runFromRow(row); - const connection = await getConnectionRow(connectionId, companyId); - if (row.remoteRunId) { - await remotePost(connection, `/api/upstream-import-runs/${encodeURIComponent(row.remoteRunId)}/cancel`, {}).catch(() => null); - } - return updateRun(row.id, { - status: "cancelled", - activeStep: "push", - progressPercent: 100, - completedAt: new Date(), - events: [ - ...row.events, - event(new Date().toISOString(), "push", "failed", "Push cancelled locally before remote apply completed."), - ], - }); - }, - - activateRunEntities: async (input: { - connectionId: string; - runId: string; - companyId: string; - entityType: CloudUpstreamActivationEntityType; - }): Promise => { - const row = await getRunRow(input.connectionId, input.runId, input.companyId); - assertActivationEntityType(input.entityType); - if (row.status !== "succeeded") { - throw badRequest("Only succeeded cloud upstream runs can activate imported entities"); - } - - const activatedAt = new Date().toISOString(); - const count = summaryCount(row.summary, input.entityType); - const nextDecision: CloudUpstreamActivationDecision = { - entityType: input.entityType, - count, - status: "activated", - activatedAt, - }; - const report = asRecord(row.report); - const activationChecklist = activationChecklistFromReport(report); - const label = activationEntityLabel(input.entityType, count); - - return updateRun(row.id, { - report: { - ...report, - activationChecklist: { - ...activationChecklist, - [input.entityType]: nextDecision, - }, - }, - events: [ - ...row.events, - event(activatedAt, "activate", "completed", `Activated ${count} imported ${label}.`), - ], - }); - }, - }; - - async function requireCompany(companyId: string) { - const row = await db.select({ id: companies.id }).from(companies).where(eq(companies.id, companyId)).then((rows) => rows[0]); - if (!row) throw notFound("Company was not found"); - } - - async function getConnectionRow(connectionId: string, companyId?: string): Promise { - const row = await db - .select() - .from(cloudUpstreamConnections) - .where(companyId - ? and(eq(cloudUpstreamConnections.id, connectionId), eq(cloudUpstreamConnections.companyId, companyId)) - : eq(cloudUpstreamConnections.id, connectionId)) - .then((rows) => rows[0]); - if (!row) throw notFound("Cloud upstream connection was not found"); - return row; - } - - async function getRunRow(connectionId: string, runId: string, companyId: string): Promise { - const row = await db - .select() - .from(cloudUpstreamRuns) - .where(and( - eq(cloudUpstreamRuns.id, runId), - eq(cloudUpstreamRuns.connectionId, connectionId), - eq(cloudUpstreamRuns.companyId, companyId), - )) - .then((rows) => rows[0]); - if (!row) throw notFound("Cloud upstream run was not found"); - return row; - } - - async function assertNoRunningRun( - connectionId: string, - companyId: string, - database: Pick, - ) { - const [running] = await database - .select({ id: cloudUpstreamRuns.id }) - .from(cloudUpstreamRuns) - .where(and( - eq(cloudUpstreamRuns.connectionId, connectionId), - eq(cloudUpstreamRuns.companyId, companyId), - eq(cloudUpstreamRuns.status, "running"), - )) - .limit(1); - if (running) { - throw conflict("A cloud upstream run is already running for this connection", { runId: running.id }); - } - } - - async function updateRun(runId: string, patch: Partial): Promise { - const [updated] = await db - .update(cloudUpstreamRuns) - .set({ ...patch, updatedAt: new Date() }) - .where(eq(cloudUpstreamRuns.id, runId)) - .returning(); - if (!updated) throw notFound("Cloud upstream run was not found"); - return runFromRow(updated); - } - - async function updateRunIfRunning(runId: string, patch: Partial): Promise { - const [updated] = await db - .update(cloudUpstreamRuns) - .set({ ...patch, updatedAt: new Date() }) - .where(and(eq(cloudUpstreamRuns.id, runId), eq(cloudUpstreamRuns.status, "running"))) - .returning(); - if (updated) return runFromRow(updated); - - const [current] = await db - .select() - .from(cloudUpstreamRuns) - .where(eq(cloudUpstreamRuns.id, runId)) - .limit(1); - if (!current) throw notFound("Cloud upstream run was not found"); - return runFromRow(current); - } - - async function localPreview(connection: ConnectionRow): Promise { - return { - connectionId: connection.id, - sourceCompanyId: connection.companyId, - target: targetFromConnectionRow(connection), - schemaCompatible: connection.targetSchemaMajor === TRANSFER_SCHEMA.major, - summary: await buildSummary(connection.companyId), - warnings: buildWarnings(connection.targetSchemaMajor), - conflicts: [], - generatedAt: new Date().toISOString(), - }; - } - - async function buildSummary(companyId: string): Promise { - const [agentCount, projectCount, goalCount, issueCount, commentCount, routineCount] = await Promise.all([ - db.select({ count: count() }).from(agents).where(eq(agents.companyId, companyId)).then((rows) => rows[0]?.count ?? 0), - db.select({ count: count() }).from(projects).where(eq(projects.companyId, companyId)).then((rows) => rows[0]?.count ?? 0), - db.select({ count: count() }).from(goals).where(eq(goals.companyId, companyId)).then((rows) => rows[0]?.count ?? 0), - db.select({ count: count() }).from(issues).where(eq(issues.companyId, companyId)).then((rows) => rows[0]?.count ?? 0), - db.select({ count: count() }).from(issueComments).where(eq(issueComments.companyId, companyId)).then((rows) => rows[0]?.count ?? 0), - db.select({ count: count() }).from(routines).where(eq(routines.companyId, companyId)).then((rows) => rows[0]?.count ?? 0), - ]); - return [ - { key: "companies", label: "Companies", count: 1 }, - { key: "goals", label: "Goals", count: goalCount }, - { key: "projects", label: "Projects", count: projectCount }, - { key: "agents", label: "Agents", count: agentCount }, - { key: "issues", label: "Issues", count: issueCount }, - { key: "comments", label: "Comments", count: commentCount }, - { key: "routines", label: "Routines", count: routineCount }, - { key: "warnings", label: "Warnings", count: buildWarnings(TRANSFER_SCHEMA.major).length }, - ]; - } - - async function buildBundle(connection: ConnectionRow, mode: "preview" | "apply"): Promise { - const exported = await portability.exportBundle(connection.companyId, { - include: { - company: true, - agents: true, - projects: true, - issues: true, - skills: true, - }, - expandReferencedSkills: true, - }); - const sourceHash = normalizedContentHash({ - manifest: exported.manifest, - files: exported.files, - }); - const source = { - sourceInstanceId: connection.sourceInstanceId, - sourceCompanyId: connection.companyId, - sourceInstanceKeyFingerprint: connection.sourceInstanceFingerprint, - exporterVersion: "paperclip-local-cloud-ui-v1", - sourceSchemaVersion: TRANSFER_SCHEMA.version, - }; - const target = { - targetStackId: connection.targetStackId, - targetCompanyId: connection.targetCompanyId, - targetOrigin: connection.targetOrigin, - supportedSchemaMajor: connection.targetSchemaMajor, - }; - const idempotencyKey = [ - mode, - connection.sourceInstanceId, - connection.companyId, - connection.targetStackId, - sourceHash, - ].join(":"); - return buildLocalUpstreamExportBundle({ - source, - target, - runId: `local-${mode}-${shortHash(idempotencyKey)}`, - idempotencyKey, - entities: buildEntitiesFromPortableExport(connection.companyId, connection.sourceInstanceId, exported), - warnings: exported.warnings.map((message): UpstreamTransferWarning => ({ - code: "local_company_export_warning", - severity: "warning", - message, - })), - featureFlags: ["cloud_sync"], - maxEntitiesPerChunk: DEFAULT_MAX_ENTITIES_PER_CHUNK, - }); - } -} - -async function fetchDiscovery(remoteUrl: string): Promise> { - const parsed = new URL(remoteUrl); - if (parsed.protocol !== "https:" && parsed.hostname !== "localhost" && parsed.hostname !== "127.0.0.1") { - throw badRequest("Cloud upstream targets require HTTPS except localhost development"); - } - const stackId = firstPathSegment(parsed.pathname); - const discoveryUrl = new URL("/.well-known/paperclip-upstream", parsed.origin); - if (stackId) { - discoveryUrl.searchParams.set("stackId", stackId); - } - const response = await fetchWithTimeout(discoveryUrl, undefined, DISCOVERY_FETCH_TIMEOUT_MS); - if (!response.ok) { - throw badRequest(`Cloud upstream discovery failed: ${response.status}`); - } - return await response.json() as Record; -} - -export async function reconcileCloudUpstreamRunsOnStartup(db: Db, now = new Date()): Promise<{ reconciled: number }> { - const runningRows = await db - .select() - .from(cloudUpstreamRuns) - .where(eq(cloudUpstreamRuns.status, "running")); - if (runningRows.length === 0) return { reconciled: 0 }; - - for (const row of runningRows) { - const report = asRecord(row.report); - await db - .update(cloudUpstreamRuns) - .set({ - status: "failed", - activeStep: row.activeStep, - progressPercent: 100, - completedAt: now, - updatedAt: now, - events: [ - ...safeRunEvents(row.events), - event( - now.toISOString(), - cloudUpstreamStep(row.activeStep), - "failed", - "Marked failed on server startup because the previous process stopped while the cloud upstream run was in progress.", - ), - ], - report: { - ...report, - error: optionalString(report.error) ?? "orphaned_running_run", - errorMessage: optionalString(report.errorMessage) - ?? "The server restarted while this cloud upstream run was running, so Paperclip marked it failed instead of leaving it stuck.", - reconciledAt: now.toISOString(), - }, - }) - .where(eq(cloudUpstreamRuns.id, row.id)); - } - - return { reconciled: runningRows.length }; -} - -function firstPathSegment(pathname: string): string | null { - const segment = pathname.split("/").find(Boolean); - return segment && segment.toLowerCase() !== "dashboard" ? segment : null; -} - -function targetFromDiscovery(discovery: Record): CloudUpstreamTarget { - const stack = objectField(discovery, "stack"); - const transfer = objectField(discovery, "transfer"); - const schema = optionalObject(transfer.schema); - const origin = stringField(stack, "origin"); - return { - stackId: stringField(stack, "id"), - stackSlug: optionalString(stack.slug), - stackDisplayName: optionalString(stack.displayName), - companyId: stringField(stack, "companyId"), - primaryHost: optionalString(stack.primaryHost) ?? new URL(origin).host, - origin, - product: optionalString(discovery.product) ?? "Paperclip Cloud", - schemaMajor: optionalNumber(schema?.major) ?? numberField(transfer, "supportedSchemaMajor"), - maxChunkBytes: optionalNumber(transfer.maxChunkBytes) ?? 8 * 1024 * 1024, - }; -} - -function targetFromConnectionRow(row: ConnectionRow): CloudUpstreamTarget { - return { - stackId: row.targetStackId, - stackSlug: row.targetStackSlug, - stackDisplayName: row.targetStackDisplayName, - companyId: row.targetCompanyId, - primaryHost: row.targetPrimaryHost, - origin: row.targetOrigin, - product: row.targetProduct, - schemaMajor: row.targetSchemaMajor, - maxChunkBytes: row.targetMaxChunkBytes, - }; -} - -function connectionFromRow(row: ConnectionRow): CloudUpstreamConnection { - return { - id: row.id, - companyId: row.companyId, - remoteUrl: row.remoteUrl, - target: targetFromConnectionRow(row), - tokenStatus: cloudUpstreamTokenStatus(row.tokenStatus), - scopes: row.scopes, - authorizedGlobalUserId: row.authorizedGlobalUserId, - expiresAt: row.tokenExpiresAt?.toISOString() ?? null, - createdAt: row.createdAt.toISOString(), - updatedAt: row.updatedAt.toISOString(), - lastRunId: row.lastRunId, - }; -} - -function runFromRow(row: RunRow): CloudUpstreamRun { - return { - id: row.id, - connectionId: row.connectionId, - companyId: row.companyId, - status: cloudUpstreamRunStatus(row.status), - activeStep: cloudUpstreamStep(row.activeStep), - progressPercent: row.progressPercent, - dryRun: row.dryRun, - summary: row.summary, - warnings: row.warnings, - conflicts: row.conflicts, - events: row.events, - targetUrl: row.targetUrl, - report: row.report, - retryOfRunId: row.retryOfRunId, - createdAt: row.createdAt.toISOString(), - updatedAt: row.updatedAt.toISOString(), - completedAt: row.completedAt?.toISOString() ?? null, - }; -} - -function scopesFromDiscovery(discovery: Record): string[] { - const auth = objectField(discovery, "auth"); - const scopes = Array.isArray(auth.scopes) ? auth.scopes.map(String).filter(Boolean) : []; - return scopes.length > 0 ? scopes : [...DEFAULT_SCOPES]; -} - -function consentUrlFromDiscovery(discovery: Record): string { - const pkce = objectField(objectField(discovery, "auth"), "pkce"); - return optionalString(pkce.consentUrl) ?? stringField(pkce, "authorizeUrl"); -} - -function tokenUrlFromDiscovery(discovery: Record): string { - return stringField(objectField(objectField(discovery, "auth"), "pkce"), "tokenUrl"); -} - -function buildWarnings(schemaMajor: number): CloudUpstreamWarning[] { - const warnings: CloudUpstreamWarning[] = [ - { - code: "imported_automations_paused", - severity: "warning", - title: "Automations stay paused", - detail: "Imported agents, routines, and monitors require explicit activation after the push.", - }, - { - code: "unmatched_users_import_as_historical_authors", - severity: "warning", - title: "Unmatched users become historical authors", - detail: "Invite now remains a secondary action after the transfer is complete.", - }, - { - code: "secret_values_redacted", - severity: "warning", - title: "Secret values are not transferred", - detail: "The push carries secret requirements only. Configure cloud secrets before activating automations.", - }, - ]; - if (schemaMajor !== TRANSFER_SCHEMA.major) { - warnings.unshift({ - code: "schema_mismatch", - severity: "blocker", - title: "Cloud stack upgrade required", - detail: `This local build uses upstream schema ${TRANSFER_SCHEMA.major}, but the cloud stack reports schema ${schemaMajor}.`, - }); - } - return warnings; -} - -type LocalUpstreamExportEntityInput = { - key: SourceEntityKey; - body: Record; - dependencies?: SourceEntityKey[]; - warnings?: UpstreamTransferWarning[]; - conflictKeys?: string[]; -}; - -function buildEntitiesFromPortableExport( - localCompanyId: string, - sourceInstanceId: string, - exported: CompanyPortabilityExportResult, -): LocalUpstreamExportEntityInput[] { - const companyKey: SourceEntityKey = { - sourceInstanceId, - sourceCompanyId: localCompanyId, - sourceEntityType: "company", - sourceEntityId: localCompanyId, - sourceNaturalKey: exported.manifest.company?.name ?? localCompanyId, - }; - const entities: LocalUpstreamExportEntityInput[] = [ - { - key: companyKey, - body: { - kind: "paperclip_company_portability_manifest", - manifest: exported.manifest, - rootPath: exported.rootPath, - paperclipExtensionPath: exported.paperclipExtensionPath, - fileCount: Object.keys(exported.files).length, - }, - conflictKeys: [`company:${companyKey.sourceNaturalKey ?? localCompanyId}`], - }, - ]; - - for (const [filePath, entry] of Object.entries(exported.files).sort(([left], [right]) => left.localeCompare(right))) { - entities.push({ - key: { - sourceInstanceId, - sourceCompanyId: localCompanyId, - sourceEntityType: "company_setting", - sourceEntityId: shortHash(filePath), - sourceNaturalKey: filePath, - }, - body: { - kind: "paperclip_portable_file", - path: filePath, - entry: normalizePortableFileEntry(entry), - }, - dependencies: [companyKey], - conflictKeys: [`portable_file:${filePath}`], - }); - } - return entities; -} - -function normalizePortableFileEntry(entry: CompanyPortabilityFileEntry): Record { - if (typeof entry === "string") { - return { encoding: "utf8", data: entry }; - } - return { ...entry }; -} - -function buildLocalUpstreamExportBundle(input: { - source: UpstreamTransferManifest["source"]; - target: UpstreamTransferManifest["target"]; - runId: string; - idempotencyKey: string; - entities: LocalUpstreamExportEntityInput[]; - warnings?: UpstreamTransferWarning[]; - featureFlags?: string[]; - maxEntitiesPerChunk?: number; -}): LocalUpstreamExportBundle { - const entities = input.entities.map((entity) => ({ - record: { - key: entity.key, - contentHash: normalizedContentHash(entity.body), - dependencies: entity.dependencies ?? [], - warnings: entity.warnings ?? [], - }, - body: entity.body, - conflictKeys: entity.conflictKeys, - })); - const chunksWithoutManifestHash = buildLocalChunks(entities, input.maxEntitiesPerChunk ?? DEFAULT_MAX_ENTITIES_PER_CHUNK); - const manifestWithoutHash = { - schema: TRANSFER_SCHEMA, - source: input.source, - target: input.target, - runId: input.runId, - idempotencyKey: input.idempotencyKey, - generatedAt: new Date(0).toISOString(), - entityCount: entities.length, - perEntityTypeCounts: countEntityTypesForManifest(entities), - entities: entities.map((entity) => entity.record), - chunks: chunksWithoutManifestHash.map(({ payload: _payload, ...chunk }) => chunk), - warnings: input.warnings ?? [], - featureFlags: (input.featureFlags ?? ["cloud_sync"]).slice().sort(), - }; - const manifestHash = normalizedContentHash(manifestWithoutHash); - return { - manifest: { - ...manifestWithoutHash, - chunks: chunksWithoutManifestHash.map(({ payload: _payload, ...chunk }) => ({ ...chunk, manifestHash })), - manifestHash, - }, - entities, - chunks: chunksWithoutManifestHash, - }; -} - -function countEntityTypesForManifest(entities: LocalUpstreamExportEntity[]): Record { - const counts: Record = {}; - for (const entity of entities) { - const entityType = entity.record.key.sourceEntityType; - counts[entityType] = (counts[entityType] ?? 0) + 1; - } - return counts; -} - -function buildLocalChunks(entities: LocalUpstreamExportEntity[], maxEntitiesPerChunk: number): LocalUpstreamExportChunk[] { - if (!Number.isInteger(maxEntitiesPerChunk) || maxEntitiesPerChunk < 1) { - throw new Error("maxEntitiesPerChunk must be a positive integer"); - } - if (entities.length === 0) return []; - - const groups: LocalUpstreamExportEntity[][] = []; - for (let index = 0; index < entities.length; index += maxEntitiesPerChunk) { - groups.push(entities.slice(index, index + maxEntitiesPerChunk)); - } - - return groups.map((group, index) => { - const payload = { - entityKeys: group.map((entity) => entity.record.key), - }; - return { - chunkIndex: index, - totalChunks: groups.length, - byteLength: Buffer.byteLength(canonicalJson(payload)), - sha256: normalizedContentHash(payload), - payload, - }; - }); -} - -async function remoteGet(connection: ConnectionRow, path: string): Promise { - const response = await fetchWithTimeout(`${connection.targetOrigin}${path}`, { - method: "GET", - headers: await proofHeaders(connection, "GET", path), - }, REMOTE_FETCH_TIMEOUT_MS); - return parseRemoteResponse(response); -} - -async function remotePost(connection: ConnectionRow, path: string, body: unknown): Promise { - const response = await fetchWithTimeout(`${connection.targetOrigin}${path}`, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...await proofHeaders(connection, "POST", path), - }, - body: JSON.stringify(body), - }, REMOTE_FETCH_TIMEOUT_MS); - return parseRemoteResponse(response); -} - -async function proofHeaders(connection: ConnectionRow, method: string, pathAndSearch: string): Promise> { - if (!connection.accessToken || !connection.tokenId) { - throw badRequest("Cloud upstream connection is missing an import token"); - } - const accessToken = await unsealCloudUpstreamCredential(connection.accessToken); - const privateKeyPem = await unsealCloudUpstreamCredential(connection.privateKeyPem); - const timestamp = new Date().toISOString(); - const nonce = crypto.randomUUID(); - const payload = [ - method, - new URL(connection.targetOrigin).host.toLowerCase(), - pathAndSearch, - connection.tokenId, - connection.sourceInstanceId, - timestamp, - nonce, - ].join("\n"); - return { - Authorization: `Bearer ${accessToken}`, - "X-Paperclip-Upstream-Source-Instance-Id": connection.sourceInstanceId, - "X-Paperclip-Upstream-Proof-Timestamp": timestamp, - "X-Paperclip-Upstream-Proof-Nonce": nonce, - "X-Paperclip-Upstream-Proof-Signature": sign( - null, - Buffer.from(payload, "utf8"), - privateKeyPem, - ).toString("base64url"), - }; -} - -async function parseRemoteResponse(response: Response): Promise { - const text = await response.text(); - const parsed = text.trim() ? safeParseJson(text) : {}; - if (!response.ok) { - const message = typeof parsed === "object" && parsed !== null && "error" in parsed - ? String((parsed as { error: unknown }).error) - : `Cloud upstream request failed: ${response.status}`; - throw badRequest(message, parsed); - } - return parsed; -} - -async function fetchWithTimeout(input: RequestInfo | URL, init: RequestInit | undefined, timeoutMs: number): Promise { - return fetch(input, { - ...init, - signal: AbortSignal.timeout(timeoutMs), - }); -} - -export async function sealCloudUpstreamCredential(value: string): Promise { - const prepared = await localEncryptedProvider.createSecret({ value }); - return `${CLOUD_CREDENTIAL_PREFIX}${JSON.stringify(prepared.material)}`; -} - -export async function unsealCloudUpstreamCredential(value: string): Promise { - if (!value.startsWith(CLOUD_CREDENTIAL_PREFIX)) return value; - const encoded = value.slice(CLOUD_CREDENTIAL_PREFIX.length); - const parsed = safeParseJson(encoded); - const material = optionalObject(parsed); - if (!material) { - throw badRequest("Invalid encrypted cloud upstream credential material"); - } - return localEncryptedProvider.resolveVersion({ - material, - externalRef: null, - }); -} - -export function cloudUpstreamRemoteFailureReport(error: unknown): { - error: string; - errorMessage?: string; - details?: unknown; -} { - const fallback = error instanceof Error ? error.message : String(error); - if (!(error instanceof HttpError)) { - return { error: fallback }; - } - const remote = remoteErrorBody(error.details); - return { - error: remote.error ?? error.message, - ...(remote.message ? { errorMessage: remote.message } : {}), - ...(error.details !== undefined ? { details: error.details } : {}), - }; -} - -function remoteErrorBody(details: unknown): { error?: string; message?: string } { - const record = optionalObject(details); - if (!record) return {}; - return { - error: optionalString(record.error) ?? undefined, - message: optionalString(record.message) ?? undefined, - }; -} - -async function postJson(url: string, body: unknown): Promise { - const response = await fetchWithTimeout(url, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(body), - }, DISCOVERY_FETCH_TIMEOUT_MS); - const payload = await response.json().catch(() => null); - if (!response.ok) { - throw badRequest((payload as { error?: string } | null)?.error ?? `Cloud upstream request failed: ${response.status}`); - } - return payload as T; -} - -function remoteRunIdFromResponse(value: unknown): string { - const record = asRecord(value); - const run = asRecord(record.run); - const id = optionalString(run.id); - if (!id) throw badRequest("Remote upstream importer did not return a run id"); - return id; -} - -function warningsFromRemote(value: unknown): CloudUpstreamWarning[] { - const record = asRecord(value); - const warnings = Array.isArray(record.warnings) ? record.warnings : []; - return warnings.map((warning, index): CloudUpstreamWarning => { - const item = asRecord(warning); - const code = optionalString(item.code) ?? `remote_warning_${index}`; - const severity = item.severity === "blocker" ? "blocker" : "warning"; - const message = optionalString(item.message) ?? optionalString(item.detail) ?? "Remote importer warning."; - return { - code, - severity, - title: titleFromCode(code), - detail: message, - }; - }); -} - -function conflictsFromRemote(value: unknown): CloudUpstreamConflict[] { - const record = asRecord(value); - const conflicts = Array.isArray(record.conflicts) ? record.conflicts : []; - return conflicts.map((conflict, index): CloudUpstreamConflict => { - const item = asRecord(conflict); - const source = asRecord(item.source); - return { - id: optionalString(item.id) ?? `remote-conflict-${index}`, - entityType: optionalString(item.entityType) ?? optionalString(source.sourceEntityType) ?? "entity", - sourceLabel: optionalString(item.sourceLabel) ?? optionalString(source.sourceNaturalKey) ?? optionalString(source.sourceEntityId) ?? "Source entity", - targetLabel: optionalString(item.targetLabel) ?? optionalString(item.targetEntityId) ?? "Cloud entity", - plannedAction: "blocked", - reason: optionalString(item.reason) ?? "Remote importer reported a conflict.", - }; - }); -} - -function eventsFromRemote(value: unknown): CloudUpstreamRunEvent[] { - const record = asRecord(value); - const events = Array.isArray(record.events) ? record.events : []; - return events.slice(-25).map((remote, index) => { - const item = asRecord(remote); - const action = optionalString(item.action) ?? "updated"; - return event( - optionalString(item.createdAt) ?? new Date().toISOString(), - "verify", - action.includes("created") ? "created" : "updated", - `Cloud importer ${action.replace(/_/g, " ")}${index >= 0 ? "." : "."}`, - ); - }); -} - -function safeRunEvents(value: unknown): CloudUpstreamRunEvent[] { - return Array.isArray(value) ? value as CloudUpstreamRunEvent[] : []; -} - -function assertActivationEntityType(value: string): asserts value is CloudUpstreamActivationEntityType { - if (value !== "agents" && value !== "routines" && value !== "monitors") { - throw badRequest("entityType must be agents, routines, or monitors"); - } -} - -function summaryCount(summary: unknown, key: CloudUpstreamActivationEntityType): number { - if (!Array.isArray(summary)) return 0; - const item = summary.find((entry) => asRecord(entry).key === key); - const count = asRecord(item).count; - return typeof count === "number" && Number.isFinite(count) ? count : 0; -} - -function activationChecklistFromReport(report: Record): Record { - const value = asRecord(report.activationChecklist); - const decisions: Record = {}; - for (const [key, decision] of Object.entries(value)) { - if (key !== "agents" && key !== "routines" && key !== "monitors") continue; - const item = asRecord(decision); - decisions[key] = { - entityType: key, - count: typeof item.count === "number" && Number.isFinite(item.count) ? item.count : 0, - status: item.status === "activated" ? "activated" : "paused", - activatedAt: optionalString(item.activatedAt), - }; - } - return decisions; -} - -function activationEntityLabel(entityType: CloudUpstreamActivationEntityType, count: number): string { - const singular = entityType === "agents" ? "agent" : entityType === "routines" ? "routine" : "monitor"; - return `${singular}${count === 1 ? "" : "s"}`; -} - -function mergeWarnings(base: CloudUpstreamWarning[], extra: CloudUpstreamWarning[]): CloudUpstreamWarning[] { - const byCode = new Map(); - for (const warning of [...base, ...extra]) byCode.set(warning.code, warning); - return [...byCode.values()]; -} - -function event( - at: string, - phase: CloudUpstreamRunEvent["phase"], - type: CloudUpstreamRunEvent["type"], - message: string, -): CloudUpstreamRunEvent { - return { - id: crypto.randomUUID(), - at, - phase, - type, - message, - }; -} - -function normalizedContentHash(value: unknown): NormalizedSha256 { - return `sha256:${crypto.createHash("sha256").update(canonicalJson(value)).digest("hex")}`; -} - -function canonicalJson(value: unknown): string { - return JSON.stringify(sortJson(value)); -} - -function sortJson(value: unknown): unknown { - if (Array.isArray(value)) return value.map(sortJson); - if (typeof value !== "object" || value === null) return value; - return Object.fromEntries( - Object.entries(value as Record) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, entry]) => [key, sortJson(entry)]), - ); -} - -function shortHash(value: string): string { - return crypto.createHash("sha256").update(value).digest("hex").slice(0, 12); -} - -function sourceEntityKeyString(key: SourceEntityKey): string { - return [key.sourceInstanceId, key.sourceCompanyId, key.sourceEntityType, key.sourceEntityId] - .map((part) => encodeURIComponent(part)) - .join("/"); -} - -function titleFromCode(code: string): string { - return code - .replace(/_/g, " ") - .replace(/\b\w/g, (letter) => letter.toUpperCase()); -} - -function objectField(value: Record, key: string): Record { - const field = value[key]; - if (!field || typeof field !== "object" || Array.isArray(field)) { - throw badRequest(`Cloud upstream discovery missing ${key}`); - } - return field as Record; -} - -function stringField(value: Record, key: string): string { - const field = value[key]; - if (typeof field !== "string" || field.length === 0) { - throw badRequest(`Cloud upstream discovery missing ${key}`); - } - return field; -} - -function numberField(value: Record, key: string): number { - const field = value[key]; - if (typeof field !== "number" || !Number.isFinite(field)) { - throw badRequest(`Cloud upstream discovery missing ${key}`); - } - return field; -} - -function optionalString(value: unknown): string | null { - return typeof value === "string" && value.length > 0 ? value : null; -} - -function optionalNumber(value: unknown): number | null { - return typeof value === "number" && Number.isFinite(value) ? value : null; -} - -function optionalObject(value: unknown): Record | null { - return value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; -} - -function asRecord(value: unknown): Record { - return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; -} - -function safeParseJson(text: string): unknown { - try { - return JSON.parse(text); - } catch { - return text; - } -} - -function cloudUpstreamTokenStatus(value: string): CloudUpstreamConnection["tokenStatus"] { - return value === "connected" || value === "expired" || value === "revoked" ? value : "pending"; -} - -function cloudUpstreamRunStatus(value: string): CloudUpstreamRun["status"] { - return value === "previewed" || value === "running" || value === "succeeded" || value === "failed" || value === "cancelled" - ? value - : "failed"; -} - -function cloudUpstreamStep(value: string): CloudUpstreamRun["activeStep"] { - return value === "connect" || value === "scan" || value === "preview" || value === "push" || value === "verify" || value === "activate" - ? value - : "push"; -} diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index 01a32c287a..0e5b5baf85 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -4,9 +4,17 @@ import { execFile } from "node:child_process"; import path from "node:path"; import { promisify } from "node:util"; import { and, eq, inArray } from "drizzle-orm"; -import { builtInManagedResources, principalPermissionGrants, type Db } from "@paperclipai/db"; +import { + builtInManagedResources, + issueRelations, + issues as issuesTable, + principalPermissionGrants, + type Db, +} from "@paperclipai/db"; import type { CompanyPortabilityAgentManifestEntry, + CompanyPortabilityBlobManifestEntry, + CompanyPortabilityEmbeddedAssetManifestEntry, CompanyPortabilityCollisionStrategy, CompanyPortabilityEnvInput, CompanyPortabilityExport, @@ -25,6 +33,10 @@ import type { CompanyPortabilityProjectWorkspaceManifestEntry, CompanyPortabilityIssueRoutineManifestEntry, CompanyPortabilityIssueRoutineTriggerManifestEntry, + CompanyPortabilityIssueDocumentManifestEntry, + CompanyPortabilityIssueWorkProductManifestEntry, + CompanyPortabilityIssueMonitorManifestEntry, + CompanyPortabilityIssueAttachmentManifestEntry, CompanyPortabilityIssueManifestEntry, CompanyPortabilitySidebarOrder, CompanyPortabilitySkillManifestEntry, @@ -52,12 +64,14 @@ import { normalizeAgentUrlKey, PERMISSION_KEYS, } from "@paperclipai/shared"; +import { sha256HexOfBytes } from "@paperclipai/shared/portability-hash"; import { readPaperclipSkillSyncPreference, writePaperclipSkillSyncPreference, } from "@paperclipai/adapter-utils/server-utils"; import { requireOpenCodeModelId } from "@paperclipai/adapter-opencode-local/server"; import { findServerAdapter } from "../adapters/index.js"; +import { normalizeIssueAttachmentMaxBytes } from "../attachment-types.js"; import { forbidden, notFound, unprocessable } from "../errors.js"; import { ghFetch, gitHubApiBase, resolveRawGitHubUrl } from "./github-fetch.js"; import type { StorageService } from "../storage/types.js"; @@ -70,8 +84,10 @@ import { renderOrgChartPng, type OrgNode } from "../routes/org-chart-svg.js"; import { companySkillService } from "./company-skills.js"; import { companyService } from "./companies.js"; import { validateCron } from "./cron.js"; +import { documentService } from "./documents.js"; import { issueService } from "./issues.js"; import { projectService } from "./projects.js"; +import { workProductService } from "./work-products.js"; import { routineService } from "./routines.js"; import { secretService } from "./secrets.js"; import { getConfiguredSecretProvider } from "../secrets/configured-provider.js"; @@ -137,6 +153,16 @@ const DEFAULT_INCLUDE: CompanyPortabilityInclude = { }; const DEFAULT_COLLISION_STRATEGY: CompanyPortabilityCollisionStrategy = "rename"; +// The bundle shape this build reads and writes. Bundles began declaring +// their schemaVersion in the .paperclip.yaml extension at 6; undeclared +// bundles are read as 5, the last unstamped shape. +const BUNDLE_SCHEMA_VERSION = 6; +const UNSTAMPED_BUNDLE_SCHEMA_VERSION = 5; +const DEFAULT_IMPORTED_LABEL_COLOR = "#6366f1"; +// Blob entries are content-addressed by sha256; the store itself is +// type-agnostic, so blob files always travel as opaque octet streams while +// each attachment entry carries the real content type. +const PORTABLE_BLOB_CONTENT_TYPE = "application/octet-stream"; const IMPORT_FORBIDDEN_ADAPTER_TYPES = new Set(["process", "http"]); const execFileAsync = promisify(execFile); let bundledSkillsCommitPromise: Promise | null = null; @@ -640,6 +666,7 @@ type ImportMode = "board_full" | "agent_safe"; type ImportBehaviorOptions = { mode?: ImportMode; sourceCompanyId?: string | null; + pauseAutomations?: boolean; }; type AgentLike = { @@ -805,6 +832,231 @@ function readPortableIssueComments( return comments; } +function normalizePortableLabelDefinitions(value: unknown): Array<{ name: string; color: string }> { + const entries: Array<{ name: string; color: string }> = []; + const seen = new Set(); + const append = (nameValue: unknown, colorValue: unknown) => { + const name = asString(nameValue); + if (!name || seen.has(name)) return; + seen.add(name); + entries.push({ name, color: asString(colorValue) ?? DEFAULT_IMPORTED_LABEL_COLOR }); + }; + if (Array.isArray(value)) { + for (const entry of value) { + if (!isPlainRecord(entry)) continue; + append(entry.name, entry.color); + } + } else if (isPlainRecord(value)) { + // Tolerate a name -> { color } (or name -> color) map from hand-edited bundles. + for (const [name, entry] of Object.entries(value)) { + append(name, isPlainRecord(entry) ? entry.color : entry); + } + } + return entries.sort((left, right) => left.name.localeCompare(right.name)); +} + +function readPortableIssueLabelNames(value: unknown): string[] { + if (!Array.isArray(value)) return []; + const names: string[] = []; + const seen = new Set(); + for (const entry of value) { + const name = asString(entry); + if (!name || seen.has(name)) continue; + seen.add(name); + names.push(name); + } + return names; +} + +function readPortableIssueBlockedBy(value: unknown): string[] { + if (!Array.isArray(value)) return []; + const slugs: string[] = []; + const seen = new Set(); + for (const entry of value) { + const slug = asString(entry); + if (!slug || seen.has(slug)) continue; + seen.add(slug); + slugs.push(slug); + } + return slugs; +} + +function normalizePortableIssueDocuments( + value: unknown, + warnings: string[], + sourceLabel: string, +): CompanyPortabilityIssueDocumentManifestEntry[] { + if (value === undefined || value === null) return []; + if (!Array.isArray(value)) { + warnings.push(`${sourceLabel} documents were ignored because they are not an array.`); + return []; + } + const documents: CompanyPortabilityIssueDocumentManifestEntry[] = []; + const seenKeys = new Set(); + for (const [index, entry] of value.entries()) { + if (!isPlainRecord(entry)) { + warnings.push(`${sourceLabel} document ${index + 1} was ignored because it is not an object.`); + continue; + } + const key = asString(entry.key); + const documentPath = asString(entry.path); + if (!key || !documentPath) { + warnings.push(`${sourceLabel} document ${index + 1} was ignored because it is missing a key or path.`); + continue; + } + if (seenKeys.has(key)) continue; + seenKeys.add(key); + documents.push({ + key, + title: asString(entry.title), + format: asString(entry.format) ?? "markdown", + path: normalizePortablePath(documentPath), + }); + } + return documents; +} + +function normalizePortableIssueWorkProducts(value: unknown): CompanyPortabilityIssueWorkProductManifestEntry[] { + if (!Array.isArray(value)) return []; + const workProducts: CompanyPortabilityIssueWorkProductManifestEntry[] = []; + for (const entry of value) { + if (!isPlainRecord(entry)) continue; + const type = asString(entry.type); + const provider = asString(entry.provider); + const title = asString(entry.title); + if (!type || !provider || !title) continue; + workProducts.push({ + type, + provider, + externalId: asString(entry.externalId), + title, + url: asString(entry.url), + status: asString(entry.status) ?? "active", + reviewState: asString(entry.reviewState) ?? "none", + isPrimary: asBoolean(entry.isPrimary) ?? false, + healthStatus: asString(entry.healthStatus) ?? "unknown", + summary: asString(entry.summary), + metadata: isPlainRecord(entry.metadata) ? entry.metadata : null, + }); + } + return workProducts; +} + +function normalizePortableIssueMonitor(value: unknown): CompanyPortabilityIssueMonitorManifestEntry | null { + if (!isPlainRecord(value)) return null; + const monitor = { + notes: asString(value.notes), + scheduledBy: asString(value.scheduledBy), + hadSchedule: asBoolean(value.hadSchedule) ?? false, + }; + return monitor.notes !== null || monitor.scheduledBy !== null || monitor.hadSchedule ? monitor : null; +} + +function portableBlobPath(sha256: string) { + return `blobs/${sha256}`; +} + +// Markdown can embed company asset images by their serving URL. The uuid is +// the asset row id; export ships the referenced bytes as blobs and import +// rewrites each reference to the asset id it minted. +const EMBEDDED_ASSET_URL_PATTERN = /\/api\/assets\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\/content/gi; + +function collectEmbeddedAssetIds(text: string): string[] { + const ids: string[] = []; + for (const match of text.matchAll(EMBEDDED_ASSET_URL_PATTERN)) { + ids.push(match[1]!.toLowerCase()); + } + return ids; +} + +export function rewriteEmbeddedAssetUrls(text: string, assetIdMap: Map): string { + if (assetIdMap.size === 0) return text; + return text.replace(EMBEDDED_ASSET_URL_PATTERN, (match, assetId: string) => { + const mapped = assetIdMap.get(assetId.toLowerCase()); + return mapped ? `/api/assets/${mapped}/content` : match; + }); +} + +function normalizePortableEmbeddedAssets(value: unknown): CompanyPortabilityEmbeddedAssetManifestEntry[] { + if (!Array.isArray(value)) return []; + const entries: CompanyPortabilityEmbeddedAssetManifestEntry[] = []; + const seen = new Set(); + for (const entry of value) { + if (!isPlainRecord(entry)) continue; + const assetId = asString(entry.assetId)?.toLowerCase() ?? null; + const sha256 = asString(entry.sha256)?.toLowerCase() ?? null; + if (!assetId || !sha256 || seen.has(assetId)) continue; + seen.add(assetId); + const ownedBy = Array.isArray(entry.ownedBy) + ? Array.from(new Set(entry.ownedBy.flatMap((owner) => { + const normalized = asString(owner); + return normalized ? [normalized] : []; + }))) + : []; + entries.push({ + assetId, + sha256, + contentType: asString(entry.contentType) ?? PORTABLE_BLOB_CONTENT_TYPE, + originalFilename: asString(entry.originalFilename), + ownedBy: ownedBy.length > 0 ? ownedBy : undefined, + }); + } + return entries; +} + +function normalizePortableBlobIndex(value: unknown): CompanyPortabilityBlobManifestEntry[] { + if (!Array.isArray(value)) return []; + const blobs: CompanyPortabilityBlobManifestEntry[] = []; + const seen = new Set(); + for (const entry of value) { + if (!isPlainRecord(entry)) continue; + const sha256 = asString(entry.sha256)?.toLowerCase() ?? null; + const byteSize = asInteger(entry.byteSize); + if (!sha256 || seen.has(sha256) || byteSize === null || byteSize < 0) continue; + seen.add(sha256); + blobs.push({ + sha256, + byteSize, + contentType: asString(entry.contentType) ?? PORTABLE_BLOB_CONTENT_TYPE, + }); + } + return blobs; +} + +function normalizePortableIssueAttachments( + value: unknown, + warnings: string[], + sourceLabel: string, +): CompanyPortabilityIssueAttachmentManifestEntry[] { + if (value === undefined || value === null) return []; + if (!Array.isArray(value)) { + warnings.push(`${sourceLabel} attachments were ignored because they are not an array.`); + return []; + } + const attachments: CompanyPortabilityIssueAttachmentManifestEntry[] = []; + for (const [index, entry] of value.entries()) { + if (!isPlainRecord(entry)) { + warnings.push(`${sourceLabel} attachment ${index + 1} was ignored because it is not an object.`); + continue; + } + const sha256 = asString(entry.sha256)?.toLowerCase() ?? null; + if (!sha256) { + warnings.push(`${sourceLabel} attachment ${index + 1} was ignored because it has no sha256.`); + continue; + } + const byteSize = asInteger(entry.byteSize); + const commentIndex = asInteger(entry.commentIndex); + attachments.push({ + sha256, + contentType: asString(entry.contentType) ?? PORTABLE_BLOB_CONTENT_TYPE, + originalFilename: asString(entry.originalFilename), + byteSize: byteSize !== null && byteSize >= 0 ? byteSize : 0, + commentIndex: commentIndex !== null && commentIndex >= 0 ? commentIndex : null, + }); + } + return attachments; +} + function appendCodexImportArg(adapterConfig: Record, arg: string) { const extraArgs = readStringArray(adapterConfig.extraArgs); if (extraArgs) { @@ -1683,7 +1935,12 @@ function sortAgentsBySidebarOrder) { +function filterPortableExtensionYaml( + yaml: string, + selectedFiles: Set, + remainingFiles: Record, + extensionPath: string, +) { const selected = collectSelectedExportSlugs(selectedFiles); const parsed = parseYamlFile(yaml); for (const section of ["agents", "projects", "tasks", "routines"] as const) { @@ -1709,6 +1966,84 @@ function filterPortableExtensionYaml(yaml: string, selectedFiles: Set) { } } + if (Array.isArray(parsed.labels)) { + const referencedLabelNames = new Set(); + const tasksSection = parsed.tasks; + if (isPlainRecord(tasksSection)) { + for (const entry of Object.values(tasksSection)) { + if (!isPlainRecord(entry)) continue; + for (const name of readPortableIssueLabelNames(entry.labels)) { + referencedLabelNames.add(name); + } + } + } + const filteredLabels = parsed.labels.filter( + (entry) => isPlainRecord(entry) && typeof entry.name === "string" && referencedLabelNames.has(entry.name.trim()), + ); + if (filteredLabels.length > 0) { + parsed.labels = filteredLabels; + } else { + delete parsed.labels; + } + } + + // Embedded-asset entries stay only while some remaining text file (or a + // remaining task's comments, which live in this yaml) still references + // their assetId; blobs owned solely by pruned entries drop below. + const keptEmbeddedAssetShas = new Set(); + if (Array.isArray(parsed.embeddedAssets)) { + const referencedAssetIds = new Set(); + for (const [filePath, content] of Object.entries(remainingFiles)) { + if (filePath === extensionPath || typeof content !== "string") continue; + for (const assetId of collectEmbeddedAssetIds(content)) { + referencedAssetIds.add(assetId); + } + } + const tasksSection = parsed.tasks; + if (isPlainRecord(tasksSection)) { + for (const entry of Object.values(tasksSection)) { + if (!isPlainRecord(entry)) continue; + for (const assetId of collectEmbeddedAssetIds(JSON.stringify(entry.comments ?? []))) { + referencedAssetIds.add(assetId); + } + } + } + const filteredEmbeddedAssets = parsed.embeddedAssets.filter((entry) => { + if (!isPlainRecord(entry)) return false; + const assetId = asString(entry.assetId)?.toLowerCase(); + if (!assetId || !referencedAssetIds.has(assetId)) return false; + const sha256 = asString(entry.sha256)?.toLowerCase(); + if (sha256) keptEmbeddedAssetShas.add(sha256); + return true; + }); + if (filteredEmbeddedAssets.length > 0) { + parsed.embeddedAssets = filteredEmbeddedAssets; + } else { + delete parsed.embeddedAssets; + } + } + + if (Array.isArray(parsed.blobs)) { + const referencedBlobShas = new Set(keptEmbeddedAssetShas); + const tasksSection = parsed.tasks; + if (isPlainRecord(tasksSection)) { + for (const entry of Object.values(tasksSection)) { + if (!isPlainRecord(entry)) continue; + for (const attachment of normalizePortableIssueAttachments(entry.attachments, [], "")) { + referencedBlobShas.add(attachment.sha256); + } + } + } + const filteredBlobs = parsed.blobs.filter( + (entry) => isPlainRecord(entry) && typeof entry.sha256 === "string" && referencedBlobShas.has(entry.sha256.trim().toLowerCase()), + ); + if (filteredBlobs.length > 0) { + parsed.blobs = filteredBlobs; + } else { + delete parsed.blobs; + } + } + const sidebarOrder = normalizePortableSidebarOrder(parsed.sidebar); if (sidebarOrder) { const filteredSidebar = stripEmptyValues({ @@ -1749,7 +2084,12 @@ function filterExportFiles( const extensionEntry = filtered[paperclipExtensionPath]; if (selectedFiles.has(paperclipExtensionPath) && typeof extensionEntry === "string") { - filtered[paperclipExtensionPath] = filterPortableExtensionYaml(extensionEntry, selectedFiles); + filtered[paperclipExtensionPath] = filterPortableExtensionYaml( + extensionEntry, + selectedFiles, + filtered, + paperclipExtensionPath, + ); } return filtered; @@ -2176,9 +2516,12 @@ async function buildSkillSourceEntry(skill: CompanySkill) { } function shouldReferenceSkillOnExport(skill: CompanySkill, expandReferencedSkills: boolean) { - if (expandReferencedSkills) return false; const metadata = isPlainRecord(skill.metadata) ? skill.metadata : null; + // Bundled Paperclip skills ship with every build and may contain executable + // scripts that import policy rejects when expanded; the target re-resolves + // them from its own catalog via the pinned reference stub instead. if (asString(metadata?.sourceKind) === "paperclip_bundled") return true; + if (expandReferencedSkills) return false; return skill.sourceType === "github" || skill.sourceType === "skills_sh" || skill.sourceType === "url"; } @@ -2603,8 +2946,18 @@ function buildManifestFromPackageFiles( const paperclipExtension = paperclipExtensionPath ? parseYamlFile(readPortableTextFile(normalizedFiles, paperclipExtensionPath) ?? "") : {}; + const declaredSchemaVersion = asInteger(paperclipExtension.schemaVersion); + const bundleSchemaVersion = declaredSchemaVersion !== null && declaredSchemaVersion > 0 + ? declaredSchemaVersion + : UNSTAMPED_BUNDLE_SCHEMA_VERSION; + if (bundleSchemaVersion > BUNDLE_SCHEMA_VERSION) { + throw unprocessable(`Company package declares schemaVersion ${bundleSchemaVersion}, which was produced by a newer Paperclip; this board reads up to schemaVersion ${BUNDLE_SCHEMA_VERSION}.`); + } const paperclipCompany = isPlainRecord(paperclipExtension.company) ? paperclipExtension.company : {}; const paperclipSidebar = normalizePortableSidebarOrder(paperclipExtension.sidebar); + const paperclipLabels = normalizePortableLabelDefinitions(paperclipExtension.labels); + const paperclipBlobs = normalizePortableBlobIndex(paperclipExtension.blobs); + const paperclipEmbeddedAssets = normalizePortableEmbeddedAssets(paperclipExtension.embeddedAssets); const paperclipAgents = isPlainRecord(paperclipExtension.agents) ? paperclipExtension.agents : {}; const paperclipProjects = isPlainRecord(paperclipExtension.projects) ? paperclipExtension.projects : {}; const paperclipTasks = isPlainRecord(paperclipExtension.tasks) ? paperclipExtension.tasks : {}; @@ -2649,7 +3002,7 @@ function buildManifestFromPackageFiles( const skillPaths = Array.from(new Set([...referencedSkillPaths, ...discoveredSkillPaths])).sort(); const manifest: CompanyPortabilityManifest = { - schemaVersion: 5, + schemaVersion: bundleSchemaVersion, generatedAt: new Date().toISOString(), source: opts?.sourceLabel ?? null, includes: { @@ -2687,6 +3040,9 @@ function buildManifestFromPackageFiles( asString(paperclipCompany.feedbackDataSharingTermsVersion), }, sidebar: paperclipSidebar, + labels: paperclipLabels, + blobs: paperclipBlobs, + embeddedAssets: paperclipEmbeddedAssets, agents: [], skills: [], projects: [], @@ -2933,6 +3289,7 @@ function buildManifestFromPackageFiles( labelIds: Array.isArray(extension.labelIds) ? extension.labelIds.filter((entry): entry is string => typeof entry === "string") : [], + labelNames: readPortableIssueLabelNames(extension.labels), billingCode: asString(extension.billingCode), executionWorkspaceSettings: isPlainRecord(extension.executionWorkspaceSettings) ? extension.executionWorkspaceSettings @@ -2941,6 +3298,11 @@ function buildManifestFromPackageFiles( ? extension.assigneeAdapterOverrides : null, comments: readPortableIssueComments(extension.comments, warnings, `Task ${slug}`), + blockedBy: readPortableIssueBlockedBy(extension.blockedBy), + documents: normalizePortableIssueDocuments(extension.documents, warnings, `Task ${slug}`), + workProducts: normalizePortableIssueWorkProducts(extension.workProducts), + monitor: normalizePortableIssueMonitor(extension.monitor), + attachments: normalizePortableIssueAttachments(extension.attachments, warnings, `Task ${slug}`), metadata: isPlainRecord(extension.metadata) ? extension.metadata : null, }); if (frontmatter.kind && frontmatter.kind !== "task") { @@ -2948,6 +3310,10 @@ function buildManifestFromPackageFiles( } } + if (bundleSchemaVersion < BUNDLE_SCHEMA_VERSION && manifest.issues.length > 0) { + warnings.push(`This package declares schemaVersion ${bundleSchemaVersion} and predates label, blocker, document, work product, monitor, attachment, and embedded image transfer; that task data imports only if the bundle carries it.`); + } + manifest.envInputs = dedupeEnvInputs(manifest.envInputs); return { manifest, @@ -3023,6 +3389,8 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const issues = issueService(db); const companySkills = companySkillService(db); const secrets = secretService(db); + const documentsSvc = documentService(db); + const workProductsSvc = workProductService(db); const strictSecretsMode = process.env.PAPERCLIP_SECRETS_STRICT_MODE === "true"; const defaultSecretProvider = getConfiguredSecretProvider(); @@ -3778,6 +4146,28 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { paperclipProjectsOut[slug] = isPlainRecord(extension) ? extension : {}; } + const referencedLabelIds = new Set(); + for (const issue of selectedIssueRows) { + for (const labelId of issue.labelIds ?? []) referencedLabelIds.add(labelId); + } + const labelNameById = new Map(); + const exportedLabels: Array<{ name: string; color: string }> = []; + if (referencedLabelIds.size > 0) { + for (const label of await issuesSvc.listLabels(companyId)) { + if (!referencedLabelIds.has(label.id)) continue; + labelNameById.set(label.id, label.name); + exportedLabels.push({ name: label.name, color: label.color }); + } + exportedLabels.sort((left, right) => left.name.localeCompare(right.name)); + const missingLabelIds = Array.from(referencedLabelIds).filter((labelId) => !labelNameById.has(labelId)); + if (missingLabelIds.length > 0) { + warnings.push(`Skipped ${missingLabelIds.length} task label reference${missingLabelIds.length === 1 ? "" : "s"} whose label definitions no longer exist.`); + } + } + + let unexportedBlockerEdgeCount = 0; + let unportableWorkProductRefCount = 0; + const exportedBlobs = new Map(); for (const issue of selectedIssueRows) { const taskSlug = taskSlugByIssueId.get(issue.id)!; const projectSlug = issue.projectId ? (projectSlugById.get(issue.projectId) ?? null) : null; @@ -3800,6 +4190,93 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { } } const comments = await issuesSvc.listComments(issue.id, { order: "asc" }); + // Blocker edges travel by task slug; only edges with both endpoints in + // the export can be carried. + const relationSummaries = await issuesSvc.getRelationSummaries(issue.id); + const blockedBySlugs: string[] = []; + for (const blocker of relationSummaries.blockedBy) { + const blockerSlug = taskSlugByIssueId.get(blocker.id); + if (blockerSlug) { + blockedBySlugs.push(blockerSlug); + } else { + unexportedBlockerEdgeCount += 1; + } + } + blockedBySlugs.sort((left, right) => left.localeCompare(right)); + unexportedBlockerEdgeCount += relationSummaries.blocks + .filter((blocked) => !taskSlugByIssueId.has(blocked.id)) + .length; + const issueDocumentRows = await documentsSvc.listIssueDocuments(issue.id, { includeSystem: true }); + const documentEntries = issueDocumentRows.map((document) => { + const documentPath = `tasks/${taskSlug}/documents/${document.key}.md`; + files[documentPath] = document.body ?? ""; + return { + key: document.key, + title: document.title ?? null, + format: document.format, + path: documentPath, + }; + }); + const workProductRows = await workProductsSvc.listForIssue(issue.id); + const workProductEntries = workProductRows.map((workProduct) => { + if (workProduct.executionWorkspaceId || workProduct.runtimeServiceId || workProduct.createdByRunId) { + unportableWorkProductRefCount += 1; + } + return stripEmptyValues({ + type: workProduct.type, + provider: workProduct.provider, + externalId: workProduct.externalId ?? null, + title: workProduct.title, + url: workProduct.url ?? null, + status: workProduct.status, + reviewState: workProduct.reviewState !== "none" ? workProduct.reviewState : undefined, + isPrimary: workProduct.isPrimary ? true : undefined, + healthStatus: workProduct.healthStatus !== "unknown" ? workProduct.healthStatus : undefined, + summary: workProduct.summary ?? null, + metadata: workProduct.metadata ?? null, + }); + }); + // Attachment bytes travel as content-addressed blobs/ entries, + // deduped across the bundle; each per-task entry references its blob by + // hash and its comment by index into the exported comments array. + const attachmentRows = (await issuesSvc.listAttachments(issue.id)) + .slice() + .sort((left, right) => new Date(left.createdAt).getTime() - new Date(right.createdAt).getTime()); + const commentIndexById = new Map(comments.map((comment, index) => [comment.id, index] as const)); + const attachmentEntries: Array> = []; + if (attachmentRows.length > 0 && !storage) { + warnings.push(`Skipped ${attachmentRows.length} attachment${attachmentRows.length === 1 ? "" : "s"} on task ${taskSlug} because storage is unavailable.`); + } else if (storage) { + for (const attachment of attachmentRows) { + let body: Buffer; + try { + const object = await storage.getObject(companyId, attachment.objectKey); + body = await streamToBuffer(object.stream); + } catch (err) { + warnings.push(`Skipped attachment ${attachment.originalFilename ?? attachment.sha256} on task ${taskSlug} because its stored object could not be read: ${err instanceof Error ? err.message : String(err)}`); + continue; + } + // Blobs are addressed by the hash of the bytes actually read; a + // stale asset-row hash loses to the recomputed one. + const sha256 = sha256HexOfBytes(body); + if (attachment.sha256 && attachment.sha256.toLowerCase() !== sha256) { + warnings.push(`Attachment ${attachment.originalFilename ?? attachment.sha256} on task ${taskSlug} was exported under its recomputed content hash because the stored hash did not match.`); + } + if (!exportedBlobs.has(sha256)) { + files[portableBlobPath(sha256)] = bufferToPortableBinaryFile(body, PORTABLE_BLOB_CONTENT_TYPE); + exportedBlobs.set(sha256, { sha256, byteSize: body.length, contentType: PORTABLE_BLOB_CONTENT_TYPE }); + } + attachmentEntries.push({ + sha256, + contentType: attachment.contentType ?? PORTABLE_BLOB_CONTENT_TYPE, + originalFilename: attachment.originalFilename ?? null, + byteSize: body.length, + commentIndex: attachment.issueCommentId != null + ? commentIndexById.get(attachment.issueCommentId) ?? null + : null, + }); + } + } files[taskPath] = buildMarkdown( { name: issue.title, @@ -3812,7 +4289,11 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { identifier: issue.identifier, status: issue.status, priority: issue.priority, - labelIds: issue.labelIds ?? undefined, + // Labels travel by name (their natural key); the bundle-level labels + // section carries the matching color definitions. + labels: (issue.labelIds ?? []) + .map((labelId) => labelNameById.get(labelId)) + .filter((name): name is string => Boolean(name)), billingCode: issue.billingCode ?? null, projectWorkspaceKey: projectWorkspaceKey ?? undefined, executionWorkspaceSettings: issue.executionWorkspaceSettings ?? undefined, @@ -3831,10 +4312,28 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { : new Date(comment.createdAt).toISOString(), })) : undefined, + blockedBy: blockedBySlugs, + documents: documentEntries, + workProducts: workProductEntries, + attachments: attachmentEntries, + monitor: { + notes: issue.monitorNotes ?? null, + scheduledBy: issue.monitorScheduledBy ?? null, + // Timestamps are not portable; the importer restores monitors + // un-armed, so only the fact that a check was scheduled travels. + hadSchedule: issue.monitorNextCheckAt != null ? true : undefined, + }, }); paperclipTasksOut[taskSlug] = isPlainRecord(extension) ? extension : {}; } + if (unexportedBlockerEdgeCount > 0) { + warnings.push(`${unexportedBlockerEdgeCount} blocker relation${unexportedBlockerEdgeCount === 1 ? " references a task" : "s reference tasks"} outside this export and ${unexportedBlockerEdgeCount === 1 ? "was" : "were"} not included.`); + } + if (unportableWorkProductRefCount > 0) { + warnings.push(`${unportableWorkProductRefCount} work product${unportableWorkProductRefCount === 1 ? " references" : "s reference"} execution workspaces or runs that are not portable; those references were omitted from the export.`); + } + for (const { workspaceId, taskSlugs } of unportableTaskWorkspaceRefs.values()) { const preview = taskSlugs.slice(0, 4).join(", "); const remainder = taskSlugs.length > 4 ? ` and ${taskSlugs.length - 4} more` : ""; @@ -3876,7 +4375,92 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { paperclipRoutinesOut[taskSlug] = isPlainRecord(extension) ? extension : {}; } + // Exported markdown can embed company asset images as + // /api/assets//content references (issue descriptions and documents, + // agent instructions, comment bodies). Ship the referenced bytes as + // content-addressed blobs plus an embeddedAssets index so imports can + // recreate the assets and rewrite every reference. Each entry records + // which export categories reference it, so selection can follow the + // toggles of the referencing files. + const embeddedAssetOwners = new Map>(); + const routineTaskSlugs = new Set(taskSlugByRoutineId.values()); + const categorizeEmbeddedAssetOwner = (filePath: string) => { + if (filePath.startsWith("agents/")) return "agents"; + if (filePath.startsWith("projects/")) return "projects"; + if (filePath.startsWith("skills/")) return "skills"; + const taskMatch = filePath.match(/^tasks\/([^/]+)\//); + if (taskMatch) return routineTaskSlugs.has(taskMatch[1]!) ? "routines" : "tasks"; + return "always"; + }; + const noteEmbeddedAssetReference = (assetId: string, owner: string) => { + const owners = embeddedAssetOwners.get(assetId) ?? new Set(); + owners.add(owner); + embeddedAssetOwners.set(assetId, owners); + }; + for (const [filePath, content] of Object.entries(files)) { + if (typeof content !== "string") continue; + for (const assetId of collectEmbeddedAssetIds(content)) { + noteEmbeddedAssetReference(assetId, categorizeEmbeddedAssetOwner(filePath)); + } + } + // Comment bodies travel in the extension yaml rather than TASK.md, so + // scan the assembled task extension entries for their references too. + for (const extension of Object.values(paperclipTasksOut)) { + for (const assetId of collectEmbeddedAssetIds(JSON.stringify(extension.comments ?? []))) { + noteEmbeddedAssetReference(assetId, "tasks"); + } + } + + const embeddedAssetIndex: CompanyPortabilityEmbeddedAssetManifestEntry[] = []; + let unownedEmbeddedAssetRefCount = 0; + const referencedEmbeddedAssetIds = Array.from(embeddedAssetOwners.keys()) + .sort((left, right) => left.localeCompare(right)); + if (referencedEmbeddedAssetIds.length > 0 && !storage) { + warnings.push(`Skipped ${referencedEmbeddedAssetIds.length} embedded image asset${referencedEmbeddedAssetIds.length === 1 ? "" : "s"} because storage is unavailable.`); + } else if (storage) { + for (const assetId of referencedEmbeddedAssetIds) { + const asset = await assetRecords.getById(assetId); + // Embedded references only pull bytes for assets owned by the + // exporting company: a crafted URL naming another company's asset id + // must not leak that asset's bytes into the bundle. + if (!asset || asset.companyId !== companyId) { + unownedEmbeddedAssetRefCount += 1; + continue; + } + let body: Buffer; + try { + const object = await storage.getObject(companyId, asset.objectKey); + body = await streamToBuffer(object.stream); + } catch (err) { + warnings.push(`Skipped embedded image asset ${asset.originalFilename ?? assetId} because its stored object could not be read: ${err instanceof Error ? err.message : String(err)}`); + continue; + } + // Blobs are addressed by the hash of the bytes actually read; a + // stale asset-row hash loses to the recomputed one. + const sha256 = sha256HexOfBytes(body); + if (!exportedBlobs.has(sha256)) { + files[portableBlobPath(sha256)] = bufferToPortableBinaryFile(body, PORTABLE_BLOB_CONTENT_TYPE); + exportedBlobs.set(sha256, { sha256, byteSize: body.length, contentType: PORTABLE_BLOB_CONTENT_TYPE }); + } + const owners = embeddedAssetOwners.get(assetId) ?? new Set(); + embeddedAssetIndex.push({ + assetId, + sha256, + contentType: asset.contentType ?? PORTABLE_BLOB_CONTENT_TYPE, + originalFilename: asset.originalFilename ?? null, + ownedBy: Array.from(owners).sort((left, right) => left.localeCompare(right)), + }); + } + } + if (unownedEmbeddedAssetRefCount > 0) { + warnings.push(unownedEmbeddedAssetRefCount === 1 + ? "1 embedded image reference points at an asset that does not belong to this company or no longer exists; its image was not exported." + : `${unownedEmbeddedAssetRefCount} embedded image references point at assets that do not belong to this company or no longer exist; their images were not exported.`); + } + const paperclipExtensionPath = ".paperclip.yaml"; + const exportedBlobIndex = Array.from(exportedBlobs.values()) + .sort((left, right) => left.sha256.localeCompare(right.sha256)); const paperclipAgents = Object.fromEntries( Object.entries(paperclipAgentsOut).filter(([, value]) => isPlainRecord(value) && Object.keys(value).length > 0), ); @@ -3892,6 +4476,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { files[paperclipExtensionPath] = buildYamlFile( { schema: "paperclip/v1", + schemaVersion: BUNDLE_SCHEMA_VERSION, company: stripEmptyValues({ brandColor: company.brandColor ?? null, logoPath: companyLogoPath, @@ -3903,6 +4488,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { feedbackDataSharingTermsVersion: company.feedbackDataSharingTermsVersion ?? null, }), sidebar: stripEmptyValues(sidebarOrder), + labels: exportedLabels.length > 0 ? exportedLabels : undefined, + blobs: exportedBlobIndex.length > 0 ? exportedBlobIndex : undefined, + embeddedAssets: embeddedAssetIndex.length > 0 ? embeddedAssetIndex : undefined, agents: Object.keys(paperclipAgents).length > 0 ? paperclipAgents : undefined, projects: Object.keys(paperclipProjects).length > 0 ? paperclipProjects : undefined, tasks: Object.keys(paperclipTasks).length > 0 ? paperclipTasks : undefined, @@ -4118,11 +4706,8 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { } } if (issue.recurring) { - if (!issue.projectSlug) { - errors.push(`Recurring task ${issue.slug} must declare a project to import as a routine.`); - } if (!issue.assigneeAgentSlug) { - errors.push(`Recurring task ${issue.slug} must declare an assignee to import as a routine.`); + warnings.push(`Recurring task ${issue.slug} has no assignee; the routine will stay paused until one is set.`); } const resolvedRoutine = resolvePortableRoutineDefinition(issue, parsed.frontmatter.schedule); warnings.push(...resolvedRoutine.warnings); @@ -4425,9 +5010,23 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { } const sourceManifest = plan.source.manifest; + const pauseAutomations = options?.pauseAutomations === true; + const importedAutomationPausedAt = pauseAutomations ? new Date() : null; const warnings = [...plan.preview.warnings]; const include = plan.include; + // Content-addressed blobs double as the bundle's tamper seal. Verify every + // blob before any row is written so a corrupted package cannot leave a + // partially imported company behind. + for (const [filePath, fileEntry] of Object.entries(plan.source.files)) { + if (!filePath.startsWith("blobs/")) continue; + const declaredSha = filePath.slice("blobs/".length); + const body = portableFileToBuffer(fileEntry, filePath); + if (sha256HexOfBytes(body) !== declaredSha) { + throw unprocessable(`Bundle blob ${filePath} does not match its declared sha256; the package is corrupted or was tampered with.`); + } + } + let targetCompany: { id: string; name: string; @@ -4591,8 +5190,57 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { } } + // Recreate embedded markdown image assets before any markdown is + // written, so issue descriptions, comments, documents, and agent + // instructions can be rewritten to the asset ids this board minted. + // References whose entry or blob cannot be imported keep their source + // ids and stay broken rather than pointing at the wrong asset. + const embeddedAssetIdMap = new Map(); + const manifestEmbeddedAssets = sourceManifest.embeddedAssets ?? []; + if (manifestEmbeddedAssets.length > 0) { + if (!storage) { + warnings.push(`Skipped ${manifestEmbeddedAssets.length} embedded image asset${manifestEmbeddedAssets.length === 1 ? "" : "s"} because storage is unavailable; their references were left unchanged.`); + } else { + for (const embeddedAsset of manifestEmbeddedAssets) { + const embeddedAssetLabel = embeddedAsset.originalFilename ?? embeddedAsset.assetId; + const blobPath = portableBlobPath(embeddedAsset.sha256); + const blobFile = plan.source.files[blobPath]; + if (blobFile === undefined) { + warnings.push(`Embedded image asset ${embeddedAssetLabel} was skipped because its blob is missing from the package: ${blobPath}; its references were left unchanged.`); + continue; + } + // The pre-apply loop above already verified that every blobs/* + // entry hashes to its path, which this entry's sha256 derives. + const body = portableFileToBuffer(blobFile, blobPath); + try { + const stored = await storage.putFile({ + companyId: targetCompany.id, + namespace: "assets/general", + originalFilename: embeddedAsset.originalFilename, + contentType: embeddedAsset.contentType, + body, + }); + const createdAsset = await assetRecords.create(targetCompany.id, { + provider: stored.provider, + objectKey: stored.objectKey, + contentType: stored.contentType, + byteSize: stored.byteSize, + sha256: stored.sha256, + originalFilename: stored.originalFilename, + createdByAgentId: null, + createdByUserId: actorUserId ?? null, + }); + embeddedAssetIdMap.set(embeddedAsset.assetId, createdAsset.id); + } catch (error) { + warnings.push(`Embedded image asset ${embeddedAssetLabel} could not be imported: ${error instanceof Error ? error.message : String(error)}; its references were left unchanged.`); + } + } + } + } + const resultAgents: CompanyPortabilityImportResult["agents"] = []; const resultProjects: CompanyPortabilityImportResult["projects"] = []; + const resultRoutines: CompanyPortabilityImportResult["routines"] = []; const importedSlugToAgentId = new Map(); const existingSlugToAgentId = new Map(); const preImportExistingSlugToAgentId = new Map(); @@ -4671,6 +5319,11 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { if (!markdownRaw && !fallbackPromptTemplate) { warnings.push(`Missing AGENTS markdown for ${manifestAgent.slug}; imported with an empty managed bundle.`); } + if (embeddedAssetIdMap.size > 0) { + for (const [relativePath, content] of Object.entries(bundleFiles)) { + bundleFiles[relativePath] = rewriteEmbeddedAssetUrls(content, embeddedAssetIdMap); + } + } // Apply adapter overrides from request if present const adapterOverride = input.adapterOverrides?.[planAgent.slug]; @@ -4700,9 +5353,19 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { permissions: manifestAgent.permissions, metadata: manifestAgent.metadata, }; + const automationPausePatch = pauseAutomations + ? { + status: "paused", + pauseReason: "system", + pausedAt: importedAutomationPausedAt, + } + : {}; if (planAgent.action === "update" && planAgent.existingAgentId) { - let updated = await agents.update(planAgent.existingAgentId, patch); + let updated = await agents.update(planAgent.existingAgentId, { + ...patch, + ...automationPausePatch, + }); if (!updated) { warnings.push(`Skipped update for missing agent ${planAgent.existingAgentId}.`); resultAgents.push({ @@ -4747,10 +5410,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { continue; } - const createdStatus = "idle"; let created = await agents.create(targetCompany.id, { ...patch, - status: createdStatus, + ...automationPausePatch, + status: pauseAutomations ? "paused" : "idle", }); await access.ensureMembership(targetCompany.id, "agent", created.id, "member", "active"); await access.setPrincipalPermission( @@ -4776,7 +5439,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { manifestAgent.permissionGrants ?? [], actorUserId ?? null, ); - agentStatusById.set(created.id, created.status ?? createdStatus); + agentStatusById.set(created.id, created.status ?? (pauseAutomations ? "paused" : "idle")); await secrets.syncEnvBindingsForTarget?.( targetCompany.id, { targetType: "agent", targetId: created.id }, @@ -4955,10 +5618,64 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { if (include.issues) { const routines = routineService(db); + + // Resolve label names against the target company before creating + // issues: reuse existing labels (the target's color wins) and create + // the rest from the bundle's definitions. Old bundles carry raw + // labelIds instead; those only survive when the id happens to exist + // in the target company. + const labelColorByName = new Map(); + for (const label of sourceManifest.labels ?? []) { + if (!labelColorByName.has(label.name)) labelColorByName.set(label.name, label.color); + } + const referencedLabelNames = new Set(); + let hasLegacyLabelIds = false; + for (const manifestIssue of sourceManifest.issues) { + if (manifestIssue.recurring) continue; + for (const name of manifestIssue.labelNames ?? []) referencedLabelNames.add(name); + if ((manifestIssue.labelIds ?? []).length > 0) hasLegacyLabelIds = true; + } + const labelIdByName = new Map(); + const existingTargetLabelIds = new Set(); + if (referencedLabelNames.size > 0 || hasLegacyLabelIds) { + const targetLabels = await issues.listLabels(targetCompany.id); + const targetLabelByName = new Map(targetLabels.map((label) => [label.name, label])); + for (const label of targetLabels) existingTargetLabelIds.add(label.id); + const keptColorNames: string[] = []; + for (const name of Array.from(referencedLabelNames).sort((left, right) => left.localeCompare(right))) { + const existing = targetLabelByName.get(name); + if (existing) { + labelIdByName.set(name, existing.id); + const exportedColor = labelColorByName.get(name); + if (exportedColor && exportedColor.toLowerCase() !== existing.color.toLowerCase()) { + keptColorNames.push(name); + } + continue; + } + const created = await issues.createLabel(targetCompany.id, { + name, + color: labelColorByName.get(name) ?? DEFAULT_IMPORTED_LABEL_COLOR, + }); + labelIdByName.set(name, created.id); + } + if (keptColorNames.length > 0) { + warnings.push(`Existing label color${keptColorNames.length === 1 ? " was" : "s were"} kept for ${keptColorNames.join(", ")}; the imported bundle used different colors.`); + } + } + + const importedIssueIdBySlug = new Map(); + const blockedByBySlug = new Map(); + let unarmedMonitorCount = 0; + let attachmentsSkippedNoStorage = 0; + const attachmentMaxBytes = normalizeIssueAttachmentMaxBytes(targetCompany.attachmentMaxBytes ?? null); + for (const manifestIssue of sourceManifest.issues) { const markdownRaw = readPortableTextFile(plan.source.files, manifestIssue.path); const parsed = markdownRaw ? parseFrontmatterMarkdown(markdownRaw) : null; - const description = parsed?.body || manifestIssue.description || null; + const rawDescription = parsed?.body || manifestIssue.description || null; + const description = rawDescription === null + ? null + : rewriteEmbeddedAssetUrls(rawDescription, embeddedAssetIdMap); const assigneeAgentId = resolveImportedAssigneeAgentId( manifestIssue.assigneeAgentSlug, importedSlugToAgentId, @@ -4979,8 +5696,11 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { warnings.push(`Task ${manifestIssue.slug} references workspace key ${manifestIssue.projectWorkspaceKey}, but that workspace was not imported.`); } if (manifestIssue.recurring) { - if (!projectId) { - throw unprocessable(`Recurring task ${manifestIssue.slug} is missing the project required to create a routine.`); + // Routines can legitimately exist without a project or assignee; + // routines.create accepts a null project and pauses an active + // routine that has no assignee to run it. + if (manifestIssue.projectSlug && !projectId) { + warnings.push(`Recurring task ${manifestIssue.slug} references project ${manifestIssue.projectSlug}, which was not imported; the routine was created without a project.`); } const resolvedRoutine = resolvePortableRoutineDefinition(manifestIssue, parsed?.frontmatter.schedule); if (resolvedRoutine.errors.length > 0) { @@ -5003,7 +5723,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { priority: manifestIssue.priority && ISSUE_PRIORITIES.includes(manifestIssue.priority as any) ? manifestIssue.priority as typeof ISSUE_PRIORITIES[number] : "medium", - status: manifestIssue.status && ROUTINE_STATUSES.includes(manifestIssue.status as any) + status: pauseAutomations + ? "paused" + : manifestIssue.status && ROUTINE_STATUSES.includes(manifestIssue.status as any) ? manifestIssue.status as typeof ROUTINE_STATUSES[number] : "active", concurrencyPolicy: @@ -5019,6 +5741,16 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { agentId: null, userId: actorUserId ?? null, }); + resultRoutines.push({ + slug: manifestIssue.slug, + id: createdRoutine.id, + action: "created", + title: createdRoutine.title, + status: createdRoutine.status, + }); + if (!assigneeAgentId) { + warnings.push(`Routine ${manifestIssue.slug} was imported without an assignee and will stay paused until one is set.`); + } for (const trigger of routineDefinition.triggers) { if (trigger.kind === "schedule") { await routines.createTrigger(createdRoutine.id, { @@ -5067,6 +5799,21 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { warnings.push(`Task ${manifestIssue.slug} was downgraded to todo because its assignee could not be imported as assignable work.`); issueStatus = "todo"; } + const resolvedLabelIds: string[] = []; + for (const name of manifestIssue.labelNames ?? []) { + const labelId = labelIdByName.get(name); + if (labelId && !resolvedLabelIds.includes(labelId)) resolvedLabelIds.push(labelId); + } + const legacyLabelIds = manifestIssue.labelIds ?? []; + const unresolvedLegacyCount = legacyLabelIds.filter((labelId) => !existingTargetLabelIds.has(labelId)).length; + if (unresolvedLegacyCount > 0) { + warnings.push(`Task ${manifestIssue.slug} dropped ${unresolvedLegacyCount} label reference${unresolvedLegacyCount === 1 ? "" : "s"} because the bundle carries raw label ids that do not exist in the target company.`); + } + for (const labelId of legacyLabelIds) { + if (existingTargetLabelIds.has(labelId) && !resolvedLabelIds.includes(labelId)) { + resolvedLabelIds.push(labelId); + } + } const createdIssue = await issues.create(targetCompany.id, { projectId, projectWorkspaceId, @@ -5080,8 +5827,11 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { billingCode: manifestIssue.billingCode, assigneeAdapterOverrides: manifestIssue.assigneeAdapterOverrides, executionWorkspaceSettings: manifestIssue.executionWorkspaceSettings, - labelIds: manifestIssue.labelIds ?? [], + labelIds: resolvedLabelIds, }); + // Created comment ids are captured positionally so attachment + // entries can resolve their commentIndex against them. + const createdCommentIds: Array = []; for (const comment of manifestIssue.comments ?? []) { const authorAgentId = comment.authorType === "agent" && comment.authorAgentSlug ? importedSlugToAgentId.get(comment.authorAgentSlug) @@ -5099,7 +5849,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { : comment.authorType === "user" && actorUserId ? "user" : "system"; - await issues.addComment(createdIssue.id, comment.body, { + const createdComment = await issues.addComment(createdIssue.id, rewriteEmbeddedAssetUrls(comment.body, embeddedAssetIdMap), { agentId: authorAgentId ?? undefined, userId: authorType === "user" ? actorUserId ?? undefined : undefined, }, { @@ -5108,7 +5858,181 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { metadata: comment.metadata, createdAt: comment.createdAt, }); + createdCommentIds.push(createdComment?.id ?? null); } + importedIssueIdBySlug.set(manifestIssue.slug, createdIssue.id); + if ((manifestIssue.blockedBy ?? []).length > 0) { + blockedByBySlug.set(manifestIssue.slug, manifestIssue.blockedBy ?? []); + } + for (const documentEntry of manifestIssue.documents ?? []) { + const documentBody = readPortableTextFile(plan.source.files, documentEntry.path); + if (documentBody === null) { + warnings.push(`Task ${manifestIssue.slug} document ${documentEntry.key} was skipped because its file is missing from the package: ${documentEntry.path}`); + continue; + } + try { + await documentsSvc.upsertIssueDocument({ + issueId: createdIssue.id, + key: documentEntry.key, + title: documentEntry.title, + format: documentEntry.format, + body: rewriteEmbeddedAssetUrls(documentBody, embeddedAssetIdMap), + createdByUserId: actorUserId ?? null, + }); + } catch (error) { + warnings.push(`Task ${manifestIssue.slug} document ${documentEntry.key} could not be imported: ${error instanceof Error ? error.message : String(error)}`); + } + } + for (const workProductEntry of manifestIssue.workProducts ?? []) { + await workProductsSvc.createForIssue(createdIssue.id, targetCompany.id, { + projectId: createdIssue.projectId ?? projectId ?? null, + type: workProductEntry.type, + provider: workProductEntry.provider, + externalId: workProductEntry.externalId, + title: workProductEntry.title, + url: workProductEntry.url, + status: workProductEntry.status, + reviewState: workProductEntry.reviewState, + isPrimary: workProductEntry.isPrimary, + healthStatus: workProductEntry.healthStatus, + summary: workProductEntry.summary, + metadata: workProductEntry.metadata, + // Workspace/run references never travel across boards. + executionWorkspaceId: null, + runtimeServiceId: null, + createdByRunId: null, + sourceTrust: null, + }); + } + if (manifestIssue.monitor) { + // Monitors land un-armed: notes and provenance are restored but + // monitorNextCheckAt stays NULL until an operator re-arms them. + if (manifestIssue.monitor.notes !== null || manifestIssue.monitor.scheduledBy !== null) { + await db + .update(issuesTable) + .set({ + monitorNotes: manifestIssue.monitor.notes, + monitorScheduledBy: manifestIssue.monitor.scheduledBy, + }) + .where(eq(issuesTable.id, createdIssue.id)); + } + unarmedMonitorCount += 1; + } + for (const attachmentEntry of manifestIssue.attachments ?? []) { + const attachmentLabel = attachmentEntry.originalFilename ?? attachmentEntry.sha256; + if (!storage) { + attachmentsSkippedNoStorage += 1; + continue; + } + const blobPath = portableBlobPath(attachmentEntry.sha256); + const blobFile = plan.source.files[blobPath]; + if (blobFile === undefined) { + warnings.push(`Task ${manifestIssue.slug} attachment ${attachmentLabel} was skipped because its blob is missing from the package: ${blobPath}`); + continue; + } + const body = portableFileToBuffer(blobFile, blobPath); + // Content-addressed blobs double as the bundle's tamper seal: + // bytes that do not hash to their declared sha256 fail closed. + if (sha256HexOfBytes(body) !== attachmentEntry.sha256) { + throw unprocessable(`Attachment blob ${blobPath} does not match its declared sha256; the package is corrupted or was tampered with.`); + } + if (body.length > attachmentMaxBytes) { + warnings.push(`Task ${manifestIssue.slug} attachment ${attachmentLabel} was skipped because it exceeds this board's attachment size limit of ${attachmentMaxBytes} bytes.`); + continue; + } + let issueCommentId: string | null = null; + if (attachmentEntry.commentIndex !== null) { + issueCommentId = createdCommentIds[attachmentEntry.commentIndex] ?? null; + if (!issueCommentId) { + warnings.push(`Task ${manifestIssue.slug} attachment ${attachmentLabel} was imported at task scope because its comment reference could not be resolved.`); + } + } + try { + const stored = await storage.putFile({ + companyId: targetCompany.id, + namespace: `issues/${createdIssue.id}`, + originalFilename: attachmentEntry.originalFilename, + contentType: attachmentEntry.contentType, + body, + }); + await issues.createAttachment({ + issueId: createdIssue.id, + issueCommentId, + provider: stored.provider, + objectKey: stored.objectKey, + contentType: stored.contentType, + byteSize: stored.byteSize, + sha256: stored.sha256, + originalFilename: stored.originalFilename, + createdByAgentId: null, + createdByUserId: actorUserId ?? null, + }); + } catch (error) { + warnings.push(`Task ${manifestIssue.slug} attachment ${attachmentLabel} could not be imported: ${error instanceof Error ? error.message : String(error)}`); + } + } + } + + if (blockedByBySlug.size > 0) { + const acceptedAdjacency = new Map(); + const wouldCreateBlockingCycle = (blockedIssueId: string, blockerIssueId: string) => { + // Mirrors assertNoBlockingCycles in the issues service: a cycle + // exists when the blocked issue already (transitively) blocks the + // prospective blocker. + const queue = [...(acceptedAdjacency.get(blockedIssueId) ?? [])]; + const visited = new Set([blockedIssueId]); + while (queue.length > 0) { + const current = queue.shift()!; + if (current === blockerIssueId) return true; + if (visited.has(current)) continue; + visited.add(current); + queue.push(...(acceptedAdjacency.get(current) ?? [])); + } + return false; + }; + const relationRows: Array<{ issueId: string; relatedIssueId: string }> = []; + for (const [slug, blockerSlugs] of blockedByBySlug) { + const blockedIssueId = importedIssueIdBySlug.get(slug); + if (!blockedIssueId) continue; + for (const blockerSlug of blockerSlugs) { + const blockerIssueId = importedIssueIdBySlug.get(blockerSlug); + if (!blockerIssueId) { + warnings.push(`Task ${slug} blocker ${blockerSlug} was skipped because that task was not imported.`); + continue; + } + if (blockerIssueId === blockedIssueId) continue; + if (wouldCreateBlockingCycle(blockedIssueId, blockerIssueId)) { + warnings.push(`Task ${slug} blocker ${blockerSlug} was skipped because it would create a blocking cycle.`); + continue; + } + const adjacency = acceptedAdjacency.get(blockerIssueId) ?? []; + adjacency.push(blockedIssueId); + acceptedAdjacency.set(blockerIssueId, adjacency); + relationRows.push({ issueId: blockerIssueId, relatedIssueId: blockedIssueId }); + } + } + if (relationRows.length > 0) { + const relationCompanyId = targetCompany.id; + await db + .insert(issueRelations) + .values(relationRows.map((row) => ({ + companyId: relationCompanyId, + issueId: row.issueId, + relatedIssueId: row.relatedIssueId, + type: "blocks" as const, + createdByAgentId: null, + createdByUserId: actorUserId ?? null, + }))) + .onConflictDoNothing(); + } + } + + if (unarmedMonitorCount > 0) { + warnings.push(`${unarmedMonitorCount} monitor${unarmedMonitorCount === 1 ? " was" : "s were"} imported un-armed; re-arm ${unarmedMonitorCount === 1 ? "it" : "them"} from the task page to resume checks.`); + } + + if (attachmentsSkippedNoStorage > 0) { + warnings.push(`Skipped ${attachmentsSkippedNoStorage} attachment${attachmentsSkippedNoStorage === 1 ? "" : "s"} because storage is unavailable.`); } } @@ -5120,6 +6044,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { }, agents: resultAgents, projects: resultProjects, + routines: resultRoutines, envInputs: sourceManifest.envInputs ?? [], warnings, }; diff --git a/server/src/services/export-fidelity.ts b/server/src/services/export-fidelity.ts new file mode 100644 index 0000000000..cdfee1e25f --- /dev/null +++ b/server/src/services/export-fidelity.ts @@ -0,0 +1,83 @@ +import { and, count, eq, sql } from "drizzle-orm"; +import { + EXPORT_FIDELITY_REPORT_SCHEMA, + buildExportFidelityWarnings, + type ExportFidelityCounts, + type ExportFidelityReport, +} from "@paperclipai/shared/portability-fidelity"; +import type { Db } from "@paperclipai/db"; +import { + activityLog, + approvals, + costEvents, + issueAttachments, + issueDocuments, + issueLabels, + issueRelations, + issueWorkProducts, + issues, + labels, +} from "@paperclipai/db"; + +export async function collectExportFidelityCounts(db: Db, companyId: string): Promise { + const [ + labelDefinitions, + issueLabelReferences, + issueBlockerRelations, + issueDocumentCount, + issueWorkProductCount, + issueAttachmentCount, + approvalCount, + costEventCount, + activityLogEntries, + issueMonitors, + ] = await Promise.all([ + db.select({ count: count() }).from(labels).where(eq(labels.companyId, companyId)).then(firstCount), + db.select({ count: count() }).from(issueLabels).where(eq(issueLabels.companyId, companyId)).then(firstCount), + db + .select({ count: count() }) + .from(issueRelations) + .where(and(eq(issueRelations.companyId, companyId), eq(issueRelations.type, "blocks"))) + .then(firstCount), + db.select({ count: count() }).from(issueDocuments).where(eq(issueDocuments.companyId, companyId)).then(firstCount), + db.select({ count: count() }).from(issueWorkProducts).where(eq(issueWorkProducts.companyId, companyId)).then(firstCount), + db.select({ count: count() }).from(issueAttachments).where(eq(issueAttachments.companyId, companyId)).then(firstCount), + db.select({ count: count() }).from(approvals).where(eq(approvals.companyId, companyId)).then(firstCount), + db.select({ count: count() }).from(costEvents).where(eq(costEvents.companyId, companyId)).then(firstCount), + db.select({ count: count() }).from(activityLog).where(eq(activityLog.companyId, companyId)).then(firstCount), + db + .select({ count: count() }) + .from(issues) + .where(and( + eq(issues.companyId, companyId), + sql`(${issues.monitorNextCheckAt} is not null or ${issues.monitorScheduledBy} is not null)`, + )) + .then(firstCount), + ]); + return { + labelDefinitions, + issueLabelReferences, + issueBlockerRelations, + issueDocuments: issueDocumentCount, + issueWorkProducts: issueWorkProductCount, + issueAttachments: issueAttachmentCount, + approvals: approvalCount, + costEvents: costEventCount, + activityLogEntries, + issueMonitors, + }; +} + +export function buildExportFidelityReport(companyId: string, counts: ExportFidelityCounts): ExportFidelityReport { + return { + schema: EXPORT_FIDELITY_REPORT_SCHEMA, + companyId, + counts, + warnings: buildExportFidelityWarnings(counts), + generatedAt: new Date().toISOString(), + }; +} + +function firstCount(rows: Array<{ count: number }>): number { + return rows[0]?.count ?? 0; +} diff --git a/server/src/services/index.ts b/server/src/services/index.ts index 08e0386c6d..c84218888d 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -121,7 +121,7 @@ export { } from "./managed-config.js"; export { bootstrapExecutionPolicyFromEnv } from "./execution-policy-bootstrap.js"; export { applyManagedEnvironments } from "./managed-environments.js"; -export { cloudUpstreamService, reconcileCloudUpstreamRunsOnStartup } from "./cloud-upstreams.js"; +export { buildExportFidelityReport, collectExportFidelityCounts } from "./export-fidelity.js"; export { companyPortabilityService } from "./company-portability.js"; export { teamsCatalogService } from "./teams-catalog.js"; export { environmentService } from "./environments.js"; diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 0a99e87ba5..2a94177c20 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -217,7 +217,6 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta enableIssuePlanDecompositions: parsed.data.enableIssuePlanDecompositions ?? false, enableExperimentalFileViewer: parsed.data.enableExperimentalFileViewer ?? false, enableTaskWatchdogs: parsed.data.enableTaskWatchdogs ?? false, - enableCloudSync: parsed.data.enableCloudSync ?? false, enableExternalObjects: parsed.data.enableExternalObjects ?? false, enableSmokeLab: parsed.data.enableSmokeLab ?? false, enableBuiltInAgents: parsed.data.enableBuiltInAgents ?? false, @@ -252,7 +251,6 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta enableTaskWatchdogs: false, enableIssuePlanDecompositions: false, enableExperimentalFileViewer: false, - enableCloudSync: false, enableExternalObjects: false, enableSmokeLab: false, enableBuiltInAgents: false, diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 7d28bd2b1a..981a0ebada 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -46,8 +46,6 @@ import { TrainingInspector, TrainingLibrary } from "./pages/Training"; import { BoardChat } from "./pages/BoardChat"; import { CompanySettings } from "./pages/CompanySettings"; import { CompanyEnvironments } from "./pages/CompanyEnvironments"; -import { CloudUpstream } from "./pages/CloudUpstream"; -import { CloudUpstreamUxLab } from "./pages/CloudUpstreamUxLab"; import { BootstrapSetupUxLab } from "./pages/BootstrapSetupUxLab"; import { ResponsibleUserDenialUxLab } from "./pages/ResponsibleUserDenialUxLab"; import { CompanySettingsPluginPage } from "./pages/CompanySettingsPluginPage"; @@ -107,7 +105,7 @@ function boardRoutes() { } /> } /> } /> - } /> + } /> } /> } /> } /> @@ -534,7 +532,6 @@ export function App() { } /> } /> } /> - } /> } /> } /> diff --git a/ui/src/api/cloudUpstreams.ts b/ui/src/api/cloudUpstreams.ts deleted file mode 100644 index 48ddff5ced..0000000000 --- a/ui/src/api/cloudUpstreams.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { - CloudUpstreamActivationEntityType, - CloudUpstreamConnectStartResponse, - CloudUpstreamConnection, - CloudUpstreamPreview, - CloudUpstreamRun, - CloudUpstreamsState, -} from "@paperclipai/shared"; -import { api } from "./client"; - -export const cloudUpstreamsApi = { - list: (companyId: string) => - api.get(`/cloud-upstreams?companyId=${encodeURIComponent(companyId)}`), - startConnect: (input: { companyId: string; remoteUrl: string; redirectUri: string }) => - api.post("/cloud-upstreams/connect/start", input), - finishConnect: (input: { pendingConnectionId: string; code: string; state: string }) => - api.post("/cloud-upstreams/connect/finish", input), - preview: (connectionId: string, input: { companyId: string }) => - api.post(`/cloud-upstreams/${encodeURIComponent(connectionId)}/push-runs/preview`, input), - createRun: (connectionId: string, input: { companyId: string; retryOfRunId?: string | null }) => - api.post(`/cloud-upstreams/${encodeURIComponent(connectionId)}/push-runs`, input ?? {}), - getRun: (connectionId: string, runId: string, companyId: string) => - api.get( - `/cloud-upstreams/${encodeURIComponent(connectionId)}/push-runs/${encodeURIComponent(runId)}?companyId=${encodeURIComponent(companyId)}`, - ), - cancelRun: (connectionId: string, runId: string, input: { companyId: string }) => - api.post( - `/cloud-upstreams/${encodeURIComponent(connectionId)}/push-runs/${encodeURIComponent(runId)}/cancel`, - input, - ), - activateEntities: ( - connectionId: string, - runId: string, - input: { companyId: string; entityType: CloudUpstreamActivationEntityType }, - ) => - api.post( - `/cloud-upstreams/${encodeURIComponent(connectionId)}/push-runs/${encodeURIComponent(runId)}/activation`, - input, - ), -}; diff --git a/ui/src/api/companies.ts b/ui/src/api/companies.ts index 3357c5d127..222262d352 100644 --- a/ui/src/api/companies.ts +++ b/ui/src/api/companies.ts @@ -9,6 +9,7 @@ import type { CompanyPortabilityPreviewResult, UpdateCompanyBranding, } from "@paperclipai/shared"; +import type { ExportFidelityReport } from "@paperclipai/shared/portability-fidelity"; import { api } from "./client"; export type CompanyStats = Record; @@ -54,6 +55,8 @@ export const companiesApi = { data: CompanyPortabilityExportRequest, ) => api.post(`/companies/${companyId}/exports/preview`, data), + exportFidelity: (companyId: string) => + api.get(`/companies/${companyId}/export/fidelity`), importPreview: (data: CompanyPortabilityPreviewRequest) => api.post("/companies/import/preview", data), importBundle: (data: CompanyPortabilityImportRequest) => diff --git a/ui/src/components/CompanySettingsSidebar.test.tsx b/ui/src/components/CompanySettingsSidebar.test.tsx index bd1156bbeb..92073fce3e 100644 --- a/ui/src/components/CompanySettingsSidebar.test.tsx +++ b/ui/src/components/CompanySettingsSidebar.test.tsx @@ -9,9 +9,6 @@ const sidebarNavItemMock = vi.hoisted(() => vi.fn()); const mockSidebarBadgesApi = vi.hoisted(() => ({ get: vi.fn(), })); -const mockInstanceSettingsApi = vi.hoisted(() => ({ - getExperimental: vi.fn(), -})); const mockPluginsApi = vi.hoisted(() => ({ list: vi.fn(), })); @@ -74,10 +71,6 @@ vi.mock("@/api/sidebarBadges", () => ({ sidebarBadgesApi: mockSidebarBadgesApi, })); -vi.mock("@/api/instanceSettings", () => ({ - instanceSettingsApi: mockInstanceSettingsApi, -})); - vi.mock("@/api/plugins", () => ({ pluginsApi: mockPluginsApi, })); @@ -114,18 +107,12 @@ describe("CompanySettingsSidebar", () => { failedRuns: 0, joinRequests: 2, }); - mockInstanceSettingsApi.getExperimental.mockResolvedValue({ - enableCloudSync: false, - }); mockPluginsApi.list.mockResolvedValue([]); mockUsePluginSlots.mockReturnValue({ slots: [], isLoading: false, errorMessage: null, }); - mockInstanceSettingsApi.getExperimental.mockResolvedValue({ - enableCloudSync: false, - }); }); afterEach(() => { @@ -155,9 +142,9 @@ describe("CompanySettingsSidebar", () => { expect(container.textContent).toContain("Instance settings"); expect(container.textContent).toContain("General"); expect(container.textContent).toContain("Environments"); - expect(container.textContent).not.toContain("Cloud upstream"); + expect(container.textContent).toContain("Export"); + expect(container.textContent).toContain("Import"); expect(container.textContent).toContain("Members"); - expect(container.textContent).not.toContain("Cloud upstream"); expect(container.textContent).toContain("Invites"); expect(container.textContent).toContain("Secrets"); expect(container.textContent).not.toContain("Tools & Access"); @@ -168,6 +155,19 @@ describe("CompanySettingsSidebar", () => { end: true, }), ); + expect(sidebarNavItemMock).toHaveBeenCalledWith( + expect.objectContaining({ + to: "/company/export", + label: "Export", + }), + ); + expect(sidebarNavItemMock).toHaveBeenCalledWith( + expect.objectContaining({ + to: "/company/import", + label: "Import", + end: true, + }), + ); expect(sidebarNavItemMock).toHaveBeenCalledWith( expect.objectContaining({ to: "/company/settings/instance/environments", @@ -234,38 +234,6 @@ describe("CompanySettingsSidebar", () => { }); }); - it("shows cloud upstream only when cloud sync is enabled", async () => { - mockInstanceSettingsApi.getExperimental.mockResolvedValue({ - enableCloudSync: true, - }); - const root = createRoot(container); - const queryClient = new QueryClient({ - defaultOptions: { queries: { retry: false } }, - }); - - await act(async () => { - root.render( - - - , - ); - }); - await flushReact(); - - expect(container.textContent).toContain("Cloud upstream"); - expect(sidebarNavItemMock).toHaveBeenCalledWith( - expect.objectContaining({ - to: "/company/settings/cloud-upstream", - label: "Cloud upstream", - end: true, - }), - ); - - await act(async () => { - root.unmount(); - }); - }); - it("renders company settings pages contributed by ready plugins", async () => { mockUsePluginSlots.mockReturnValue({ slots: [ @@ -312,38 +280,6 @@ describe("CompanySettingsSidebar", () => { }); }); - it("shows cloud upstream only when cloud sync is enabled", async () => { - mockInstanceSettingsApi.getExperimental.mockResolvedValue({ - enableCloudSync: true, - }); - const root = createRoot(container); - const queryClient = new QueryClient({ - defaultOptions: { queries: { retry: false } }, - }); - - await act(async () => { - root.render( - - - , - ); - }); - await flushReact(); - - expect(container.textContent).toContain("Cloud upstream"); - expect(sidebarNavItemMock).toHaveBeenCalledWith( - expect.objectContaining({ - to: "/company/settings/cloud-upstream", - label: "Cloud upstream", - end: true, - }), - ); - - await act(async () => { - root.unmount(); - }); - }); - it("renders instance plugin links while filtering sandbox-provider-only plugins", async () => { mockPluginsApi.list.mockResolvedValue([ { diff --git a/ui/src/components/CompanySettingsSidebar.tsx b/ui/src/components/CompanySettingsSidebar.tsx index eda2a8edbd..2d1cec667c 100644 --- a/ui/src/components/CompanySettingsSidebar.tsx +++ b/ui/src/components/CompanySettingsSidebar.tsx @@ -2,8 +2,8 @@ import { useQuery } from "@tanstack/react-query"; import { ChevronLeft, Clock3, - CloudUpload, Cpu, + Download, FlaskConical, KeyRound, MailPlus, @@ -12,12 +12,12 @@ import { Settings, Shield, SlidersHorizontal, + Upload, UserRoundPen, Users, } from "lucide-react"; import type { PluginRecord } from "@paperclipai/shared"; import { sidebarBadgesApi } from "@/api/sidebarBadges"; -import { instanceSettingsApi } from "@/api/instanceSettings"; import { pluginsApi } from "@/api/plugins"; import { ApiError } from "@/api/client"; import { Link, NavLink } from "@/lib/router"; @@ -67,15 +67,10 @@ export function CompanySettingsSidebar() { retry: false, refetchInterval: 15_000, }); - const { data: experimentalSettings } = useQuery({ - queryKey: queryKeys.instance.experimentalSettings, - queryFn: () => instanceSettingsApi.getExperimental(), - }); const { data: plugins } = useQuery({ queryKey: queryKeys.plugins.all, queryFn: () => pluginsApi.list(), }); - const showCloudUpstream = experimentalSettings?.enableCloudSync === true; const sidebarPlugins = (plugins ?? []).filter((plugin) => !isSandboxProviderOnly(plugin)); return ( @@ -105,14 +100,8 @@ export function CompanySettingsSidebar() {
- {showCloudUpstream ? ( - - ) : null} + + { expect(selector?.value).toBe("secrets"); const selectorText = selector?.textContent?.toLowerCase() ?? ""; expect(selectorText).toContain("general"); - expect(selectorText).toContain("cloud upstream"); + expect(selectorText).toContain("export"); + expect(selectorText).toContain("import"); expect(selectorText).toContain("members"); expect(selectorText).toContain("invites"); expect(selectorText).toContain("secrets"); diff --git a/ui/src/components/access/CompanySettingsNav.test.tsx b/ui/src/components/access/CompanySettingsNav.test.tsx index a45cca9be0..40f2eeed59 100644 --- a/ui/src/components/access/CompanySettingsNav.test.tsx +++ b/ui/src/components/access/CompanySettingsNav.test.tsx @@ -67,7 +67,10 @@ describe("CompanySettingsNav", () => { expect(getCompanySettingsTab("/company/settings")).toBe("general"); expect(getCompanySettingsTab("/PAP/company/settings")).toBe("general"); expect(getCompanySettingsTab("/company/settings/environments")).toBe("instance-environments"); - expect(getCompanySettingsTab("/company/settings/cloud-upstream")).toBe("cloud-upstream"); + expect(getCompanySettingsTab("/company/export")).toBe("export"); + expect(getCompanySettingsTab("/PAP/company/export")).toBe("export"); + expect(getCompanySettingsTab("/company/import")).toBe("import"); + expect(getCompanySettingsTab("/PAP/company/import")).toBe("import"); expect(getCompanySettingsTab("/company/settings/members")).toBe("members"); expect(getCompanySettingsTab("/PAP/company/settings/members")).toBe("members"); expect(getCompanySettingsTab("/company/settings/access")).toBe("members"); @@ -98,7 +101,8 @@ describe("CompanySettingsNav", () => { value: "members", items: [ { value: "general", label: "General" }, - { value: "cloud-upstream", label: "Cloud upstream" }, + { value: "export", label: "Export" }, + { value: "import", label: "Import" }, { value: "members", label: "Members" }, { value: "invites", label: "Invites" }, { value: "secrets", label: "Secrets" }, diff --git a/ui/src/components/access/CompanySettingsNav.tsx b/ui/src/components/access/CompanySettingsNav.tsx index 876c2b45bc..d06b913f8d 100644 --- a/ui/src/components/access/CompanySettingsNav.tsx +++ b/ui/src/components/access/CompanySettingsNav.tsx @@ -5,7 +5,8 @@ import { useLocation, useNavigate } from "@/lib/router"; const items = [ { value: "general", label: "General", href: "/company/settings" }, - { value: "cloud-upstream", label: "Cloud upstream", href: "/company/settings/cloud-upstream" }, + { value: "export", label: "Export", href: "/company/export" }, + { value: "import", label: "Import", href: "/company/import" }, { value: "members", label: "Members", href: "/company/settings/members" }, { value: "invites", label: "Invites", href: "/company/settings/invites" }, { value: "secrets", label: "Secrets", href: "/company/settings/secrets" }, @@ -58,8 +59,12 @@ export function getCompanySettingsTab(pathname: string): CompanySettingsTab { return "instance-environments"; } - if (pathname.includes("/company/settings/cloud-upstream")) { - return "cloud-upstream"; + if (pathname.includes("/company/export")) { + return "export"; + } + + if (pathname.includes("/company/import")) { + return "import"; } if (pathname.includes("/company/settings/members") || pathname.includes("/company/settings/access")) { diff --git a/ui/src/index.css b/ui/src/index.css index 01f8f60600..c7570059de 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -1834,10 +1834,7 @@ span.paperclip-mention-chip[data-mention-kind="external-object"] { --gtc-17: 1fr auto; /* Extracted from ui/src/components/ui/card.tsx (grid-cols-[1fr_auto]). */ --gtc-18: auto minmax(0,1fr) minmax(12rem,0.65fr); /* Extracted from ui/src/pages/AgentDetail.tsx (grid-cols-[auto_minmax(0,1fr)_minmax(12rem,0.65fr)]). */ --gtc-19: 260px minmax(0,1fr); /* Extracted from ui/src/pages/AgentDetail.tsx (grid-cols-[260px_minmax(0,1fr)]). */ - --gtc-20: 7rem 8rem 1fr; /* Extracted from ui/src/pages/CloudUpstream.tsx (grid-cols-[7rem_8rem_1fr]). */ - --gtc-21: 1.25rem 12rem 1fr; /* Extracted from ui/src/pages/CloudUpstream.tsx (grid-cols-[1.25rem_12rem_1fr]). */ - --gtc-22: 8rem 1fr 1fr 8rem; /* Extracted from ui/src/pages/CloudUpstream.tsx (grid-cols-[8rem_1fr_1fr_8rem]). */ - --gtc-23: 8rem 1fr auto; /* Extracted from ui/src/pages/CloudUpstream.tsx (grid-cols-[8rem_1fr_auto]). */ + /* --gtc-20 through --gtc-23 retired with the removed Cloud Upstream page. */ --gtc-24: minmax(0,1.5fr) 120px 120px 180px; /* Extracted from ui/src/pages/CompanyAccess.tsx (grid-cols-[minmax(0,1.5fr)_120px_120px_180px]). */ --gtc-25: 19rem minmax(0,1fr); /* Extracted from ui/src/pages/CompanyExport.tsx (grid-cols-[19rem_minmax(0,1fr)]). */ --gtc-26: 7rem minmax(0,1fr); /* Extracted from ui/src/pages/CompanySkills.tsx (grid-cols-[7rem_minmax(0,1fr)]). */ diff --git a/ui/src/lib/company-export-selection.test.ts b/ui/src/lib/company-export-selection.test.ts index 91828e4aa9..5096e9209e 100644 --- a/ui/src/lib/company-export-selection.test.ts +++ b/ui/src/lib/company-export-selection.test.ts @@ -1,41 +1,308 @@ import { describe, expect, it } from "vitest"; -import { buildInitialExportCheckedFiles } from "./company-export-selection"; +import { + type ExportCategorySelection, + type ExportSelectionEmbeddedAsset, + type ExportSelectionIssue, + buildDefaultExportCategorySelection, + buildExportCheckedFiles, + countExportFilesByCategory, + isAttachmentsCategoryEnabled, +} from "./company-export-selection"; -describe("buildInitialExportCheckedFiles", () => { - it("checks non-task files and recurring task packages by default", () => { - const checked = buildInitialExportCheckedFiles( - [ - "README.md", - ".paperclip.yaml", - "tasks/one-off/TASK.md", - "tasks/recurring/TASK.md", - "tasks/recurring/notes.md", - ], - [ - { path: "tasks/one-off/TASK.md", recurring: false }, - { path: "tasks/recurring/TASK.md", recurring: true }, - ], - new Set(), - ); +function attachment(sha256: string) { + return { + sha256, + contentType: "application/octet-stream", + originalFilename: null, + byteSize: 1, + commentIndex: null, + }; +} - expect(Array.from(checked).sort()).toEqual([ - ".paperclip.yaml", - "README.md", - "tasks/recurring/TASK.md", - "tasks/recurring/notes.md", - ]); +const ROOT_FILES = [ + "README.md", + "COMPANY.md", + ".paperclip.yaml", + "images/org-chart.png", + "images/logo.png", +]; + +const FILE_PATHS = [ + ...ROOT_FILES, + "agents/ceo/AGENT.md", + "agents/ceo/instructions.md", + "projects/growth/PROJECT.md", + "skills/research/SKILL.md", + "tasks/one-off/TASK.md", + "tasks/one-off/documents/spec.md", + "tasks/weekly-report/TASK.md", + "blobs/aaa111", + "blobs/bbb222", +]; + +const ISSUES: ExportSelectionIssue[] = [ + { + path: "tasks/one-off/TASK.md", + recurring: false, + attachments: [attachment("aaa111")], + }, + { + path: "tasks/weekly-report/TASK.md", + recurring: true, + attachments: [attachment("bbb222")], + }, +]; + +function selection(overrides: Partial = {}): ExportCategorySelection { + return { ...buildDefaultExportCategorySelection(), ...overrides }; +} + +function checkedWith(overrides: Partial = {}): string[] { + return Array.from( + buildExportCheckedFiles({ + filePaths: FILE_PATHS, + issues: ISSUES, + categories: selection(overrides), + }), + ).sort(); +} + +describe("buildExportCheckedFiles", () => { + it("checks every file by default", () => { + expect(checkedWith()).toEqual([...FILE_PATHS].sort()); }); - it("preserves previous manual selections for one-time tasks", () => { - const checked = buildInitialExportCheckedFiles( - ["README.md", "tasks/one-off/TASK.md"], - [{ path: "tasks/one-off/TASK.md", recurring: false }], - new Set(["tasks/one-off/TASK.md"]), + it("excludes agent files when agents is off", () => { + const checked = checkedWith({ agents: false }); + expect(checked).not.toContain("agents/ceo/AGENT.md"); + expect(checked).not.toContain("agents/ceo/instructions.md"); + expect(checked).toEqual( + [...FILE_PATHS].filter((p) => !p.startsWith("agents/")).sort(), ); + }); - expect(Array.from(checked).sort()).toEqual([ - "README.md", - "tasks/one-off/TASK.md", - ]); + it("excludes project files when projects is off", () => { + expect(checkedWith({ projects: false })).toEqual( + [...FILE_PATHS].filter((p) => !p.startsWith("projects/")).sort(), + ); + }); + + it("excludes skill files when skills is off", () => { + expect(checkedWith({ skills: false })).toEqual( + [...FILE_PATHS].filter((p) => !p.startsWith("skills/")).sort(), + ); + }); + + it("splits recurring task files from one-off task files", () => { + const withoutTasks = checkedWith({ tasks: false }); + expect(withoutTasks).not.toContain("tasks/one-off/TASK.md"); + expect(withoutTasks).not.toContain("tasks/one-off/documents/spec.md"); + expect(withoutTasks).toContain("tasks/weekly-report/TASK.md"); + + const withoutRoutines = checkedWith({ routines: false }); + expect(withoutRoutines).toContain("tasks/one-off/TASK.md"); + expect(withoutRoutines).toContain("tasks/one-off/documents/spec.md"); + expect(withoutRoutines).not.toContain("tasks/weekly-report/TASK.md"); + }); + + it("drops a blob when its owning one-off task is excluded", () => { + const checked = checkedWith({ tasks: false }); + expect(checked).not.toContain("blobs/aaa111"); + // Routine-owned blob still travels with its routine. + expect(checked).toContain("blobs/bbb222"); + }); + + it("drops a blob when its owning routine is excluded", () => { + const checked = checkedWith({ routines: false }); + expect(checked).toContain("blobs/aaa111"); + expect(checked).not.toContain("blobs/bbb222"); + }); + + it("excludes all blobs when attachments is off, keeping task files", () => { + const checked = checkedWith({ attachments: false }); + expect(checked).not.toContain("blobs/aaa111"); + expect(checked).not.toContain("blobs/bbb222"); + expect(checked).toContain("tasks/one-off/TASK.md"); + expect(checked).toContain("tasks/weekly-report/TASK.md"); + }); + + it("forces attachments off when both tasks and routines are off", () => { + const checked = checkedWith({ tasks: false, routines: false, attachments: true }); + expect(checked.some((p) => p.startsWith("blobs/"))).toBe(false); + expect(checked.some((p) => p.startsWith("tasks/"))).toBe(false); + }); + + it("always includes root files, even with every category off", () => { + const checked = checkedWith({ + agents: false, + projects: false, + skills: false, + routines: false, + tasks: false, + attachments: false, + }); + expect(checked).toEqual([...ROOT_FILES].sort()); + }); + + it("keeps a blob referenced by both a one-off task and a routine while either is on", () => { + const issues: ExportSelectionIssue[] = [ + { path: "tasks/one-off/TASK.md", recurring: false, attachments: [attachment("shared")] }, + { path: "tasks/weekly-report/TASK.md", recurring: true, attachments: [attachment("shared")] }, + ]; + const filePaths = ["tasks/one-off/TASK.md", "tasks/weekly-report/TASK.md", "blobs/shared"]; + + for (const overrides of [{ tasks: false }, { routines: false }] as const) { + const checked = buildExportCheckedFiles({ + filePaths, + issues, + categories: selection(overrides), + }); + expect(checked.has("blobs/shared")).toBe(true); + } + + const bothOff = buildExportCheckedFiles({ + filePaths, + issues, + categories: selection({ tasks: false, routines: false }), + }); + expect(bothOff.has("blobs/shared")).toBe(false); + }); + + it("gates unreferenced blobs on the attachments toggle alone", () => { + const filePaths = ["tasks/one-off/TASK.md", "blobs/orphan"]; + const issues: ExportSelectionIssue[] = [ + { path: "tasks/one-off/TASK.md", recurring: false, attachments: [] }, + ]; + + expect( + buildExportCheckedFiles({ filePaths, issues, categories: selection() }).has("blobs/orphan"), + ).toBe(true); + expect( + buildExportCheckedFiles({ filePaths, issues, categories: selection({ attachments: false }) }).has("blobs/orphan"), + ).toBe(false); + expect( + buildExportCheckedFiles({ + filePaths, + issues, + categories: selection({ tasks: false, routines: false }), + }).has("blobs/orphan"), + ).toBe(false); + }); +}); + +describe("embedded asset blob ownership", () => { + const embeddedFilePaths = [ + "COMPANY.md", + "agents/ceo/AGENT.md", + "tasks/one-off/TASK.md", + "blobs/embed-task", + "blobs/embed-agent", + "blobs/embed-root", + ]; + const embeddedIssues: ExportSelectionIssue[] = [ + { path: "tasks/one-off/TASK.md", recurring: false, attachments: [] }, + ]; + const embeddedAssets: ExportSelectionEmbeddedAsset[] = [ + { sha256: "embed-task", ownedBy: ["tasks"] }, + { sha256: "embed-agent", ownedBy: ["agents"] }, + { sha256: "embed-root", ownedBy: ["always"] }, + ]; + + function checkedEmbedded(overrides: Partial = {}): Set { + return buildExportCheckedFiles({ + filePaths: embeddedFilePaths, + issues: embeddedIssues, + categories: selection(overrides), + embeddedAssets, + }); + } + + it("keeps embedded blobs while their owning category is selected", () => { + const checked = checkedEmbedded(); + expect(checked.has("blobs/embed-task")).toBe(true); + expect(checked.has("blobs/embed-agent")).toBe(true); + expect(checked.has("blobs/embed-root")).toBe(true); + }); + + it("drops an embedded blob when all its owners are off, even with attachments on", () => { + const checked = checkedEmbedded({ tasks: false, attachments: true }); + expect(checked.has("blobs/embed-task")).toBe(false); + expect(checked.has("blobs/embed-agent")).toBe(true); + + const agentsOff = checkedEmbedded({ agents: false }); + expect(agentsOff.has("blobs/embed-agent")).toBe(false); + expect(agentsOff.has("blobs/embed-task")).toBe(true); + }); + + it("keeps embedded blobs independently of the attachments toggle", () => { + const checked = checkedEmbedded({ attachments: false }); + expect(checked.has("blobs/embed-task")).toBe(true); + expect(checked.has("blobs/embed-agent")).toBe(true); + expect(checked.has("blobs/embed-root")).toBe(true); + }); + + it("always keeps root-owned and ownerless embedded blobs", () => { + const checked = buildExportCheckedFiles({ + filePaths: ["blobs/embed-root", "blobs/no-owner"], + issues: [], + categories: selection({ + agents: false, + projects: false, + skills: false, + routines: false, + tasks: false, + attachments: false, + }), + embeddedAssets: [ + { sha256: "embed-root", ownedBy: ["always"] }, + { sha256: "no-owner" }, + ], + }); + expect(checked.has("blobs/embed-root")).toBe(true); + expect(checked.has("blobs/no-owner")).toBe(true); + }); + + it("lets a shared blob qualify through its attachment owner when its embedded owners are off", () => { + const issues: ExportSelectionIssue[] = [ + { path: "tasks/one-off/TASK.md", recurring: false, attachments: [] }, + { path: "tasks/weekly-report/TASK.md", recurring: true, attachments: [attachment("shared-sha")] }, + ]; + const checked = buildExportCheckedFiles({ + filePaths: ["tasks/weekly-report/TASK.md", "blobs/shared-sha"], + issues, + categories: selection({ tasks: false }), + embeddedAssets: [{ sha256: "shared-sha", ownedBy: ["tasks"] }], + }); + expect(checked.has("blobs/shared-sha")).toBe(true); + + const bothOff = buildExportCheckedFiles({ + filePaths: ["tasks/weekly-report/TASK.md", "blobs/shared-sha"], + issues, + categories: selection({ tasks: false, routines: false }), + embeddedAssets: [{ sha256: "shared-sha", ownedBy: ["tasks"] }], + }); + expect(bothOff.has("blobs/shared-sha")).toBe(false); + }); +}); + +describe("isAttachmentsCategoryEnabled", () => { + it("is enabled while tasks or routines remain selected", () => { + expect(isAttachmentsCategoryEnabled(selection())).toBe(true); + expect(isAttachmentsCategoryEnabled(selection({ tasks: false }))).toBe(true); + expect(isAttachmentsCategoryEnabled(selection({ routines: false }))).toBe(true); + expect(isAttachmentsCategoryEnabled(selection({ tasks: false, routines: false }))).toBe(false); + }); +}); + +describe("countExportFilesByCategory", () => { + it("counts files per category, leaving root files uncounted", () => { + expect(countExportFilesByCategory(FILE_PATHS, ISSUES)).toEqual({ + agents: 2, + projects: 1, + skills: 1, + routines: 1, + tasks: 2, + attachments: 2, + }); }); }); diff --git a/ui/src/lib/company-export-selection.ts b/ui/src/lib/company-export-selection.ts index 2b4d59be80..7065771a26 100644 --- a/ui/src/lib/company-export-selection.ts +++ b/ui/src/lib/company-export-selection.ts @@ -1,12 +1,80 @@ -import type { CompanyPortabilityIssueManifestEntry } from "@paperclipai/shared"; +import type { + CompanyPortabilityEmbeddedAssetManifestEntry, + CompanyPortabilityIssueManifestEntry, +} from "@paperclipai/shared"; -function isTaskPath(filePath: string): boolean { - return /(?:^|\/)tasks\//.test(filePath); +/** + * Export selection is category-driven: instead of per-file checkboxes the + * export page exposes a small set of toggles, and the checked-file set is + * derived from them. Root-level files (README.md, COMPANY.md, + * .paperclip.yaml, images, company logo) always export and have no toggle. + */ +export type ExportCategoryKey = + | "agents" + | "projects" + | "skills" + | "routines" + | "tasks" + | "attachments"; + +export type ExportCategorySelection = Record; + +export type ExportSelectionIssue = Pick< + CompanyPortabilityIssueManifestEntry, + "path" | "recurring" | "attachments" +>; + +export type ExportSelectionEmbeddedAsset = Pick< + CompanyPortabilityEmbeddedAssetManifestEntry, + "sha256" | "ownedBy" +>; + +export const EXPORT_CATEGORY_ORDER: ExportCategoryKey[] = [ + "agents", + "projects", + "skills", + "routines", + "tasks", + "attachments", +]; + +export const EXPORT_CATEGORY_LABELS: Record = { + agents: "Agents", + projects: "Projects", + skills: "Skills", + routines: "Routines", + tasks: "Tasks", + attachments: "Attachments", +}; + +/** Everything exports by default. */ +export function buildDefaultExportCategorySelection(): ExportCategorySelection { + return { + agents: true, + projects: true, + skills: true, + routines: true, + tasks: true, + attachments: true, + }; } -function buildRecurringTaskPrefixes( - issues: Array>, -): Set { +/** + * Attachment blobs only travel with the task (or routine) that references + * them — a blob without its owning task is an orphan the importer cannot + * attach. The Attachments toggle is therefore only meaningful while at + * least one of Tasks or Routines is still selected. Routines count because + * the bundle schema lets a recurring task manifest entry carry attachments + * (e.g. bundles that were imported and re-exported), even though natively + * exported routines emit only their TASK.md. + */ +export function isAttachmentsCategoryEnabled( + categories: Pick, +): boolean { + return categories.tasks || categories.routines; +} + +function buildRecurringTaskPrefixes(issues: ExportSelectionIssue[]): Set { const prefixes = new Set(); for (const issue of issues) { @@ -33,24 +101,172 @@ function isRecurringTaskFile(filePath: string, recurringTaskPrefixes: Set>, - previousCheckedFiles: Set, -): Set { - const next = new Set(); - const recurringTaskPrefixes = buildRecurringTaskPrefixes(issues); +/** + * Map each attachment blob sha to the kind of task that references it, so + * blob inclusion can follow its owning task's toggle. + */ +function buildAttachmentShaOwnership(issues: ExportSelectionIssue[]): { + recurring: Set; + oneOff: Set; +} { + const recurring = new Set(); + const oneOff = new Set(); - for (const filePath of filePaths) { - if (previousCheckedFiles.has(filePath)) { - next.add(filePath); - continue; - } - - if (!isTaskPath(filePath) || isRecurringTaskFile(filePath, recurringTaskPrefixes)) { - next.add(filePath); + for (const issue of issues) { + for (const attachment of issue.attachments ?? []) { + const sha = attachment.sha256.trim().toLowerCase(); + if (!sha) continue; + (issue.recurring ? recurring : oneOff).add(sha); } } - return next; + return { recurring, oneOff }; +} + +const EMBEDDED_ASSET_OWNER_CATEGORY_KEYS = new Set([ + "agents", + "projects", + "skills", + "routines", + "tasks", +]); + +/** + * Map each embedded-asset blob sha to the export categories whose files + * reference it (from the manifest's embeddedAssets ownedBy field). Multiple + * asset entries can share one content-addressed blob; ownership unions. + * Entries without ownership data count as always included. + */ +function buildEmbeddedAssetShaOwnership( + embeddedAssets: ExportSelectionEmbeddedAsset[], +): Map> { + const ownersBySha = new Map>(); + for (const entry of embeddedAssets) { + const sha = entry.sha256.trim().toLowerCase(); + if (!sha) continue; + const owners = ownersBySha.get(sha) ?? new Set(); + const ownedBy = entry.ownedBy && entry.ownedBy.length > 0 ? entry.ownedBy : ["always"]; + for (const owner of ownedBy) owners.add(owner); + ownersBySha.set(sha, owners); + } + return ownersBySha; +} + +/** + * An embedded-asset blob follows the toggles of the files that reference it, + * independent of the Attachments toggle: it stays while any owning category + * is still selected. "always" owners (root-file references) and owners this + * build does not recognize keep the bytes rather than silently dropping them. + */ +function isEmbeddedAssetBlobIncluded( + owners: Set, + categories: ExportCategorySelection, +): boolean { + for (const owner of owners) { + if (!EMBEDDED_ASSET_OWNER_CATEGORY_KEYS.has(owner as ExportCategoryKey)) return true; + if (categories[owner as ExportCategoryKey]) return true; + } + return false; +} + +/** + * Classify a bundle file path into its export category, or null for + * root-level files that always export (README.md, COMPANY.md, + * .paperclip.yaml, images, company logo). + */ +export function categorizeExportFile( + filePath: string, + recurringTaskPrefixes: Set, +): ExportCategoryKey | null { + if (filePath.startsWith("agents/")) return "agents"; + if (filePath.startsWith("projects/")) return "projects"; + if (filePath.startsWith("skills/")) return "skills"; + if (filePath.startsWith("tasks/")) { + return isRecurringTaskFile(filePath, recurringTaskPrefixes) ? "routines" : "tasks"; + } + if (filePath.startsWith("blobs/")) return "attachments"; + return null; +} + +/** Count bundle files per category (root always-exported files are not counted). */ +export function countExportFilesByCategory( + filePaths: string[], + issues: ExportSelectionIssue[], +): Record { + const recurringTaskPrefixes = buildRecurringTaskPrefixes(issues); + const counts: Record = { + agents: 0, + projects: 0, + skills: 0, + routines: 0, + tasks: 0, + attachments: 0, + }; + for (const filePath of filePaths) { + const category = categorizeExportFile(filePath, recurringTaskPrefixes); + if (category) counts[category] += 1; + } + return counts; +} + +/** + * Compute the set of files the export will include for the given category + * selection. Pure: same inputs, same set. The Tasks/Routines → Attachments + * dependency is enforced here, so callers cannot produce orphan blobs even + * with an inconsistent selection. + */ +export function buildExportCheckedFiles(input: { + filePaths: string[]; + issues: ExportSelectionIssue[]; + categories: ExportCategorySelection; + embeddedAssets?: ExportSelectionEmbeddedAsset[]; +}): Set { + const { filePaths, issues, categories } = input; + const recurringTaskPrefixes = buildRecurringTaskPrefixes(issues); + const blobOwnership = buildAttachmentShaOwnership(issues); + const embeddedOwnership = buildEmbeddedAssetShaOwnership(input.embeddedAssets ?? []); + const attachmentsOn = categories.attachments && isAttachmentsCategoryEnabled(categories); + + const checked = new Set(); + for (const filePath of filePaths) { + const category = categorizeExportFile(filePath, recurringTaskPrefixes); + + if (category === null) { + // Root files always export. + checked.add(filePath); + continue; + } + + if (category === "attachments") { + const sha = filePath.slice("blobs/".length).trim().toLowerCase(); + // Embedded-asset blobs travel with the files that reference them, + // regardless of the Attachments toggle. A blob shared with an + // attachment can still qualify through the attachment path below. + const embeddedOwners = embeddedOwnership.get(sha); + if (embeddedOwners && isEmbeddedAssetBlobIncluded(embeddedOwners, categories)) { + checked.add(filePath); + continue; + } + if (!attachmentsOn) continue; + const ownedByRecurring = blobOwnership.recurring.has(sha); + const ownedByOneOff = blobOwnership.oneOff.has(sha); + if (!ownedByRecurring && !ownedByOneOff) { + // Unreferenced blob: no owning task or embedded reference to + // follow, gate on the attachments toggle alone (already dependent + // on tasks/routines). Embedded-owned blobs whose owners are all + // disabled stay dropped instead of falling back to this gate. + if (!embeddedOwners) checked.add(filePath); + } else if ( + (ownedByOneOff && categories.tasks) + || (ownedByRecurring && categories.routines) + ) { + checked.add(filePath); + } + continue; + } + + if (categories[category]) checked.add(filePath); + } + + return checked; } diff --git a/ui/src/lib/company-routes.test.ts b/ui/src/lib/company-routes.test.ts index cce73805ea..bff8f3a8b4 100644 --- a/ui/src/lib/company-routes.test.ts +++ b/ui/src/lib/company-routes.test.ts @@ -39,9 +39,6 @@ describe("company routes", () => { it("rewrites company package paths with the active prefix", () => { expect(applyCompanyPrefix("/company/export", "NEU")).toBe("/NEU/company/export"); expect(applyCompanyPrefix("/company/import", "NEU")).toBe("/NEU/company/import"); - expect(applyCompanyPrefix("/company/settings/cloud-upstream", "NEU")).toBe( - "/NEU/company/settings/cloud-upstream", - ); expect(applyCompanyPrefix("/org", "NEU")).toBe("/NEU/org"); }); diff --git a/ui/src/lib/import-preflight.test.ts b/ui/src/lib/import-preflight.test.ts new file mode 100644 index 0000000000..ff7e6e0381 --- /dev/null +++ b/ui/src/lib/import-preflight.test.ts @@ -0,0 +1,92 @@ +// @vitest-environment node + +import { describe, expect, it } from "vitest"; +import { + INLINE_IMPORT_MAX_BYTES, + REQUEST_ENVELOPE_ALLOWANCE_BYTES, + buildInlineImportPreflight, + estimateInlineImportBytes, + isBlobStoreFilePath, + stripBlobFiles, +} from "./import-preflight"; + +const megabyte = 1024 * 1024; + +describe("import preflight", () => { + it("recognizes blob store paths at the package root only", () => { + expect(isBlobStoreFilePath("blobs/4f2d1c9a")).toBe(true); + expect(isBlobStoreFilePath("paperclip-demo/blobs/4f2d1c9a")).toBe(true); + expect(isBlobStoreFilePath("tasks/pap-1/TASK.md")).toBe(false); + expect(isBlobStoreFilePath("blobs/nested/file")).toBe(false); + }); + + it("estimates inline bytes from JSON-escaped text, base64 entries, keys, and the envelope", () => { + const base = estimateInlineImportBytes({}); + expect(base).toBe(REQUEST_ENVELOPE_ALLOWANCE_BYTES); + // "COMPANY.md" key = 12 serialized bytes + 2 separators; "12345" = 7 with quotes. + expect(estimateInlineImportBytes({ "COMPANY.md": "12345" }) - base).toBe(12 + 2 + 7); + // Base64 entries count their object structure and contentType, not just the payload. + expect(estimateInlineImportBytes({ + "blobs/abc": { encoding: "base64", data: "QUJDRA==", contentType: "application/octet-stream" }, + }) - base).toBe(11 + 2 + (48 + 8 + 24)); + }); + + it("counts UTF-8 bytes of serialized text, not UTF-16 code units", () => { + const base = estimateInlineImportBytes({}); + // Two CJK characters: String.length is 2, but each is 3 bytes of UTF-8, + // plus the surrounding JSON quotes the request body will carry. + expect(estimateInlineImportBytes({ "COMPANY.md": "汉字" }) - base).toBe(12 + 2 + 8); + // A newline serializes as the two-character escape sequence \n. + expect(estimateInlineImportBytes({ "NOTES.md": "a\nb" }) - base).toBe(10 + 2 + 6); + }); + + it("counts file-path keys so path-heavy packages cannot undercount", () => { + const shortKey = estimateInlineImportBytes({ B: "x" }); + const longKey = estimateInlineImportBytes({ ["A".repeat(100)]: "x" }); + expect(longKey - shortKey).toBe(99); + }); + + it("never estimates below the actual serialized files map", () => { + const files = { + "COMPANY.md": '汉字 with\nnewlines and "quotes"', + "tasks/pap-1/TASK.md": "plain text", + "blobs/abc": { encoding: "base64" as const, data: "QUJDRA==", contentType: "image/png" }, + }; + const serialized = new TextEncoder().encode(JSON.stringify(files)).length; + expect(estimateInlineImportBytes(files)).toBeGreaterThanOrEqual(serialized); + }); + + it("passes packages under the inline limit", () => { + const preflight = buildInlineImportPreflight({ "COMPANY.md": "x".repeat(megabyte) }); + expect(preflight.tooLarge).toBe(false); + expect(preflight.canDropAttachments).toBe(false); + }); + + it("blocks oversized packages and offers to drop attachments when that fits", () => { + const preflight = buildInlineImportPreflight({ + "COMPANY.md": "x".repeat(megabyte), + "blobs/abc": { + encoding: "base64", + data: "A".repeat(INLINE_IMPORT_MAX_BYTES), + contentType: "application/octet-stream", + }, + }); + expect(preflight.tooLarge).toBe(true); + expect(preflight.canDropAttachments).toBe(true); + }); + + it("blocks oversized packages without the attachment escape hatch when text alone is too big", () => { + const preflight = buildInlineImportPreflight({ + "COMPANY.md": "x".repeat(INLINE_IMPORT_MAX_BYTES + 1), + }); + expect(preflight.tooLarge).toBe(true); + expect(preflight.canDropAttachments).toBe(false); + }); + + it("strips only blob files from the package", () => { + expect(stripBlobFiles({ + "COMPANY.md": "text", + "blobs/abc": { encoding: "base64", data: "QQ==", contentType: "application/octet-stream" }, + })).toEqual({ "COMPANY.md": "text" }); + }); +}); diff --git a/ui/src/lib/import-preflight.ts b/ui/src/lib/import-preflight.ts new file mode 100644 index 0000000000..df8c143df3 --- /dev/null +++ b/ui/src/lib/import-preflight.ts @@ -0,0 +1,80 @@ +import type { CompanyPortabilityFileEntry } from "@paperclipai/shared"; + +// Inline imports post the whole parsed package as one JSON body, so oversized +// packages must be blocked before the request is built. Packages past this +// limit go through the CLI folder import today; a blob-store relay is planned. +export const INLINE_IMPORT_MAX_BYTES = 56 * 1024 * 1024; + +export function isBlobStoreFilePath(filePath: string): boolean { + return /(^|\/)blobs\/[^/]+$/.test(filePath); +} + +const utf8 = new TextEncoder(); + +// Fixed serialization overhead of a base64 entry object around its data and +// contentType values: {"encoding":"base64","data":"…","contentType":"…"}. +const BASE64_ENTRY_STRUCTURE_BYTES = '{"encoding":"base64","data":"","contentType":""}'.length; + +// Allowance for everything in the request body besides the files map itself +// (rootPath, include flags, target, collision strategy, adapter overrides, +// braces and commas). Deliberately generous so the estimate never undercounts. +export const REQUEST_ENVELOPE_ALLOWANCE_BYTES = 256 * 1024; + +// The server enforces its body limit on raw request bytes, so estimate each +// entry the way it actually travels: JSON-escaped UTF-8 for text (multi-byte +// characters and escape sequences both inflate past `String.length`), and the +// base64 payload plus its object structure for blobs (base64 and MIME types +// are ASCII, one byte per character). +function fileEntryInlineBytes(entry: CompanyPortabilityFileEntry): number { + if (typeof entry === "string") return utf8.encode(JSON.stringify(entry)).length; + return BASE64_ENTRY_STRUCTURE_BYTES + entry.data.length + (entry.contentType?.length ?? 0); +} + +/** + * Approximate JSON request size of an inline import: JSON-escaped UTF-8 text + * bytes, base64 payloads with their entry structure, the serialized file-path + * keys (thousands of paths are real bytes), and an envelope allowance for the + * rest of the request body. + */ +export function estimateInlineImportBytes(files: Record): number { + let total = REQUEST_ENVELOPE_ALLOWANCE_BYTES; + for (const [filePath, entry] of Object.entries(files)) { + // "path": entry, → key bytes + colon + comma. + total += utf8.encode(JSON.stringify(filePath)).length + 2 + fileEntryInlineBytes(entry); + } + return total; +} + +export function stripBlobFiles( + files: Record, +): Record { + return Object.fromEntries( + Object.entries(files).filter(([filePath]) => !isBlobStoreFilePath(filePath)), + ); +} + +export interface InlineImportPreflight { + estimatedBytes: number; + tooLarge: boolean; + /** Dropping blobs/** attachment payloads would bring the package under the limit. */ + canDropAttachments: boolean; +} + +export function buildInlineImportPreflight( + files: Record, +): InlineImportPreflight { + const estimatedBytes = estimateInlineImportBytes(files); + if (estimatedBytes <= INLINE_IMPORT_MAX_BYTES) { + return { estimatedBytes, tooLarge: false, canDropAttachments: false }; + } + const bytesWithoutBlobs = estimateInlineImportBytes(stripBlobFiles(files)); + return { + estimatedBytes, + tooLarge: true, + canDropAttachments: bytesWithoutBlobs < estimatedBytes && bytesWithoutBlobs <= INLINE_IMPORT_MAX_BYTES, + }; +} + +export function formatMegabytes(bytes: number): string { + return `${Math.max(1, Math.round(bytes / (1024 * 1024)))} MB`; +} diff --git a/ui/src/lib/queryKeys.ts b/ui/src/lib/queryKeys.ts index 9738a432fd..149ce2c6b9 100644 --- a/ui/src/lib/queryKeys.ts +++ b/ui/src/lib/queryKeys.ts @@ -3,6 +3,7 @@ export const queryKeys = { all: ["companies"] as const, detail: (id: string) => ["companies", id] as const, stats: ["companies", "stats"] as const, + exportFidelity: (companyId: string) => ["companies", companyId, "export-fidelity"] as const, }, apps: { gallery: (companyId: string) => ["apps", companyId, "gallery"] as const, @@ -337,7 +338,6 @@ export const queryKeys = { schedulerHeartbeats: ["instance", "scheduler-heartbeats"] as const, experimentalSettings: ["instance", "experimental-settings"] as const, }, - cloudUpstreams: (companyId: string) => ["cloud-upstreams", companyId] as const, health: ["health"] as const, secrets: { list: (companyId: string) => ["secrets", companyId] as const, diff --git a/ui/src/lib/zip.test.ts b/ui/src/lib/zip.test.ts index 60258a8d65..ae2201de11 100644 --- a/ui/src/lib/zip.test.ts +++ b/ui/src/lib/zip.test.ts @@ -2,7 +2,7 @@ import { deflateRawSync } from "node:zlib"; import { describe, expect, it } from "vitest"; -import { createZipArchive, readZipArchive } from "./zip"; +import { createZipArchive, estimateZipArchiveSize, readZipArchive } from "./zip"; function readUint16(bytes: Uint8Array, offset: number) { return bytes[offset]! | (bytes[offset + 1]! << 8); @@ -257,6 +257,51 @@ describe("createZipArchive", () => { }); }); + it("round-trips extensionless blobs/ entries as base64 octet streams", async () => { + const bytes = new Uint8Array([0x00, 0x01, 0x80, 0xfe, 0xff]); + const entry = { + encoding: "base64" as const, + data: Buffer.from(bytes).toString("base64"), + contentType: "application/octet-stream", + }; + const archive = createZipArchive( + { + "COMPANY.md": "# Company\n", + "blobs/4f2d1c9a": entry, + }, + "paperclip-demo", + ); + + await expect(readZipArchive(archive)).resolves.toEqual({ + rootPath: "paperclip-demo", + files: { + "COMPANY.md": "# Company\n", + "blobs/4f2d1c9a": entry, + }, + }); + }); + + it("falls back to base64 for invalid UTF-8 bytes instead of mangling them", async () => { + const invalidUtf8 = new Uint8Array([0x68, 0x69, 0xff, 0xfe, 0xc0]); + const archive = createZipArchive( + { + "tasks/pap-1/raw-notes": { + encoding: "base64", + data: Buffer.from(invalidUtf8).toString("base64"), + contentType: null, + }, + }, + "paperclip-demo", + ); + + const result = await readZipArchive(archive); + expect(result.files["tasks/pap-1/raw-notes"]).toEqual({ + encoding: "base64", + data: Buffer.from(invalidUtf8).toString("base64"), + contentType: "application/octet-stream", + }); + }); + it("reads standard DEFLATE zip archives created outside Paperclip", async () => { const archive = createDeflatedZipArchive( { @@ -287,3 +332,60 @@ describe("createZipArchive", () => { }); }); }); + +describe("estimateZipArchiveSize", () => { + it("matches the real archive byte length for mixed text and binary entries", () => { + const files = { + "COMPANY.md": "# Company\n", + "agents/céo/AGENT.md": "# Héllo wörld — 日本語 🚀\n", + "images/logo.png": { + encoding: "base64" as const, + data: Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]).toString("base64"), + contentType: "image/png", + }, + "blobs/4f2d1c9a": { + encoding: "base64" as const, + data: Buffer.from([0x00, 0x01, 0x80, 0xfe]).toString("base64"), + contentType: "application/octet-stream", + }, + "notes/empty.txt": "", + }; + + expect(estimateZipArchiveSize(files, "paperclip-demo")).toBe( + createZipArchive(files, "paperclip-demo").byteLength, + ); + }); + + it("matches the real archive byte length for every base64 padding variant", () => { + for (const byteCount of [1, 2, 3, 4, 5, 6]) { + const files = { + "blobs/blob": { + encoding: "base64" as const, + data: Buffer.alloc(byteCount, 0xab).toString("base64"), + contentType: "application/octet-stream", + }, + }; + + expect(estimateZipArchiveSize(files, "root")).toBe( + createZipArchive(files, "root").byteLength, + ); + } + }); + + it("mirrors the writer's path normalization for messy paths", () => { + const files = { + "agents//ceo\\AGENT.md": "# CEO\n", + }; + + expect(estimateZipArchiveSize(files, "demo/")).toBe( + createZipArchive(files, "demo/").byteLength, + ); + }); + + it("returns the 22-byte end-of-central-directory record for an empty map", () => { + expect(estimateZipArchiveSize({}, "paperclip-demo")).toBe(22); + expect(estimateZipArchiveSize({}, "paperclip-demo")).toBe( + createZipArchive({}, "paperclip-demo").byteLength, + ); + }); +}); diff --git a/ui/src/lib/zip.ts b/ui/src/lib/zip.ts index 509bba32ef..654259821b 100644 --- a/ui/src/lib/zip.ts +++ b/ui/src/lib/zip.ts @@ -2,6 +2,9 @@ import type { CompanyPortabilityFileEntry } from "@paperclipai/shared"; const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); +// ignoreBOM keeps a leading BOM in the decoded text so text entries +// re-encode to their original bytes; fatal surfaces invalid UTF-8. +const strictTextDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); const crcTable = new Uint32Array(256); for (let i = 0; i < 256; i++) { @@ -121,14 +124,39 @@ function base64ToBytes(base64: string) { return bytes; } +function isBlobStorePath(pathValue: string) { + return /(^|\/)blobs\/[^/]+$/.test(normalizeArchivePath(pathValue)); +} + +function decodeStrictUtf8(bytes: Uint8Array): string | null { + let text: string; + try { + text = strictTextDecoder.decode(bytes); + } catch { + return null; + } + const reEncoded = textEncoder.encode(text); + if (reEncoded.length !== bytes.length) return null; + for (let index = 0; index < bytes.length; index += 1) { + if (reEncoded[index] !== bytes[index]) return null; + } + return text; +} + function bytesToPortableFileEntry(pathValue: string, bytes: Uint8Array): CompanyPortabilityFileEntry { + // Content-addressed blob entries are opaque bytes regardless of extension. + if (isBlobStorePath(pathValue)) { + return { encoding: "base64", data: bytesToBase64(bytes), contentType: "application/octet-stream" }; + } const contentType = inferBinaryContentType(pathValue); - if (!contentType) return textDecoder.decode(bytes); - return { - encoding: "base64", - data: bytesToBase64(bytes), - contentType, - }; + if (contentType) { + return { encoding: "base64", data: bytesToBase64(bytes), contentType }; + } + const text = decodeStrictUtf8(bytes); + if (text !== null) return text; + // Bytes that are not valid UTF-8 must not be decoded lossily; fall back + // to base64 so they round-trip exactly. + return { encoding: "base64", data: bytesToBase64(bytes), contentType: "application/octet-stream" }; } function portableFileEntryToBytes(entry: CompanyPortabilityFileEntry): Uint8Array { @@ -281,3 +309,67 @@ export function createZipArchive(files: Record= 0xd800 && code < 0xdc00 && index + 1 < text.length) { + const next = text.charCodeAt(index + 1); + if (next >= 0xdc00 && next < 0xe000) { + bytes += 4; + index += 1; + } else { + bytes += 3; + } + } else { + bytes += 3; + } + } + return bytes; +} + +/** + * Decoded byte length of a base64 payload, mirroring atob's forgiving-base64 + * handling in base64ToBytes: ASCII whitespace is stripped and trailing "=" + * padding carries no data. + */ +function base64ByteLength(data: string): number { + const stripped = data.replace(/[\t\n\f\r ]+/g, ""); + let end = stripped.length; + if (end > 0 && stripped[end - 1] === "=") end -= 1; + if (end > 0 && stripped[end - 1] === "=") end -= 1; + return Math.floor((end * 3) / 4); +} + +/** + * Exact byte size of the archive createZipArchive(files, rootPath) would + * produce, without building it. Every entry is STOREd (never compressed), so + * the size is fully determined by the raw body bytes plus fixed overhead: per + * entry a 30-byte local file header and a 46-byte central-directory record + * (each followed by the archive path), and one 22-byte end-of-central-directory + * record. The writer emits no data descriptors, extra fields, or comments. + */ +export function estimateZipArchiveSize( + files: Record, + rootPath: string, +): number { + const normalizedRoot = normalizeArchivePath(rootPath); + let size = 22; + for (const [relativePath, contents] of Object.entries(files)) { + const fileNameLength = utf8ByteLength(normalizeArchivePath(`${normalizedRoot}/${relativePath}`)); + const bodyLength = + typeof contents === "string" ? utf8ByteLength(contents) : base64ByteLength(contents.data); + size += 30 + fileNameLength + bodyLength + 46 + fileNameLength; + } + return size; +} diff --git a/ui/src/pages/Agents.test.tsx b/ui/src/pages/Agents.test.tsx index f58a759ddb..7e01249a00 100644 --- a/ui/src/pages/Agents.test.tsx +++ b/ui/src/pages/Agents.test.tsx @@ -243,7 +243,6 @@ function makeInstanceSettings({ enableTaskWatchdogs: true, enableIssuePlanDecompositions: true, enableExperimentalFileViewer: false, - enableCloudSync: false, enableExternalObjects: false, enableBuiltInAgents, autoRestartDevServerWhenIdle: false, diff --git a/ui/src/pages/CloudUpstream.test.tsx b/ui/src/pages/CloudUpstream.test.tsx deleted file mode 100644 index bbc67088ae..0000000000 --- a/ui/src/pages/CloudUpstream.test.tsx +++ /dev/null @@ -1,413 +0,0 @@ -// @vitest-environment jsdom - -import { createRoot } from "react-dom/client"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import type { CloudUpstreamRun, CloudUpstreamsState } from "@paperclipai/shared"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { CloudUpstream, buildActivationRows } from "./CloudUpstream"; - -const mockCloudUpstreamsApi = vi.hoisted(() => ({ - list: vi.fn(), - startConnect: vi.fn(), - finishConnect: vi.fn(), - preview: vi.fn(), - createRun: vi.fn(), - getRun: vi.fn(), - cancelRun: vi.fn(), - activateEntities: vi.fn(), -})); -const mockInstanceSettingsApi = vi.hoisted(() => ({ - getExperimental: vi.fn(), -})); -const mockSetBreadcrumbs = vi.hoisted(() => vi.fn()); -const mockCompanyState = vi.hoisted(() => ({ - selectedCompany: { id: "company-1", name: "Paperclip", issuePrefix: "PAP" } as - | { id: string; name: string; issuePrefix: string | null } - | null, - selectedCompanyId: "company-1" as string | null, -})); -const mockLocationState = vi.hoisted(() => ({ - pathname: "/PAP/company/settings/cloud-upstream", - search: "", -})); - -vi.mock("@/api/cloudUpstreams", () => ({ - cloudUpstreamsApi: mockCloudUpstreamsApi, -})); - -vi.mock("@/api/instanceSettings", () => ({ - instanceSettingsApi: mockInstanceSettingsApi, -})); - -vi.mock("@/context/BreadcrumbContext", () => ({ - useBreadcrumbs: () => ({ - setBreadcrumbs: mockSetBreadcrumbs, - }), -})); - -vi.mock("@/context/CompanyContext", () => ({ - useCompany: () => ({ - selectedCompany: mockCompanyState.selectedCompany, - selectedCompanyId: mockCompanyState.selectedCompanyId, - }), -})); - -vi.mock("@/lib/router", () => ({ - Link: ({ children, to, className }: { children: React.ReactNode; to: string; className?: string }) => ( - - {children} - - ), - useLocation: () => ({ pathname: mockLocationState.pathname, search: mockLocationState.search }), -})); - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -(globalThis as any).IS_REACT_ACT_ENVIRONMENT = true; - -async function act(callback: () => void | Promise) { - await callback(); - await Promise.resolve(); - await new Promise((resolve) => window.setTimeout(resolve, 0)); -} - -async function flushReact() { - await act(async () => { - await Promise.resolve(); - await new Promise((resolve) => window.setTimeout(resolve, 0)); - }); -} - -describe("CloudUpstream", () => { - let container: HTMLDivElement; - - beforeEach(() => { - container = document.createElement("div"); - document.body.appendChild(container); - mockCompanyState.selectedCompany = { id: "company-1", name: "Paperclip", issuePrefix: "PAP" }; - mockCompanyState.selectedCompanyId = "company-1"; - mockLocationState.pathname = "/PAP/company/settings/cloud-upstream"; - mockLocationState.search = ""; - mockInstanceSettingsApi.getExperimental.mockResolvedValue({ enableCloudSync: true }); - mockCloudUpstreamsApi.list.mockResolvedValue(stateWithRun(buildRun({ status: "succeeded" }))); - mockCloudUpstreamsApi.activateEntities.mockImplementation((_connectionId, _runId, input) => - Promise.resolve(buildRun({ - status: "succeeded", - report: { - activationChecklist: { - [input.entityType]: { - entityType: input.entityType, - count: input.entityType === "agents" ? 2 : 1, - status: "activated", - activatedAt: "2026-05-18T19:00:00.000Z", - }, - }, - }, - })), - ); - mockCloudUpstreamsApi.createRun.mockResolvedValue(buildRun({ status: "running" })); - }); - - afterEach(() => { - container.remove(); - document.body.innerHTML = ""; - vi.clearAllMocks(); - }); - - it("binds the succeeded run activation checklist to imported category counts", async () => { - const root = createRoot(container); - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - - await act(async () => { - root.render( - - - , - ); - }); - await flushReact(); - await flushReact(); - - expect(container.textContent).toContain("Re-run"); - expect(container.textContent).not.toContain("Retry"); - expect(container.textContent).toContain("Activation checklist"); - expect(container.textContent).toContain("2 paused"); - expect(container.textContent).toContain("1 paused"); - expect(container.textContent).toContain("0 imported monitors in this run."); - expect(container.textContent).toContain("Keep paused"); - - const activateButton = Array.from(container.querySelectorAll("button")) - .find((button) => button.textContent?.trim() === "Activate") as HTMLButtonElement | undefined; - expect(activateButton).toBeTruthy(); - - await act(async () => { - activateButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); - }); - await flushReact(); - - expect(mockCloudUpstreamsApi.activateEntities).toHaveBeenCalledWith( - "connection-1", - "run-1", - { companyId: "company-1", entityType: "agents" }, - ); - - await act(async () => { - root.unmount(); - }); - }); - - it("sends a company-prefixed redirectUri when starting Connect", async () => { - mockCloudUpstreamsApi.list.mockResolvedValue({ connections: [], runs: [] }); - mockCloudUpstreamsApi.startConnect.mockResolvedValue({ - pendingConnectionId: "pending-1", - authorizationUrl: "https://cloud.example/upstream-consent?state=abc", - }); - const root = createRoot(container); - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - - await act(async () => { - root.render( - - - , - ); - }); - await flushReact(); - await flushReact(); - - const input = container.querySelector("input[aria-label='Paperclip Cloud stack URL']"); - expect(input).toBeTruthy(); - await act(async () => { - const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")!.set!; - setter.call(input!, "https://cloud.example/PAP/dashboard"); - input!.dispatchEvent(new Event("input", { bubbles: true })); - }); - await flushReact(); - - const connectButton = Array.from(container.querySelectorAll("button")) - .find((button) => button.textContent?.trim() === "Connect") as HTMLButtonElement | undefined; - expect(connectButton).toBeTruthy(); - - await act(async () => { - connectButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); - }); - await flushReact(); - - expect(mockCloudUpstreamsApi.startConnect).toHaveBeenCalledWith({ - companyId: "company-1", - remoteUrl: "https://cloud.example/PAP/dashboard", - redirectUri: `${window.location.origin}/PAP/company/settings/cloud-upstream`, - }); - - await act(async () => { - root.unmount(); - }); - }); - - it("uses the URL pathname prefix when cleaning up the callback URL with no company context", async () => { - mockCompanyState.selectedCompany = null; - mockCompanyState.selectedCompanyId = null; - mockLocationState.pathname = "/PAP/company/settings/cloud-upstream"; - mockLocationState.search = "?code=cb-code&state=cb-state"; - mockCloudUpstreamsApi.list.mockResolvedValue({ connections: [], runs: [] }); - mockCloudUpstreamsApi.finishConnect.mockResolvedValue({ - id: "connection-1", - companyId: "company-1", - remoteUrl: "https://cloud.example/PAP", - target: { - stackId: "stack-1", - stackSlug: "stack", - stackDisplayName: "Paperclip Cloud", - companyId: "cloud-company-1", - primaryHost: "cloud.example", - origin: "https://cloud.example", - product: "Paperclip Cloud", - schemaMajor: 1, - maxChunkBytes: 1024, - }, - tokenStatus: "connected", - scopes: ["upstream_import:write"], - authorizedGlobalUserId: "user-1", - expiresAt: null, - createdAt: "2026-05-18T18:00:00.000Z", - updatedAt: "2026-05-18T18:00:00.000Z", - lastRunId: null, - }); - window.localStorage.setItem("paperclip-cloud-upstream-pending-connection", "pending-1"); - const replaceStateSpy = vi.spyOn(window.history, "replaceState"); - - try { - const root = createRoot(container); - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - - await act(async () => { - root.render( - - - , - ); - }); - await flushReact(); - await flushReact(); - - expect(mockCloudUpstreamsApi.finishConnect).toHaveBeenCalledWith({ - pendingConnectionId: "pending-1", - code: "cb-code", - state: "cb-state", - }); - expect(replaceStateSpy).toHaveBeenCalledWith(null, "", "/PAP/company/settings/cloud-upstream"); - - await act(async () => { - root.unmount(); - }); - } finally { - replaceStateSpy.mockRestore(); - window.localStorage.removeItem("paperclip-cloud-upstream-pending-connection"); - } - }); - - it("does not retry the OAuth callback finish mutation after an error", async () => { - mockLocationState.pathname = "/PAP/company/settings/cloud-upstream"; - mockLocationState.search = "?code=cb-code&state=cb-state"; - mockCloudUpstreamsApi.list.mockResolvedValue({ connections: [], runs: [] }); - mockCloudUpstreamsApi.finishConnect.mockRejectedValue(new Error("state expired")); - window.localStorage.setItem("paperclip-cloud-upstream-pending-connection", "pending-1"); - - try { - const root = createRoot(container); - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - - await act(async () => { - root.render( - - - , - ); - }); - await flushReact(); - await flushReact(); - await flushReact(); - - expect(mockCloudUpstreamsApi.finishConnect).toHaveBeenCalledTimes(1); - expect(container.textContent).toContain("state expired"); - - await act(async () => { - root.unmount(); - }); - } finally { - window.localStorage.removeItem("paperclip-cloud-upstream-pending-connection"); - } - }); - - it("keeps retry only for failed or cancelled runs", async () => { - mockCloudUpstreamsApi.list.mockResolvedValue(stateWithRun(buildRun({ status: "failed" }))); - const root = createRoot(container); - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - - await act(async () => { - root.render( - - - , - ); - }); - await flushReact(); - await flushReact(); - - expect(container.textContent).toContain("Retry"); - expect(container.textContent).not.toContain("Re-run"); - expect(container.textContent).not.toContain("Activation checklist"); - - await act(async () => { - root.unmount(); - }); - }); -}); - -describe("buildActivationRows", () => { - it("reads activation decisions from the run report", () => { - const rows = buildActivationRows(buildRun({ - status: "succeeded", - report: { - activationChecklist: { - agents: { - entityType: "agents", - count: 2, - status: "activated", - activatedAt: "2026-05-18T19:00:00.000Z", - }, - }, - }, - })); - - expect(rows[0]).toMatchObject({ key: "agents", count: 2, status: "activated", statusLabel: "2 activated" }); - expect(rows[2]).toMatchObject({ key: "monitors", count: 0, status: "paused", statusLabel: "0 imported" }); - }); -}); - -function stateWithRun(run: CloudUpstreamRun): CloudUpstreamsState { - return { - connections: [ - { - id: "connection-1", - companyId: "company-1", - remoteUrl: "https://paperclip.example/PAP", - target: { - stackId: "stack-1", - stackSlug: "stack", - stackDisplayName: "Paperclip Cloud", - companyId: "cloud-company-1", - primaryHost: "paperclip.example", - origin: "https://paperclip.example", - product: "Paperclip Cloud", - schemaMajor: 1, - maxChunkBytes: 1024, - }, - tokenStatus: "connected", - scopes: ["upstream_import:write"], - authorizedGlobalUserId: "user-1", - expiresAt: null, - createdAt: "2026-05-18T18:00:00.000Z", - updatedAt: "2026-05-18T18:00:00.000Z", - lastRunId: run.id, - }, - ], - runs: [run], - }; -} - -function buildRun(input: { - status: CloudUpstreamRun["status"]; - report?: Record; -}): CloudUpstreamRun { - return { - id: "run-1", - connectionId: "connection-1", - companyId: "company-1", - status: input.status, - activeStep: input.status === "succeeded" ? "activate" : "push", - progressPercent: input.status === "running" ? 70 : 100, - dryRun: false, - summary: [ - { key: "agents", label: "Agents", count: 2 }, - { key: "routines", label: "Routines", count: 1 }, - { key: "issues", label: "Issues", count: 7 }, - ], - warnings: [], - conflicts: [], - events: [ - { - id: "event-1", - at: "2026-05-18T18:30:00.000Z", - phase: input.status === "succeeded" ? "activate" : "push", - type: input.status === "failed" ? "failed" : "completed", - message: input.status === "failed" ? "Push failed." : "Activation checklist is ready.", - }, - ], - targetUrl: "https://paperclip.example", - report: input.report ?? {}, - retryOfRunId: null, - createdAt: "2026-05-18T18:00:00.000Z", - updatedAt: "2026-05-18T18:30:00.000Z", - completedAt: input.status === "running" ? null : "2026-05-18T18:30:00.000Z", - }; -} diff --git a/ui/src/pages/CloudUpstream.tsx b/ui/src/pages/CloudUpstream.tsx deleted file mode 100644 index 012ead2f7d..0000000000 --- a/ui/src/pages/CloudUpstream.tsx +++ /dev/null @@ -1,649 +0,0 @@ -import { useEffect, useMemo, useState } from "react"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { - AlertTriangle, - CheckCircle2, - CloudUpload, - ExternalLink, - FileJson, - History, - Loader2, - RefreshCcw, - ShieldAlert, -} from "lucide-react"; -import type { - CloudUpstreamActivationDecision, - CloudUpstreamActivationEntityType, - CloudUpstreamPreview, - CloudUpstreamRun, - CloudUpstreamStep, -} from "@paperclipai/shared"; -import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { cloudUpstreamsApi } from "@/api/cloudUpstreams"; -import { instanceSettingsApi } from "@/api/instanceSettings"; -import { useBreadcrumbs } from "@/context/BreadcrumbContext"; -import { useCompany } from "@/context/CompanyContext"; -import { applyCompanyPrefix, extractCompanyPrefixFromPath } from "@/lib/company-routes"; -import { Link, useLocation } from "@/lib/router"; -import { queryKeys } from "@/lib/queryKeys"; - -const PENDING_CONNECTION_KEY = "paperclip-cloud-upstream-pending-connection"; -const STEPS: Array<{ key: CloudUpstreamStep; label: string }> = [ - { key: "connect", label: "Connect" }, - { key: "scan", label: "Scan" }, - { key: "preview", label: "Preview" }, - { key: "push", label: "Push" }, - { key: "verify", label: "Verify" }, - { key: "activate", label: "Activate" }, -]; -const ACTIVATION_CATEGORIES: Array<{ - key: CloudUpstreamActivationEntityType; - label: string; - singular: string; - detail: string; -}> = [ - { - key: "agents", - label: "Agents", - singular: "agent", - detail: "Confirm cloud secrets and adapter credentials before unpausing imported agents.", - }, - { - key: "routines", - label: "Routines", - singular: "routine", - detail: "Review schedules and trigger settings before enabling imported routines.", - }, - { - key: "monitors", - label: "Monitors", - singular: "monitor", - detail: "Activate after the target stack has been smoke tested.", - }, -]; - -export function CloudUpstream() { - const { selectedCompany, selectedCompanyId } = useCompany(); - const { setBreadcrumbs } = useBreadcrumbs(); - const queryClient = useQueryClient(); - const location = useLocation(); - const [remoteUrl, setRemoteUrl] = useState(""); - const [preview, setPreview] = useState(null); - const [activeRun, setActiveRun] = useState(null); - const [notice, setNotice] = useState(null); - const [actionError, setActionError] = useState(null); - - useEffect(() => { - setBreadcrumbs([ - { label: selectedCompany?.name ?? "Company", href: "/dashboard" }, - { label: "Settings", href: "/company/settings" }, - { label: "Cloud upstream" }, - ]); - }, [selectedCompany?.name, setBreadcrumbs]); - - const experimentalQuery = useQuery({ - queryKey: queryKeys.instance.experimentalSettings, - queryFn: () => instanceSettingsApi.getExperimental(), - }); - const cloudSyncEnabled = experimentalQuery.data?.enableCloudSync === true; - - const upstreamQuery = useQuery({ - queryKey: selectedCompanyId ? queryKeys.cloudUpstreams(selectedCompanyId) : ["cloud-upstreams", "__disabled__"], - queryFn: () => cloudUpstreamsApi.list(selectedCompanyId!), - enabled: !!selectedCompanyId && cloudSyncEnabled, - }); - - const connection = upstreamQuery.data?.connections[0] ?? null; - const latestRun = activeRun ?? upstreamQuery.data?.runs[0] ?? null; - - const callbackParams = useMemo(() => new URLSearchParams(location.search), [location.search]); - const code = callbackParams.get("code"); - const state = callbackParams.get("state"); - const callbackError = callbackParams.get("error"); - - const settingsPath = useMemo(() => { - const pathPrefix = extractCompanyPrefixFromPath(location.pathname); - return applyCompanyPrefix("/company/settings/cloud-upstream", pathPrefix ?? selectedCompany?.issuePrefix ?? null); - }, [location.pathname, selectedCompany?.issuePrefix]); - - const finishMutation = useMutation({ - mutationFn: (input: { pendingConnectionId: string; code: string; state: string }) => - cloudUpstreamsApi.finishConnect(input), - onSuccess: async () => { - localStorage.removeItem(PENDING_CONNECTION_KEY); - setNotice("Cloud upstream connection approved."); - setActionError(null); - await invalidateUpstreams(); - window.history.replaceState(null, "", settingsPath); - }, - onError: (error) => setActionError(error instanceof Error ? error.message : "Failed to finish connection."), - }); - const { - mutate: finishConnect, - isError: finishConnectFailed, - isPending: finishConnectPending, - isSuccess: finishConnectSucceeded, - } = finishMutation; - - useEffect(() => { - if (!cloudSyncEnabled || !code || !state || finishConnectPending || finishConnectSucceeded || finishConnectFailed) return; - const pendingConnectionId = localStorage.getItem(PENDING_CONNECTION_KEY); - if (!pendingConnectionId) { - setActionError("No pending cloud upstream connection was found. Start the connection again."); - return; - } - finishConnect({ pendingConnectionId, code, state }); - }, [cloudSyncEnabled, code, finishConnect, finishConnectFailed, finishConnectPending, finishConnectSucceeded, state]); - - useEffect(() => { - if (callbackError) { - setActionError(`Cloud upstream connection was not approved: ${callbackError}`); - } - }, [callbackError]); - - const startMutation = useMutation({ - mutationFn: () => - cloudUpstreamsApi.startConnect({ - companyId: selectedCompanyId!, - remoteUrl, - redirectUri: `${window.location.origin}${settingsPath}`, - }), - onSuccess: (result) => { - localStorage.setItem(PENDING_CONNECTION_KEY, result.pendingConnectionId); - setActionError(null); - window.location.assign(result.authorizationUrl); - }, - onError: (error) => setActionError(error instanceof Error ? error.message : "Failed to start connection."), - }); - - const previewMutation = useMutation({ - mutationFn: (input: { connectionId: string; companyId: string }) => - cloudUpstreamsApi.preview(input.connectionId, { companyId: input.companyId }), - onSuccess: (nextPreview) => { - setPreview(nextPreview); - setActionError(null); - }, - onError: (error) => setActionError(previewErrorMessage(error)), - }); - - const runMutation = useMutation({ - mutationFn: (input: { connectionId: string; companyId: string; retryOfRunId?: string | null }) => - cloudUpstreamsApi.createRun(input.connectionId, { - companyId: input.companyId, - retryOfRunId: input.retryOfRunId ?? null, - }), - onSuccess: async (run) => { - setActiveRun(run); - setNotice(run.status === "succeeded" - ? "Push run completed. Review activation before unpausing automations." - : "Push run failed. Review the run events and retry after correcting the issue."); - setActionError(null); - await invalidateUpstreams(); - }, - onError: (error) => setActionError(error instanceof Error ? error.message : "Failed to run push."), - }); - const activationMutation = useMutation({ - mutationFn: (input: { run: CloudUpstreamRun; entityType: CloudUpstreamActivationEntityType }) => - cloudUpstreamsApi.activateEntities(input.run.connectionId, input.run.id, { - companyId: input.run.companyId, - entityType: input.entityType, - }), - onSuccess: async (run) => { - setActiveRun(run); - setNotice("Activation checklist updated."); - setActionError(null); - await invalidateUpstreams(); - }, - onError: (error) => setActionError(error instanceof Error ? error.message : "Failed to activate imported entities."), - }); - - async function invalidateUpstreams() { - if (!selectedCompanyId) return; - await queryClient.invalidateQueries({ queryKey: queryKeys.cloudUpstreams(selectedCompanyId) }); - } - - if (!selectedCompanyId || !selectedCompany) { - return
Select a company to configure cloud upstream.
; - } - - if (experimentalQuery.isLoading) { - return
Loading experimental settings...
; - } - - if (!cloudSyncEnabled) { - return ( -
-
- -

Cloud upstream

-
-
- Cloud sync is disabled. Enable it in{" "} - - Instance Settings - {" "} - to show upstream connection and push tools. -
-
- ); - } - - return ( -
-
-
-
- -

Cloud upstream

-
-

- Push {selectedCompany.name} into a Paperclip Cloud stack. Automations stay paused until activation. -

-
- {connection?.target.origin ? ( - - ) : null} -
- - {notice ? ( -
- {notice} -
- ) : null} - {actionError ? ( -
- {actionError} -
- ) : null} - - - -
-
Connection
-
- {connection ? ( -
-
-
- {connection.target.stackDisplayName ?? connection.target.stackSlug ?? connection.target.stackId} -
-
- {connection.target.product} · {connection.target.origin} · token {connection.tokenStatus} -
-
- Schema {connection.target.schemaMajor}. Max chunk {formatBytes(connection.target.maxChunkBytes)}. -
-
-
- - {previewMutation.isPending ? : null} -
-
- ) : ( -
- setRemoteUrl(event.target.value)} - placeholder="https://paperclip.paperclip.app/PC521D/dashboard" - aria-label="Paperclip Cloud stack URL" - /> - -
- )} -
-
- - {preview ? ( -
-
-
Preview
- -
- - - -
- ) : null} - - {latestRun ? ( -
-
-
Progress and finish
-
- - {latestRun.status === "failed" || latestRun.status === "cancelled" ? ( - - ) : latestRun.status === "succeeded" ? ( - - ) : null} -
-
-
-
-
-
{latestRun.status}
-
- Run {latestRun.id.slice(0, 8)} · {latestRun.completedAt ? `completed ${formatDate(latestRun.completedAt)}` : "in progress"} -
-
-
{latestRun.progressPercent}%
-
-
-
-
-
- {latestRun.events.map((event) => ( -
- {formatDate(event.at)} - {event.phase} - {event.message} -
- ))} -
-
- - {latestRun.status === "succeeded" ? ( - activationMutation.mutate({ run: latestRun, entityType })} - /> - ) : null} -
- ) : null} - - {upstreamQuery.data?.runs.length ? ( -
-
- - History -
-
- {upstreamQuery.data.runs.map((run) => ( - - ))} -
-
- ) : null} -
- ); -} - -function PreviewProgressHint() { - const [elapsed, setElapsed] = useState(0); - useEffect(() => { - const startedAt = Date.now(); - const interval = window.setInterval(() => setElapsed(Math.round((Date.now() - startedAt) / 1000)), 1000); - return () => window.clearInterval(interval); - }, []); - const message = elapsed < 15 - ? "Building manifest..." - : elapsed < 45 - ? `Building manifest... ${elapsed}s. Large companies can take up to a minute.` - : `Still building manifest... ${elapsed}s. PAP-scale companies routinely take ~60s.`; - return
{message}
; -} - -function Stepper({ activeStep }: { activeStep: CloudUpstreamStep }) { - const activeIndex = STEPS.findIndex((step) => step.key === activeStep); - return ( -
- {STEPS.map((step, index) => { - const complete = index < activeIndex; - const active = index === activeIndex; - return ( -
- {complete ? ( - - ) : ( - - )} - {step.label} -
- ); - })} -
- ); -} - -function SummaryGrid({ summary }: { summary: CloudUpstreamPreview["summary"] }) { - return ( -
- {summary.map((item) => ( -
-
{item.count}
-
{item.label}
-
- ))} -
- ); -} - -function WarningsPanel({ warnings }: { warnings: CloudUpstreamPreview["warnings"] }) { - return ( -
-
- - Warnings -
-
- {warnings.map((warning) => ( -
- -
{warning.title}
-
{warning.detail}
-
- ))} -
-
- ); -} - -function ConflictTable({ conflicts }: { conflicts: CloudUpstreamPreview["conflicts"] }) { - return ( -
-
Conflicts
- {conflicts.length === 0 ? ( -
No target conflicts detected for this preview.
- ) : ( -
- {conflicts.map((conflict) => ( -
- {conflict.entityType} - {conflict.sourceLabel} - {conflict.targetLabel} - {conflict.plannedAction} -
- ))} -
- )} -
- ); -} - -function ActivationChecklist({ - run, - pendingEntityType, - isPending, - onActivate, -}: { - run: CloudUpstreamRun; - pendingEntityType: CloudUpstreamActivationEntityType | null; - isPending: boolean; - onActivate: (entityType: CloudUpstreamActivationEntityType) => void; -}) { - const rows = buildActivationRows(run); - return ( -
-
Activation checklist
-
- {rows.map((row) => { - const pending = isPending && pendingEntityType === row.key; - const activated = row.status === "activated"; - return ( -
-
-
{row.label}
-
{row.statusLabel}
-
-
- {row.count === 0 ? `0 imported ${row.pluralLabel} in this run.` : row.detail} -
-
- - -
-
- ); - })} -
-
- ); -} - -export function buildActivationRows(run: CloudUpstreamRun) { - const activationChecklist = activationChecklistFromReport(run.report); - return ACTIVATION_CATEGORIES.map((category) => { - const decision = activationChecklist[category.key]; - const count = summaryCount(run.summary, category.key); - const status = decision?.status === "activated" ? "activated" : "paused"; - const pluralLabel = `${category.singular}${count === 1 ? "" : "s"}`; - return { - ...category, - count, - pluralLabel, - status, - detail: `${count} imported ${pluralLabel} are paused by default. ${category.detail}`, - statusLabel: status === "activated" - ? `${count} activated` - : count === 0 - ? "0 imported" - : `${count} paused`, - }; - }); -} - -function summaryCount(summary: CloudUpstreamRun["summary"], key: CloudUpstreamActivationEntityType): number { - return summary.find((item) => item.key === key)?.count ?? 0; -} - -function activationChecklistFromReport(report: CloudUpstreamRun["report"]): Partial> { - const value = optionalRecord(report.activationChecklist); - const decisions: Partial> = {}; - for (const key of ["agents", "routines", "monitors"] as const) { - const item = optionalRecord(value[key]); - if (!item) continue; - decisions[key] = { - entityType: key, - count: typeof item.count === "number" ? item.count : 0, - status: item.status === "activated" ? "activated" : "paused", - activatedAt: typeof item.activatedAt === "string" ? item.activatedAt : null, - }; - } - return decisions; -} - -function optionalRecord(value: unknown): Record { - return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; -} - -function downloadRunReport(run: CloudUpstreamRun) { - const blob = new Blob([JSON.stringify(run.report, null, 2)], { type: "application/json" }); - const url = URL.createObjectURL(blob); - const anchor = document.createElement("a"); - anchor.href = url; - anchor.download = `cloud-upstream-run-${run.id}.json`; - anchor.click(); - URL.revokeObjectURL(url); -} - -function formatDate(value: string) { - return new Date(value).toLocaleString(undefined, { - month: "short", - day: "numeric", - hour: "2-digit", - minute: "2-digit", - }); -} - -function formatBytes(value: number) { - if (value >= 1024 * 1024) return `${Math.round(value / (1024 * 1024))} MiB`; - if (value >= 1024) return `${Math.round(value / 1024)} KiB`; - return `${value} B`; -} - -function previewErrorMessage(error: unknown): string { - const code = error instanceof Error ? error.message : null; - if (code === "payload_too_large" || code === "bad_request") { - return "Local company is too large to preview as a single request. Click Push to continue (the Push step uploads in chunks), or see the docs for chunked-preview options."; - } - return code ?? "Failed to preview push."; -} diff --git a/ui/src/pages/CloudUpstreamUxLab.tsx b/ui/src/pages/CloudUpstreamUxLab.tsx deleted file mode 100644 index ecc1f20bac..0000000000 --- a/ui/src/pages/CloudUpstreamUxLab.tsx +++ /dev/null @@ -1,822 +0,0 @@ -import { useMemo } from "react"; -import { - AlertTriangle, - CheckCircle2, - CloudUpload, - ExternalLink, - FileJson, - History, - Loader2, - RefreshCcw, - ShieldAlert, -} from "lucide-react"; -import type { - CloudUpstreamActivationDecision, - CloudUpstreamActivationEntityType, - CloudUpstreamConflict, - CloudUpstreamConnection, - CloudUpstreamPreview, - CloudUpstreamRun, - CloudUpstreamStep, - CloudUpstreamSummaryCount, - CloudUpstreamWarning, -} from "@paperclipai/shared"; -import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { useLocation } from "@/lib/router"; - -type FixtureStateKey = - | "settings-pane" - | "connect-wizard" - | "schema-mismatch" - | "preview" - | "preview-clean" - | "progress" - | "retry" - | "finish"; - -const STEPS: Array<{ key: CloudUpstreamStep; label: string }> = [ - { key: "connect", label: "Connect" }, - { key: "scan", label: "Scan" }, - { key: "preview", label: "Preview" }, - { key: "push", label: "Push" }, - { key: "verify", label: "Verify" }, - { key: "activate", label: "Activate" }, -]; - -const ACTIVATION_CATEGORIES: Array<{ - key: CloudUpstreamActivationEntityType; - label: string; - singular: string; - detail: string; -}> = [ - { - key: "agents", - label: "Agents", - singular: "agent", - detail: "Keep paused until cloud secrets and adapter credentials are verified.", - }, - { - key: "routines", - label: "Routines", - singular: "routine", - detail: "Review schedules before enabling triggers.", - }, - { - key: "monitors", - label: "Monitors", - singular: "monitor", - detail: "Activate after the target instance has been smoke tested.", - }, -]; - -const FIXTURE_LABELS: Record = { - "settings-pane": "1 · Settings → Cloud upstream pane (enabled)", - "connect-wizard": "2 · Connect wizard — remote URL entry + PKCE launch", - "schema-mismatch": "3 · Connect wizard — schema-mismatch hard block", - preview: "4 · Preview — conflicts, warnings, planned actions", - "preview-clean": "5 · Preview — clean run with no conflicts", - progress: "6 · Durable progress — mid-run from run events", - retry: "7 · Retry without duplicating ledger entries", - finish: "8 · Finish / activation checklist with run report", -}; - -const PARSE_ORDER: FixtureStateKey[] = [ - "settings-pane", - "connect-wizard", - "schema-mismatch", - "preview", - "preview-clean", - "progress", - "retry", - "finish", -]; - -export function CloudUpstreamUxLab() { - const location = useLocation(); - const { state, showChrome } = useMemo(() => { - const params = new URLSearchParams(location.search); - const raw = (params.get("state") ?? "settings-pane") as FixtureStateKey; - return { - state: PARSE_ORDER.includes(raw) ? raw : "settings-pane", - showChrome: params.get("chrome") === "on", - }; - }, [location.search]); - - const fixture = useMemo(() => buildFixture(state), [state]); - - return ( -
- {showChrome ? : null} - -
- ); -} - -function FixtureNav({ active }: { active: FixtureStateKey }) { - return ( -
-
UX lab · cloud upstream
-
- {PARSE_ORDER.map((key) => ( - - {FIXTURE_LABELS[key]} - - ))} -
-
- ); -} - -interface Fixture { - selectedCompanyName: string; - connection: CloudUpstreamConnection | null; - preview: CloudUpstreamPreview | null; - latestRun: CloudUpstreamRun | null; - history: CloudUpstreamRun[]; - notice: string | null; - actionError: string | null; -} - -function CloudUpstreamRender({ fixture }: { fixture: Fixture }) { - const { connection, preview, latestRun, history, notice, actionError, selectedCompanyName } = fixture; - const activeStep: CloudUpstreamStep = latestRun?.activeStep - ?? (preview ? "preview" : connection?.tokenStatus === "connected" ? "scan" : "connect"); - return ( -
-
-
-
- -

Cloud upstream

-
-

- Push {selectedCompanyName} into a Paperclip Cloud stack. Automations stay paused until activation. -

-
- {connection?.target.origin ? ( - - ) : null} -
- - {notice ? ( -
- {notice} -
- ) : null} - {actionError ? ( -
- {actionError} -
- ) : null} - - - -
-
Connection
-
- {connection ? ( -
-
-
- {connection.target.stackDisplayName ?? connection.target.stackSlug ?? connection.target.stackId} -
-
- {connection.target.product} · {connection.target.origin} · token {connection.tokenStatus} -
-
- Schema {connection.target.schemaMajor}. Max chunk {formatBytes(connection.target.maxChunkBytes)}. -
-
- -
- ) : ( -
- - -
- )} -
-
- - {preview ? ( -
-
-
Preview
- -
- - - -
- ) : null} - - {latestRun ? ( -
-
-
Progress and finish
-
- - {latestRun.status === "failed" || latestRun.status === "cancelled" ? ( - - ) : latestRun.status === "succeeded" ? ( - - ) : null} -
-
-
-
-
-
{latestRun.status}
-
- Run {latestRun.id.slice(0, 8)} · {latestRun.completedAt - ? `completed ${formatDate(latestRun.completedAt)}` - : latestRun.status === "running" - ? "in progress" - : "in progress"} -
-
-
{latestRun.progressPercent}%
-
-
-
-
-
- {latestRun.events.map((event) => ( -
- {formatDate(event.at)} - {event.phase} - {event.message} -
- ))} -
-
- - {latestRun.status === "succeeded" ? : null} -
- ) : null} - - {history.length ? ( -
-
- - History -
-
- {history.map((run) => ( -
- Run {run.id.slice(0, 8)} · {run.status} - {formatDate(run.createdAt)} -
- ))} -
-
- ) : null} -
- ); -} - -function Stepper({ activeStep }: { activeStep: CloudUpstreamStep }) { - const activeIndex = STEPS.findIndex((step) => step.key === activeStep); - return ( -
- {STEPS.map((step, index) => { - const complete = index < activeIndex; - const active = index === activeIndex; - return ( -
- {complete ? ( - - ) : ( - - )} - {step.label} -
- ); - })} -
- ); -} - -function SummaryGrid({ summary }: { summary: CloudUpstreamSummaryCount[] }) { - return ( -
- {summary.map((item) => ( -
-
{item.count}
-
{item.label}
-
- ))} -
- ); -} - -function WarningsPanel({ warnings }: { warnings: CloudUpstreamWarning[] }) { - return ( -
-
- - Warnings -
-
- {warnings.map((warning) => ( -
- -
{warning.title}
-
{warning.detail}
-
- ))} -
-
- ); -} - -function ConflictTable({ conflicts }: { conflicts: CloudUpstreamConflict[] }) { - return ( -
-
Conflicts
- {conflicts.length === 0 ? ( -
No target conflicts detected for this preview.
- ) : ( -
- {conflicts.map((conflict) => ( -
- {conflict.entityType} - {conflict.sourceLabel} - {conflict.targetLabel} - {conflict.plannedAction} -
- ))} -
- )} -
- ); -} - -function ActivationChecklist({ run }: { run: CloudUpstreamRun }) { - const rows = buildActivationRows(run); - return ( -
-
Activation checklist
-
- {rows.map((row) => { - const activated = row.status === "activated"; - return ( -
-
-
{row.label}
-
{row.statusLabel}
-
-
- {row.count === 0 ? `0 imported ${row.pluralLabel} in this run.` : row.detail} -
-
- - -
-
- ); - })} -
-
- ); -} - -function buildActivationRows(run: CloudUpstreamRun) { - const decisions = decisionsFromReport(run.report); - return ACTIVATION_CATEGORIES.map((category) => { - const decision = decisions[category.key]; - const count = summaryCount(run.summary, category.key); - const status = decision?.status === "activated" ? "activated" : "paused"; - const pluralLabel = `${category.singular}${count === 1 ? "" : "s"}`; - return { - ...category, - count, - pluralLabel, - status, - detail: `${count} imported ${pluralLabel} are paused by default. ${category.detail}`, - statusLabel: status === "activated" - ? `${count} activated` - : count === 0 - ? "0 imported" - : `${count} paused`, - }; - }); -} - -function decisionsFromReport(report: Record): Partial> { - const value = optionalRecord(report.activationChecklist); - const decisions: Partial> = {}; - for (const key of ["agents", "routines", "monitors"] as const) { - const item = optionalRecord(value[key]); - if (!item) continue; - decisions[key] = { - entityType: key, - count: typeof item.count === "number" ? item.count : 0, - status: item.status === "activated" ? "activated" : "paused", - activatedAt: typeof item.activatedAt === "string" ? item.activatedAt : null, - }; - } - return decisions; -} - -function summaryCount(summary: CloudUpstreamSummaryCount[], key: CloudUpstreamActivationEntityType): number { - return summary.find((item) => item.key === key)?.count ?? 0; -} - -function optionalRecord(value: unknown): Record { - return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; -} - -function formatDate(value: string) { - return new Date(value).toLocaleString(undefined, { - month: "short", - day: "numeric", - hour: "2-digit", - minute: "2-digit", - timeZone: "UTC", - }); -} - -function formatBytes(value: number) { - if (value >= 1024 * 1024) return `${Math.round(value / (1024 * 1024))} MiB`; - if (value >= 1024) return `${Math.round(value / 1024)} KiB`; - return `${value} B`; -} - -const STACK_TARGET = { - stackId: "stk_2vKqz9D8mNFqQ7Rp", - stackSlug: "paperclip-prod", - stackDisplayName: "Paperclip Prod", - companyId: "co_4hT2yX", - primaryHost: "paperclip.paperclip.app", - origin: "https://paperclip.paperclip.app", - product: "paperclip-cloud", - schemaMajor: 7, - maxChunkBytes: 5 * 1024 * 1024, -}; - -const STACK_TARGET_SCHEMA_BEHIND = { - ...STACK_TARGET, - schemaMajor: 5, -}; - -function connectedConnection(target = STACK_TARGET): CloudUpstreamConnection { - return { - id: "cu_conn_8d3f1b6a", - companyId: "co_4hT2yX", - remoteUrl: "https://paperclip.paperclip.app/PC521D/dashboard", - target, - tokenStatus: "connected", - scopes: ["upstream.push", "upstream.preview"], - authorizedGlobalUserId: "user_9pXqYzAbCdEf", - expiresAt: "2026-08-18T19:00:00.000Z", - createdAt: "2026-05-18T18:45:00.000Z", - updatedAt: "2026-05-18T19:02:18.000Z", - lastRunId: null, - }; -} - -const PREVIEW_SUMMARY: CloudUpstreamSummaryCount[] = [ - { key: "users", label: "Users", count: 14 }, - { key: "agents", label: "Agents", count: 6 }, - { key: "routines", label: "Routines", count: 4 }, - { key: "monitors", label: "Monitors", count: 2 }, -]; - -const PREVIEW_WARNINGS_NORMAL: CloudUpstreamWarning[] = [ - { - code: "imported_automations_paused", - severity: "warning", - title: "Automations stay paused", - detail: "Imported agents, routines, and monitors require explicit activation after the push.", - }, - { - code: "unmatched_users_import_as_historical_authors", - severity: "warning", - title: "Unmatched users become historical authors", - detail: "Invite now remains a secondary action after the transfer is complete.", - }, - { - code: "secret_values_redacted", - severity: "warning", - title: "Secret values are not transferred", - detail: "The push carries secret requirements only. Configure cloud secrets before activating automations.", - }, -]; - -const PREVIEW_WARNINGS_SCHEMA: CloudUpstreamWarning[] = [ - { - code: "schema_mismatch", - severity: "blocker", - title: "Cloud stack upgrade required", - detail: "This local build uses upstream schema 7, but the cloud stack reports schema 5.", - }, - ...PREVIEW_WARNINGS_NORMAL, -]; - -const PREVIEW_CONFLICTS: CloudUpstreamConflict[] = [ - { - id: "conflict_user_serena", - entityType: "user", - sourceLabel: "serena@magicmachine.co (unmatched)", - targetLabel: "→ historical author Serena R.", - plannedAction: "create", - reason: "Target stack has no matching identity. Will arrive as historical author; invite available after push.", - }, - { - id: "conflict_user_dotta", - entityType: "user", - sourceLabel: "dotta@magicmachine.co", - targetLabel: "↦ dotta@magicmachine.co (cloud)", - plannedAction: "update", - reason: "Existing cloud identity matches local user; will be merged.", - }, - { - id: "conflict_agent_qa", - entityType: "agent", - sourceLabel: "QA · qa-bot", - targetLabel: "↦ QA · qa-bot (cloud)", - plannedAction: "update", - reason: "Mapped to existing cloud agent. Imported run history will be appended.", - }, - { - id: "conflict_routine_nightly_reports", - entityType: "routine", - sourceLabel: "Nightly status report", - targetLabel: "(new in cloud)", - plannedAction: "create", - reason: "Routine does not exist in the target stack and will be created in paused state.", - }, -]; - -function basePreview(): CloudUpstreamPreview { - return { - connectionId: "cu_conn_8d3f1b6a", - sourceCompanyId: "co_local_pc521d", - target: STACK_TARGET, - schemaCompatible: true, - summary: PREVIEW_SUMMARY, - warnings: PREVIEW_WARNINGS_NORMAL, - conflicts: PREVIEW_CONFLICTS, - generatedAt: "2026-05-18T19:03:14.000Z", - }; -} - -function schemaMismatchPreview(): CloudUpstreamPreview { - return { - ...basePreview(), - target: STACK_TARGET_SCHEMA_BEHIND, - schemaCompatible: false, - summary: [], - conflicts: [], - warnings: PREVIEW_WARNINGS_SCHEMA, - }; -} - -function cleanPreview(): CloudUpstreamPreview { - return { - ...basePreview(), - conflicts: [], - warnings: PREVIEW_WARNINGS_NORMAL.slice(0, 1), - }; -} - -const PROGRESS_EVENTS = [ - { id: "evt_01", at: "2026-05-18T19:10:02.000Z", phase: "scan" as CloudUpstreamStep, type: "completed" as const, message: "Scanned 14 users, 6 agents, 4 routines, 2 monitors." }, - { id: "evt_02", at: "2026-05-18T19:10:11.000Z", phase: "preview" as CloudUpstreamStep, type: "completed" as const, message: "Preview generated with 4 conflicts and 3 warnings." }, - { id: "evt_03", at: "2026-05-18T19:10:31.000Z", phase: "push" as CloudUpstreamStep, type: "created" as const, message: "users · 8 created, 6 mapped to existing identities." }, - { id: "evt_04", at: "2026-05-18T19:10:48.000Z", phase: "push" as CloudUpstreamStep, type: "updated" as const, message: "agents · 4 created paused, 2 updated paused." }, - { id: "evt_05", at: "2026-05-18T19:10:58.000Z", phase: "push" as CloudUpstreamStep, type: "updated" as const, message: "routines · 3 created paused, 1 updated." }, - { id: "evt_06", at: "2026-05-18T19:11:09.000Z", phase: "push" as CloudUpstreamStep, type: "created" as const, message: "monitors · 2 created paused." }, - { id: "evt_07", at: "2026-05-18T19:11:18.000Z", phase: "verify" as CloudUpstreamStep, type: "updated" as const, message: "Verifying transferred ledger checksums…" }, -]; - -function runningRun(): CloudUpstreamRun { - return { - id: "run_3kQ8mNpW9bX2zL4Y", - connectionId: "cu_conn_8d3f1b6a", - companyId: "co_local_pc521d", - status: "running", - activeStep: "push", - progressPercent: 62, - dryRun: false, - summary: PREVIEW_SUMMARY, - warnings: PREVIEW_WARNINGS_NORMAL, - conflicts: PREVIEW_CONFLICTS, - events: PROGRESS_EVENTS, - targetUrl: "https://paperclip.paperclip.app/PC521D/dashboard", - report: {}, - retryOfRunId: null, - createdAt: "2026-05-18T19:10:01.000Z", - updatedAt: "2026-05-18T19:11:18.000Z", - completedAt: null, - }; -} - -function failedRun(): CloudUpstreamRun { - return { - id: "run_5fXqR2bT7aD8zP1K", - connectionId: "cu_conn_8d3f1b6a", - companyId: "co_local_pc521d", - status: "failed", - activeStep: "push", - progressPercent: 78, - dryRun: false, - summary: PREVIEW_SUMMARY, - warnings: PREVIEW_WARNINGS_NORMAL, - conflicts: PREVIEW_CONFLICTS, - events: [ - ...PROGRESS_EVENTS, - { - id: "evt_08", - at: "2026-05-18T19:11:30.000Z", - phase: "push", - type: "failed", - message: "Apply rejected: cloud rejected chunk 4 of 6 (HTTP 502). Ledger entries from chunks 1–3 retained; chunk 4 not committed.", - }, - ], - targetUrl: "https://paperclip.paperclip.app/PC521D/dashboard", - report: { ledgerCheckpoint: "chunk-3" }, - retryOfRunId: null, - createdAt: "2026-05-18T19:10:01.000Z", - updatedAt: "2026-05-18T19:11:30.000Z", - completedAt: null, - }; -} - -function succeededRun(): CloudUpstreamRun { - return { - id: "run_7aBcD9eFgH2iJ3kL", - connectionId: "cu_conn_8d3f1b6a", - companyId: "co_local_pc521d", - status: "succeeded", - activeStep: "activate", - progressPercent: 100, - dryRun: false, - summary: PREVIEW_SUMMARY, - warnings: PREVIEW_WARNINGS_NORMAL, - conflicts: PREVIEW_CONFLICTS, - events: [ - ...PROGRESS_EVENTS, - { - id: "evt_08", - at: "2026-05-18T19:11:25.000Z", - phase: "verify", - type: "completed", - message: "Ledger checksums match. Push committed.", - }, - { - id: "evt_09", - at: "2026-05-18T19:11:31.000Z", - phase: "activate", - type: "completed", - message: "Activation checklist pending operator approval — automations remain paused.", - }, - ], - targetUrl: "https://paperclip.paperclip.app/PC521D/dashboard", - report: { - activationChecklist: { - agents: { count: 6, status: "paused", activatedAt: null }, - routines: { count: 4, status: "paused", activatedAt: null }, - monitors: { count: 2, status: "paused", activatedAt: null }, - }, - }, - retryOfRunId: null, - createdAt: "2026-05-18T19:10:01.000Z", - updatedAt: "2026-05-18T19:11:31.000Z", - completedAt: "2026-05-18T19:11:31.000Z", - }; -} - -function buildFixture(state: FixtureStateKey): Fixture { - switch (state) { - case "settings-pane": - return { - selectedCompanyName: "Paperclip · PC521D", - connection: connectedConnection(), - preview: null, - latestRun: null, - history: [], - notice: "Cloud upstream connection approved.", - actionError: null, - }; - case "connect-wizard": - return { - selectedCompanyName: "Paperclip · PC521D", - connection: null, - preview: null, - latestRun: null, - history: [], - notice: null, - actionError: null, - }; - case "schema-mismatch": - return { - selectedCompanyName: "Paperclip · PC521D", - connection: connectedConnection(STACK_TARGET_SCHEMA_BEHIND), - preview: schemaMismatchPreview(), - latestRun: null, - history: [], - notice: null, - actionError: "Cloud stack is on schema 5 but this local build pushes schema 7. Upgrade the cloud stack to continue.", - }; - case "preview": - return { - selectedCompanyName: "Paperclip · PC521D", - connection: connectedConnection(), - preview: basePreview(), - latestRun: null, - history: [], - notice: null, - actionError: null, - }; - case "preview-clean": - return { - selectedCompanyName: "Paperclip · PC521D", - connection: connectedConnection(), - preview: cleanPreview(), - latestRun: null, - history: [], - notice: "Preview completed. No target conflicts detected.", - actionError: null, - }; - case "progress": - return { - selectedCompanyName: "Paperclip · PC521D", - connection: connectedConnection(), - preview: null, - latestRun: runningRun(), - history: [], - notice: null, - actionError: null, - }; - case "retry": - return { - selectedCompanyName: "Paperclip · PC521D", - connection: connectedConnection(), - preview: null, - latestRun: failedRun(), - history: [ - { ...failedRun(), id: "run_9pYqXwVtSrQ" }, - ], - notice: null, - actionError: "Push run failed. Review the events. Retry resumes from ledger checkpoint chunk-3 — chunks 1–3 will not be re-applied.", - }; - case "finish": - return { - selectedCompanyName: "Paperclip · PC521D", - connection: connectedConnection(), - preview: null, - latestRun: succeededRun(), - history: [ - { ...succeededRun(), id: "run_aZcXvBnMqWeR" }, - ], - notice: "Push run completed. Review activation before unpausing automations.", - actionError: null, - }; - } -} diff --git a/ui/src/pages/CompanyExport.test.tsx b/ui/src/pages/CompanyExport.test.tsx new file mode 100644 index 0000000000..4dcdb46556 --- /dev/null +++ b/ui/src/pages/CompanyExport.test.tsx @@ -0,0 +1,353 @@ +// @vitest-environment jsdom + +import type { ReactNode } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { ExportFidelityReport } from "@paperclipai/shared/portability-fidelity"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CompanyExport } from "./CompanyExport"; + +const mockCompaniesApi = vi.hoisted(() => ({ + exportPreview: vi.fn(), + exportBundle: vi.fn(), + exportFidelity: vi.fn(), +})); +const mockAgentsApi = vi.hoisted(() => ({ + list: vi.fn(), +})); +const mockProjectsApi = vi.hoisted(() => ({ + list: vi.fn(), +})); +const mockAuthApi = vi.hoisted(() => ({ + getSession: vi.fn(), +})); + +vi.mock("../api/companies", () => ({ + companiesApi: mockCompaniesApi, +})); + +vi.mock("../api/agents", () => ({ + agentsApi: mockAgentsApi, +})); + +vi.mock("../api/projects", () => ({ + projectsApi: mockProjectsApi, +})); + +vi.mock("../api/auth", () => ({ + authApi: mockAuthApi, +})); + +vi.mock("../context/BreadcrumbContext", () => ({ + useBreadcrumbs: () => ({ setBreadcrumbs: vi.fn() }), +})); + +vi.mock("../context/CompanyContext", () => ({ + useCompany: () => ({ + selectedCompanyId: "company-1", + selectedCompany: { id: "company-1", name: "Paperclip" }, + }), + useOptionalCompany: () => null, +})); + +vi.mock("../context/ToastContext", () => ({ + useToastActions: () => ({ pushToast: vi.fn() }), +})); + +vi.mock("@/lib/router", () => ({ + useNavigate: () => vi.fn(), + useLocation: () => ({ pathname: "/PAP/company/export", search: "" }), +})); + +vi.mock("../components/MarkdownBody", () => ({ + MarkdownBody: ({ children }: { children: ReactNode }) =>
{children}
, +})); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +(globalThis as any).IS_REACT_ACT_ENVIRONMENT = true; + +async function act(callback: () => void | Promise) { + await callback(); + await Promise.resolve(); + await new Promise((resolve) => window.setTimeout(resolve, 0)); +} + +async function flushReact() { + await act(async () => { + await Promise.resolve(); + await new Promise((resolve) => window.setTimeout(resolve, 0)); + }); +} + +function buildExportPreviewResult() { + return { + rootPath: "paperclip", + manifest: { + agents: [], + skills: [], + projects: [], + issues: [], + envInputs: [], + includes: { company: true, agents: true, projects: true, issues: true, skills: false }, + company: { name: "Paperclip", description: null }, + schemaVersion: 1, + generatedAt: "2026-01-01T00:00:00.000Z", + source: null, + }, + files: { "README.md": "# Paperclip\n" }, + fileInventory: [], + counts: { files: 1, agents: 0, skills: 0, projects: 0, issues: 0 }, + warnings: [], + paperclipExtensionPath: ".paperclip.yaml", + }; +} + +function buildTaskAttachment(sha256: string) { + return { + sha256, + contentType: "application/octet-stream", + originalFilename: null, + byteSize: 1, + commentIndex: null, + }; +} + +function buildRichExportPreviewResult() { + const base = buildExportPreviewResult(); + return { + ...base, + files: { + "README.md": "# Paperclip\n", + ".paperclip.yaml": "schema: paperclip/v1\n", + "agents/ceo/AGENT.md": "# CEO\n", + "tasks/one-off/TASK.md": "# One-off\n", + "tasks/weekly-report/TASK.md": "# Weekly report\n", + "blobs/aaa111": "binary", + }, + manifest: { + ...base.manifest, + issues: [ + { + slug: "one-off", + title: "One-off", + path: "tasks/one-off/TASK.md", + recurring: false, + attachments: [buildTaskAttachment("aaa111")], + }, + { + slug: "weekly-report", + title: "Weekly report", + path: "tasks/weekly-report/TASK.md", + recurring: true, + attachments: [], + }, + ], + }, + counts: { files: 6, agents: 1, skills: 0, projects: 0, issues: 2 }, + }; +} + +function buildFidelityReport(warnings: ExportFidelityReport["warnings"]): ExportFidelityReport { + return { + schema: "paperclip-export-fidelity-v1", + companyId: "company-1", + counts: { + labelDefinitions: 0, + issueLabelReferences: 0, + issueBlockerRelations: 0, + issueDocuments: 0, + issueWorkProducts: 0, + issueAttachments: 0, + approvals: 0, + costEvents: 0, + activityLogEntries: 0, + issueMonitors: 0, + }, + warnings, + generatedAt: "2026-01-01T00:00:00.000Z", + }; +} + +describe("CompanyExport", () => { + let container: HTMLDivElement; + let root: Root | null = null; + + beforeEach(() => { + container = document.createElement("div"); + document.body.appendChild(container); + mockAuthApi.getSession.mockResolvedValue({ user: { id: "user-1" } }); + mockAgentsApi.list.mockResolvedValue([]); + mockProjectsApi.list.mockResolvedValue([]); + mockCompaniesApi.exportPreview.mockResolvedValue(buildExportPreviewResult()); + mockCompaniesApi.exportBundle.mockResolvedValue({ rootPath: "paperclip", files: {} }); + mockCompaniesApi.exportFidelity.mockResolvedValue(buildFidelityReport([])); + Object.assign(URL, { + createObjectURL: vi.fn(() => "blob:mock"), + revokeObjectURL: vi.fn(), + }); + }); + + afterEach(async () => { + if (root) { + const currentRoot = root; + await act(async () => { + currentRoot.unmount(); + }); + root = null; + } + container.remove(); + document.body.innerHTML = ""; + vi.clearAllMocks(); + }); + + async function renderPage() { + root = createRoot(container); + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const currentRoot = root; + + await act(async () => { + currentRoot.render( + + + , + ); + }); + await flushReact(); + await flushReact(); + await flushReact(); + } + + function categoryInput(key: string): HTMLInputElement { + const input = container.querySelector(`input[data-export-category="${key}"]`); + if (!input) throw new Error(`No category toggle for ${key}`); + return input; + } + + function exportButton(): HTMLButtonElement { + const button = Array.from(container.querySelectorAll("button")).find((candidate) => + candidate.textContent?.trim().startsWith("Export "), + ); + if (!button) throw new Error("Export button not found"); + return button as HTMLButtonElement; + } + + async function clickElement(element: HTMLElement) { + await act(async () => { + element.click(); + }); + await flushReact(); + } + + it("selects every file by default and requests them all on download", async () => { + mockCompaniesApi.exportPreview.mockResolvedValue(buildRichExportPreviewResult()); + + await renderPage(); + + expect(container.textContent).toContain("Exporting 6 of 6 files"); + // The tree is a pure browser now — no per-file checkboxes. + expect(container.querySelector('[role="tree"] input[type="checkbox"]')).toBeNull(); + + await clickElement(exportButton()); + + expect(mockCompaniesApi.exportBundle).toHaveBeenCalledTimes(1); + const request = mockCompaniesApi.exportBundle.mock.calls[0]![1]; + expect(request.selectedFiles).toEqual([ + ".paperclip.yaml", + "README.md", + "agents/ceo/AGENT.md", + "blobs/aaa111", + "tasks/one-off/TASK.md", + "tasks/weekly-report/TASK.md", + ]); + }); + + it("toggling Tasks off drops one-off task files and their blobs but keeps routines", async () => { + mockCompaniesApi.exportPreview.mockResolvedValue(buildRichExportPreviewResult()); + + await renderPage(); + await clickElement(categoryInput("tasks")); + + expect(container.textContent).toContain("Exporting 4 of 6 files"); + // Routines can carry attachments, so Attachments stays enabled while routines remain. + expect(categoryInput("attachments").disabled).toBe(false); + // Excluded files render dimmed in the tree browser. + expect(container.querySelector('[data-file-tree-path="blobs/aaa111"]')?.className).toContain("opacity-50"); + expect(container.querySelector('[data-file-tree-path="blobs"]')?.className).toContain("opacity-50"); + + await clickElement(exportButton()); + + expect(mockCompaniesApi.exportBundle).toHaveBeenCalledTimes(1); + const request = mockCompaniesApi.exportBundle.mock.calls[0]![1]; + expect(request.selectedFiles).toEqual([ + ".paperclip.yaml", + "README.md", + "agents/ceo/AGENT.md", + "tasks/weekly-report/TASK.md", + ]); + }); + + it("disables Attachments once both Tasks and Routines are off", async () => { + mockCompaniesApi.exportPreview.mockResolvedValue(buildRichExportPreviewResult()); + + await renderPage(); + await clickElement(categoryInput("tasks")); + await clickElement(categoryInput("routines")); + + const attachments = categoryInput("attachments"); + expect(attachments.disabled).toBe(true); + expect(attachments.checked).toBe(false); + expect(container.textContent).toContain("Exporting 3 of 6 files"); + expect(container.querySelector('[data-file-tree-path="tasks"]')?.className).toContain("opacity-50"); + }); + + it("shows the estimated download size and updates it when a category toggles off", async () => { + mockCompaniesApi.exportPreview.mockResolvedValue(buildRichExportPreviewResult()); + + await renderPage(); + + const sizeText = () => + container.textContent?.match(/Exporting [\d,]+ of [\d,]+ files \(~([\d.]+ [KMGT]?B)\)/)?.[1] ?? null; + + const initialSize = sizeText(); + expect(initialSize).not.toBeNull(); + + await clickElement(categoryInput("tasks")); + + expect(container.textContent).toContain("Exporting 4 of 6 files"); + const toggledSize = sizeText(); + expect(toggledSize).not.toBeNull(); + // Dropping the one-off task and its blob shrinks the estimated zip. + expect(toggledSize).not.toBe(initialSize); + }); + + it("renders the export fidelity panel with blocker and warning messages", async () => { + mockCompaniesApi.exportFidelity.mockResolvedValue(buildFidelityReport([ + { + code: "bundle_incompatible", + severity: "blocker", + message: "Importing this export will fail because the bundle references data this board cannot restore.", + }, + { + code: "approvals_not_exported", + severity: "warning", + message: "3 approvals are not included in the export bundle.", + }, + ])); + + await renderPage(); + + expect(mockCompaniesApi.exportFidelity).toHaveBeenCalledWith("company-1"); + expect(container.textContent).toContain("Not included in this export"); + expect(container.textContent).toContain( + "Importing this export will fail because the bundle references data this board cannot restore.", + ); + expect(container.textContent).toContain("3 approvals are not included in the export bundle."); + }); + + it("renders no fidelity panel when the report has no warnings", async () => { + await renderPage(); + + expect(mockCompaniesApi.exportFidelity).toHaveBeenCalledWith("company-1"); + expect(container.textContent).not.toContain("Not included in this export"); + }); +}); diff --git a/ui/src/pages/CompanyExport.tsx b/ui/src/pages/CompanyExport.tsx index dd8c61c818..27f82642f7 100644 --- a/ui/src/pages/CompanyExport.tsx +++ b/ui/src/pages/CompanyExport.tsx @@ -23,8 +23,18 @@ import { MarkdownBody } from "../components/MarkdownBody"; import { toCompanyRelativePath } from "@/lib/company-routes"; import { cn } from "../lib/utils"; import { queryKeys } from "../lib/queryKeys"; -import { createZipArchive } from "../lib/zip"; -import { buildInitialExportCheckedFiles } from "../lib/company-export-selection"; +import { formatBytes } from "../lib/issue-output"; +import { createZipArchive, estimateZipArchiveSize } from "../lib/zip"; +import { + type ExportCategoryKey, + type ExportCategorySelection, + EXPORT_CATEGORY_LABELS, + EXPORT_CATEGORY_ORDER, + buildDefaultExportCategorySelection, + buildExportCheckedFiles, + countExportFilesByCategory, + isAttachmentsCategoryEnabled, +} from "../lib/company-export-selection"; import { useAgentOrder } from "../hooks/useAgentOrder"; import { useProjectOrder } from "../hooks/useProjectOrder"; import { buildPortableSidebarOrder } from "../lib/company-portability-sidebar"; @@ -36,6 +46,7 @@ import { } from "lucide-react"; import { type FileTreeNode, + type FileTreeTone, type FrontmatterData, buildFileTree, countFiles, @@ -248,36 +259,19 @@ function collectMatchedParentDirs(nodes: FileTreeNode[], query: string): Set): FileTreeNode[] { - return nodes.map((node) => { - if (node.kind === "dir") { - return { ...node, children: sortByChecked(node.children, checkedFiles) }; - } - return node; - }).sort((a, b) => { - if (a.kind !== b.kind) return a.kind === "file" ? -1 : 1; - if (a.kind === "file" && b.kind === "file") { - const aChecked = checkedFiles.has(a.path); - const bChecked = checkedFiles.has(b.path); - if (aChecked !== bChecked) return aChecked ? -1 : 1; - } - return a.name.localeCompare(b.name); - }); -} - const TASKS_PAGE_SIZE = 10; /** * Paginate children of `tasks/` directories: show up to `limit` entries, - * but always include children that are checked or match the search query. + * but always include children that match the search query or contain the + * currently previewed file (so deep links stay visible). * Returns the paginated tree and the total count of task children. */ function paginateTaskNodes( nodes: FileTreeNode[], limit: number, - checkedFiles: Set, searchQuery: string, + selectedFile: string | null, ): { nodes: FileTreeNode[]; totalTaskChildren: number; visibleTaskChildren: number } { let totalTaskChildren = 0; let visibleTaskChildren = 0; @@ -287,20 +281,20 @@ function paginateTaskNodes( if (node.kind === "dir" && node.name === "tasks") { totalTaskChildren = node.children.length; - // Partition children: pinned (checked or search-matched) vs rest + // Partition children: pinned (search-matched or previewed) vs rest const pinned: FileTreeNode[] = []; const rest: FileTreeNode[] = []; const lower = searchQuery.toLowerCase(); for (const child of node.children) { const childFiles = collectAllPaths([child], "file"); - const isChecked = [...childFiles].some((p) => checkedFiles.has(p)); + const containsSelected = selectedFile !== null && childFiles.has(selectedFile); const isSearchMatch = searchQuery && ( child.name.toLowerCase().includes(lower) || child.path.toLowerCase().includes(lower) || [...childFiles].some((p) => p.toLowerCase().includes(lower)) ); - if (isChecked || isSearchMatch) { + if (containsSelected || isSearchMatch) { pinned.push(child); } else { rest.push(child); @@ -320,15 +314,31 @@ function paginateTaskNodes( return { nodes: result, totalTaskChildren, visibleTaskChildren }; } +/** + * Build the file map the zip download will contain: the exported files + * restricted to the selected set, preferring the client-side effective + * content (regenerated README.md, filtered .paperclip.yaml) when present. + * The download size estimate runs this same filter so the number shown + * matches what actually gets zipped. + */ +function filterExportForDownload( + files: Record, + selectedFiles: Set, + effectiveFiles: Record, +): Record { + const filteredFiles: Record = {}; + for (const path of Object.keys(files)) { + if (selectedFiles.has(path)) filteredFiles[path] = effectiveFiles[path] ?? files[path]!; + } + return filteredFiles; +} + function downloadZip( exported: CompanyPortabilityExportResult, selectedFiles: Set, effectiveFiles: Record, ) { - const filteredFiles: Record = {}; - for (const [path] of Object.entries(exported.files)) { - if (selectedFiles.has(path)) filteredFiles[path] = effectiveFiles[path] ?? exported.files[path]; - } + const filteredFiles = filterExportForDownload(exported.files, selectedFiles, effectiveFiles); const zipBytes = createZipArchive(filteredFiles, exported.rootPath); const zipBuffer = new ArrayBuffer(zipBytes.byteLength); new Uint8Array(zipBuffer).set(zipBytes); @@ -599,11 +609,16 @@ export function CompanyExport() { queryFn: () => projectsApi.list(selectedCompanyId!), enabled: !!selectedCompanyId, }); + const { data: fidelityReport } = useQuery({ + queryKey: queryKeys.companies.exportFidelity(selectedCompanyId!), + queryFn: () => companiesApi.exportFidelity(selectedCompanyId!), + enabled: !!selectedCompanyId, + }); const [exportData, setExportData] = useState(null); const [selectedFile, setSelectedFile] = useState(null); const [expandedDirs, setExpandedDirs] = useState>(new Set()); - const [checkedFiles, setCheckedFiles] = useState>(new Set()); + const [categories, setCategories] = useState(buildDefaultExportCategorySelection); const [treeSearch, setTreeSearch] = useState(""); const [taskLimit, setTaskLimit] = useState(TASKS_PAGE_SIZE); const savedExpandedRef = useRef | null>(null); @@ -674,10 +689,11 @@ export function CompanyExport() { useEffect(() => { setBreadcrumbs([ - { label: "Org Chart", href: "/org" }, + { label: selectedCompany?.name ?? "Company", href: "/dashboard" }, + { label: "Settings", href: "/company/settings" }, { label: "Export" }, ]); - }, [setBreadcrumbs]); + }, [selectedCompany?.name, setBreadcrumbs]); const exportPreviewMutation = useMutation({ mutationFn: () => @@ -687,13 +703,6 @@ export function CompanyExport() { }), onSuccess: (result) => { setExportData(result); - setCheckedFiles((prev) => - buildInitialExportCheckedFiles( - Object.keys(result.files), - result.manifest.issues, - prev, - ), - ); // Expand top-level dirs (except tasks — collapsed by default) const tree = buildFileTree(result.files); const topDirs = new Set(); @@ -765,17 +774,62 @@ export function CompanyExport() { [exportData], ); + // The checked set is derived from the category toggles; the file tree is + // a pure browser on top of it. + const checkedFiles = useMemo( + () => + exportData + ? buildExportCheckedFiles({ + filePaths: Object.keys(exportData.files), + issues: exportData.manifest.issues, + categories, + embeddedAssets: exportData.manifest.embeddedAssets ?? [], + }) + : new Set(), + [exportData, categories], + ); + + const categoryCounts = useMemo( + () => + exportData + ? countExportFilesByCategory(Object.keys(exportData.files), exportData.manifest.issues) + : null, + [exportData], + ); + const { displayTree, totalTaskChildren, visibleTaskChildren } = useMemo(() => { let result = tree; if (treeSearch) result = filterTree(result, treeSearch); - result = sortByChecked(result, checkedFiles); - const paginated = paginateTaskNodes(result, taskLimit, checkedFiles, treeSearch); + const paginated = paginateTaskNodes(result, taskLimit, treeSearch, selectedFile); return { displayTree: paginated.nodes, totalTaskChildren: paginated.totalTaskChildren, visibleTaskChildren: paginated.visibleTaskChildren, }; - }, [tree, treeSearch, checkedFiles, taskLimit]); + }, [tree, treeSearch, taskLimit, selectedFile]); + + // Files the current toggles exclude render dimmed, so a toggle's effect is + // visible in the tree. Directories dim once none of their files export. + const fileTones = useMemo(() => { + const tones: Record = {}; + + function walk(node: FileTreeNode): boolean { + if (node.kind === "file") { + const included = checkedFiles.has(node.path); + if (!included) tones[node.path] = "muted"; + return included; + } + let anyIncluded = false; + for (const child of node.children) { + if (walk(child)) anyIncluded = true; + } + if (!anyIncluded) tones[node.path] = "muted"; + return anyIncluded; + } + + for (const node of tree) walk(node); + return tones; + }, [tree, checkedFiles]); // Recompute .paperclip.yaml and README.md content whenever checked files // change so the preview & download always reflect the current selection. @@ -804,6 +858,17 @@ export function CompanyExport() { return filtered; }, [exportData, checkedFiles, selectedCompany?.name]); + // The zip is STORE-only, so its size is exactly computable from the filtered + // file map. Depends only on the selection (not e.g. the tree search), so it + // recomputes per toggle change rather than per keystroke. + const estimatedZipBytes = useMemo(() => { + if (!exportData) return 0; + return estimateZipArchiveSize( + filterExportForDownload(exportData.files, checkedFiles, effectiveFiles), + exportData.rootPath, + ); + }, [exportData, checkedFiles, effectiveFiles]); + const totalFiles = useMemo(() => countFiles(tree), [tree]); const selectedCount = checkedFiles.size; @@ -822,40 +887,8 @@ export function CompanyExport() { }); } - function handleToggleCheck(path: string, kind: "file" | "dir") { - if (!exportData) return; - setCheckedFiles((prev) => { - const next = new Set(prev); - if (kind === "file") { - if (next.has(path)) next.delete(path); - else next.add(path); - } else { - // Find all child file paths under this dir - const dirTree = buildFileTree(exportData.files); - const findNode = (nodes: FileTreeNode[], target: string): FileTreeNode | null => { - for (const n of nodes) { - if (n.path === target) return n; - const found = findNode(n.children, target); - if (found) return found; - } - return null; - }; - const dirNode = findNode(dirTree, path); - if (dirNode) { - const childFiles = collectAllPaths(dirNode.children, "file"); - // Add the dir's own file children - for (const child of dirNode.children) { - if (child.kind === "file") childFiles.add(child.path); - } - const allChecked = [...childFiles].every((p) => next.has(p)); - for (const f of childFiles) { - if (allChecked) next.delete(f); - else next.add(f); - } - } - } - return next; - }); + function handleToggleCategory(key: ExportCategoryKey) { + setCategories((prev) => ({ ...prev, [key]: !prev[key] })); } function handleSearchChange(query: string) { @@ -940,7 +973,8 @@ export function CompanyExport() { {selectedCompany?.name ?? "Company"} export - {selectedCount} / {totalFiles} file{totalFiles === 1 ? "" : "s"} selected + Exporting {selectedCount.toLocaleString()} of {totalFiles.toLocaleString()} file{totalFiles === 1 ? "" : "s"} + {selectedCount > 0 && ` (~${formatBytes(estimatedZipBytes)})`} {warnings.length > 0 && ( @@ -956,7 +990,7 @@ export function CompanyExport() { {downloadMutation.isPending ? "Building export..." - : `Export ${selectedCount} file${selectedCount === 1 ? "" : "s"}`} + : `Export ${selectedCount.toLocaleString()} file${selectedCount === 1 ? "" : "s"}`}
@@ -970,12 +1004,66 @@ export function CompanyExport() { )} + {/* Export fidelity: data the bundle will not carry */} + {fidelityReport && fidelityReport.warnings.length > 0 && ( +
+

Not included in this export

+ {fidelityReport.warnings.map((warning) => ( +
+ {warning.message} +
+ ))} +
+ )} + {/* Two-column layout */}