diff --git a/packages/shared/src/portability-zip.test.ts b/packages/shared/src/portability-zip.test.ts index f128b701b8..a6b20335ed 100644 --- a/packages/shared/src/portability-zip.test.ts +++ b/packages/shared/src/portability-zip.test.ts @@ -270,7 +270,7 @@ describe("readZipArchive", () => { it("bounds a highly compressible DEFLATE entry at the per-entry decompressed limit", async () => { // Compresses tiny but expands to 8 KiB; a 1 KiB cap must reject it before it - // materializes. Real packages sit far under the 256 MB production default. + // materializes. Real packages sit far under the 512 MB production default. const bomb = new TextEncoder().encode("a".repeat(8 * 1024)); const archive = buildZip([{ path: "bomb.txt", bytes: bomb, method: 8 }], "paperclip-demo"); await expect( diff --git a/packages/shared/src/portability-zip.ts b/packages/shared/src/portability-zip.ts index ea531512ed..f4b77e7e5d 100644 --- a/packages/shared/src/portability-zip.ts +++ b/packages/shared/src/portability-zip.ts @@ -20,10 +20,13 @@ const strictTextDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: tru // malicious highly-compressible package cannot exhaust server memory: a // per-entry ceiling (passed to zlib as maxOutputLength, so it fails before // over-allocating) plus an aggregate ceiling across all entries. Both sit far -// above any real company package (inline JSON was historically capped at 64MB) -// yet far below what a bomb would need. -export const MAX_ZIP_ENTRY_DECOMPRESSED_BYTES = 256 * 1024 * 1024; -export const MAX_ZIP_TOTAL_DECOMPRESSED_BYTES = 512 * 1024 * 1024; +// above any real company package yet far below what a bomb would need. These +// are only fallback defaults: the server import route passes explicit limits +// scaled from its configured upload cap. The per-entry ceiling stays at 512 MB +// even for large caps because a text or base64 entry beyond that cannot +// materialize as a JS string (V8's string length limit) regardless. +export const MAX_ZIP_ENTRY_DECOMPRESSED_BYTES = 512 * 1024 * 1024; +export const MAX_ZIP_TOTAL_DECOMPRESSED_BYTES = 1024 * 1024 * 1024; export const binaryContentTypeByExtension: Record = { ".gif": "image/gif", diff --git a/server/src/__tests__/body-limits.test.ts b/server/src/__tests__/body-limits.test.ts index 8164e32bd8..2749edcf7f 100644 --- a/server/src/__tests__/body-limits.test.ts +++ b/server/src/__tests__/body-limits.test.ts @@ -1,12 +1,18 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_JSON_BODY_LIMIT, PORTABLE_JSON_BODY_LIMIT, PORTABLE_JSON_BODY_LIMIT_BYTES, + PORTABLE_ZIP_UPLOAD_LIMIT_BYTES, } from "../http/body-limits.js"; describe("HTTP body limits", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + it("keeps the global JSON parser at the established ceiling", () => { expect(DEFAULT_JSON_BODY_LIMIT).toBe("10mb"); }); @@ -16,4 +22,34 @@ describe("HTTP body limits", () => { expect(PORTABLE_JSON_BODY_LIMIT_BYTES).toBe(64 * 1024 * 1024); expect(PORTABLE_JSON_BODY_LIMIT_BYTES).toBeGreaterThan(10 * 1024 * 1024); }); + + it("accepts large real-world compressed company packages by default", () => { + expect(PORTABLE_ZIP_UPLOAD_LIMIT_BYTES).toBe(1024 * 1024 * 1024); + }); + + it("lets operators override the zip upload cap via PAPERCLIP_IMPORT_ZIP_MAX_BYTES", async () => { + vi.stubEnv("PAPERCLIP_IMPORT_ZIP_MAX_BYTES", String(2 * 1024 * 1024 * 1024)); + vi.resetModules(); + const reloaded = await import("../http/body-limits.js"); + expect(reloaded.PORTABLE_ZIP_UPLOAD_LIMIT_BYTES).toBe(2 * 1024 * 1024 * 1024); + }); + + it("falls back to the default when the env override is not a usable number", async () => { + for (const raw of ["unlimited", "0", "-5", "0.5"]) { + vi.stubEnv("PAPERCLIP_IMPORT_ZIP_MAX_BYTES", raw); + vi.resetModules(); + const reloaded = await import("../http/body-limits.js"); + expect(reloaded.PORTABLE_ZIP_UPLOAD_LIMIT_BYTES, `override ${JSON.stringify(raw)}`).toBe( + 1024 * 1024 * 1024, + ); + } + }); + + it("clamps an oversized override so derived guard math cannot overflow", async () => { + vi.stubEnv("PAPERCLIP_IMPORT_ZIP_MAX_BYTES", String(Number.MAX_VALUE)); + vi.resetModules(); + const reloaded = await import("../http/body-limits.js"); + expect(reloaded.PORTABLE_ZIP_UPLOAD_LIMIT_BYTES).toBe(64 * 1024 * 1024 * 1024); + expect(Number.isSafeInteger(reloaded.PORTABLE_ZIP_UPLOAD_LIMIT_BYTES * 4)).toBe(true); + }); }); diff --git a/server/src/http/body-limits.ts b/server/src/http/body-limits.ts index 29c92b2baf..6601dadb77 100644 --- a/server/src/http/body-limits.ts +++ b/server/src/http/body-limits.ts @@ -6,4 +6,18 @@ export const PORTABLE_JSON_BODY_LIMIT_BYTES = 64 * 1024 * 1024; // application/zip) instead of an inflated inline JSON body. The compressed zip // is roughly a third of the inline size, but the limit is kept generous so a // large company package uploads in one request rather than truncating in transit. -export const PORTABLE_ZIP_UPLOAD_LIMIT_BYTES = 128 * 1024 * 1024; +// The whole upload is buffered in memory and unzipped in one pass, so this cap +// bounds peak per-import memory (roughly the compressed size plus the inflated +// package). Operators who need more (or less) headroom can override it with +// PAPERCLIP_IMPORT_ZIP_MAX_BYTES; the import route scales its decompression-bomb +// guards from whatever value is in effect. +// The override is clamped to [1 byte, 64 GiB]: a fractional value would floor +// to a zero-byte limit that rejects every upload, and an astronomically large +// one would overflow the derived 4x decompression guard. 64 GiB is far beyond +// what the in-memory import pipeline can serve anyway. +const MAX_ZIP_UPLOAD_LIMIT_OVERRIDE_BYTES = 64 * 1024 * 1024 * 1024; +const zipUploadLimitOverride = Math.floor(Number(process.env.PAPERCLIP_IMPORT_ZIP_MAX_BYTES)); +export const PORTABLE_ZIP_UPLOAD_LIMIT_BYTES = + Number.isFinite(zipUploadLimitOverride) && zipUploadLimitOverride >= 1 + ? Math.min(zipUploadLimitOverride, MAX_ZIP_UPLOAD_LIMIT_OVERRIDE_BYTES) + : 1024 * 1024 * 1024; diff --git a/server/src/routes/companies.ts b/server/src/routes/companies.ts index 19686def29..186e05db56 100644 --- a/server/src/routes/companies.ts +++ b/server/src/routes/companies.ts @@ -6,7 +6,11 @@ import { z } from "zod"; import type { Db } from "@paperclipai/db"; import { agents as agentsTable } from "@paperclipai/db"; import type { CompanyPortabilityImportResult } from "@paperclipai/shared"; -import { readZipArchive } from "@paperclipai/shared/portability-zip"; +import { + MAX_ZIP_ENTRY_DECOMPRESSED_BYTES, + MAX_ZIP_TOTAL_DECOMPRESSED_BYTES, + readZipArchive, +} from "@paperclipai/shared/portability-zip"; import { DEFAULT_FEEDBACK_DATA_SHARING_TERMS_VERSION, companyArtifactsQuerySchema, @@ -131,7 +135,9 @@ async function resolveImportPayload(req: Request, res: Response): Promise>; try { - archive = await readZipArchive(zipBytes); + // Scale the bomb guards from the configured upload cap so a legitimately + // compressible package under the cap never trips them (~4x covers dense + // text; the per-entry ceiling is bounded by V8's string limit regardless). + archive = await readZipArchive(zipBytes, { + maxEntryDecompressedBytes: MAX_ZIP_ENTRY_DECOMPRESSED_BYTES, + maxTotalDecompressedBytes: Math.max( + MAX_ZIP_TOTAL_DECOMPRESSED_BYTES, + PORTABLE_ZIP_UPLOAD_LIMIT_BYTES * 4, + ), + }); } catch (error) { throw badRequest(`Import package could not be read: ${errorMessage(error)}`); } diff --git a/ui/src/lib/import-preflight.ts b/ui/src/lib/import-preflight.ts index df8c143df3..fcc9e42bb7 100644 --- a/ui/src/lib/import-preflight.ts +++ b/ui/src/lib/import-preflight.ts @@ -2,7 +2,8 @@ import type { CompanyPortabilityFileEntry } from "@paperclipai/shared"; // Inline imports post the whole parsed package as one JSON body, so oversized // packages must be blocked before the request is built. Packages past this -// limit go through the CLI folder import today; a blob-store relay is planned. +// limit go through the zip upload path (a far higher, operator-configurable +// server cap); a blob-store relay is planned for arbitrarily large migrations. export const INLINE_IMPORT_MAX_BYTES = 56 * 1024 * 1024; export function isBlobStoreFilePath(filePath: string): boolean { diff --git a/ui/src/pages/CompanyImport.test.tsx b/ui/src/pages/CompanyImport.test.tsx index dd2fefcb97..310f16203c 100644 --- a/ui/src/pages/CompanyImport.test.tsx +++ b/ui/src/pages/CompanyImport.test.tsx @@ -420,7 +420,7 @@ describe("CompanyImport", () => { await flushReact(); expect(container.textContent).toContain("Preview failed: stream disconnected"); - expect(container.textContent).toContain("Retry, or use the CLI folder import"); + expect(container.textContent).toContain("Retry, or re-export the package without large attachments"); expect(mockPushToast).toHaveBeenCalledWith(expect.objectContaining({ tone: "error" })); // Changing the package supersedes the failed request: the error panel resets. diff --git a/ui/src/pages/CompanyImport.tsx b/ui/src/pages/CompanyImport.tsx index 3339867d6d..a227894a68 100644 --- a/ui/src/pages/CompanyImport.tsx +++ b/ui/src/pages/CompanyImport.tsx @@ -1779,7 +1779,7 @@ export function CompanyImport() { {previewMutation.error instanceof Error ? previewMutation.error.message : "the request did not complete."}{" "} - Retry, or use the CLI folder import for very large packages. + Retry, or re-export the package without large attachments to shrink it.

)}