From 53f09cb818b934beadc2a58897604c9ddca1d42e Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:00:53 -0500 Subject: [PATCH] fix: prevent duplicate task creation and recovery loops (#9648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source control plane people use to coordinate AI-agent companies > - Agents, routines, productivity review, and recovery services can all create or re-trigger work > - Repeated heartbeats or catch-up cycles can produce duplicate tasks or repeat recovery actions before prior work is visible > - The base duplicate-create and routine catch-up coalescing work has now landed on `master` via related PRs while this PR was being prepared > - This pull request carries the remaining hardening: bounded idempotency retention, recovery cooldown/throttle fixes, productivity-review query batching and ordering fixes, and regression coverage > - The benefit is fewer duplicate tasks, safer retries, and enough provenance to diagnose any future recurrence ## Linked Issues or Issue Description Agents can retry issue creation after ambiguous responses or independently recreate the same child title, while recovery and short-interval routine catch-up paths can repeat before prior work settles. This can produce visible duplicate tasks and makes the originating heartbeat difficult to identify. Related work: Refs #8356 for caller-supplied issue-create idempotency and Refs #9224 for plugin-scoped issue-create idempotency. Prior related PR: #6936. The base issue-create deduplication and routine catch-up coalescing pieces have since landed on `master` via #9650 and #9649; this PR remains as the follow-up hardening stack on top of those changes. ## What Changed - Add 7-day retention for issue-create idempotency claims with indexed, batch-limited cleanup so the claim table does not grow forever. - Preserve recovery cooldown intent after terminal recovery actions are closed, and throttle repeated source-scoped recovery work. - Batch productivity-review source-activity checks to avoid repeated per-source queries while keeping the no-action suppression behavior. - Order productivity-review no-action streak windows by review creation time, matching the window semantics even when completion timestamps are out of order. - Preserve generated issue IDs in route mocks used by backlog/assignment contract tests. - Document PR-gardening task deduplication expectations in the company skill. - Add focused regression tests for idempotency retention, liveness recovery cooldowns, and productivity-review batching/suppression/ordering behavior. ## Verification - `pnpm --filter @paperclipai/server typecheck` — passed on latest head. - `pnpm exec vitest run --project @paperclipai/server server/src/__tests__/productivity-review-service.test.ts server/src/__tests__/issue-create-deduplication-routes.test.ts --reporter verbose` — 2 files, 23 tests passed on latest head. - `pnpm --filter @paperclipai/adapter-utils build && pnpm exec vitest run --project @paperclipai/adapter-utils --reporter dot` — 30 files passed, 476 tests passed, 8 skipped. - `pnpm build` — passed on latest head. - `pnpm -r typecheck` — passed on the rebased head before the final productivity-review ordering commit; the latest touched server code is covered by the server typecheck above. - `pnpm check:token-gates` — passed. - `pnpm test:run` — progressed through server, UI, CLI, shared, skills-catalog, and DB sections, then exposed an adapter-utils compiled-test fingerprint mismatch before rebuilding adapter-utils; the adapter-utils project passed after rebuild, and the GitHub split PR checks passed on the pushed head. ## Risks - Caller-supplied idempotency replay is now bounded to 7 days; reusing an old key after retention can create new work, which matches retry-oriented idempotency semantics. - Recovery and productivity-review timing changes may suppress redundant follow-up work; focused tests cover the intended boundaries. - Advisory locking and idempotency cleanup rely on PostgreSQL-compatible transaction semantics already used by the production data layer. - The migration extends the private claim table indexes without rewriting existing issue rows. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex based on GPT-5, with reasoning, repository editing, shell/tool execution, and test execution. Exact model ID and context-window size are not exposed by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .agents/skills/pr-gardening/SKILL.md | 13 +++++ ...76_issue_create_idempotency_key_expiry.sql | 2 + packages/db/src/migrations/meta/_journal.json | 7 +++ .../schema/issue_create_idempotency_keys.ts | 4 ++ ...e-assigned-backlog-contract-routes.test.ts | 4 +- .../issue-create-deduplication-routes.test.ts | 44 ++++++++++++++- .../productivity-review-service.test.ts | 56 +++++++++++++++++++ server/src/services/issues.ts | 16 ++++++ server/src/services/productivity-review.ts | 2 +- 9 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 packages/db/src/migrations/0176_issue_create_idempotency_key_expiry.sql diff --git a/.agents/skills/pr-gardening/SKILL.md b/.agents/skills/pr-gardening/SKILL.md index 925ca6db55..5819c3d595 100644 --- a/.agents/skills/pr-gardening/SKILL.md +++ b/.agents/skills/pr-gardening/SKILL.md @@ -64,6 +64,19 @@ For every candidate, the script re-fetches the current head SHA and records: Verdicts are `ready`, `needs_gardening`, or `report_only` for drafts. Always rerun this stage after any wake or claim that a PR was fixed. Never trust issue comments as proof of readiness. +## Follow-up Create-PR Task Deduplication + +If gardening decides a branch needs a follow-up task to create a single pull request, deduplicate before creating anything. + +For each branch, process one branch at a time and do this serially: + +1. Search open Paperclip issues for the exact branch name with statuses `backlog`, `todo`, `in_progress`, `in_review`, and `blocked`. +2. Inspect matching issue titles, descriptions, and recent comments for an equivalent open "create PR from this branch" task for the same branch. +3. If an equivalent open task exists, reuse it: add a concise comment with the current PR/head/reason context and link it from the gardening issue or blocker list. Do not create another task. +4. Only if no equivalent open task exists, create exactly one follow-up task for that branch. + +Never fan out follow-up task creation in parallel. Do not issue concurrent `POST /api/companies/:companyId/issues` calls for create-PR tasks. After P1's issue-create idempotency support is available, every create-PR follow-up task creation must include `idempotencyKey: "pr-gardening:create-pr:{branch}"`, where `{branch}` is the exact branch name. + ## Stage C — Comment on Originating Issues Skip this stage in `--dry-run` mode and for `ready` or `report_only` entries. diff --git a/packages/db/src/migrations/0176_issue_create_idempotency_key_expiry.sql b/packages/db/src/migrations/0176_issue_create_idempotency_key_expiry.sql new file mode 100644 index 0000000000..e75cf2805f --- /dev/null +++ b/packages/db/src/migrations/0176_issue_create_idempotency_key_expiry.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS "issue_create_idempotency_keys_company_created_at_idx" + ON "issue_create_idempotency_keys" USING btree ("company_id", "created_at"); diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 0e0c76170c..2e32c8fb8a 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -1219,6 +1219,13 @@ "when": 1784210755027, "tag": "0175_nested_skill_folders", "breakpoints": true + }, + { + "idx": 176, + "version": "7", + "when": 1784211956161, + "tag": "0176_issue_create_idempotency_key_expiry", + "breakpoints": true } ] } diff --git a/packages/db/src/schema/issue_create_idempotency_keys.ts b/packages/db/src/schema/issue_create_idempotency_keys.ts index c428d9943d..8a4f918e1f 100644 --- a/packages/db/src/schema/issue_create_idempotency_keys.ts +++ b/packages/db/src/schema/issue_create_idempotency_keys.ts @@ -17,5 +17,9 @@ export const issueCreateIdempotencyKeys = pgTable( table.idempotencyKey, ), issueIdx: index("issue_create_idempotency_keys_issue_idx").on(table.issueId), + companyCreatedAtIdx: index("issue_create_idempotency_keys_company_created_at_idx").on( + table.companyId, + table.createdAt, + ), }), ); diff --git a/server/src/__tests__/issue-assigned-backlog-contract-routes.test.ts b/server/src/__tests__/issue-assigned-backlog-contract-routes.test.ts index 52e8224fc2..ac80ef32d5 100644 --- a/server/src/__tests__/issue-assigned-backlog-contract-routes.test.ts +++ b/server/src/__tests__/issue-assigned-backlog-contract-routes.test.ts @@ -178,7 +178,7 @@ describe("assigned backlog creation contract", () => { })); mockIssueService.create.mockImplementation(async (_companyId: string, data: Record) => makeIssue({ - id: "issue-1", + id: String(data.id), title: String(data.title), status: String(data.status), assigneeAgentId: data.assigneeAgentId as string | null | undefined, @@ -326,7 +326,7 @@ describe("assigned backlog creation contract", () => { expect.anything(), expect.objectContaining({ action: "issue.created", - entityId: "issue-1", + entityId: expect.any(String), details: expect.objectContaining({ status: "backlog", statusDefaulted: false, diff --git a/server/src/__tests__/issue-create-deduplication-routes.test.ts b/server/src/__tests__/issue-create-deduplication-routes.test.ts index 3958a6dd28..9a7701cba5 100644 --- a/server/src/__tests__/issue-create-deduplication-routes.test.ts +++ b/server/src/__tests__/issue-create-deduplication-routes.test.ts @@ -19,7 +19,10 @@ import { import { actorMiddleware } from "../middleware/auth.js"; import { errorHandler } from "../middleware/index.js"; import { issueRoutes } from "../routes/issues.js"; -import { issueService } from "../services/issues.js"; +import { + ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_DAYS, + issueService, +} from "../services/issues.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -112,6 +115,45 @@ describeEmbeddedPostgres("issue create deduplication routes", () => { expect(await db.select().from(issueCreateIdempotencyKeys)).toHaveLength(1); }); + it("expires old idempotency keys before replay lookup", async () => { + const companyId = await seedCompany(); + const parent = await seedParent(companyId); + const app = createApp(); + const oldIssueId = randomUUID(); + const idempotencyKey = "run-1:expired-retry"; + const expiredCreatedAt = new Date( + Date.now() - (ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_DAYS + 1) * 24 * 60 * 60 * 1000, + ); + await db.insert(issues).values({ + id: oldIssueId, + companyId, + parentId: parent.id, + title: "Expired retry target", + status: "todo", + priority: "medium", + }); + await db.insert(issueCreateIdempotencyKeys).values({ + companyId, + idempotencyKey, + issueId: oldIssueId, + createdAt: expiredCreatedAt, + }); + + const recreated = await request(app) + .post(`/api/companies/${companyId}/issues`) + .send({ parentId: parent.id, title: "Expired retry creates new work", idempotencyKey }) + .expect(201); + + const rows = await db.select().from(issueCreateIdempotencyKeys); + expect(recreated.body.id).not.toBe(oldIssueId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + companyId, + idempotencyKey, + issueId: recreated.body.id, + }); + }); + it("returns a recent open sibling whose normalized title matches", async () => { const companyId = await seedCompany(); const parent = await seedParent(companyId); diff --git a/server/src/__tests__/productivity-review-service.test.ts b/server/src/__tests__/productivity-review-service.test.ts index 9dbf784923..12b3bdd18c 100644 --- a/server/src/__tests__/productivity-review-service.test.ts +++ b/server/src/__tests__/productivity-review-service.test.ts @@ -385,6 +385,62 @@ describeEmbeddedPostgres("productivity review service", () => { expect(await listProductivityReviews(seeded.companyId)).toHaveLength(4); }); + it("uses review creation order for no-action streak windows", async () => { + const now = new Date("2026-04-28T12:00:00.000Z"); + const seeded = await seedAssignedIssue(); + await insertRuns({ + companyId: seeded.companyId, + agentId: seeded.coderId, + issueId: seeded.issueId, + count: DEFAULT_PRODUCTIVITY_REVIEW_NO_COMMENT_STREAK_RUNS, + now, + }); + const reviewWindows = [ + { hoursAgo: 96, updatedAt: new Date(now.getTime() - 95 * 60 * 60 * 1000) }, + { hoursAgo: 72, updatedAt: new Date(now.getTime() - 7 * 60 * 60 * 1000) }, + { hoursAgo: 48, updatedAt: new Date(now.getTime() - 47 * 60 * 60 * 1000) }, + ].map((window, index) => { + const createdAt = new Date(now.getTime() - window.hoursAgo * 60 * 60 * 1000); + return { + id: randomUUID(), + companyId: seeded.companyId, + title: `Productivity review ordered window ${index + 1}`, + status: "done" as const, + priority: "high" as const, + originKind: PRODUCTIVITY_REVIEW_ORIGIN_KIND, + originId: seeded.issueId, + originFingerprint: `productivity-review:${seeded.issueId}`, + parentId: seeded.issueId, + issueNumber: index + 2, + identifier: `${seeded.issuePrefix}-${index + 2}`, + createdAt, + updatedAt: window.updatedAt, + }; + }); + const middleReviewCreatedAt = reviewWindows[1]!.createdAt; + await db.insert(issues).values(reviewWindows); + await db.insert(activityLog).values({ + companyId: seeded.companyId, + actorType: "agent", + actorId: seeded.coderId, + agentId: seeded.coderId, + action: "issue.updated", + entityType: "issue", + entityId: seeded.issueId, + createdAt: new Date(middleReviewCreatedAt.getTime() + 60_000), + }); + + const result = await productivityReviewService(db).reconcileProductivityReviews({ + now, + companyId: seeded.companyId, + thresholds: { maxConsecutiveNoActionReviews: 1 }, + }); + + expect(result.created).toBe(0); + expect(result.noActionSuppressed).toBe(1); + expect(await listProductivityReviews(seeded.companyId)).toHaveLength(3); + }); + it("does not count cancelled productivity reviews toward the creation cap", async () => { const now = new Date("2026-04-28T12:00:00.000Z"); const seeded = await seedAssignedIssue(); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 6184683d01..3130570c1c 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -134,6 +134,9 @@ const ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_LOG_BYTES = 2_000_000; const ISSUE_COMMENT_RUN_LOG_DERIVATION_CHUNK_BYTES = 256_000; const ISSUE_COMMENT_RUN_LOG_DERIVATION_END_SLACK_MS = 60_000; const ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_PARALLEL_READS = 8; +export const ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_DAYS = 7; +const ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_MS = ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_DAYS * 24 * 60 * 60 * 1000; +const ISSUE_CREATE_IDEMPOTENCY_KEY_CLEANUP_BATCH_SIZE = 500; const DELETED_ISSUE_COMMENT_BODY = ""; const ISSUE_WAKE_DIAGNOSTICS_ACTIVITY_ACTIONS = ["issue.tree_hold_wakeup_deferred"] as const; @@ -6152,6 +6155,19 @@ export function issueService(db: Db) { let existingIssue: typeof issues.$inferSelect | undefined; let deduplicationReason: "idempotency_key" | "recent_open_title" | null = null; if (idempotencyKey) { + const idempotencyKeyRetentionCutoff = new Date(Date.now() - ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_MS); + await tx.execute(sql` + delete from ${issueCreateIdempotencyKeys} + where ${issueCreateIdempotencyKeys.id} in ( + select ${issueCreateIdempotencyKeys.id} + from ${issueCreateIdempotencyKeys} + where ${issueCreateIdempotencyKeys.companyId} = ${companyId} + and ${issueCreateIdempotencyKeys.createdAt} < ${idempotencyKeyRetentionCutoff.toISOString()}::timestamptz + order by ${issueCreateIdempotencyKeys.createdAt} asc, ${issueCreateIdempotencyKeys.id} asc + limit ${ISSUE_CREATE_IDEMPOTENCY_KEY_CLEANUP_BATCH_SIZE} + ) + `); + [existingIssue] = await tx .select() .from(issueCreateIdempotencyKeys) diff --git a/server/src/services/productivity-review.ts b/server/src/services/productivity-review.ts index 8301ea6e08..e694f2e222 100644 --- a/server/src/services/productivity-review.ts +++ b/server/src/services/productivity-review.ts @@ -333,7 +333,7 @@ export function productivityReviewService(db: Db, deps?: { enqueueWakeup?: Enque visibleIssueCondition(), ), ) - .orderBy(desc(issues.updatedAt), desc(issues.id)) + .orderBy(desc(issues.createdAt), desc(issues.id)) .limit(thresholds.maxConsecutiveNoActionReviews); const earliestReviewCreatedAt = completedReviews.at(-1)?.createdAt;