diff --git a/packages/shared/src/telemetry/events.ts b/packages/shared/src/telemetry/events.ts index 15315719b5..88f50aae0a 100644 --- a/packages/shared/src/telemetry/events.ts +++ b/packages/shared/src/telemetry/events.ts @@ -145,6 +145,7 @@ export function trackInteractionResolved( questionCount?: number; answeredQuestionCount?: number; createdTaskCount?: number; + skippedTaskCount?: number; }, ): void { client.track("interaction.resolved", { @@ -161,5 +162,6 @@ export function trackInteractionResolved( ...(dims.questionCount === undefined ? {} : { question_count: dims.questionCount }), ...(dims.answeredQuestionCount === undefined ? {} : { answered_question_count: dims.answeredQuestionCount }), ...(dims.createdTaskCount === undefined ? {} : { created_task_count: dims.createdTaskCount }), + ...(dims.skippedTaskCount === undefined ? {} : { skipped_task_count: dims.skippedTaskCount }), }); } diff --git a/server/src/__tests__/issue-thread-interactions-telemetry.test.ts b/server/src/__tests__/issue-thread-interactions-telemetry.test.ts new file mode 100644 index 0000000000..0fe45ac507 --- /dev/null +++ b/server/src/__tests__/issue-thread-interactions-telemetry.test.ts @@ -0,0 +1,505 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + agents, + companies, + createDb, + documentRevisions, + documents, + goals, + issueComments, + issueDocuments, + issueThreadInteractions, + issues, +} from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; + +const telemetryMocks = vi.hoisted(() => ({ + track: vi.fn(), + getTelemetryClient: vi.fn(), +})); + +vi.mock("../telemetry.js", () => ({ + getTelemetryClient: telemetryMocks.getTelemetryClient, +})); + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +describeEmbeddedPostgres("issueThreadInteractionService telemetry", () => { + let db!: ReturnType; + let interactionsSvc!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-interaction-telemetry-"); + db = createDb(tempDb.connectionString); + interactionsSvc = issueThreadInteractionService(db); + }, 20_000); + + beforeEach(() => { + telemetryMocks.track.mockClear(); + telemetryMocks.getTelemetryClient.mockReturnValue({ + track: telemetryMocks.track, + hashPrivateRef: vi.fn((value: string) => `hashed:${value}`), + }); + }); + + afterEach(async () => { + await db.delete(issueThreadInteractions); + await db.delete(issueComments); + await db.delete(issueDocuments); + await db.delete(documentRevisions); + await db.delete(documents); + await db.delete(issues); + await db.delete(goals); + await db.delete(agents); + await db.delete(companies); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + async function seedIssue(title = "Interaction telemetry") { + const companyId = randomUUID(); + const goalId = randomUUID(); + const issueId = randomUUID(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(goals).values({ + id: goalId, + companyId, + title, + level: "task", + status: "active", + }); + await db.insert(issues).values({ + id: issueId, + companyId, + goalId, + title, + status: "in_progress", + priority: "medium", + }); + + return { companyId, goalId, issueId }; + } + + async function seedAgent(companyId: string, role: string) { + const agentId = randomUUID(); + await db.insert(agents).values({ + id: agentId, + companyId, + name: role, + role, + status: "active", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + return agentId; + } + + function expectNoRawInteractionIds(dimensions: Record) { + expect(dimensions).not.toHaveProperty("interaction_id"); + expect(dimensions).not.toHaveProperty("created_by_agent_id"); + expect(dimensions).not.toHaveProperty("source_run_id"); + } + + function lastInteractionResolvedDimensions() { + expect(telemetryMocks.track).toHaveBeenCalledWith("interaction.resolved", expect.any(Object)); + const calls = telemetryMocks.track.mock.calls.filter((call) => call[0] === "interaction.resolved"); + return calls.at(-1)?.[1] as Record; + } + + it("emits accepted suggested-task telemetry with created and skipped task counts", async () => { + const { companyId, goalId, issueId } = await seedIssue("Accept suggested tasks telemetry"); + + const created = await interactionsSvc.create({ + id: issueId, + companyId, + }, { + kind: "suggest_tasks", + continuationPolicy: "wake_assignee", + payload: { + version: 1, + tasks: [ + { + clientKey: "root", + title: "Create the root follow-up", + }, + { + clientKey: "child", + parentClientKey: "root", + title: "Create the nested follow-up", + }, + { + clientKey: "sibling", + title: "Create the sibling follow-up", + }, + ], + }, + }, { + userId: "local-board", + }); + + await interactionsSvc.acceptInteraction({ + id: issueId, + companyId, + goalId, + projectId: null, + }, created.id, { + selectedClientKeys: ["root"], + }, { + userId: "local-board", + }); + + const dimensions = lastInteractionResolvedDimensions(); + expect(dimensions).toMatchObject({ + interaction_kind: "suggest_tasks", + status: "accepted", + resolved_by_kind: "user", + resolution_reason: "accepted", + created_by_kind: "user", + continuation_policy: "wake_assignee", + target_type: "none", + created_task_count: 1, + skipped_task_count: 2, + }); + expectNoRawInteractionIds(dimensions); + }); + + it("emits accepted checkbox telemetry with raw role, target, and counts", async () => { + const { companyId, goalId, issueId } = await seedIssue("Accept checkbox telemetry"); + const creatorAgentId = await seedAgent(companyId, "Backend Engineer"); + + const created = await interactionsSvc.create({ + id: issueId, + companyId, + }, { + kind: "request_checkbox_confirmation", + continuationPolicy: "wake_assignee", + payload: { + version: 1, + prompt: "Which files should be deleted?", + options: [ + { id: "file-a", label: "a.txt" }, + { id: "file-b", label: "b.txt" }, + ], + target: { + type: "custom", + key: "cleanup-plan", + }, + }, + }, { + agentId: creatorAgentId, + }); + + await interactionsSvc.acceptInteraction({ + id: issueId, + companyId, + goalId, + projectId: null, + }, created.id, { + selectedOptionIds: ["file-b"], + }, { + userId: "local-board", + }); + + const dimensions = lastInteractionResolvedDimensions(); + expect(dimensions).toMatchObject({ + interaction_kind: "request_checkbox_confirmation", + status: "accepted", + resolved_by_kind: "user", + resolution_reason: "accepted", + created_by_kind: "agent", + creator_agent_role: "Backend Engineer", + continuation_policy: "wake_assignee", + target_type: "custom", + option_count: 2, + selected_option_count: 1, + }); + expectNoRawInteractionIds(dimensions); + }); + + it("emits rejected confirmation telemetry and omits creator_agent_role for user-created interactions", async () => { + const { companyId, issueId } = await seedIssue("Reject confirmation telemetry"); + const resolverAgentId = await seedAgent(companyId, "SecurityEngineer"); + + const created = await interactionsSvc.create({ + id: issueId, + companyId, + }, { + kind: "request_confirmation", + payload: { + version: 1, + prompt: "Approve this?", + }, + }, { + userId: "local-board", + }); + + await interactionsSvc.rejectInteraction({ + id: issueId, + companyId, + }, created.id, { + reason: "Needs edits before approval.", + }, { + agentId: resolverAgentId, + }); + + const dimensions = lastInteractionResolvedDimensions(); + expect(dimensions).toMatchObject({ + interaction_kind: "request_confirmation", + status: "rejected", + resolved_by_kind: "agent", + resolution_reason: "rejected", + created_by_kind: "user", + continuation_policy: "none", + target_type: "none", + }); + expect(dimensions).not.toHaveProperty("creator_agent_role"); + expect(dimensions).not.toHaveProperty("reason"); + expectNoRawInteractionIds(dimensions); + }); + + it("emits answered question telemetry with system resolver and raw creator role", async () => { + const { companyId, issueId } = await seedIssue("Answer question telemetry"); + const creatorAgentId = await seedAgent(companyId, "Wizard"); + + const created = await interactionsSvc.create({ + id: issueId, + companyId, + }, { + kind: "ask_user_questions", + continuationPolicy: "wake_assignee", + payload: { + version: 1, + questions: [ + { + id: "scope", + prompt: "Choose the scope", + selectionMode: "single", + required: true, + options: [{ id: "phase-1", label: "Phase 1" }], + }, + { + id: "extras", + prompt: "Optional extras", + selectionMode: "multi", + options: [{ id: "docs", label: "Docs" }], + }, + ], + }, + }, { + agentId: creatorAgentId, + }); + await db + .update(issueThreadInteractions) + .set({ continuationPolicy: "" }) + .where(eq(issueThreadInteractions.id, created.id)); + + await interactionsSvc.answerQuestions({ + id: issueId, + companyId, + }, created.id, { + answers: [{ questionId: "scope", optionIds: ["phase-1"] }], + summaryMarkdown: "Do not emit this free text.", + }, {}); + + const dimensions = lastInteractionResolvedDimensions(); + expect(dimensions).toMatchObject({ + interaction_kind: "ask_user_questions", + status: "answered", + resolved_by_kind: "system", + created_by_kind: "agent", + creator_agent_role: "Wizard", + target_type: "none", + question_count: 2, + answered_question_count: 1, + }); + expect(dimensions).not.toHaveProperty("continuation_policy"); + expect(dimensions).not.toHaveProperty("summaryMarkdown"); + expect(dimensions).not.toHaveProperty("answers"); + expectNoRawInteractionIds(dimensions); + }); + + it("emits expired question telemetry with zero answered question count", async () => { + const { companyId, issueId } = await seedIssue("Expired question telemetry"); + const commentId = randomUUID(); + + const created = await interactionsSvc.create({ + id: issueId, + companyId, + }, { + kind: "ask_user_questions", + continuationPolicy: "wake_assignee", + payload: { + version: 1, + questions: [ + { + id: "scope", + prompt: "Choose the scope", + selectionMode: "single", + required: true, + options: [{ id: "phase-1", label: "Phase 1" }], + }, + ], + }, + }, { + userId: "local-board", + }); + + const expired = await interactionsSvc.expireRequestConfirmationsSupersededByComment({ + id: issueId, + companyId, + }, { + id: commentId, + createdAt: new Date(new Date(created.createdAt).getTime() + 1_000), + authorUserId: "local-board", + }, { + userId: "local-board", + }); + + expect(expired).toHaveLength(1); + const dimensions = lastInteractionResolvedDimensions(); + expect(dimensions).toMatchObject({ + interaction_kind: "ask_user_questions", + status: "expired", + resolved_by_kind: "user", + resolution_reason: "superseded_by_comment", + created_by_kind: "user", + continuation_policy: "wake_assignee", + target_type: "none", + question_count: 1, + answered_question_count: 0, + }); + expectNoRawInteractionIds(dimensions); + }); + + it("emits expired stale-target telemetry without stale target identifiers", async () => { + const { companyId, issueId } = await seedIssue("Stale target telemetry"); + const documentId = randomUUID(); + const revisionId = randomUUID(); + + await db.insert(documents).values({ + id: documentId, + companyId, + title: "Plan", + format: "markdown", + latestBody: "v1", + latestRevisionId: revisionId, + latestRevisionNumber: 1, + }); + await db.insert(issueDocuments).values({ + companyId, + issueId, + documentId, + key: "plan", + }); + await db.insert(documentRevisions).values({ + id: revisionId, + companyId, + documentId, + revisionNumber: 1, + title: "Plan", + format: "markdown", + body: "v1", + }); + + await interactionsSvc.create({ + id: issueId, + companyId, + }, { + kind: "request_confirmation", + payload: { + version: 1, + prompt: "Approve the plan?", + target: { + type: "issue_document", + issueId, + documentId, + key: "plan", + revisionId, + revisionNumber: 1, + }, + }, + }, { + userId: "local-board", + }); + + const expired = await interactionsSvc.expireStaleRequestConfirmationsForIssueDocument({ + id: issueId, + companyId, + }, null, {}); + + expect(expired).toHaveLength(1); + const dimensions = lastInteractionResolvedDimensions(); + expect(dimensions).toMatchObject({ + interaction_kind: "request_confirmation", + status: "expired", + resolved_by_kind: "system", + resolution_reason: "stale_target", + created_by_kind: "user", + continuation_policy: "none", + target_type: "issue_document", + }); + expect(dimensions).not.toHaveProperty("staleTarget"); + expect(dimensions).not.toHaveProperty("revisionId"); + expectNoRawInteractionIds(dimensions); + }); + + it("emits superseded expiration telemetry without comment identifiers", async () => { + const { companyId, issueId } = await seedIssue("Superseded telemetry"); + const commentId = randomUUID(); + + const created = await interactionsSvc.create({ + id: issueId, + companyId, + }, { + kind: "request_confirmation", + payload: { + version: 1, + prompt: "Approve this plan?", + }, + }, { + userId: "local-board", + }); + + const expired = await interactionsSvc.expireRequestConfirmationsSupersededByComment({ + id: issueId, + companyId, + }, { + id: commentId, + createdAt: new Date(new Date(created.createdAt).getTime() + 1_000), + authorUserId: "local-board", + }, { + userId: "local-board", + }); + + expect(expired).toHaveLength(1); + const dimensions = lastInteractionResolvedDimensions(); + expect(dimensions).toMatchObject({ + interaction_kind: "request_confirmation", + status: "expired", + resolved_by_kind: "user", + resolution_reason: "superseded_by_comment", + created_by_kind: "user", + target_type: "none", + }); + expect(dimensions).not.toHaveProperty("commentId"); + expectNoRawInteractionIds(dimensions); + }); +}); diff --git a/server/src/__tests__/shared-telemetry-events.test.ts b/server/src/__tests__/shared-telemetry-events.test.ts index 8150910ae9..19341fdd20 100644 --- a/server/src/__tests__/shared-telemetry-events.test.ts +++ b/server/src/__tests__/shared-telemetry-events.test.ts @@ -110,6 +110,7 @@ describe("shared telemetry agent events", () => { targetType: "issue_document", optionCount: 2, selectedOptionCount: 1, + skippedTaskCount: 3, }); expect(client.track).toHaveBeenCalledWith("interaction.resolved", { @@ -123,6 +124,7 @@ describe("shared telemetry agent events", () => { target_type: "issue_document", option_count: 2, selected_option_count: 1, + skipped_task_count: 3, }); }); }); diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index ef05fa4bf8..62550db0dd 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -2,6 +2,7 @@ import { isDeepStrictEqual } from "node:util"; import { and, asc, eq, inArray, isNotNull } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { + agents, documents, heartbeatRuns, issueComments, @@ -9,6 +10,7 @@ import { issueThreadInteractions, issues, } from "@paperclipai/db"; +import { trackInteractionResolved } from "@paperclipai/shared/telemetry"; import type { AcceptIssueThreadInteraction, AskUserQuestionsAnswer, @@ -39,6 +41,7 @@ import { suggestTasksResultSchema, } from "@paperclipai/shared"; import { conflict, notFound, unprocessable } from "../errors.js"; +import { getTelemetryClient } from "../telemetry.js"; import { issueService, runWorkspaceIsFinalized } from "./issues.js"; type InteractionActor = { @@ -247,6 +250,151 @@ function buildSupersededByCommentResult(row: IssueThreadInteractionRow, commentI } as const; } +function resolveActorKind(interaction: Pick) { + if (interaction.resolvedByAgentId) return "agent"; + if (interaction.resolvedByUserId) return "user"; + return "system"; +} + +function resolveCreatorKind(interaction: Pick) { + if (interaction.createdByAgentId) return "agent"; + if (interaction.createdByUserId) return "user"; + return undefined; +} + +function deriveTargetType(interaction: IssueThreadInteraction) { + if (interaction.kind !== "request_confirmation" && interaction.kind !== "request_checkbox_confirmation") { + return "none"; + } + return interaction.payload.target?.type ?? "none"; +} + +function deriveResolutionReason(interaction: IssueThreadInteraction) { + switch (interaction.status) { + case "accepted": + return "accepted"; + case "rejected": + return "rejected"; + case "cancelled": + return "cancelled"; + case "expired": { + if (interaction.kind === "ask_user_questions") { + return interaction.result?.expirationReason ?? "expired"; + } + if (interaction.kind === "request_confirmation" || interaction.kind === "request_checkbox_confirmation") { + return interaction.result?.outcome ?? "expired"; + } + return "expired"; + } + default: + return undefined; + } +} + +function nonNegativeInteger(value: number) { + if (!Number.isFinite(value)) return 0; + return Math.max(0, Math.trunc(value)); +} + +function buildInteractionResolvedCounts(interaction: IssueThreadInteraction, args?: { + createdTaskCount?: number; +}) { + switch (interaction.kind) { + case "suggest_tasks": + return { + createdTaskCount: nonNegativeInteger(args?.createdTaskCount ?? 0), + skippedTaskCount: nonNegativeInteger(interaction.result?.skippedClientKeys?.length ?? 0), + }; + case "request_checkbox_confirmation": + return { + optionCount: nonNegativeInteger(interaction.payload.options.length), + selectedOptionCount: nonNegativeInteger(interaction.result?.selectedOptionIds?.length ?? 0), + }; + case "ask_user_questions": + return { + questionCount: nonNegativeInteger(interaction.payload.questions.length), + answeredQuestionCount: nonNegativeInteger(interaction.result?.answers?.length ?? 0), + }; + default: + return {}; + } +} + +async function fetchCreatorAgentRoleById( + db: Pick, + interactions: readonly IssueThreadInteraction[], +) { + const creatorAgentIds = [...new Set(interactions + .map((interaction) => interaction.createdByAgentId) + .filter((value): value is string => Boolean(value)))]; + if (creatorAgentIds.length === 0) return new Map(); + + const rows = await db + .select({ + id: agents.id, + role: agents.role, + }) + .from(agents) + .where(inArray(agents.id, creatorAgentIds)); + + return new Map(rows.map((row) => [row.id, row.role] as const)); +} + +async function emitInteractionResolvedTelemetry( + db: Pick, + interaction: IssueThreadInteraction, + args?: { createdTaskCount?: number; creatorRoleByAgentId?: ReadonlyMap }, +) { + const telemetryClient = getTelemetryClient(); + if (!telemetryClient) return; + + try { + let roleByAgentId = args?.creatorRoleByAgentId ?? new Map(); + if (!args?.creatorRoleByAgentId) { + try { + roleByAgentId = await fetchCreatorAgentRoleById(db, [interaction]); + } catch (error) { + console.error("[paperclip] Failed to load interaction.resolved creator role", error); + } + } + const creatorAgentRole = interaction.createdByAgentId + ? roleByAgentId.get(interaction.createdByAgentId) ?? undefined + : undefined; + + trackInteractionResolved(telemetryClient, { + interactionKind: interaction.kind, + status: interaction.status, + resolvedByKind: resolveActorKind(interaction), + resolutionReason: deriveResolutionReason(interaction), + createdByKind: resolveCreatorKind(interaction), + creatorAgentRole, + continuationPolicy: interaction.continuationPolicy, + targetType: deriveTargetType(interaction), + ...buildInteractionResolvedCounts(interaction, { + createdTaskCount: args?.createdTaskCount, + }), + }); + } catch (error) { + console.error("[paperclip] Failed to emit interaction.resolved telemetry", error); + } +} + +async function emitResolvedInteractionsTelemetry( + db: Pick, + interactions: readonly IssueThreadInteraction[], +) { + if (interactions.length === 0 || !getTelemetryClient()) return; + let roleByAgentId = new Map(); + try { + roleByAgentId = await fetchCreatorAgentRoleById(db, interactions); + } catch (error) { + console.error("[paperclip] Failed to load interaction.resolved creator roles", error); + } + await Promise.all(interactions.map((interaction) => + emitInteractionResolvedTelemetry(db, interaction, { creatorRoleByAgentId: roleByAgentId }) + )); +} + function isCommentAtOrAfterInteraction(args: { commentCreatedAt: Date | string; interactionCreatedAt: Date | string; @@ -557,7 +705,9 @@ async function expireStaleRequestConfirmationTarget(db: Db | any, args: { throw conflict("Interaction has already been resolved"); } await touchIssue(db, args.row.issueId); - return hydrateInteraction(updated); + const expired = hydrateInteraction(updated); + await emitInteractionResolvedTelemetry(db, expired); + return expired; } export function issueThreadInteractionService(db: Db) { @@ -654,7 +804,7 @@ export function issueThreadInteractionService(db: Db) { : undefined; const now = new Date(); - return db.transaction(async (tx) => { + const result = await db.transaction(async (tx) => { const [updated] = await tx .update(issueThreadInteractions) .set({ @@ -727,6 +877,8 @@ export function issueThreadInteractionService(db: Db) { continuationIssue, }; }); + await emitInteractionResolvedTelemetry(db, result.interaction); + return result; } async function rejectRequestConfirmation(args: { @@ -774,7 +926,9 @@ export function issueThreadInteractionService(db: Db) { throw conflict("Interaction has already been resolved"); } await touchIssue(db, args.issue.id); - return hydrateInteraction(updated); + const rejected = hydrateInteraction(updated); + await emitInteractionResolvedTelemetry(db, rejected); + return rejected; } return { @@ -1086,8 +1240,12 @@ export function issueThreadInteractionService(db: Db) { current.updatedAt = updated.updatedAt; }); + const accepted = hydrateInteraction(current); + await emitInteractionResolvedTelemetry(db, accepted, { + createdTaskCount: createdWakeTargets.length, + }); return { - interaction: hydrateInteraction(current), + interaction: accepted, createdIssues: createdWakeTargets, }; }, @@ -1157,7 +1315,9 @@ export function issueThreadInteractionService(db: Db) { } await touchIssue(db, issue.id); - return hydrateInteraction(updated); + const rejected = hydrateInteraction(updated); + await emitInteractionResolvedTelemetry(db, rejected); + return rejected; }, expireRequestConfirmationsSupersededByComment: async ( @@ -1214,6 +1374,7 @@ export function issueThreadInteractionService(db: Db) { if (expired.length > 0) { await touchIssue(db, issue.id); + await emitResolvedInteractionsTelemetry(db, expired); } return expired; }, @@ -1332,6 +1493,7 @@ export function issueThreadInteractionService(db: Db) { if (expired.length > 0) { await touchIssue(db, issue.id); + await emitResolvedInteractionsTelemetry(db, expired); } return expired; }, @@ -1407,6 +1569,7 @@ export function issueThreadInteractionService(db: Db) { if (expired.length > 0) { await touchIssue(db, issue.id); + await emitResolvedInteractionsTelemetry(db, expired); } return expired; }, @@ -1465,7 +1628,9 @@ export function issueThreadInteractionService(db: Db) { } await touchIssue(db, issue.id); - return hydrateInteraction(updated); + const answered = hydrateInteraction(updated); + await emitInteractionResolvedTelemetry(db, answered); + return answered; }, cancelQuestions: async ( @@ -1520,7 +1685,9 @@ export function issueThreadInteractionService(db: Db) { } await touchIssue(db, issue.id); - return hydrateInteraction(updated); + const cancelled = hydrateInteraction(updated); + await emitInteractionResolvedTelemetry(db, cancelled); + return cancelled; }, }; }